Merge pull request FederatedAI#808 from FederatedAI/develop

release v1.1.1

.gitmodules

@@ -1,8 +1,8 @@
 [submodule "fateboard"]
 	path = fateboard
 	url = https://github.com/FederatedAI/FATE-Board.git
-	branch = v1.1
+	branch = v1.1.1
 [submodule "eggroll"]
 	path = eggroll
 	url = https://github.com/WeBankFinTech/eggroll.git
-	branch = v1.1.1
+	branch = 1.1.2

README.md

@@ -25,23 +25,42 @@ FATE already supports a number of federated learning algorithms, including verti
 
 
 ## Install
-FATE can be installed on Linux or Mac. Now, FATE can support standalone and cluster deployments.
 
+FATE can be installed on Linux or Mac. Now, FATE can support：
+
+* Native installation: standalone and cluster deployments;
+
+* KubeFATE installation:
+
+	- Multipal parties deployment by docker-compose, which for devolopment and test purpose;
+
+	- Cluster (multi-node) deployment by Kubernetes
+
+### Native installation: 
 Software environment :jdk1.8+、Python3.6、python virtualenv、mysql5.6+、redis-5.0.2
 
-#### Standalone
+##### Standalone
 FATE provides Standalone runtime architecture for developers. It can help developers quickly test FATE. Standalone support two types of deployment: Docker version and Manual version. Please refer to Standalone deployment guide: [standalone-deploy](./standalone-deploy/)
 
-#### Cluster
+##### Cluster
 FATE also provides a distributed runtime architecture for Big Data scenario. Migration from standalone to cluster requires configuration change only. No algorithm change is needed. 
 
 To deploy FATE on a cluster, please refer to cluster deployment guide: [cluster-deploy](./cluster-deploy).
 
-#### Get source
+##### Get source
 ```shell
 git clone --recursive [email protected]:FederatedAI/FATE.git
 ```
 
+### KubeFATE installation:
+With KubeFATE, FATE can be deployed with docker-compose or Kubernetes:
+
+* For development or test purpose, we recommend using docker-compose, which only Docker enviroment required. For more detail, please refer to [Deployment by Docker Compose](https://github.com/FederatedAI/KubeFATE/tree/master/docker-deploy)
+
+* For product or large scale deployment, we recommend using Kubernetes cluster as an underlaying infrastructure to manage FATE system. For more detail, please refer to: [Deployment on Kubernetes](https://github.com/FederatedAI/KubeFATE/blob/master/k8s-deploy)
+
+Verfication steps, please refer to related instructions in [KubeFATE](https://github.com/FederatedAI/KubeFATE).
+
 ## Running Tests
 
 A script to run all the unittests has been provided in ./federatedml/test folder. 

RELEASE.md

@@ -1,3 +1,17 @@
+# Release 1.1.1
+## Major Features and Improvements
+* Add cluster deployment support based on ubuntu operating system。
+* Add union component which support data merging. 
+* Support indicating partial columns in Onehot Encoder
+* Support intermediate data cleanup after the task ends
+
+
+## Bug Fixes
+* Fix a bug of secureboost' early stop 
+* Fix a bug in download api
+* Fix bugs of spark-backend
+
+
 # Release 1.1
 ## Major Features and Improvements
 >FederatedML

arch/api/table/eggroll_util.py

@@ -19,7 +19,7 @@
 _EGGROLL_CLIENT = "_eggroll_client"
 
 
-def build_eggroll_session(work_mode:WorkMode, job_id=None, server_conf_path="eggroll/conf/server_conf.json"):
+def build_eggroll_session(work_mode: WorkMode, job_id=None, server_conf_path="eggroll/conf/server_conf.json"):
     if work_mode.is_standalone():
         from eggroll.api.core import EggrollSession
         import uuid
@@ -42,7 +42,8 @@ def build_eggroll_runtime(work_mode: WorkMode, eggroll_session: EggrollSession):
         from eggroll.api.cluster.eggroll import eggroll_init, _EggRoll
         if _EggRoll.instance is None:
             return eggroll_init(eggroll_session)
-    raise ValueError(f"work_mode: {work_mode} not supported!")
+    else:
+        raise ValueError(f"work_mode: {work_mode} not supported!")
 
 
 def broadcast_eggroll_session(sc, work_mode, eggroll_session):
@@ -60,4 +61,13 @@ def maybe_create_eggroll_client():
     import pickle
     from pyspark.taskcontext import TaskContext
     mode, eggroll_session = pickle.loads(bytes.fromhex(TaskContext.get().getLocalProperty(_EGGROLL_CLIENT)))
-    build_eggroll_runtime(WorkMode(mode), eggroll_session)
+    if mode == 1:
+        from eggroll.api.cluster.eggroll import _EggRoll
+        if _EggRoll.instance is None:
+            from eggroll.api import ComputingEngine
+            from eggroll.api.cluster.eggroll import _EggRoll
+            eggroll_runtime = _EggRoll(eggroll_session=eggroll_session)
+            eggroll_session.set_runtime(ComputingEngine.EGGROLL_DTABLE, eggroll_runtime)
+    else:
+        from eggroll.api.standalone.eggroll import Standalone
+        Standalone(eggroll_session)

arch/api/table/pyspark/rdd_func.py

@@ -28,7 +28,7 @@ def _save_as_func(rdd: RDD, name, namespace, partition, persistent):
 
     def _func(_, it):
         eggroll_util.maybe_create_eggroll_client()
-        dup.put_all(it)
+        dup.put_all(list(it))
         return 1,
 
     rdd.mapPartitionsWithIndex(_func, preservesPartitioning=False).collect()

arch/api/table/pyspark/session_impl.py

@@ -32,10 +32,10 @@ class FateSessionImpl(FateSession):
     def __init__(self, eggroll_session, work_mode: WorkMode):
         self._session_id = eggroll_session.get_session_id()
         self._eggroll_session = eggroll_session
-        self._eggroll = eggroll_util.build_eggroll_runtime(work_mode, eggroll_session)
 
         self._sc = self._build_spark_context()
         eggroll_util.broadcast_eggroll_session(self._sc, work_mode, eggroll_session)
+        self._eggroll = eggroll_util.build_eggroll_runtime(work_mode, eggroll_session)
 
         FateSession.set_instance(self)
 

arch/api/utils/dtable_utils.py

@@ -36,11 +36,17 @@ def get_table_info(config, create=False):
             table_name = get_latest_commit(data_table_namespace=namespace, branch='master')
     return table_name, namespace
 
+def gen_party_version(namespace, branch='master', create=False):
+    if create:
+        table_name = get_commit_id()
+    else:
+        table_name = get_latest_commit(data_table_namespace=namespace, branch=branch)
+
+    return table_name
 
 def gen_party_namespace(all_party, data_type, role, party_id):
     return gen_namespace_separator.join([role, str(party_id), all_party_key(all_party), data_type])
 
-
 def gen_party_namespace_by_federated_namespace(federated_namespace, role, party_id):
     return gen_namespace_separator.join([role, str(party_id), federated_namespace])
 

arch/api/utils/splitable.py

@@ -46,7 +46,7 @@ def __call__(cls, *args, **kwargs):
     return _AttrInjected
 
 
-def segment_transfer_enabled(max_part_size=0x100000000):
+def segment_transfer_enabled(max_part_size=0x1fffc00):
     """
     a metaclass, indicate objects in this class should be transfer in segments
     Args:
@@ -56,7 +56,7 @@ def segment_transfer_enabled(max_part_size=0x100000000):
 
 
 def split_remote(obj):
-    obj_bytes = Pickle.dumps(obj)
+    obj_bytes = Pickle.dumps(obj, protocol=4)
     byte_size = len(obj_bytes)
     num_slice = num_split_parts(obj, byte_size)
     if num_slice <= 1:

cluster-deploy/scripts/default_configurations.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 fate_cos_address=https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com
+version=1.1.1
+egg_version=1.1
+meta_service_version=1.1
+roll_version=1.1
+federation_version=1.1
+proxy_version=1.1
+fateboard_version=1.1
+fateflow_version=1.1
 python_version=1.1
 jdk_version=8u192
 mysql_version=8.0.13

cluster-deploy/scripts/deploy/eggroll/egg/configurations.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 #federation
 
-version=1.1
+version=
+egg_version=
 java_dir=
 source_code_dir=
 output_packages_dir=
@@ -19,3 +20,4 @@ roll_ip=
 roll_port=8011
 proxy_ip=
 proxy_port=9370
+clustercomm_ip=

cluster-deploy/scripts/deploy/eggroll/egg/deploy.sh

@@ -146,12 +146,29 @@ install(){
     cp -r ${deploy_packages_dir}/source/${module_name}/egg-manager/* ${deploy_dir}/${module_name}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}/
     cd ${deploy_dir}/${module_name}
-    ln -s eggroll-${module_name}-${version}.jar eggroll-${module_name}.jar
+    ln -s eggroll-${module_name}-${egg_version}.jar eggroll-${module_name}.jar
     mv ./services.sh ${deploy_dir}/
 
     cd ${deploy_packages_dir}/source/${module_name}/egg-services
 
     mkdir -p ${deploy_dir}/storage-service-cxx
+
+    system=`sed -e '/"/s/"//g' /etc/os-release | awk -F= '/^NAME/{print $2}'`
+    echo ${system}
+    case "${system}" in
+        "CentOS Linux")
+                echo "CentOS System"
+                rm -rf ./storage-service-cxx/third_party_eggrollv1_ubuntu
+                ;;
+        "Ubuntu")
+                echo "Ubuntu System"
+                rm -rf ./storage-service-cxx/third_party
+                mv ./storage-service-cxx/third_party_eggrollv1_ubuntu  ./storage-service-cxx/third_party
+                ;;
+        *)
+                echo "Not support this system."
+    esac
+
     cp -r ./storage-service-cxx/* ${deploy_dir}/storage-service-cxx/
 
     mkdir -p ${deploy_dir}/python/eggroll/computing
@@ -163,6 +180,17 @@ install(){
     mkdir -p ${deploy_dir}/python/eggroll/conf
     cp -r ./eggroll-conf/* ${deploy_dir}/python/eggroll/conf/
 
+    system=`sed -e '/"/s/"//g' /etc/os-release | awk -F= '/^NAME/{print $2}'`
+    echo ${system}
+    pwd
+    if [[ "${system}" == "Ubuntu" ]];then
+        cd ${deploy_dir}/storage-service-cxx/third_party
+        cd rocksdb
+        sudo cp librocksdb.so /usr/local/lib
+        sudo mkdir -p /usr/local/include/rocksdb/
+        sudo cp -r ./include/* /usr/local/include/
+    fi
+
     cd ${deploy_dir}/storage-service-cxx
 	sed -i.bak "20s#-I. -I.*#-I. -I${deploy_dir}/storage-service-cxx/third_party/include#g" ./Makefile
 	sed -i.bak "34s#LDFLAGS += -L.*#LDFLAGS += -L${deploy_dir}/storage-service-cxx/third_party/lib -llmdb -lboost_system -lboost_filesystem -lglog -lgpr#g" ./Makefile
@@ -176,6 +204,7 @@ install(){
 	sed -i.bak "s/rollport=.*/rollport=${roll_port}/g" ./modify_json.py
 	sed -i.bak "s/proxyip=.*/proxyip=\"${proxy_ip}\"/g" ./modify_json.py
 	sed -i.bak "s/proxyport=.*/proxyport=${proxy_port}/g" ./modify_json.py
+	sed -i.bak "s/clustercommip=.*/clustercommip=\"${clustercomm_ip}\"/g" ./modify_json.py
 	python ./modify_json.py python ./server_conf.json
 }
 
@@ -195,4 +224,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac

cluster-deploy/scripts/deploy/eggroll/meta-service/configurations.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 #federation
 
-version=1.1
+version=
+meta_service_version=
 java_dir=
 source_code_dir=
 output_packages_dir=

cluster-deploy/scripts/deploy/eggroll/meta-service/deploy.sh

@@ -79,7 +79,7 @@ install(){
     cp -r ${deploy_packages_dir}/source/${module_name} ${deploy_dir}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}
     cd ${deploy_dir}/${module_name}
-    ln -s eggroll-${module_name}-${version}.jar eggroll-${module_name}.jar
+    ln -s eggroll-${module_name}-${meta_service_version}.jar eggroll-${module_name}.jar
     mv ./services.sh ${deploy_dir}/
 }
 
@@ -99,4 +99,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac

cluster-deploy/scripts/deploy/eggroll/roll/configurations.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 #federation
 
-version=1.1
+version=
+roll_version=
 java_dir=
 source_code_dir=
 output_packages_dir=

cluster-deploy/scripts/deploy/eggroll/roll/deploy.sh

@@ -77,7 +77,7 @@ install(){
     cp -r ${deploy_packages_dir}/source/${module_name} ${deploy_dir}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}
     cd ${deploy_dir}/${module_name}
-    ln -s eggroll-${module_name}-${version}.jar eggroll-${module_name}.jar
+    ln -s eggroll-${module_name}-${roll_version}.jar eggroll-${module_name}.jar
     mv ./services.sh ${deploy_dir}/
 }
 
@@ -97,4 +97,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac

cluster-deploy/scripts/deploy/fate_base/env.sh

@@ -1,3 +1,23 @@
 #!/bin/bash
-sudo yum -y install gcc gcc-c++ make openssl-devel supervisor gmp-devel mpfr-devel libmpc-devel libaio numactl autoconf automake libtool libffi-devel snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan lsof
+#sudo yum -y install gcc gcc-c++ make openssl-devel supervisor gmp-devel mpfr-devel libmpc-devel libaio numactl autoconf automake libtool libffi-devel snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan lsof
 
+system=`sed -e '/"/s/"//g' /etc/os-release | awk -F= '/^NAME/{print $2}'`
+echo ${system}
+case "${system}" in
+    "CentOS Linux")
+            echo "CentOS System"
+            sudo yum -y install gcc gcc-c++ make openssl-devel supervisor gmp-devel mpfr-devel libmpc-devel libaio numactl autoconf automake libtool libffi-devel snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan lsof
+            ;;
+    "Ubuntu")
+            echo "Ubuntu System"
+            sudo apt-get install -y gcc g++ make  openssl supervisor libgmp-dev  libmpfr-dev libmpc-dev libaio1 libaio-dev numactl autoconf automake libtool libffi-dev libssl1.0.0 libssl-dev  liblz4-1 liblz4-dev liblz4-1-dbg liblz4-tool  zlib1g zlib1g-dbg zlib1g-dev
+            cd /usr/lib/x86_64-linux-gnu
+            if [ ! -f "libssl.so.10" ];then
+                 sudo ln -s libssl.so.1.0.0 libssl.so.10
+                 sudo ln -s libcrypto.so.1.0.0 libcrypto.so.10
+            fi
+            ;;
+    *)
+            echo "Not support this system."
+            exit -1
+esac

cluster-deploy/scripts/deploy/fate_flow/configurations.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 #fate_flow
 
-version=1.1
+version=
+fateflow_version=
 source_code_dir=
 output_packages_dir=
 deploy_dir=
@@ -14,4 +15,4 @@ db_user=
 db_password=
 db_name=
 redis_ip=
-redis_password=
+redis_password=

cluster-deploy/scripts/deploy/fateboard/configurations.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 #fateboard
 
-version=1.1
+version=
+fateboard_version=
 java_dir=
 source_code_dir=
 output_packages_dir=
@@ -14,4 +15,4 @@ db_ip=
 db_user=
 db_password=
 db_name=fate_flow
-node_list=()
+node_list=()

cluster-deploy/scripts/deploy/fateboard/deploy.sh

@@ -38,7 +38,7 @@ source ${config_path}
 packaging() {
     source ../../default_configurations.sh
     package_init ${output_packages_dir} ${module_name}
-    get_module_package ${source_code_dir} ${module_name} ${module_name}-${version}.jar
+    get_module_package ${source_code_dir} ${module_name} ${module_name}-${fateboard_version}.jar
 }
 
 config() {
@@ -75,7 +75,7 @@ install() {
     cp -r ${deploy_packages_dir}/source/${module_name} ${deploy_dir}/
     cp -r ${deploy_packages_dir}/config/${module_name}/conf/* ${deploy_dir}/${module_name}
     cd ${deploy_dir}/${module_name}
-    ln -s ${module_name}-${version}.jar ${module_name}.jar
+    ln -s ${module_name}-${fateboard_version}.jar ${module_name}.jar
 }
 
 init() {
@@ -98,4 +98,4 @@ case "$2" in
  *)
      usage
         exit -1
-esac
+esac