Merge pull request FederatedAI#2036 from FederatedAI/develop-1.5

Develop 1.5

.readthedocs.yml

@@ -18,4 +18,8 @@ formats: all
 python:
   version: 3.6
   install:
-    - requirements: doc/readthedoc/requirements.txt
+    - requirements: doc/readthedoc/requirements.txt
+    - method: pip
+      path: python/fate_client
+    - method: pip
+      path: python/fate_test

doc/readthedoc/conf.py

@@ -25,6 +25,8 @@
 from recommonmark.parser import CommonMarkParser
 
 sys.path.insert(0, os.path.abspath('_build_temp/python'))
+sys.path.insert(0, os.path.abspath('_build_temp/python/fate_client'))
+sys.path.insert(0, os.path.abspath('_build_temp/python/fate_test'))
 
 # -- Project information -----------------------------------------------------
 
@@ -42,7 +44,8 @@
     'sphinx.ext.autodoc',
     'sphinx.ext.napoleon',
     'autodocsumm',
-    'recommonmark'
+    'recommonmark',
+    'sphinx_click.ext'
 ]
 
 autosummary_generate = True

doc/readthedoc/index.rst

@@ -35,6 +35,21 @@ https://fate.fedai.org
    CLI API <_build_temp/python/fate_flow/doc/fate_flow_cli>
    REST API <_build_temp/python/fate_flow/doc/fate_flow_rest_api>
 
+.. toctree::
+   :maxdepth: 3
+   :caption: FATE Clients
+
+   Flow SDK <_build_temp/python/fate_client/flow_sdk/README>
+   Flow Client <_build_temp/python/fate_client/flow_client/README>
+   Pipeline <_build_temp/python/fate_client/pipeline/README>
+
+.. toctree::
+   :maxdepth: 3
+   :caption: FATE Test
+
+   FATE TEST <_build_temp/python/fate_test/README>
+
+
 .. toctree::
    :maxdepth: 3
    :caption: Develop

doc/readthedoc/requirements.txt

@@ -3,6 +3,7 @@ sphinx>=2.4.4
 sphinx_rtd_theme>=0.4.3
 autodocsumm
 recommonmark
+sphinx-click
 
 apsw==3.9.2.post1
 beautifultable==1.0.0

examples/benchmark_quality/hetero_linear_regression/pipeline-linr.py → examples/benchmark_quality/hetero_linear_regression/fate-linr.py

@@ -112,12 +112,16 @@ def main(config="../../config.yaml", param="./linr_config.yaml", namespace=""):
 
     # fit model
     pipeline.fit(backend=backend, work_mode=work_mode)
-    # query component summary
-    return pipeline.get_component("evaluation_0").get_summary()
+
+    metric_summary = pipeline.get_component("evaluation_0").get_summary()
+    data_summary = {"train": {"guest": guest_train_data["name"], "host": host_train_data["name"]},
+                    "test": {"guest": guest_train_data["name"], "host": host_train_data["name"]}
+                    }
+    return data_summary, metric_summary
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser("BENCHMARK-QUALITY PIPELINE JOB")
+    parser = argparse.ArgumentParser("BENCHMARK-QUALITY FATE JOB")
     parser.add_argument("-config", type=str,
                         help="config file")
     parser.add_argument("-param", type=str,

examples/benchmark_quality/hetero_linear_regression/hetero_linr_sklearn_benchmark.json → examples/benchmark_quality/hetero_linear_regression/hetero_linr_benchmark.json

@@ -1,33 +1,33 @@
 {
     "data": [
         {
-            "file": "../../data/motor_hetero_host.csv",
+            "file": "examples/data/motor_hetero_host.csv",
             "head": 1,
             "partition": 16,
             "table_name": "motor_hetero_host",
             "namespace": "experiment",
             "role": "host_0"
         },
         {
-            "file": "../../data/motor_hetero_guest.csv",
+            "file": "examples/data/motor_hetero_guest.csv",
             "head": 1,
             "partition": 16,
             "table_name": "motor_hetero_guest",
             "namespace": "experiment",
             "role": "guest_0"
         }
     ],
-    "linear-regression": {
+    "hetero_linear_regression-regression": {
         "local": {
-            "script": "./sklearn-linr.py",
+            "script": "./local-linr.py",
             "conf": "./linr_config.yaml"
         },
-        "pipeline": {
-            "script": "./pipeline-linr.py",
+        "FATE": {
+            "script": "./fate-linr.py",
             "conf": "./linr_config.yaml"
         },
         "compare_setting": {
-            "relative_tol": 0.05
+            "relative_tol": 0.01
         }
     }
 }

examples/benchmark_quality/hetero_linear_regression/linr_config.yaml

@@ -3,10 +3,10 @@ data_host: "examples/data/motor_hetero_host.csv"
 idx: "idx"
 label_name: "motor_speed"
 penalty: "L2"
-max_iter: 5
-optimizer: "adagrad"
+max_iter: 6
+optimizer: "sgd"
 batch_size: 100
-learning_rate: 0.15
+learning_rate: 0.13
 alpha: 0.01
 tol: 1e-4
 init_param:

examples/benchmark_quality/hetero_linear_regression/sklearn-linr.py → examples/benchmark_quality/hetero_linear_regression/local-linr.py

@@ -47,15 +47,16 @@ def main(param="./linr_config.yaml"):
     rmse = np.sqrt(mse)
     r2 = r2_score(y, y_pred)
     explained_var = explained_variance_score(y, y_pred)
-    result = {"r2_score": r2,
+    metric_summary = {"r2_score": r2,
               "mean_squared_error": mse,
               "root_mean_squared_error": rmse,
               "explained_variance": explained_var}
-    return result
+    data_summary = {}
+    return data_summary, metric_summary
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser("BENCHMARK-QUALITY SKLEARN JOB")
+    parser = argparse.ArgumentParser("BENCHMARK-QUALITY LOCAL JOB")
     parser.add_argument("-param", type=str,
                         help="config file for params")
     args = parser.parse_args()

examples/benchmark_quality/hetero_lr/breast_config.yaml

@@ -0,0 +1,11 @@
+data_guest: "/home/mingchao/python/examples/data/breast_hetero_guest.csv"
+data_host: "/home/mingchao/python/examples/data/breast_hetero_host.csv"
+idx: "id"
+label_name: "y"
+penalty: "L2"
+max_iter: 30
+alpha: 0.01
+learning_rate: 0.15
+optimizer: "rmsprop"
+batch_size: 5000
+early_stop: "diff"

examples/benchmark_quality/hetero_lr/lr_config.yaml → examples/benchmark_quality/hetero_lr/default_credit_config.yaml

@@ -3,6 +3,9 @@ data_host: "/home/mingchao/python/examples/data/default_credit_hetero_host.csv"
 idx: "id"
 label_name: "y"
 penalty: "L2"
-max_iter: 100
-alpha: 0.01
+max_iter: 30
+alpha: 0.001
 learning_rate: 0.15
+optimizer: "nesterov_momentum_sgd"
+batch_size: -1
+early_stop: "diff"

examples/benchmark_quality/hetero_lr/epsilon_5k_config.yaml

@@ -0,0 +1,11 @@
+data_guest: "/home/mingchao/python/examples/data/epsilon_5k_hetero_guest.csv"
+data_host: "/home/mingchao/python/examples/data/epsilon_5k_hetero_host.csv"
+idx: "id"
+label_name: "y"
+penalty: "L2"
+max_iter: 15
+alpha: 0.001
+learning_rate: 0.15
+optimizer: "rmsprop"
+batch_size: 3200
+early_stop: "diff"

examples/benchmark_quality/hetero_lr/give_credit_config.yaml

@@ -0,0 +1,12 @@
+data_guest: "/home/mingchao/python/examples/data/give_credit_hetero_guest.csv"
+data_host: "/home/mingchao/python/examples/data/give_credit_hetero_host.csv"
+idx: "id"
+label_name: "y"
+penalty: "L2"
+max_iter: 30
+alpha: 100
+learning_rate: 0.15
+optimizer: "rmsprop"
+batch_size: -1
+early_stop: "diff"
+init_method: "zeros"

examples/benchmark_quality/hetero_lr/hetero_lr_sklearn_benchmark.json

@@ -1,33 +1,62 @@
 {
     "data": [
+                {
+            "file": "../../data/breast_hetero_guest.csv",
+            "head": 1,
+            "partition": 16,
+            "table_name": "breast_hetero_guest",
+            "namespace": "experiment",
+            "role": "guest_0"
+        },
         {
-            "file": "../../data/vehicle_scale_hetero_host.csv",
+            "file": "../../data/breast_hetero_host.csv",
             "head": 1,
             "partition": 16,
-            "table_name": "vehicle_scale_hetero_host",
+            "table_name": "breast_hetero_host",
             "namespace": "experiment",
             "role": "host_0"
         },
         {
-            "file": "../../data/vehicle_scale_hetero_guest.csv",
+            "file": "examples/data/vehicle_scale_hetero_guest.csv",
             "head": 1,
             "partition": 16,
             "table_name": "vehicle_scale_hetero_guest",
             "namespace": "experiment",
             "role": "guest_0"
+        },
+        {
+            "file": "examples/data/vehicle_scale_hetero_host.csv",
+            "head": 1,
+            "partition": 16,
+            "table_name": "vehicle_scale_hetero_host",
+            "namespace": "experiment",
+            "role": "host_0"
         }
     ],
     "binary": {
         "local": {
-            "script": "./sklearn-lr-multi.py",
+            "script": "./sklearn-lr-binary.py",
             "conf": "./lr_config.yaml"
         },
         "pipeline": {
-            "script": "./pipeline-lr-multi.py",
+            "script": "./pipeline-lr-binary.py",
             "conf": "./lr_config.yaml"
         },
         "compare_setting": {
-            "relative_tol": 1e-9
+            "relative_tol": 1e-2
+        }
+    },
+    "multi": {
+        "local": {
+            "script": "./sklearn-lr-multi.py",
+            "conf": "./lr_multi_config.yaml"
+        },
+        "pipeline": {
+            "script": "./pipeline-lr-multi.py",
+            "conf": "./lr_multi_config.yaml"
+        },
+        "compare_setting": {
+            "relative_tol": 1e-2
         }
     }
 }

examples/benchmark_quality/hetero_lr/lr_bench_mark_data.json

@@ -0,0 +1,68 @@
+{
+  "data": [
+    {
+      "file": "examples/data/breast_hetero_guest.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "breast_hetero_guest",
+      "namespace": "experiment",
+      "count": 569
+    },
+    {
+      "file": "examples/data/breast_hetero_host.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "breast_hetero_host",
+      "namespace": "experiment",
+      "count": 569
+    },
+    {
+      "file": "examples/data/default_credit_hetero_guest.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "default_credit_hetero_guest",
+      "namespace": "experiment",
+      "count": 30000
+    },
+    {
+      "file": "examples/data/default_credit_hetero_host.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "default_credit_hetero_host",
+      "namespace": "experiment",
+      "count": 30000
+    },
+    {
+      "file": "examples/data/give_credit_hetero_guest.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "give_credit_hetero_guest",
+      "namespace": "experiment",
+      "count": 150000
+    },
+    {
+      "file": "examples/data/give_credit_hetero_host.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "give_credit_hetero_host",
+      "namespace": "experiment",
+      "count": 150000
+    },
+    {
+      "file": "examples/data/epsilon_5k_hetero_guest.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "epsilon_5k_hetero_guest",
+      "namespace": "experiment",
+      "count": 5000
+    },
+    {
+      "file": "examples/data/epsilon_5k_hetero_host.csv",
+      "head": 1,
+      "partition": 16,
+      "table_name": "epsilon_5k_hetero_host",
+      "namespace": "experiment",
+      "count": 5000
+    }
+  ]
+}

examples/benchmark_quality/hetero_lr/lr_multi_config.yaml

@@ -0,0 +1,9 @@
+data_guest: "/home/mingchao/python/examples/data/vehicle_scale_hetero_guest.csv"
+data_host: "/home/mingchao/python/examples/data/vehicle_scale_hetero_host.csv"
+idx: "id"
+label_name: "y"
+penalty: "L2"
+max_iter: 30
+alpha: 0.01
+learning_rate: 0.15
+optimizer: "rmsporp"