Get cluster config from configmap when managing service with paictl. (…

…microsoft#1489)

Jenkinsfile

@@ -169,9 +169,17 @@ sed -i "42s/.*/    zkid: "1"/" /cluster-configuration/cluster-configuration.yaml
 # ! TODO wait for cluster ready
 sleep 6s
 
-# Step 3. Start all PAI services
+# Step 3. Upload cluster-configuration into kubernetes cluster. And set cluster-id
+./paictl.py config push -p /cluster-configuration << EOF
+openpai-test
+EOF
+
+
+# Step 4. Start all PAI services
 # start pai services
-./paictl.py service start -p /cluster-configuration
+./paictl.py service start << EOF
+openpai-test
+EOF
 EOF_DEV_BOX
 sudo chown core:core -R $JENKINS_HOME
 sudo chown core:core -R /pathHadoop/
@@ -261,9 +269,16 @@ sed -i "42s/.*/    zkid: "2"/" /cluster-configuration/cluster-configuration.yaml
 # ! TODO wait for cluster ready
 sleep 6s
 
-# Step 3. Start all PAI services
+# Step 3. Upload cluster configuration into kubernetes cluster. And set cluster-id
+./paictl.py config push -p /cluster-configuration << EOF
+openpai-test
+EOF
+
+# Step 4. Start all PAI services
 # start pai services
-./paictl.py service start -p /cluster-configuration
+./paictl.py service start << EOF
+openpai-test
+EOF
 EOF_DEV_BOX
 sudo chown core:core -R $JENKINS_HOME
 sudo chown core:core -R /pathHadoop/
@@ -556,12 +571,13 @@ else
     git reset --hard origin/${GIT_BRANCH}
 fi
 # delete service for next install
-./paictl.py service start -p /cluster-configuration -n cluster-configuration << EOF
-Y
+./paictl.py service start -n cluster-configuration << EOF
+openpai-test
 EOF
 
-./paictl.py service delete -p /cluster-configuration << EOF
+./paictl.py service delete << EOF
 Y
+openpai-test
 EOF
 
 # clean k8s
@@ -599,12 +615,13 @@ else
     git reset --hard origin/${GIT_BRANCH}
 fi
 # delete service for next install
-./paictl.py service start -p /cluster-configuration -n cluster-configuration << EOF
-Y
+./paictl.py service start -n cluster-configuration << EOF
+openpai-test
 EOF
 
-./paictl.py service delete -p /cluster-configuration << EOF
+./paictl.py service delete << EOF
 Y
+openpai-test
 EOF
 
 # clean k8s

deployment/paiLibrary/paiService/service_management_configuration.py

@@ -0,0 +1,56 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+import time
+
+from ...confStorage.download import download_configuration
+from ..clusterObjectModel.objectModelFactory import objectModelFactory
+
+
+
+class service_management_configuration:
+
+    def __init__(self, **kwargs):
+
+        self.KUBE_CONFIG_LOCATION = None
+        if "kube_config_path" in kwargs and kwargs["kube_config_path"] != None:
+            self.KUBE_CONFIG_LOCATION = kwargs["kube_config_path"]
+
+        self.time = str(int(time.time()))
+        self.tmp_path = "./tmp-service-config-{0}".format(self.time)
+
+        self.cluster_object_service = None
+
+
+
+    def get_latest_cluster_configuration_from_pai(self):
+        config_get_handler = download_configuration(config_output_path = self.tmp_path, kube_config_path = self.KUBE_CONFIG_LOCATION)
+        config_get_handler.run()
+
+
+
+    def get_cluster_object_model_service(self):
+        objectModelFactoryHandler = objectModelFactory(configurationPath = self.tmp_path)
+        self.cluster_object_service = objectModelFactoryHandler.objectModelPipeLine()["service"]
+
+
+
+    def run(self):
+        self.get_latest_cluster_configuration_from_pai()
+        self.get_cluster_object_model_service()
+        return self.cluster_object_service

deployment/paiLibrary/paiService/service_management_delete.py

@@ -24,6 +24,7 @@
 from . import service_delete
 from . import service_template_generate
 from . import service_template_clean
+from . import service_management_configuration
 #
 from ..common import directory_handler
 from ..common import file_handler
@@ -34,10 +35,14 @@ class service_management_delete:
 
 
 
-    def __init__(self, cluster_object_model, service_list = None, **kwargs):
+    def __init__(self, kube_config_path = None, service_list = None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = cluster_object_model
+        self.cluster_object_model = None
+
+        self.kube_config_path = None
+        if kube_config_path != None:
+            self.kube_config_path = kube_config_path
 
         if service_list == None:
             self.service_list = self.get_service_list()
@@ -90,6 +95,9 @@ def start(self, serv):
 
     def run(self):
 
+        config_handler = service_management_configuration.service_management_configuration(kube_config_path=self.kube_config_path)
+        self.cluster_object_model = config_handler.run()
+
         for serv in self.service_list:
             if serv == "cluster-configuration":
                 continue

deployment/paiLibrary/paiService/service_management_refresh.py

@@ -22,6 +22,7 @@
 from . import service_refresh
 from . import service_template_generate
 from . import service_template_clean
+from . import service_management_configuration
 
 from ..common import directory_handler
 from ..common import file_handler
@@ -31,10 +32,14 @@
 class service_management_refresh:
 
 
-    def __init__(self, cluster_object_model, service_list = None, **kwargs):
+    def __init__(self, kube_config_path = None, service_list = None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = cluster_object_model
+        self.cluster_object_model = None
+
+        self.kube_config_path = None
+        if kube_config_path != None:
+            self.kube_config_path = kube_config_path
 
         if service_list == None:
             self.service_list = self.get_service_list()
@@ -129,6 +134,9 @@ def start(self, serv):
 
     def run(self):
 
+        config_handler = service_management_configuration.service_management_configuration(kube_config_path=self.kube_config_path)
+        self.cluster_object_model = config_handler.run()
+
         self.done_dict = dict()
         self.refresh_all_label()
         for serv in self.service_list:

deployment/paiLibrary/paiService/service_management_start.py

@@ -24,6 +24,7 @@
 from . import service_start
 from . import service_template_generate
 from . import service_template_clean
+from . import service_management_configuration
 #
 from ..common import directory_handler
 from ..common import file_handler
@@ -33,10 +34,14 @@ class serivce_management_start:
 
 
 
-    def __init__(self, cluster_object_model, service_list = None, **kwargs):
+    def __init__(self, kube_config_path = None, service_list = None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = cluster_object_model
+        self.cluster_object_model = None
+
+        self.kube_config_path = None
+        if kube_config_path != None:
+            self.kube_config_path = kube_config_path
 
         if service_list == None:
             self.service_list = self.get_service_list()
@@ -125,6 +130,9 @@ def start(self, serv):
 
     def run(self):
 
+        config_handler = service_management_configuration.service_management_configuration(kube_config_path = self.kube_config_path)
+        self.cluster_object_model = config_handler.run()
+
         self.done_dict = dict()
 
         for serv in self.service_list:

deployment/paiLibrary/paiService/service_management_stop.py

@@ -23,6 +23,7 @@
 from . import service_stop
 from . import service_template_generate
 from . import service_template_clean
+from . import service_management_configuration
 
 from ..common import directory_handler
 from ..common import file_handler
@@ -32,10 +33,14 @@
 class service_management_stop:
 
 
-    def __init__(self, cluster_object_model, service_list = None, **kwargs):
+    def __init__(self, kube_config_path = None, service_list = None, **kwargs):
         self.logger = logging.getLogger(__name__)
 
-        self.cluster_object_model = cluster_object_model
+        self.cluster_object_model = None
+
+        self.kube_config_path = None
+        if kube_config_path != None:
+            self.kube_config_path = kube_config_path
 
         if service_list == None:
             self.service_list = self.get_service_list()
@@ -88,6 +93,9 @@ def start(self, serv):
 
     def run(self):
 
+        config_handler = service_management_configuration.service_management_configuration(kube_config_path = self.kube_config_path)
+        self.cluster_object_model = config_handler.run()
+
         for serv in self.service_list:
             if serv == "cluster-configuration":
                 continue

docs/etcd/etcd.md

@@ -59,7 +59,7 @@ Sometimes the etcd nodes may not healthy, it can be repaired with command
 ```bash
 python paictl.py machine etcd-fix -p /path/to/configuration/directory -l /path/to/your/errornodelist.yaml
 ```
-Please follow instructions in [machine maintenance](../pai-management/doc/machine-maintenance.md) for the details.
+Please follow instructions in [machine maintenance](../paictl/paictl-manual.md#Machine) for the details.
 
 ## Data Stored on Etcd
 

docs/grafana/README.md

@@ -42,7 +42,7 @@ Then redeploy Grafana only:
 ```sh
 python paictl.py service start -p /path/to/cluster-configuration/dir -n grafana
 ```
-For more details, please refer to [Maintain your service](../pai-management/doc/service-maintain.md).
+For more details, please refer to [Maintain your service](../paictl/paictl-manual.md#Service).
 # Upgrading
 
 System will automatically pull the latest Grafana image, there is no need to upgrade. If you want to use the specific version of Grafana, you can change the version configuration at the [grafana.yaml.template](../../src/grafana/deploy/grafana.yaml.template#L44) and then redeploy it.

docs/pai-management/README.md

@@ -41,7 +41,7 @@ Note: Please working on dev-box when managing your cluster.
 
 - Please maintain the PAI cluster from within a dev-box described.
 
-- [A Guide For PAI Machine Maintenance](doc/machine-maintenance.md)
+- [A Guide For PAI Machine Maintenance](../paictl/paictl-manual.md#Machine)
 
 #### Service Maintenance
 
@@ -50,7 +50,7 @@ Note: Please working on dev-box when managing your cluster.
 
 
 
-- [A Guide For Service Maintenance](doc/service-maintain.md)
+- [A Guide For Service Maintenance](../paictl/paictl-manual.md#Service)
 
 - [A Guide For Pulling Image From the Public Registry](doc/how-to-write-pai-configuration.md#serivices-configurationyaml)