Merge pull request rundeck-plugins#79 from karann-jain/master

Fixing Issue rundeck-plugins#75, rundeck-plugins#76, rundeck-plugins#77, rundeck-plugins#78
alizdavoodi · Sep 3, 2020 · add7cf5 · add7cf5
2 parents 0699db2 + 51147dd
commit add7cf5
Showing 1 changed file with 55 additions and 40 deletions.
diff --git a/contents/job-wait.py b/contents/job-wait.py
@@ -27,15 +27,18 @@ def wait():
         sleep = float(environ.get("RD_CONFIG_SLEEP"))
         show_log = environ.get("RD_CONFIG_SHOW_LOG") == "true"
 
-        batch_v1 = client.BatchV1Api()
-        core_v1 = client.CoreV1Api()
-
         # Poll for completion if retries
         retries_count = 0
         completed = False
 
 
         while True:
+
+            common.connect()
+
+            batch_v1 = client.BatchV1Api()
+            core_v1 = client.CoreV1Api()
+
             api_response = batch_v1.read_namespaced_job_status(
                 name,
                 namespace,
@@ -59,49 +62,61 @@ def wait():
 
             if api_response.status.completion_time:
                 completed = True
-            else:
-                if show_log:
-                    log.debug("Searching for pod associated with job")
-
-                    pod_list = core_v1.list_namespaced_pod(
-                        namespace,
-                        label_selector="job-name==" + name
-                    )
-                    first_item = pod_list.items[0]
-                    pod_name = first_item.metadata.name
-                    log.info("Fetching logs from pod: {0}".format(pod_name))
-
-                    # time.sleep(15)
-                    log.info("========================== job log start ==========================")
-                    start_time = time.time()
-                    timeout = 60
-                    while True:
-                        try:
-                            core_v1.read_namespaced_pod_log(name=pod_name,
-                                                            namespace=namespace)
+
+            if show_log:
+                log.debug("Searching for pod associated with job")
+
+                schedule_start_time = time.time()
+                schedule_timeout = 600
+                while True:
+                    try:
+                        pod_list = core_v1.list_namespaced_pod(
+                            namespace,
+                            label_selector="job-name==" + name
+                        )
+                        first_item = pod_list.items[0]
+                        pod_name = first_item.metadata.name
+                        break
+                    except IndexError as IndexEx:
+                        log.warning("Still Waiting for Pod to be Scheduled")
+                        time.sleep(60)
+                        if schedule_timeout and time.time() - schedule_start_time > schedule_timeout:  # pragma: no cover
+                            raise TimeoutError
+
+                log.info("Fetching logs from pod: {0}".format(pod_name))
+
+                # time.sleep(15)
+                log.info("========================== job log start ==========================")
+                start_time = time.time()
+                timeout = 300
+                while True:
+                    try:
+                        core_v1.read_namespaced_pod_log(name=pod_name,
+                                                        namespace=namespace)
+                        break
+                    except ApiException as ex:
+                        log.warning("Pod is not ready, status: {}".format(ex.status))
+                        if ex.status == 200:
                             break
-                        except ApiException as ex:
-                            log.warning("Pod is not ready, status: {}".format(ex.status))
-                            if ex.status == 200:
-                                break
-                            else:
-                                log.info("waiting for log")
-                                time.sleep(15)
-                                if timeout and time.time() - start_time > timeout:  # pragma: no cover
-                                    raise TimeoutError
-
-                    w = watch.Watch()
-                    for line in w.stream(core_v1.read_namespaced_pod_log,
-                                         name=pod_name,
-                                         namespace=namespace):
-                        print(line)
-
-                    log.info("=========================== job log end ===========================")
+                        else:
+                            log.info("waiting for log")
+                            time.sleep(15)
+                            if timeout and time.time() - start_time > timeout:  # pragma: no cover
+                                raise TimeoutError
+
+                w = watch.Watch()
+                for line in w.stream(core_v1.read_namespaced_pod_log,
+                                        name=pod_name,
+                                        namespace=namespace):
+                    print(line)
+
+                log.info("=========================== job log end ===========================")
 
             if completed:
                 break
 
             log.info("Waiting for job completion")
+            show_log = False
             time.sleep(sleep)