From eaebd054813ed792a71af34f648aa8b4067d28b6 Mon Sep 17 00:00:00 2001
From: Kristaps Ermanis <ke291@cam.ac.uk>
Date: Wed, 19 Jun 2019 13:20:34 +0100
Subject: [PATCH] Convergence checking and Ziggy queue checking added.

---
 GaussianDarwin.py |  2 ++
 GaussianZiggy.py  | 22 +++++++++++++++++++++-
 PyDP4.py          | 19 ++++++++++++++++---
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/GaussianDarwin.py b/GaussianDarwin.py
index 1db3c53..06c3062 100755
--- a/GaussianDarwin.py
+++ b/GaussianDarwin.py
@@ -32,6 +32,8 @@
 
 IsGausCompleted = Gaussian.IsGausCompleted
 
+Converged = Gaussian.Converged
+
 def RunNMRCalcs(Isomers, settings):
     print('\nRunning Gaussian NMR calculations on Darwin...')
 
diff --git a/GaussianZiggy.py b/GaussianZiggy.py
index e45a900..39fded2 100755
--- a/GaussianZiggy.py
+++ b/GaussianZiggy.py
@@ -32,6 +32,8 @@
 
 IsGausCompleted = Gaussian.IsGausCompleted
 
+Converged = Gaussian.Converged
+
 def RunNMRCalcs(Isomers, settings):
     print('\nRunning Gaussian NMR calculations on Ziggy...')
 
@@ -171,10 +173,14 @@ def RunBatchOnZiggy(findex, queue, GausFiles, settings):
 
     print(str(len(GausFiles)) + ' .com and slurm files uploaded to ziggy')
 
+    JobIDs = []
     #Launch the calculations
     for f in GausFiles:
         job = '~/' + folder + '/' + f[:-4]
         outp = subprocess.check_output('ssh ziggy sbatch ' + job + 'slurm', shell=True)
+        status = outp.decode()[:-1]
+        print(status)
+        JobIDs.append(status.split('job ')[1])
 
     print(str(len(GausFiles)) + ' jobs submitted to the queue on ziggy')
 
@@ -182,6 +188,10 @@ def RunBatchOnZiggy(findex, queue, GausFiles, settings):
     if settings.user in outp.decode():
         print("Jobs are running on ziggy")
 
+    time.sleep(60)
+    OldQRes = CheckZiggyQueue(JobIDs, settings)
+    print('Pending: ' + str(OldQRes[0]) + ', Running: ' + str(OldQRes[1]) + ', Not in queue: ' + str(OldQRes[2]))
+
     Jobs2Complete = list(GausFiles)
     n2complete = len(Jobs2Complete)
 
@@ -195,7 +205,17 @@ def RunBatchOnZiggy(findex, queue, GausFiles, settings):
             n2complete = len(Jobs2Complete)
             print(str(n2complete) + " remaining.")
 
-        time.sleep(60)
+        QRes = CheckZiggyQueue(JobIDs, settings)
+        if QRes != OldQRes:
+            OldQRes = QRes
+            print('Pending: ' + str(OldQRes[0]) + ', Running: ' + str(OldQRes[1]) + ', Not in queue: ' + str(OldQRes[2]))
+
+        if QRes[2] == len(JobIDs):
+            #check each gaussian file to ascertain the status of individual gaus jobs
+            print('No jobs left in Ziggy queue')
+            break
+
+        time.sleep(120)
 
     #When done, copy the results back
     print("\nCopying the output files back to localhost...")
diff --git a/PyDP4.py b/PyDP4.py
index 57f62a5..9b697bd 100755
--- a/PyDP4.py
+++ b/PyDP4.py
@@ -233,6 +233,10 @@ def main(settings):
 
         # Run DFT optimizations, if requested
         if ('o' in settings.Workflow):
+
+            now = datetime.datetime.now()
+            settings.StartTime = now.strftime('%d%b%H%M')
+
             print('\nSetting up geometry optimization calculations...')
             Isomers = DFT.SetupOptCalcs(Isomers, settings)
             print('\nRunning geometry optimization calculations...')
@@ -245,9 +249,14 @@ def main(settings):
             #Add convergence check here before continuing with calcs!
             if DFT.Converged(Isomers) == False:
                 print('Some of the conformers did not converge, quitting...')
+                quit()
 
         # Run DFT single-point energy calculations, if requested
         if ('e' in settings.Workflow):
+
+            now = datetime.datetime.now()
+            settings.StartTime = now.strftime('%d%b%H%M')
+
             print('\nSetting up energy calculations...')
             Isomers = DFT.SetupECalcs(Isomers, settings)
             print('\nRunning energy calculations...')
@@ -260,6 +269,10 @@ def main(settings):
 
         # Run DFT NMR calculations, if requested
         if ('n' in settings.Workflow):
+
+            now = datetime.datetime.now()
+            settings.StartTime = now.strftime('%d%b%H%M')
+
             print('\nSetting up NMR calculations...')
             Isomers = DFT.SetupNMRCalcs(Isomers, settings)
             print('\nRunning NMR calculations...')
@@ -278,15 +291,15 @@ def main(settings):
                 print(iso.InputFile + ": " + str(iso.DFTEnergies))
 
     else:
-        # Run DFT optimizations, if requested
+        # Read DFT optimized geometries, if requested
         if ('o' in settings.Workflow):
             Isomers = DFT.ReadDFTGeometries(Isomers, settings)
 
-        # Run DFT single-point energy calculations, if requested
+        # Read DFT single-point energies, if requested
         if ('e' in settings.Workflow):
             Isomers = DFT.ReadDFTEnergies(Isomers, settings)
 
-        # Run DFT NMR calculations, if requested
+        # Read DFT NMR data, if requested
         if ('n' in settings.Workflow):
             Isomers = DFT.ReadShieldings(Isomers)
             Isomers = DFT.ReadDFTEnergies(Isomers)