test 1 cpu request and 1 job request

ec-jrc · Dec 3, 2024 · 96441b2 · 96441b2
1 parent eff5e0e
commit 96441b2
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 90 deletions.
diff --git a/A_config/a10_config.py b/A_config/a10_config.py
@@ -80,7 +80,8 @@ def __init__(self, forecastingMonths):
     self.forecastingMonths = forecastingMonths
 
     # scikit, numbers of cores to be used when multi-thread is possible, at least 4
-    self.nJobsForGridSearchCv = 4
+    # self.nJobsForGridSearchCv = 4 # attempt to solve overuse of cpu in condor
+    self.nJobsForGridSearchCv = 1
 
     # Input data scaling. Admitted values:
     # z_f: z-score features

diff --git a/E_viz/e100_eval_figs.py b/E_viz/e100_eval_figs.py
@@ -54,7 +54,11 @@ def AU_error(b1, config, outputDir):
         fn_mRes_out = os.path.join(config.models_out_dir, 'ID_' + str(myID) +
                                    '_crop_' + row['Crop'] + '_Yield_' + row['Estimator'] +
                                    '_mres.csv')
-        mRes = pd.read_csv(fn_mRes_out)
+        if os.path.exists(fn_mRes_out):
+            mRes = pd.read_csv(fn_mRes_out)
+        else:
+            fn_spec = os.path.join(config.models_spec_dir, str(myID) + '_' + row['Crop'] + '_' + row['Estimator'] + '.json')
+            mRes = d090_model_wrapper.fit_and_validate_single_model(fn_spec, config, 'tuning', run2get_mres_only=True)
         rRMSE_pByAdmin = d140_modelStats.statsByAdmin(mRes)
         rRMSE_pByAdmin = rRMSE_pByAdmin.merge(df_regNames, how='left', left_on='adm_id', right_on='adm_id')
         rRMSE_pByAdmin = rRMSE_pByAdmin.merge(df_Stats5yrs[df_Stats5yrs['Crop_name|first'] == row['Crop']], how='left',

diff --git a/E_viz/e110_ope_figs.py b/E_viz/e110_ope_figs.py
@@ -36,7 +36,7 @@ def map(b1, config, var4time, OutputDir, fn_shape_gaul1, country_name_in_shp_fil
         lbl = "YF % difference with last avail. 5 years"
         minmax = [-df_c['fyield_diff_pct (last 5 yrs in data avail)'].abs().max(), df_c['fyield_diff_pct (last 5 yrs in data avail)'].abs().max()]
         e50_yield_data_analysis.mapDfColumn(df_c, 'adm_id', 'fyield_diff_pct (last 5 yrs in data avail)', 'Region_name', gdf, gdf_gaul1_id,
-                    gdf_gaul0_column, country_name_in_shp_file, lbl, cmap='bwr', fn_fig=None, ax=axs[1], minmax=minmax)
+                    gdf_gaul0_column, country_name_in_shp_file, lbl, cmap='bwr_r', fn_fig=None, ax=axs[1], minmax=minmax)
         fig.suptitle(title, fontsize=14)
         fig.tight_layout()
         fig.savefig(fig_name)

diff --git a/F_post_processsing/F110_process_opeForecast_output.py b/F_post_processsing/F110_process_opeForecast_output.py
diff --git a/G_HTCondor/condor.submit_template b/G_HTCondor/condor.submit_template
@@ -5,7 +5,7 @@ Executable=/bin/bash
 Docker_image = harbor.cidsn.jrc.it/jeodpp-htcondor/debian12-slim:0.1
 initialdir = {root_dir}
 arguments                = /eos/jeodpp/data/projects/ML4CAST/ml4cast-ml/G_HTCondor/run.sh $(Cluster) $(Process) $(tasks)
-request_cpus             = 4
+request_cpus             = 1
 request_memory           = 10GB
 transfer_input_files     = /eos/jeodpp/data/projects/ML4CAST/ml4cast-ml/condor_launcher.py
 should_transfer_files    = YES

diff --git a/aaa_debugger.py b/aaa_debugger.py
@@ -1,9 +1,14 @@
 from D_modelling import d090_model_wrapper
 from A_config import a10_config
+import os
 
-fn = r'V:\foodsec\Projects\SNYF\stable_input_data\ZA\summer\RUN_Maize_(corn)_WC-South_Africa-ASAP\TUNE_20241118\Specs\[email protected]'
-config_fn = r'V:\foodsec\Projects\SNYF\stable_input_data\ZA\summer\ZAsummer_Maize_(corn)_WC-South_Africa-ASAP_config.json'
+# limit multithreat (even setting njobs = 4, undelying libriaries were using more, see https://github.com/joblib/joblib/issues/793)
+os.environ['MKL_NUM_THREADS'] = '1'
+os.environ['OPENBLAS_NUM_THREADS'] = '1'
+os.environ['OMP_NUM_THREADS'] = '1'
+fn = r'V:\foodsec\Projects\SNYF\stable_input_data\DZ\RUN_Multiple_WC-Algeria-ASAP\TUNE_debug\Specs\000483_Durum_wheat_GPR.json'
+config_fn = r'V:\foodsec\Projects\SNYF\stable_input_data\DZ\DZMultiple_WC-Algeria-ASAP_config.json'
 runType = 'fast_tuning'
-run_name = '20241118'
+run_name = 'debug'
 config = a10_config.read(config_fn, run_name)
 d090_model_wrapper.fit_and_validate_single_model(fn, config, runType, run2get_mres_only=False)
diff --git a/manager_0_user_params.py b/manager_0_user_params.py
@@ -3,7 +3,7 @@
 #############################
 # 1. Config file, run type, name, and where to tune
 #############################
-run_name = '20241118'# '20241016_75_100_maize_sunflower_soybeans_NorthernCape' #'20241004_75_100_maize_sunflower_soybeans'
+run_name = '20241203_test'# '20241016_75_100_maize_sunflower_soybeans_NorthernCape' #'20241004_75_100_maize_sunflower_soybeans'
 # runtype is overwritten when called by manager_50_ope
 runType = 'fast_tuning' #'fast_tuning'     # this is fixed for tuning ['tuning', 'fast_tuning', 'opeForecast']
 

diff --git a/manager_20_tune.py b/manager_20_tune.py
@@ -156,6 +156,8 @@ def monitor_condor_q(time_step_minutes, submitter, config, run_name):
     config = a10_config.read(config_fn, run_name, run_type=runType)
     # forecastingMonths = config.forecastingMonths
     spec_files_list = tuner.tuneA(run_name, config_fn, tune_on_condor, runType)
+    # condor debug
+    spec_files_list = spec_files_list[0:500]
     tuner.tuneB(run_name, config_fn, tune_on_condor, runType, spec_files_list)
     if tune_on_condor:
         print('Condor runs launched, start the monitoring')