Finetuning REANALYSS procedures

RENCI · Jan 16, 2022 · 366b042 · 366b042
1 parent 955c784
commit 366b042
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 14 deletions.
diff --git a/compute_error_field/computeErrorField.py b/compute_error_field/computeErrorField.py
@@ -63,7 +63,9 @@ def __init__(self, obsf, adcf, meta, inputcfg=None, yamlname=os.path.join(os.pat
             raise IOError("Failed to read %s" % self.meta_filename)
         utilities.log.debug('input ADC wl data are {}'.format(str(self.df_adc_wl)))
         utilities.log.debug('input OBS wl data are {}'.format(str(self.df_obs_wl)))
-        #print(self.df_adc_wl)
+
+        #print('ADCIRC temp')
+        #print(self.df_adc_wl[8536110])
         #print(self.df_obs_wl)
 
     def _intersectionStations(self):

diff --git a/reanalysis/runInterpolate_parallel_hsofs.sh b/reanalysis/runInterpolate_parallel_hsofs.sh
@@ -13,21 +13,18 @@ OBSNAME="/projects/sequence_analysis/vol1/prediction_work/ADCIRCSupportTools/ADC
 export CODEBASE=/projects/sequence_analysis/vol1/prediction_work/ADCIRCSupportTools/ADCIRCSupportTools/reanalysis
 export PYTHONPATH=/projects/sequence_analysis/vol1/prediction_work/ADCIRCSupportTools/ADCIRCSupportTools
 export BASEDIREXTRA=
-
 export KNOCKOUT=/projects/sequence_analysis/vol1/prediction_work/ADCIRCSupportTools/ADCIRCSupportTools/reanalysis/knockoutStation.json
 
 ###
 export YEAR=$1
 
-export RUNTIMEDIR=./HSOFS-DA/YEARLY-$YEAR
+export RUNTIMEDIR=./HSOFS/YEARLY-$YEAR
+#export RUNTIMEDIR=./HSOFS-DA/YEARLY-$YEAR
 export LOG_PATH=$RUNTIMEDIR
-URL="/projects/reanalysis/ADCIRC/ERA5/hsofs/$YEAR-post/fort.63.nc"
-#URL="/projects/reanalysis/ADCIRC/ERA5/hsofs/$YEAR/fort.63.nc"
-
-#URL="C/ERA5/hsofs/$YEAR/fort.63.nc"
+#URL="/projects/reanalysis/ADCIRC/ERA5/hsofs/$YEAR-post/fort.63.nc"
+URL="/projects/reanalysis/ADCIRC/ERA5/hsofs/$YEAR/fort.63.nc"
 
-#DAILY=DAILY-2018YEAR-12MONTH-REGION3-RANGE$RANGE-SILL$SILL-NUGGET$NUGGET-LP48
-DAILY=TEST_DAILY-$GRID-LP24
+DAILY=DAILY-$GRID-LP24
 
 echo "xxxxxx"
 echo $YEAR
@@ -36,23 +33,25 @@ echo $RUNTIMEDIR
 echo "xxxxxx"
 
 ####
-
 # Build the yearly error file store in $RUNTIMEDIR/BASEDIREXTRA
 python $CODEBASE/yearlyReanalysisRoundHourly.py --obsfile $OBSNAME --grid $GRID --url $URL --knockout $KNOCKOUT
-mv $RUNTIMEDIR/AdcircSupportTools.log $RUNTIMEDIR/$BASEDIREXTRA/log-yearly
+#mv $RUNTIMEDIR/AdcircSupportTools.log $RUNTIMEDIR/$BASEDIREXTRA/log-yearly
 
 # Store files in $RUNTIMEDIR/DAILY/errorfield
 export INDIR=$RUNTIMEDIR/
 export OUTROOT=$RUNTIMEDIR/$DAILY
-python $CODEBASE/dailyLowpassSampledError_hsofs.py --inyear $YEAR  --inDir $INDIR --outroot $OUTROOT # --stationarity
+python $CODEBASE/dailyLowpassSampledError.py --inyear $YEAR  --inDir $INDIR --outroot $OUTROOT # --stationarity
 mv $RUNTIMEDIR/AdcircSupportTools.log $OUTROOT/log-daily
 
 # Interpolate a single specific file
 export ADCJSON=$INDIR/adc_coord.json
-export CLAMPFILE=$PYTHONPATH/config/clamp_list_hsofs.dat
+export CLAMPFILE=$PYTHONPATH/config/water_control_list.dat
+export CONTROLFILE=$PYTHONPATH/config/land_control_list.dat
+
 export YAMLNAME=$PYTHONPATH/config/int.REANALYSIS.HSOFS.yml
 export OUTROOT=$RUNTIMEDIR/$DAILY
 export ERRDIR=$OUTROOT/errorfield
-python  $CODEBASE/runInterpolate_parallel.py  --outroot $OUTROOT --yamlname $YAMLNAME --errordir $ERRDIR --clampfile $CLAMPFILE --gridjsonfile $ADCJSON
+python  $CODEBASE/runInterpolate_parallel.py  --outroot $OUTROOT --yamlname $YAMLNAME --errordir $ERRDIR --clampfile $CLAMPFILE --controlfile $CONTROLFILE --gridjsonfile $ADCJSON
 mv $RUNTIMEDIR/AdcircSupportTools.log $OUTROOT/log-interpolate
 
+
diff --git a/reanalysis/yearlyReanalysisRoundHourly.py b/reanalysis/yearlyReanalysisRoundHourly.py
@@ -83,6 +83,8 @@ def exec_adcirc_url(urls, rootdir, iometadata, adc_yamlname, node_idx, station_i
     df.index = df.index.round('60min').to_pydatetime()
     adc.T1 = df.index[0] # Optional update to actual times fetched form ADC
     adc.T2 = df.index[-1]
+    df.dropna(how='all', axis=1)
+    utilities.log.info('ADCIRC: after drop completely empty stations. Remaining is {}'.format(df.shape))
     ADCfile = utilities.writePickle(df, rootdir=rootdir,subdir='',fileroot='adc_wl',iometadata=iometadata)
     print('write new json')
     ADCjson=writeToJSON(df, rootdir, iometadata,fileroot='adc_wl_forecast')