Test different HERON libraries

dependencies.xml

@@ -38,17 +38,17 @@ Note all install methods after "main" take
 <dependencies>
   <main>
     <h5py/>
-    <numpy>1.24</numpy>
-    <scipy>1.9</scipy>
-    <scikit-learn>1.0</scikit-learn>
+    <numpy>1.26</numpy>
+    <scipy>1.12</scipy>
+    <scikit-learn>1.1</scikit-learn>
     <pandas/>
     <!-- Note most versions of xarray work, but some (such as 0.20) don't -->
-    <xarray>2023</xarray>
+    <xarray/>
     <netcdf4 source="pip">1.6</netcdf4>
-    <matplotlib>3.5</matplotlib>
+    <matplotlib>3.6</matplotlib>
     <statsmodels>0.13</statsmodels>
     <cloudpickle/>
-    <tensorflow source="pip">2.13</tensorflow>
+    <tensorflow source="pip">2.14</tensorflow>
     <grpcio source="pip" />
     <!-- conda is really slow on windows if the version is not specified.-->
     <python skip_check='True' os='windows'>3.10</python>
@@ -66,14 +66,14 @@ Note all install methods after "main" take
     <nomkl os='linux' skip_check='True'/>
     <cmake skip_check='True' optional='True'/>
     <dask source="pip" pip_extra="[complete]"/>
-    <ray source="pip" pip_extra="[default]">2.6</ray>
+    <ray source="pip" pip_extra="[default]">2.38</ray>
     <!-- redis is needed by ray, but on windows, this seems to need to be explicitly stated -->
     <redis source="pip" os='windows'/>
     <imageio source="pip">2.22</imageio>
     <line_profiler optional='True'/>
     <!-- <ete3 optional='True'/> -->
     <statsforecast/>
-    <pywavelets>1.2</pywavelets>
+    <pywavelets>1.4</pywavelets>
     <python-sensors source="pip"/>
     <numdifftools source="pip">0.9</numdifftools>
     <fmpy optional='True'/>
@@ -83,7 +83,7 @@ Note all install methods after "main" take
     <ipopt skip_check='True' optional='True'/>
     <cyipopt optional='True'/>
     <pyomo-extensions source="pyomo" skip_check='True' optional='True'/>
-    <setuptools>69</setuptools> <!-- ray 2.6 can't be installed with setuptools 70 -->
+    <setuptools />
     <!-- source="mamba" are the ones installed when mamba is installed -->
     <!-- mamba version 2.0.0 causes failures on mac: critical libmamba filesystem error: in permissions: Operation not permitted-->
     <mamba source="mamba" skip_check='True' os='mac'>1.5</mamba>

ravenframework/DataObjects/HistorySet.py

@@ -275,7 +275,7 @@ def _toCSV(self,fileName,start=0,**kwargs):
       if startIndex > 0:
         data = self._data.isel(**{self.sampleTag:slice(startIndex,None,None)})
 
-    data = data.drop(toDrop)
+    data = data.drop_vars(toDrop)
     self.raiseADebug('Printing data to CSV: "{}"'.format(fileName+'.csv'))
     # specific implementation
     ## write input space CSV with pointers to history CSVs

ravenframework/Metrics/metrics/ScipyMetric.py

@@ -45,9 +45,6 @@ class ScipyMetric(MetricInterface):
   availMetrics['boolean']['dice']               = spatialDistance.dice
   availMetrics['boolean']['hamming']            = spatialDistance.hamming
   availMetrics['boolean']['jaccard']            = spatialDistance.jaccard
-  #Note in scipy 1.12 this needs to be changed to
-  #availMetrics['boolean']['kulsinski']           = spatialDistance.kulczynski1
-  availMetrics['boolean']['kulsinski']           = spatialDistance.kulsinski
   availMetrics['boolean']['russellrao']         = spatialDistance.russellrao
   availMetrics['boolean']['sokalmichener']      = spatialDistance.sokalmichener
   availMetrics['boolean']['sokalsneath']        = spatialDistance.sokalsneath

ravenframework/Models/PostProcessors/BasicStatistics.py

@@ -1257,6 +1257,7 @@ def getCovarianceSubset(desired):
         pivotCoords = reducedCovar.coords[self.pivotParameter].values
         ds = None
         for label, group in reducedCovar.groupby(self.pivotParameter):
+          group = group.squeeze()
           corrMatrix = self.corrCoeff(group.values)
           da = xr.DataArray(corrMatrix, dims=('targets','features'), coords={'targets':targCoords,'features':targCoords})
           ds = da if ds is None else xr.concat([ds,da], dim=self.pivotParameter)
@@ -1315,6 +1316,7 @@ def getCovarianceSubset(desired):
         pivotCoords = reducedCovar.coords[self.pivotParameter].values
         ds = None
         for label, group in reducedCovar.groupby(self.pivotParameter):
+          group = group.squeeze()
           da = self.varianceDepSenCalculation(targCoords,group.values)
           ds = da if ds is None else xr.concat([ds,da], dim=self.pivotParameter)
         ds.coords[self.pivotParameter] = pivotCoords

ravenframework/Models/PostProcessors/EconomicRatio.py

@@ -552,6 +552,7 @@ def __runLocal(self, inputData):
           if self.pivotParameter in targDa.sizes.keys():
             subCVaR = []
             for label, group in targDa.groupby(self.pivotParameter):
+              group = group.squeeze()
               sortedWeightsAndPoints, indexL = self._computeSortedWeightsAndPoints(group.values, targWeight,thd)
               quantile = self._computeWeightedPercentile(group.values, targWeight, needed[metric]['interpolation'], percent=[thd])[0]
               lowerPartialE = np.sum(sortedWeightsAndPoints[:indexL, 0]*sortedWeightsAndPoints[:indexL,1])

ravenframework/Models/PostProcessors/LimitSurface.py

@@ -410,9 +410,9 @@ def run(self, inputIn = None, returnListSurfCoord = False, exceptionGrid = None,
     if self.name != exceptionGrid:
       self.listSurfPointNegative, self.listSurfPointPositive = listSurfPoint[self.name][:nNegPoints-1],listSurfPoint[self.name][nNegPoints:]
     if merge == True:
-      evals = np.hstack(evaluations.values())
-      listSurfPoints = np.hstack(listSurfPoint.values())
-      surfPoint = np.hstack(self.surfPoint.values())
+      evals = np.hstack(tuple(evaluations.values()))
+      listSurfPoints = np.hstack(tuple(listSurfPoint.values()))
+      surfPoint = np.hstack(tuple(self.surfPoint.values()))
       returnSurface = (surfPoint, evals, listSurfPoints) if returnListSurfCoord else (surfPoint, evals)
     else:
       returnSurface = (self.surfPoint, evaluations, listSurfPoint) if returnListSurfCoord else (self.surfPoint, evaluations)

ravenframework/Models/PostProcessors/Validations/PPDSS.py

@@ -19,7 +19,7 @@
 #External Modules------------------------------------------------------------------------------------
 import numpy as np
 from scipy.interpolate import interp1d
-from scipy.integrate import simps
+from scipy.integrate import simpson
 import xarray as xr
 import os
 from collections import OrderedDict
@@ -360,7 +360,7 @@ def _evaluate(self, datasets, **kwargs):
           else:
             featureD[cnt2][i] = 0
       #
-      featureProcessAction = simps(featureIntNew, interpGridNew)
+      featureProcessAction = simpson(featureIntNew, x=interpGridNew)
       featureProcessTimeNorm[cnt2] = featureProcessTime/featureProcessAction
       featureOmegaNorm[cnt2] = featureProcessAction*featureOmega
     #
@@ -406,7 +406,7 @@ def _evaluate(self, datasets, **kwargs):
           else:
             targetD[cnt2][i] = 0
       #
-      targetProcessAction = simps(targetIntNew, interpGridNew)
+      targetProcessAction = simpson(targetIntNew, x=interpGridNew)
       targetProcessTimeNorm[cnt2] = targetProcessTime/targetProcessAction
       targetOmegaNorm[cnt2] = targetProcessAction*targetOmega
     #

ravenframework/SupervisedLearning/ARMA.py

@@ -1628,7 +1628,10 @@ def getFundamentalFeatures(self, requestedFeatures, featureTemplate=None):
           ## IND
           #most probabble index
           if len(group['Ind']):
-            modeInd = stats.mode(group['Ind'])[0][0]
+            try:
+              modeInd = stats.mode(group['Ind'])[0][0]
+            except IndexError:
+              modeInd = stats.mode(group['Ind'])[0]
           else:
             modeInd = 0
           ID = 'gp_{}_modeInd'.format(g)

ravenframework/SupervisedLearning/KerasBase.py

@@ -2348,7 +2348,10 @@ def __returnInitialParametersLocal__(self):
       @ In, None
       @ Out, params, dict,  dictionary of parameter names and initial values
     """
-    params = copy.deepcopy(self.__dict__)
+    selfDict = copy.copy(self.__dict__)
+    # labelEncoder can't be deepcopied so remove if it exists
+    selfDict.pop("labelEncoder", None)
+    params = copy.deepcopy(selfDict)
     return params
 
   def __returnCurrentSettingLocal__(self):

ravenframework/SupervisedLearning/ScikitLearn/ScikitLearnBase.py

@@ -212,7 +212,12 @@ def __evaluateLocal__(self,featureVals):
       except TypeError:
         outcomes = self.model.predict(featureVals)
     outcomes = np.atleast_1d(outcomes)
-    if len(outcomes.shape) == 1:
+    #possibilities for predict results are:
+    # (n_samples,) or (n_samples, n_targets)
+    if len(outcomes.shape) == 1 and len(self.target) == 1:
+      returnDict = {self.target[0]:outcomes}
+    elif len(outcomes.shape) == 1:
+      #this might only be possible for scikitlearn bugs
       returnDict = {key:value for (key,value) in zip(self.target,outcomes)}
     else:
       returnDict = {key: outcomes[:, i] for i, key in enumerate(self.target)}

rook/NumTextDiff.py

@@ -90,7 +90,7 @@ def check_output(self):
         same = False
       # if either file did not exist, clean up and go to next outfile
       if not same:
-        self.finalize_message(same, msg, test_filename)
+        self.finalize_message(same, msg, test_filename, gold_filename)
         continue
       cswf = DU.compare_strings_with_floats
       same, message = cswf(test_file.read(),
@@ -100,18 +100,21 @@ def check_output(self):
                            rel_err=self.__text_opts['rel_err'])
       if not same:
         msg.append(message)
-      self.finalize_message(same, msg, test_filename)
+      self.finalize_message(same, msg, test_filename, gold_filename)
     return self.__same, self.__message
 
 
-  def finalize_message(self, same, msg, filename):
+  def finalize_message(self, same, msg, test_filename, gold_filename):
     """
       Compiles useful messages to print, prepending with file paths.
       @ In, same, bool, True if files are the same
       @ In, msg, list(str), messages that explain differences
-      @ In, filename, str, test filename/path
+      @ In, test_filename, str, test filename/path
+      @ In, gold_filename, str, gold filename/path
       @ Out, None
     """
     if not same:
       self.__same = False
-      self.__message += '\nDIFF in {}: \n  {}'.format(filename, '\n  '.join(msg))
+      self.__message += '\nDIFF in {} and\n{}: \n  {}'.format(test_filename,
+                                                              gold_filename,
+                                                              '\n  '.join(msg))

rook/OrderedCSVDiffer.py

@@ -68,17 +68,20 @@ def __init__(self, out_files, gold_files, relative_error=1e-10,
       print('abs check:', self.__check_absolute_values)
       print('zero thr :', self.__zero_threshold)
 
-  def finalize_message(self, same, msg, filename):
+  def finalize_message(self, same, msg, test_filename, gold_filename):
     """
       Compiles useful messages to print, prepending with file paths.
       @ In, same, bool, True if files are the same
       @ In, msg, list(str), messages that explain differences
-      @ In, filename, str, test filename/path
+      @ In, test_filename, str, test filename/path
+      @ In, gold_filename, str, gold filename/path
       @ Out, None
     """
     if not same:
       self.__same = False
-      self.__message += '\nDIFF in {}: \n  {}'.format(filename, '\n  '.join(msg))
+      self.__message += '\nDIFF in {} and\n{}: \n  {}'.format(test_filename,
+                                                              gold_filename,
+                                                              '\n  '.join(msg))
 
   def matches(self, a_obj, b_obj, is_number, tol):
     """
@@ -143,7 +146,7 @@ def diff(self):
         same = False
       # if either file did not exist, clean up and go to next outfile
       if not same:
-        self.finalize_message(same, msg, test_filename)
+        self.finalize_message(same, msg, test_filename, gold_filename)
         continue
       # at this point, we've loaded both files (even if they're empty), so compare them.
       ## first, cover the case when both files are empty.
@@ -157,14 +160,14 @@ def diff(self):
       if len(diff_columns) > 0:
         same = False
         msg.append('Columns are not the same! Different: {}'.format(', '.join(diff_columns)))
-        self.finalize_message(same, msg, test_filename)
+        self.finalize_message(same, msg, test_filename, gold_filename)
         continue
       ## check index length
       if len(gold_rows) != len(test_rows):
         same = False
         msg.append('Different number of entires in Gold ({}) versus Test ({})!'
                    .format(len(gold_rows), len(test_rows)))
-        self.finalize_message(same, msg, test_filename)
+        self.finalize_message(same, msg, test_filename, gold_filename)
         continue
       ## at this point both CSVs have the same shape, with the same header contents.
       ## figure out column indexs
@@ -179,6 +182,12 @@ def diff(self):
       for idx in range(1, len(gold_rows)):
         gold_row = gold_rows[idx]
         test_row = test_rows[idx]
+        if len(gold_row) != len(test_row):
+          same = False
+          msg.append("Different row lengths"+
+                     f" {len(gold_row)} != {len(test_row)} "+
+                     f" in {gold_row} and {test_row}")
+          continue
         for column in range(len(gold_row)):
           gold_value = to_float(gold_row[column])
           test_value = to_float(test_row[test_indexes[column]])
@@ -201,7 +210,7 @@ def diff(self):
         msg.append('| Difference | statistics:')
         msg.append('  MEAN    diff.: {:1.9e}'.format(sum(diffs)/float(len(diffs))))
         msg.append('  LARGEST diff.: {:1.9e}'.format(max(diffs)))
-      self.finalize_message(same, msg, test_filename)
+      self.finalize_message(same, msg, test_filename, gold_filename)
     return self.__same, self.__message
 
 

tests/framework/Optimizers/BayesianOptimizer/Matyas/BayesianMatyas.xml

@@ -68,7 +68,7 @@
       </variable>
       <TargetEvaluation class="DataObjects" type="PointSet">optOut</TargetEvaluation>
       <samplerInit>
-        <limit>50</limit>
+        <limit>100</limit>
         <initialSeed>42</initialSeed>
         <writeSteps>every</writeSteps>
       </samplerInit>
@@ -80,7 +80,7 @@
       </ModelSelection>
       <convergence>
         <acquisition>1e-8</acquisition>
-        <persistence>5</persistence>
+        <persistence>6</persistence>
       </convergence>
       <Acquisition>
         <ProbabilityOfImprovement>

tests/framework/PostProcessors/Metrics/gold/scipyMetricsBoolean/pp1_print.csv

@@ -1,2 +1,2 @@
-rogerstanimoto_ans2_ans,dice_ans2_ans,hamming_ans2_ans,jaccard_ans2_ans,kulsinski_ans2_ans,russellrao_ans2_ans,sokalmichener_ans2_ans,sokalsneath_ans2_ans,yule_ans2_ans
-0.666666666667,0.384615384615,0.5,0.555555555556,0.733333333333,0.6,0.666666666667,0.714285714286,1.2
+rogerstanimoto_ans2_ans,dice_ans2_ans,hamming_ans2_ans,jaccard_ans2_ans,russellrao_ans2_ans,sokalmichener_ans2_ans,sokalsneath_ans2_ans,yule_ans2_ans
+0.666666666667,0.384615384615,0.5,0.555555555556,0.6,0.666666666667,0.714285714286,1.2

tests/framework/PostProcessors/Metrics/test_metric_scipy_boolean.xml

@@ -31,7 +31,6 @@
       <Metric class="Metrics" type="Metric">dice</Metric>
       <Metric class="Metrics" type="Metric">hamming</Metric>
       <Metric class="Metrics" type="Metric">jaccard</Metric>
-      <Metric class="Metrics" type="Metric">kulsinski</Metric>
       <Metric class="Metrics" type="Metric">russellrao</Metric>
       <Metric class="Metrics" type="Metric">sokalmichener</Metric>
       <Metric class="Metrics" type="Metric">sokalsneath</Metric>
@@ -50,7 +49,6 @@
             dice_ans2_ans,
             hamming_ans2_ans,
             jaccard_ans2_ans,
-            kulsinski_ans2_ans,
             russellrao_ans2_ans,
             sokalmichener_ans2_ans,
             sokalsneath_ans2_ans,
@@ -88,9 +86,6 @@
     <Metric name="jaccard" subType="ScipyMetric">
       <metricType>boolean|jaccard</metricType>
     </Metric>
-    <Metric name="kulsinski" subType="ScipyMetric">
-      <metricType>boolean|kulsinski</metricType>
-    </Metric>
     <Metric name="russellrao" subType="ScipyMetric">
       <metricType>boolean|russellrao</metricType>
     </Metric>

tests/framework/PostProcessors/TSACharacterizer/basic.xml

@@ -25,6 +25,7 @@
       <Model class="Models" type="PostProcessor">tsa_chz</Model>
       <Output class="DataObjects" type="PointSet">chz</Output>
       <Output class="OutStreams" type="Print">chz</Output>
+      <Output class="OutStreams" type="Print">chz_full</Output>
     </PostProcess>
   </Steps>
 
@@ -49,6 +50,11 @@
     <Print name="chz">
       <type>csv</type>
       <source>chz</source>
+      <what>metadata|ARMA__signal_fa__AR__0,metadata|ARMA__signal_fa__MA__0,metadata|ARMA__signal_a__variance,metadata|Fourier__signal_f__period2.0__amplitude,metadata|ARMA__signal_a__MA__0,metadata|ARMA__signal_a__MA__2,metadata|Fourier__signal_f__fit_intercept,metadata|Fourier__signal_fa__period10.0__amplitude,metadata|Fourier__signal_f__period5.0__amplitude,metadata|Fourier__signal_fa__period2.0__amplitude,metadata|Fourier__signal_f__period5.0__phase,metadata|ARMA__signal_fa__variance,metadata|Fourier__signal_fa__period10.0__phase,metadata|ARMA__signal_fa__AR__1,metadata|Fourier__signal_fa__period2.0__phase,metadata|ARMA__signal_a__MA__1,metadata|ARMA__signal_fa__constant,metadata|Fourier__signal_f__period2.0__phase,metadata|ARMA__signal_a__constant,metadata|Fourier__signal_fa__period5.0__amplitude,metadata|ARMA__signal_fa__MA__1,metadata|ARMA__signal_a__AR__1,metadata|ARMA__signal_fa__MA__2,metadata|ARMA__signal_a__AR__0,metadata|Fourier__signal_f__period10.0__amplitude,metadata|Fourier__signal_fa__fit_intercept,metadata|Fourier__signal_fa__period5.0__phase</what>
+    </Print>
+    <Print name="chz_full">
+      <type>csv</type>
+      <source>chz</source>
     </Print>
   </OutStreams>