Add tests: storage level processing in marketmodel

pomato/data/results.py

@@ -1,15 +1,16 @@
 import json
 import logging
-
-import numpy as np
-import pandas as pd
+import shutil
 import types
 from copy import deepcopy
 from zipfile import ZipFile
 
+import numpy as np
+import pandas as pd
 import pomato.tools as tools
 from pomato.visualization.geoplot_functions import line_coordinates
 
+
 class Results():
     """Results of POMATO makes market results available to the user.
 
@@ -197,6 +198,15 @@ def read_cached_result(self, name):
             df = pd.read_feather(file).set_index("index")
             return df
 
+    def _clear_cached_results(self):
+        self._cached_results = {}
+        self._cached_result_structs = types.SimpleNamespace(
+            result_data=None,
+            averaged_result_data=None,
+        )
+        folder = self.wdir.joinpath("data_temp/results_cache").joinpath(str(id(self)))
+        shutil.rmtree(folder, ignore_errors=True)
+
     def result_data_struct(self):
         """Data struct, as a standart template for result processing.
         

pomato/market_model/market_model.py

@@ -77,19 +77,21 @@ def _start_julia_daemon(self):
         """Start julia subprocess."""
         self.julia_model = tools.JuliaDaemon(self.logger, self.wdir, self.package_dir, "market_model")
 
+    @property
+    def model_horizon(self):
+        model_horizon_range = range(self.options["model_horizon"][0],
+                                    self.options["model_horizon"][1])
+        timesteps = self.data.demand_el.timestep.unique()
+        model_horizon = [str(x) for x in timesteps[model_horizon_range]]
+        return model_horizon
+
     def update_data(self):
         """Initialise or update the underlying data of the market model.
 
         Updates all data used to run the market model: input data, grid representation, options and
         model horizon by running :meth:`~data_to_csv`.
         """
-
-        model_horizon_range = range(self.options["model_horizon"][0],
-                                    self.options["model_horizon"][1])
-
-        timesteps = self.data.demand_el.timestep.unique()
-        model_horizon = [str(x) for x in timesteps[model_horizon_range]]
-        self.data_to_csv(model_horizon)
+        self.data_to_csv()
 
     def run(self):
         """Run the julia program via command Line.
@@ -138,20 +140,17 @@ def run(self):
             self.logger.warning("Process not terminated successfully!")
             self.status = 'error'
 
-    def save_rolling_horizon_storage_levels(self, model_horizon):
+    def save_rolling_horizon_storage_levels(self):
         """Set start/end storage levels for rolling horizon market clearing.
         
         This method alters the *storage_level* attribute of the :class:`~pomato.data.DataManagement`
         instance, when market model horizon shorter than total model horizon. Per default storage levels 
         are 65% of total capacity, however the storage_level attribute can be edited manually. 
         
-        Parameters
-        ----------
-        model_horizon : list
-            List of timesteps that are the model horizon
         """
-
-        splits = int(len(model_horizon)/self.options["timeseries"]["market_horizon"])
+
+        model_horizon = self.model_horizon
+        splits = round(len(model_horizon)/self.options["timeseries"]["market_horizon"])
         market_model_horizon = self.options["timeseries"]["market_horizon"]
         splits_start = [model_horizon[t*market_model_horizon] for t in range(0, splits)]
 
@@ -177,7 +176,7 @@ def add_to_timestep(t):
         data = []
         for plant in self.data.plants[self.data.plants.plant_type.isin(self.options["plant_types"]["es"])].index:
             for t_start, t_end in zip(splits_start, splits_end):
-                if ((t_split_map[t_start], plant) in storage_level.index) and (self.data.plants.loc[plant, "plant_type"] == "hydro_res"):
+                if ((t_split_map[t_start], plant) in storage_level.index): # and (self.data.plants.loc[plant, "plant_type"] == "hydro_res"):
                     data.append([t_start, plant, 
                                 storage_level.loc[(t_split_map[t_start], plant), "storage_level"],
                                 storage_level.loc[(t_split_map[t_end], plant), "storage_level"]])
@@ -208,11 +207,9 @@ def add_to_timestep(t):
                     tmp_storage_level.loc[condition, "storage_start"] = tmp_storage_level.loc[condition, "storage_start"].rolling(window, min_periods=1).mean()
                     tmp_storage_level.loc[condition, "storage_end"] = tmp_storage_level.loc[condition, "storage_end"].rolling(window, min_periods=1).mean()
 
-        # tmp store for debug and then save into data folder for MarketModel
-        self.storage_levels_used = tmp_storage_level
-        self.storage_levels_used.to_csv(str(self.data_dir.joinpath('storage_level.csv')), index_label='index')
+        return tmp_storage_level
 
-    def data_to_csv(self, model_horizon):
+    def data_to_csv(self):
         """Export input data to csv files in the data_dir sub-directory.
 
         Writes all data specified in the *model structure* attribute of DataManagement to csv.
@@ -225,10 +222,12 @@ def data_to_csv(self, model_horizon):
         model_horizon : list
             List of timesteps that are the model horizon
         """
+
         if not self.data_dir.is_dir():
             self.data_dir.mkdir()
-
-
+
+        model_horizon = self.model_horizon
+
         for data in [d for d in self.data.model_structure]:
             cols = [col for col in self.data.model_structure[data].keys() if col != "index"]
             if "timestep" in cols:
@@ -237,7 +236,8 @@ def data_to_csv(self, model_horizon):
             else:
                 getattr(self.data, data)[cols].to_csv(str(self.data_dir.joinpath(f'{data}.csv')), index_label='index')
 
-        self.save_rolling_horizon_storage_levels(model_horizon)
+        storage_level = self.save_rolling_horizon_storage_levels()
+        storage_level.to_csv(str(self.data_dir.joinpath('storage_level.csv')), index_label='index')
 
         plant_types = pd.DataFrame(index=self.data.plants.plant_type.unique())
         for ptype in self.options["plant_types"]:

pomato/visualization/visualization.py

@@ -223,8 +223,6 @@ def create_geo_plot(self, market_result, show_redispatch=False, show_prices=Fals
             hovertemplate_lines.replace("<br>N-0 Flow %{customdata[2]:.2f} MW<br>N-1 Flow %{customdata[3]:.2f} MW", "")
         if isinstance(highlight_lines, list) and len(highlight_lines) > 0:
             lines["alpha"] = [1 if l in highlight_lines else 0.2 for l in lines.index]
-
-
 
         # Add Lines for each color
         line_coords = np.array(result_data.line_coordinates)

tests/test_data.py

@@ -163,7 +163,8 @@ def setUpClass(cls):
         cls.market_result, cls.redispatch_result = mato.data.return_results()
 
     def setUp(self):
-        pass
+        self.market_result._clear_cached_results()
+        self.redispatch_result._clear_cached_results()
 
     @classmethod
     def tearDownClass(cls):

tests/test_grid_model.py

@@ -41,7 +41,7 @@ def setUpClass(cls):
         cls.grid = pomato.grid.GridTopology()
         cls.grid.calculate_parameters(cls.data.nodes, cls.data.lines)
         cls.grid_model = pomato.grid.GridModel(cls.wdir, cls.grid, cls.data, cls.options)
-        cls.grid_model.logger.setLevel(logging.INFO)
+        cls.grid_model.logger.setLevel(logging.ERROR)
 
     def setUp(self):
         pass

tests/test_market.py

@@ -33,7 +33,7 @@ def setUpClass(cls):
         pomato.tools.create_folder_structure(cls.wdir)
         cls.options = pomato.tools.default_options()
         cls.data = pomato.data.DataManagement(cls.options, cls.wdir)
-        cls.data.load_data('data_input/pglib_opf_case118_ieee.m')
+        cls.data.load_data('data_input/nrel_118.zip')
         cls.grid = pomato.grid.GridTopology()
         cls.grid.calculate_parameters(cls.data.nodes, cls.data.lines)
         cls.grid_model = pomato.grid.GridModel(cls.wdir, cls.grid, cls.data, cls.options)
@@ -74,3 +74,37 @@ def test_market_model_missing_result(self):
         self.grid_model.create_grid_representation()
         self.market_model.update_data()
         self.assertRaises(FileNotFoundError, self.market_model.run)
+
+
+    def test_market_model_rolling_horizon_storage_levels(self):
+
+        self.options["timeseries"]["market_horizon"] = 20
+        self.options["model_horizon"] = [0, 168]
+        self.options["plant_types"]["es"] = ["es"]
+        model_horizon = self.market_model.model_horizon
+        # Add Storage Plant 
+        es = [["bus026", 0, 100, "es", "water", 0, 0, 100, 1, 1, None, 1000]]
+        self.data.plants = self.data.plants.append(
+            pd.DataFrame(index=["es1"], 
+            columns=self.data.plants.columns, 
+            data=es)
+        ) 
+        self.data.storage_level = pd.DataFrame(
+            columns=self.data.storage_level.columns,
+            data=[["t0001", "es1", 0.5], ["t0168", "es1", 1]]
+        )
+
+        # print("asd")
+        storage_level = self.market_model.save_rolling_horizon_storage_levels()
+        self.assertEqual(
+            len(storage_level), 
+            round(self.options["model_horizon"][1]/self.options["timeseries"]["market_horizon"])
+        )
+        self.assertTrue(all([s in [0.5, 1] for s in storage_level.storage_start]))
+        self.assertTrue(all([s in [0.5, 1] for s in storage_level.storage_end]))
+
+        self.options["timeseries"]["smooth_storage_level"] = True
+        storage_level = self.market_model.save_rolling_horizon_storage_levels()
+
+        self.assertFalse(all([s in [0.5, 1] for s in storage_level.storage_start]))
+        self.assertFalse(all([s in [0.5, 1] for s in storage_level.storage_end]))