add alberta power

Author: eschechter

dweather_client/client.py

@@ -12,7 +12,7 @@
 from timezonefinder import TimezoneFinder
 from dweather_client import gridded_datasets
 from dweather_client.storms_datasets import IbtracsDataset, AtcfDataset, SimulatedStormsDataset
-from dweather_client.ipfs_queries import StationDataset, YieldDatasets, AemoPowerDataset, AemoGasDataset
+from dweather_client.ipfs_queries import StationDataset, YieldDatasets, AemoPowerDataset, AemoGasDataset, AesoPowerDataset
 from dweather_client.ipfs_errors import *
 import ipfshttpclient
 
@@ -238,9 +238,20 @@ def get_yield_history(commodity, state, county, dataset="sco-yearly", ipfs_timeo
 def get_power_history(ipfs_timeout=None):
     """
     return:
-        dict with datetime keys and values that are two member dicts with keys 'demand' and 'price'
+        dict with datetime keys and values that are dicts with keys 'demand' and 'price'
     """
     return AemoPowerDataset(ipfs_timeout=ipfs_timeout).get_data()
 
 def get_gas_history(ipfs_timeout=None):
+    """
+    return:
+        dict with date keys and float values
+    """
     return AemoGasDataset(ipfs_timeout=ipfs_timeout).get_data()
+
+def get_alberta_power_history(ipfs_timeout=None):
+    """
+    return:
+        dict with datetime keys and values that are dicts with keys 'price' 'ravg' and 'demand'
+    """
+    return AesoPowerDataset(ipfs_timeout=ipfs_timeout).get_data()

dweather_client/ipfs_queries.py

@@ -412,7 +412,7 @@ def get_data(self, commodity, state, county):
             file_name = f"{self.head}/{commodity}-{state}-{county}.csv"
         return self.get_file_object(file_name).read().decode("utf-8")
 
-class AemoDataset(IpfsDataset):
+class PowerDataset(IpfsDataset):
     """
     Abstract class from which all AEMO datasets inherit
     """
@@ -431,7 +431,10 @@ def get_date_range_from_metadata(self, h):
         return: list of [start_time, end_time]
         """
         metadata = self.get_metadata(h)
-        str_dates = (metadata["date_range"][0], metadata["date_range"][1])
+        try:
+            str_dates = (metadata["date_range"][0], metadata["date_range"][1])
+        except KeyError:
+             str_dates = (metadata["date range"][0], metadata["date range"][1])
         return [datetime.datetime.fromisoformat(dt) for dt in str_dates]
 
     def get_data(self):
@@ -444,7 +447,7 @@ def get_data(self):
             ret_dict = {**ret_dict, **new_dict}
         return ret_dict
 
-class AemoPowerDataset(AemoDataset):
+class AemoPowerDataset(PowerDataset):
     """
     Instantiable class for AEMO Victoria power data
     """
@@ -472,7 +475,7 @@ def extract_data_from_gz(self, date_range, ipfs_hash):
         return data_dict
 
 
-class AemoGasDataset(AemoDataset):
+class AemoGasDataset(PowerDataset):
     """
     Instantiable class for AEMO Victoria gas data
     """
@@ -494,3 +497,30 @@ def extract_data_from_gz(self, date_range, ipfs_hash):
                 data_dict[date_itr] = float(day_data)
                 date_itr = date_itr + datetime.timedelta(days=1)
         return data_dict
+
+class AesoPowerDataset(PowerDataset):
+    """
+    Instantiable class for AEMO Victoria gas data
+    """
+    @property
+    def dataset(self):
+        return "alberta_power-hourly"
+
+    @property
+    def data_file_name(self):
+        return "aeso_update.gz"
+
+    def extract_data_from_gz(self, date_range, ipfs_hash):
+        with gzip.open(self.get_file_object(f"{ipfs_hash}/{self.data_file_name}")) as gz:
+            cell_text = gz.read().decode('utf-8')
+        time_itr = date_range[0]
+        data_dict = {}
+        for year_data in cell_text.split('\n'):
+            for hour_data in year_data.split(','):
+                if not hour_data:
+                    price, ravg, demand = -9999, -9999, -9999
+                else:
+                    price, ravg, demand = hour_data.split("_")
+                data_dict[time_itr] = {"price": float(price), "ravg": float(ravg), "demand": float(demand)}
+                time_itr = time_itr + datetime.timedelta(hours=1)
+        return data_dict

dweather_client/tests/test_client.py

@@ -1,5 +1,5 @@
 from dweather_client.client import get_station_history, get_gridcell_history, get_tropical_storms,\
-    get_yield_history, get_power_history, get_gas_history, GRIDDED_DATASETS
+    get_yield_history, get_power_history, get_gas_history, get_alberta_power_history, GRIDDED_DATASETS
 from dweather_client.aliases_and_units import snotel_to_ghcnd
 import numpy as np
 import pandas as pd
@@ -142,6 +142,17 @@ def test_gas():
 
     assert date_diff.days + 1 == len(power_dict) 
 
+def test_aeso_power():
+    power_dict = get_alberta_power_history(ipfs_timeout=IPFS_TIMEOUT)
+    dict_length = len(power_dict) 
+    assert dict_length >= 188098
+
+    first_date, last_date = sorted(power_dict)[0], sorted(power_dict)[-1]
+    time_diff = last_date - first_date
+    time_diff_hours = time_diff.days * 24 + time_diff.seconds // 3600
+
+    assert time_diff_hours + 1 == len(power_dict) 
+
 ''' TODO some tests for RTMA behavior to be integrated into the new system
 def test_lat_lon_to_grid():
     heads = http_client.get_heads()