Merge pull request dClimate#82 from dClimate/national-grid

national-grid retrieval code

Author: PrettyHertz

dweather_client/client.py

@@ -19,7 +19,7 @@
 from dweather_client import gridded_datasets
 from dweather_client.storms_datasets import IbtracsDataset, AtcfDataset, SimulatedStormsDataset
 from dweather_client.ipfs_queries import AustraliaBomStations, CedaBiomass, CmeStationsDataset, DutchStationsDataset, DwdStationsDataset, DwdHourlyStationsDataset, GlobalHourlyStationsDataset, JapanStations, StationDataset, EauFranceDataset,\
-    YieldDatasets, FsaIrrigationDataset, AemoPowerDataset, AemoGasDataset, AesoPowerDataset, ForecastDataset, AfrDataset, DroughtMonitor, CwvStations, SpeedwellStations, TeleconnectionsDataset, CsvStationDataset, StationForecastDataset
+    YieldDatasets, FsaIrrigationDataset, AemoPowerDataset, AemoGasDataset, AesoPowerDataset, ForecastDataset, AfrDataset, DroughtMonitor, CwvStations, SpeedwellStations, TeleconnectionsDataset, CsvStationDataset, StationForecastDataset, SapStations
 from dweather_client.slice_utils import DateRangeRetriever, has_changed
 from dweather_client.ipfs_errors import *
 from io import StringIO
@@ -771,6 +771,19 @@ def get_cwv_station_history(station_name, as_of=None, ipfs_timeout=None):
     return (resp_series * u.dimensionless_unscaled).to_dict()
 
 
+def get_sap_station_history(as_of=None, ipfs_timeout=None):
+    """
+    return:
+        dict with datetime keys and sap Quantities as values
+    """
+    metadata = get_metadata(get_heads()["sap-daily"])
+    with SapStations(ipfs_timeout=ipfs_timeout, as_of=as_of) as dataset_obj:
+        str_resp_series = dataset_obj.get_data()
+    resp_series = str_resp_series.astype(float)
+    # SAP uses financial units, best to return unscaled
+    return (resp_series * u.dimensionless_unscaled).to_dict()
+
+
 def get_australia_station_history(station_name, weather_variable, desired_units=None, as_of=None, ipfs_timeout=None):
     """
     return:
@@ -875,7 +888,7 @@ def get_teleconnections_history(weather_variable, ipfs_timeout=None):
         headers = next(reader)
         date_col = headers.index('DATE')
         try:
-            data_col=headers.index("value")
+            data_col = headers.index("value")
         except ValueError:
             raise WeatherVariableNotFoundError(
                 "Invalid weather variable for this station")

dweather_client/ipfs_queries.py

@@ -926,6 +926,43 @@ def extract_data_from_text(self, date_range, ipfs_hash, station_name):
         return data_dict
 
 
+class SapStations(GriddedDataset):
+    """
+    Instantiable class for Composite Weather Variable Station Data
+    """
+    @property
+    def dataset(self):
+        return "sap-daily"
+
+    @property
+    def data_file_format(self):
+        """
+        format string requires station name eg 'EM'
+        """
+        return "sap_update_UK.txt"
+
+    def get_data(self):
+        super().get_data()
+        hashes = self.get_hashes()
+        ret_dict = {}
+        for h in hashes:
+            date_range = self.get_date_range_from_metadata(h)
+            new_dict = self.extract_data_from_text(date_range, h)
+            ret_dict = {**ret_dict, **new_dict}
+        return pd.Series(ret_dict)
+
+    def extract_data_from_text(self, date_range, ipfs_hash):
+        byte_obj = self.get_file_object(
+            f"{ipfs_hash}/{self.data_file_format}")
+        data = byte_obj.read().decode("utf-8").split(",")
+        day_itr = date_range[0].date()
+        data_dict = {}
+        for point in data:
+            data_dict[day_itr] = point
+            day_itr += datetime.timedelta(days=1)
+        return data_dict
+
+
 class AustraliaBomStations(GriddedDataset):
     """
     Instantiable class for Australia BOM Data
@@ -1250,7 +1287,7 @@ def __init__(self, ipfs_timeout=None):
     def get_data(self, station):
         super().get_data()
         metadata = self.get_metadata(self.head)
-        
+
         file_name = f"{self.head}/{station}.csv"
         return self.get_file_object(file_name).read().decode("utf-8")
 

dweather_client/tests/test_client.py

@@ -3,7 +3,7 @@
 from dweather_client.client import get_australia_station_history, get_station_history, get_gridcell_history, get_tropical_storms,\
     get_yield_history, get_irrigation_data, get_power_history, get_gas_history, get_alberta_power_history, GRIDDED_DATASETS, has_dataset_updated,\
     get_forecast_datasets, get_forecast, get_cme_station_history, get_european_station_history, get_hourly_station_history, get_drought_monitor_history, get_japan_station_history,\
-    get_afr_history, get_cwv_station_history, get_teleconnections_history, get_station_forecast_history, get_station_forecast_stations, get_eaufrance_history
+    get_afr_history, get_cwv_station_history, get_teleconnections_history, get_station_forecast_history, get_station_forecast_stations, get_eaufrance_history, get_sap_station_history
 from dweather_client.aliases_and_units import snotel_to_ghcnd
 import pandas as pd
 from io import StringIO
@@ -362,7 +362,7 @@ def test_historical_storms():
 
 def test_hist_storm_as_of():
     df_all_na = get_tropical_storms(
-        'historical', 'NA', as_of=datetime.date(2023, 5, 20), ipfs_timeout=IPFS_TIMEOUT) # This will throw an exception if there's ever a break in the chain
+        'historical', 'NA', as_of=datetime.date(2023, 5, 20), ipfs_timeout=IPFS_TIMEOUT)  # This will throw an exception if there's ever a break in the chain
 
 
 def test_yields():
@@ -398,6 +398,12 @@ def test_cwv():
     assert data[sorted(data)[0]].unit == u.dimensionless_unscaled
 
 
+def test_sap():
+    data = get_sap_station_history(ipfs_timeout=IPFS_TIMEOUT)
+    assert len(data) == (sorted(data)[-1] - sorted(data)[0]).days + 1
+    assert data[sorted(data)[0]].unit == u.dimensionless_unscaled
+
+
 def test_australia():
     data = get_australia_station_history(
         "Adelaide Airport", weather_variable="TMAX", ipfs_timeout=IPFS_TIMEOUT)
@@ -489,13 +495,17 @@ def test_has_dataset_updated_false():
 
 
 def test_forecast_station_history():
-    history = get_station_forecast_history("cme_futures-daily", "D2", datetime.date(2023, 1, 31))
+    history = get_station_forecast_history(
+        "cme_futures-daily", "D2", datetime.date(2023, 1, 31))
     assert history[datetime.date(2023, 2, 28)] == 369.0
 
+
 def test_forecast_station_stations():
-    stations = get_station_forecast_stations("cme_futures-daily", datetime.date(2023, 1, 31))
+    stations = get_station_forecast_stations(
+        "cme_futures-daily", datetime.date(2023, 1, 31))
     assert stations["features"][0]["properties"]["station name"] == "D2"
 
+
 def test_eaufrance_station():
     history = get_eaufrance_history("V720001002", "FLOWRATE")
-    assert history[datetime.date(2022,4,2)].value == 749
+    assert history[datetime.date(2022, 4, 2)].value == 749