Merge pull request dClimate#34 from dClimate/hotfix/simulated-storm-col

add sim column to simulated trop storms

Author: eschechter

dweather_client/storms_datasets.py

@@ -85,14 +85,19 @@ def get_data(self, basin, **kwargs):
 
         if basin not in {'EP', 'NA', 'NI', 'SI', 'SP', 'WP'}:
             raise ValueError("Invalid basin ID")
-        metadata  = self.get_metadata(self.head)
-
-        files = [self.get_file_object(f"{self.head}/{f}") for f in metadata['files'] if basin in f]
-        dfs = [pd.read_csv(f, header=None, compression="gzip")[range(10)] for f in files]
 
-        df = pd.concat(dfs).reset_index(drop=True)
-        columns = ['year', 'month', 'tc_num', 'time_step', 'basin', 'lat', 'lon', 'min_press', 'max_wind', 'rmw']
-        df.columns = columns
-        df.loc[df.lon > 180, 'lon'] = df.lon - 360
-
-        return process_df(df, **kwargs)
+        metadata  = self.get_metadata(self.head)
+        dfs = []
+        for f in metadata["files"]:
+            if basin in f:
+                file_obj = self.get_file_object(f"{self.head}/{f}")
+                df = pd.read_csv(file_obj, header=None, compression="gzip")[range(10)]
+                columns = ['year', 'month', 'tc_num', 'time_step', 'basin', 'lat', 'lon', 'min_press', 'max_wind', 'rmw']
+                df.columns = columns
+                df["sim"] = f[-8]
+                dfs.append(df)
+
+        big_df = pd.concat(dfs).reset_index(drop=True)
+        big_df.loc[big_df.lon > 180, 'lon'] = big_df.lon - 360
+
+        return process_df(big_df, **kwargs)

dweather_client/tests/test_client.py

@@ -2,7 +2,6 @@
 from dweather_client.client import get_station_history, get_gridcell_history, get_tropical_storms,\
     get_yield_history, get_power_history, get_gas_history, get_alberta_power_history, GRIDDED_DATASETS
 from dweather_client.aliases_and_units import snotel_to_ghcnd
-import numpy as np
 import pandas as pd
 from io import StringIO
 import datetime
@@ -90,7 +89,7 @@ def test_simulated_storms():
     df_subset_circle_ni = get_tropical_storms('simulated', 'NI', radius=500, lat=21, lon=65, ipfs_timeout=IPFS_TIMEOUT)
     df_subset_box_ni = get_tropical_storms('simulated', 'NI', min_lat=21, max_lat=22, min_lon=65, max_lon=66, ipfs_timeout=IPFS_TIMEOUT)
 
-    assert len(df_all_ni.columns) == len(df_subset_circle_ni.columns) == len(df_subset_box_ni.columns) == 10
+    assert len(df_all_ni.columns) == len(df_subset_circle_ni.columns) == len(df_subset_box_ni.columns) == 11
     assert len(df_subset_circle_ni) < len(df_all_ni)
     assert len(df_subset_box_ni) < len(df_all_ni)