Merge pull request dClimate#27 from dClimate/mock-tests

mock out slow ipfs calls

Author: eschechter

dweather_client/df_loader.py

@@ -0,0 +1,24 @@
+from dweather_client.client import GRIDDED_DATASETS
+import pickle
+import os
+
+def constructor(self, ipfs_timeout):
+    pass
+
+def get_data(self, lat, lon):
+    to_open = os.path.join(os.path.dirname(__file__), "etc", f"{self.dataset}_{lat}_{lon}.p")
+    with open(to_open, "rb") as f:
+        return pickle.load(f)
+
+def get_patched_datasets():
+    patched_datasets = {}
+    for k in GRIDDED_DATASETS:
+        old_class = GRIDDED_DATASETS[k]
+        new_class = type(old_class.__name__, (object, ), {
+            "dataset": k,
+            "__init__": constructor,
+            "get_data": get_data
+        })
+        patched_datasets[k] = new_class
+    return patched_datasets
+        

dweather_client/tests/etc/chirpsc_final_05-daily_37_-83.p

@@ -1,3 +1,4 @@
+from dweather_client.tests.mock_fixtures import get_patched_datasets
 from dweather_client.client import get_station_history, get_gridcell_history, get_tropical_storms,\
     get_yield_history, get_power_history, get_gas_history, get_alberta_power_history, GRIDDED_DATASETS
 from dweather_client.aliases_and_units import snotel_to_ghcnd
@@ -14,7 +15,8 @@
 HOURLY_DATASETS = [ds for ds in GRIDDED_DATASETS if "hourly" in ds]
 IPFS_TIMEOUT = 60
 
-def test_get_gridcell_history_units():
+def test_get_gridcell_history_units(mocker):
+    mocker.patch("dweather_client.client.GRIDDED_DATASETS", get_patched_datasets())
     for s in DAILY_DATASETS + HOURLY_DATASETS:
         for use_imperial in [True, False]:
             res = get_gridcell_history(37, -83, s, use_imperial_units=use_imperial, ipfs_timeout=IPFS_TIMEOUT)
@@ -41,19 +43,8 @@ def test_get_gridcell_history_units():
                     else:
                         assert res[k].unit in (u.deg_C, u.K)
 
-def test_get_gridcell_history_snap():
-    lat_range = np.linspace(35, 40, 3)
-    lon_range = np.linspace(-100, -80, 3)
-
-    for s in DAILY_DATASETS:
-        for lat in lat_range:
-            for lon in lon_range:
-                res = get_gridcell_history(lat, lon, s, also_return_snapped_coordinates=True, also_return_metadata=True, ipfs_timeout=IPFS_TIMEOUT)
-                resolution, (snapped_lat, snapped_lon) = res[1]["metadata"]["resolution"], res[2]["snapped to"]
-                assert abs(snapped_lat - lat) <= resolution
-                assert abs(snapped_lon - lon) <= resolution
-
-def test_get_gridcell_history_date_range():
+def test_get_gridcell_history_date_range(mocker):
+    mocker.patch("dweather_client.client.GRIDDED_DATASETS", get_patched_datasets())
     for s in DAILY_DATASETS:
         res = get_gridcell_history(37, -83, s, ipfs_timeout=IPFS_TIMEOUT)
         first_date, last_date = sorted(res)[0], sorted(res)[-1]
@@ -66,7 +57,8 @@ def test_get_gridcell_history_date_range():
         time_diff_hours = time_diff.days * 24 + time_diff.seconds // 3600
         assert time_diff_hours + 1 == len(res)
 
-def test_get_gridcell_nans():
+def test_get_gridcell_nans(mocker):
+    mocker.patch("dweather_client.client.GRIDDED_DATASETS", get_patched_datasets())
     prism_r = get_gridcell_history(31.083, -120, "prismc-precip-daily", ipfs_timeout=IPFS_TIMEOUT)
     assert prism_r[datetime.date(1981, 8, 29)] is None
 
@@ -152,45 +144,3 @@ def test_aeso_power():
     time_diff_hours = time_diff.days * 24 + time_diff.seconds // 3600
 
     assert time_diff_hours + 1 == len(power_dict) 
-
-''' TODO some tests for RTMA behavior to be integrated into the new system
-def test_lat_lon_to_grid():
-    heads = http_client.get_heads()
-    rtma_hash = heads['rtma_pcp-hourly']
-    r = requests.get('https://gateway.arbolmarket.com/ipfs/%s/grid_history.txt.gz' % rtma_hash)
-    r.raise_for_status()
-    with gzip.GzipFile(fileobj=io.BytesIO(r.content)) as grid_history_file:
-        grid_history = grid_history_file.read().decode('utf-8')
-    assert utils.lat_lon_to_rtma_grid('40.752907470419586', '247.66162774628384', grid_history) == {'2011-01-01T00:00:00': ((491, 841), (491, 841)), '2016-01-06T14:00:00': (None, None)}
-    assert utils.lat_lon_to_rtma_grid('20.191999000000006', '238.445999', grid_history) == {'2011-01-01T00:00:00': ((0 ,0), (0, 0)), '2016-01-06T14:00:00': ((0, 0), (0, 0))}
-
-def test_rtma_grid_to_lat_lon():
-    heads = http_client.get_heads()
-    rtma_hash = heads['rtma_pcp-hourly']
-    r = requests.get('https://gateway.arbolmarket.com/ipfs/%s/grid_history.txt.gz' % rtma_hash)
-    r.raise_for_status()
-    with gzip.GzipFile(fileobj=io.BytesIO(r.content)) as grid_history_file:
-        grid_history = grid_history_file.read().decode('utf-8')
-
-        # case where lat/lon are the same
-        assert utils.rtma_grid_to_lat_lon(0, 0, grid_history) == [('20.191999000000006', '238.445999'), ('20.191999000000006', '238.445999')]
-
-        # random cases where lat/lon are different
-        assert utils.rtma_grid_to_lat_lon(50, 54, grid_history) == [('21.61726877222153', '239.39106426923487'), ('21.617275250933048', '239.39106861956924')]
-        assert utils.rtma_grid_to_lat_lon(130, 42, grid_history) == [('21.677552644312303', '241.3744282380296'), ('21.67755927656665', '241.37444172371673')]
-        assert utils.rtma_grid_to_lat_lon(491, 841, grid_history) == [('40.752907470419586', '247.66162774628384'), ('40.75299702642884', '247.66167780662005')]
-
-def test_rtma_lookup():
-    heads = http_client.get_heads()
-    rtma_hash = heads['rtma_pcp-hourly']
-    r = requests.get('https://gateway.arbolmarket.com/ipfs/%s/grid_history.txt.gz' % rtma_hash)
-    r.raise_for_status()
-    with gzip.GzipFile(fileobj=io.BytesIO(r.content)) as grid_history_file:
-        grid_history = grid_history_file.read().decode('utf-8')
-    lookup = utils.build_rtma_lookup(grid_history)
-    reverse_lookup = utils.build_rtma_reverse_lookup(grid_history)
-    for rev_lookup_lon in reverse_lookup['2016-01-06T14:00:00']['lon']:
-        rev_lookup_x, rev_lookup_y = reverse_lookup['2016-01-06T14:00:00']['lon'][rev_lookup_lon]
-        assert (rev_lookup_x, rev_lookup_y) == reverse_lookup['2016-01-06T14:00:00']['lon'][rev_lookup_lon]
-        assert lookup['2016-01-06T14:00:00'][1][rev_lookup_y][rev_lookup_x] == rev_lookup_lon
-'''

dweather_client/tests/etc/chirpsc_final_25-daily_37_-83.p

@@ -1,6 +1,7 @@
 ipfshttpclient==0.7.0
 pandas>=1.1.4
 pytest>=6.1.2
+pytest-mock==3.6.1
 requests
 geopy
 zeep