Merge pull request st-tech#61 from st-tech/add-obd-dl-option

Add default option to download the small sized Open Bandit Dataset with the dataset module

MANIFEST.in

@@ -1,4 +1,5 @@
 include *.txt
 include *.md
 recursive-include obp/policy/conf *
+recursive-include obp/dataset/obd *
 include LICENSE

README.md

@@ -279,7 +279,6 @@ We also prepare a script to conduct the evaluation of OPE experiment with multi-
 ## Example with Open Bandit Dataset
 
 Here is an example of conducting OPE of the performance of BernoulliTS as an evaluation policy using Inverse Probability Weighting (IPW) and logged bandit feedback generated by the Random policy (behavior policy) on the ZOZOTOWN platform.
-(Please download the small sized Open Bandit Dataset to run this example)
 
 ```python
 # a case for implementing OPE of the BernoulliTS policy using log data generated by the Random policy

examples/obd/README.md

@@ -2,10 +2,7 @@
 
 ## Description
 
-Here, we use the open bandit dataset and pipeline to implement and evaluate OPE.
-Specifically, we evaluate the estimation performances of well-known off-policy estimators using the ground-truth policy value of an evaluation policy, which is calculable with our data using on-policy estimation.
-
-Please clone [the obp repository](https://github.com/st-tech/zr-obp) and download [the small sized Open Bandit Dataset](https://github.com/st-tech/zr-obp/tree/master/obd) to run this example.
+Here, we use the open bandit dataset and pipeline to implement and evaluate OPE. Specifically, we evaluate the estimation performances of well-known off-policy estimators using the ground-truth policy value of an evaluation policy, which is calculable with our data using on-policy estimation.
 
 ## Evaluating Off-Policy Estimators
 
@@ -55,9 +52,9 @@ python evaluate_off_policy_estimators.py\
 # random_state=12345
 # ------------------------------
 #          mean       std
-# dm   0.180305  0.114693
+# dm   0.180288  0.114694
 # ipw  0.333113  0.350425
-# dr   0.304421  0.347771
+# dr   0.304401  0.347842
 # ==============================
 ```
 

examples/obd/evaluate_off_policy_estimators.py

@@ -96,10 +96,10 @@
     n_jobs = args.n_jobs
     random_state = args.random_state
     np.random.seed(random_state)
-    data_path = Path(".").resolve().parents[1] / "obd"
 
     obd = OpenBanditDataset(
-        behavior_policy=behavior_policy, campaign=campaign, data_path=data_path
+        behavior_policy=behavior_policy,
+        campaign=campaign,
     )
     # compute action distribution by evaluation policy
     kwargs = dict(
@@ -115,7 +115,8 @@
     # ground-truth policy value of an evaluation policy
     # , which is estimated with factual (observed) rewards (on-policy estimation)
     ground_truth_policy_value = OpenBanditDataset.calc_on_policy_policy_value_estimate(
-        behavior_policy=evaluation_policy, campaign=campaign, data_path=data_path
+        behavior_policy=evaluation_policy,
+        campaign=campaign,
     )
 
     def process(b: int):
@@ -139,7 +140,8 @@ def process(b: int):
         )
         # evaluate estimators' performances using relative estimation error (relative-ee)
         ope = OffPolicyEvaluation(
-            bandit_feedback=bandit_feedback, ope_estimators=ope_estimators,
+            bandit_feedback=bandit_feedback,
+            ope_estimators=ope_estimators,
         )
         action_dist = np.tile(
             action_dist_single_round, (bandit_feedback["n_rounds"], 1, 1)
@@ -152,12 +154,17 @@ def process(b: int):
 
         return relative_ee_b
 
-    processed = Parallel(backend="multiprocessing", n_jobs=n_jobs, verbose=50,)(
-        [delayed(process)(i) for i in np.arange(n_runs)]
-    )
+    processed = Parallel(
+        backend="multiprocessing",
+        n_jobs=n_jobs,
+        verbose=50,
+    )([delayed(process)(i) for i in np.arange(n_runs)])
     relative_ee_dict = {est.estimator_name: dict() for est in ope_estimators}
     for b, relative_ee_b in enumerate(processed):
-        for (estimator_name, relative_ee_,) in relative_ee_b.items():
+        for (
+            estimator_name,
+            relative_ee_,
+        ) in relative_ee_b.items():
             relative_ee_dict[estimator_name][b] = relative_ee_
     relative_ee_df = DataFrame(relative_ee_dict).describe().T.round(6)
 

obp/dataset/obd/bts/all/item_context.csv

@@ -0,0 +1,81 @@
+,item_id,item_feature_0,item_feature_1,item_feature_2,item_feature_3
+0,0,-0.49917162609493676,527a325f46248058e3c5df9fb548f05f,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+1,1,-0.5437753679553131,84c5238bc7dfedff64df3e71e39456d3,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+2,2,0.9727518552974822,84c5238bc7dfedff64df3e71e39456d3,b913be25b9f7b764dcdeeac826fbd94b,78469ccb0d9544e55354632ff51ed422
+3,3,-0.5214734970251249,ab5678265d1b6b884ddc30cac35dc0b7,1c3a19a4f1fb7e108afd14f43393a79d,3c8f79fff6b73fd5a3eaebdc20b37078
+4,4,1.9094304343653852,31a4dca305b5aae308a77f24d2568951,8d83a426afeac570837704062f5bbe37,78469ccb0d9544e55354632ff51ed422
+5,5,-0.49917162609493676,ab5678265d1b6b884ddc30cac35dc0b7,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+6,6,0.34829946925221356,527a325f46248058e3c5df9fb548f05f,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+7,7,-0.2761529167930551,9a2a41dfb2af3d9b20b5e58eed23f4f8,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+8,8,-0.6552847226062539,ab5678265d1b6b884ddc30cac35dc0b7,a472426cf770b94bf774a0efb89080bd,84fd569b1b2309788c39aced8f88d084
+9,9,-0.38766227144399595,527a325f46248058e3c5df9fb548f05f,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+10,10,0.34829946925221356,31a4dca305b5aae308a77f24d2568951,229ddd5c1687175cd18d3e3b1c816465,84fd569b1b2309788c39aced8f88d084
+11,11,1.9094304343653852,527a325f46248058e3c5df9fb548f05f,8dffa1b0f662ed25cf44b4c0daff9207,78469ccb0d9544e55354632ff51ed422
+12,12,3.1583352064559223,31a4dca305b5aae308a77f24d2568951,b913be25b9f7b764dcdeeac826fbd94b,78469ccb0d9544e55354632ff51ed422
+13,13,1.5972042413427507,31a4dca305b5aae308a77f24d2568951,a54e4753d385aad2954d8c2cc7f4ee00,78469ccb0d9544e55354632ff51ed422
+14,14,1.4410911448314336,ab5678265d1b6b884ddc30cac35dc0b7,b913be25b9f7b764dcdeeac826fbd94b,78469ccb0d9544e55354632ff51ed422
+15,15,1.8425248215748207,ab5678265d1b6b884ddc30cac35dc0b7,b913be25b9f7b764dcdeeac826fbd94b,78469ccb0d9544e55354632ff51ed422
+16,16,0.6605256622748479,527a325f46248058e3c5df9fb548f05f,02f313f24a15548a9d0c2948d584f370,36163e60414fb83e275f7a3cff00ccbc
+17,17,1.4410911448314336,527a325f46248058e3c5df9fb548f05f,50b2b3c1fbfe11227c13f51b876a87a2,78469ccb0d9544e55354632ff51ed422
+18,18,0.8389406297163532,ab5678265d1b6b884ddc30cac35dc0b7,50b2b3c1fbfe11227c13f51b876a87a2,78469ccb0d9544e55354632ff51ed422
+19,19,-0.8336996900477592,ab5678265d1b6b884ddc30cac35dc0b7,7ee018fa21054e085847b83adcde16ea,84fd569b1b2309788c39aced8f88d084
+20,20,-0.18694543307230246,527a325f46248058e3c5df9fb548f05f,0a13a288ab79c3e522be11f1763e9fb8,3c8f79fff6b73fd5a3eaebdc20b37078
+21,21,-0.4099641423741841,527a325f46248058e3c5df9fb548f05f,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+22,22,-0.5214734970251249,ab5678265d1b6b884ddc30cac35dc0b7,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+23,23,-0.7444922063270066,84c5238bc7dfedff64df3e71e39456d3,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+24,24,-0.5883791098156894,84c5238bc7dfedff64df3e71e39456d3,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+25,25,-0.43226601330437225,84c5238bc7dfedff64df3e71e39456d3,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+26,26,-0.5214734970251249,ab5678265d1b6b884ddc30cac35dc0b7,1c3a19a4f1fb7e108afd14f43393a79d,3c8f79fff6b73fd5a3eaebdc20b37078
+27,27,-0.5214734970251249,84640f156cd8cf32ecf2fa3aa9904dfc,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+28,28,-0.38766227144399595,1c3ca9815f059a8c26c6d1fae6071105,5cb8a4e490e08332e9b3a4773a5cfef6,78469ccb0d9544e55354632ff51ed422
+29,29,0.6382237913446597,bf9100bc60ba7ae103ce1297f9e7187b,a54e4753d385aad2954d8c2cc7f4ee00,78469ccb0d9544e55354632ff51ed422
+30,30,-0.12003982028173796,527a325f46248058e3c5df9fb548f05f,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+31,31,-0.2761529167930551,527a325f46248058e3c5df9fb548f05f,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+32,32,-0.947439231791719,a4752221d5657e31030d04a82d0ed4b3,0a13a288ab79c3e522be11f1763e9fb8,3c8f79fff6b73fd5a3eaebdc20b37078
+33,33,-0.947439231791719,a4752221d5657e31030d04a82d0ed4b3,0a13a288ab79c3e522be11f1763e9fb8,3c8f79fff6b73fd5a3eaebdc20b37078
+34,34,-0.7244205224898372,a4752221d5657e31030d04a82d0ed4b3,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+35,35,-0.7244205224898372,a4752221d5657e31030d04a82d0ed4b3,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+36,36,0.19218637274089637,fd5c8b8aa48f0f39108b280da9c3bb99,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+37,37,0.19218637274089637,fd5c8b8aa48f0f39108b280da9c3bb99,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+38,38,2.689995916921971,ab5678265d1b6b884ddc30cac35dc0b7,8dffa1b0f662ed25cf44b4c0daff9207,78469ccb0d9544e55354632ff51ed422
+39,39,1.1288649518087994,9a2a41dfb2af3d9b20b5e58eed23f4f8,8dffa1b0f662ed25cf44b4c0daff9207,78469ccb0d9544e55354632ff51ed422
+40,40,-0.38766227144399595,527a325f46248058e3c5df9fb548f05f,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+41,41,0.5044125657635307,9a2a41dfb2af3d9b20b5e58eed23f4f8,02f313f24a15548a9d0c2948d584f370,36163e60414fb83e275f7a3cff00ccbc
+42,42,2.0655435308767025,9a2a41dfb2af3d9b20b5e58eed23f4f8,577f2100c2b03c70ba37174fcc43261f,78469ccb0d9544e55354632ff51ed422
+43,43,-0.5214734970251249,527a325f46248058e3c5df9fb548f05f,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+44,44,-0.9675109156288882,ab5678265d1b6b884ddc30cac35dc0b7,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+45,45,0.6605256622748479,fd5c8b8aa48f0f39108b280da9c3bb99,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+46,46,-0.7444922063270066,9a2a41dfb2af3d9b20b5e58eed23f4f8,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+47,47,0.6605256622748479,fd5c8b8aa48f0f39108b280da9c3bb99,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+48,48,-0.43226601330437225,ab5678265d1b6b884ddc30cac35dc0b7,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+49,49,-0.8336996900477592,527a325f46248058e3c5df9fb548f05f,647fb6b80c46798ce5a1596eeea761b8,3c8f79fff6b73fd5a3eaebdc20b37078
+50,50,-1.056718399349641,539e71c4caa0d99daa99307259cce47a,647fb6b80c46798ce5a1596eeea761b8,3c8f79fff6b73fd5a3eaebdc20b37078
+51,51,-0.7444922063270066,539e71c4caa0d99daa99307259cce47a,1c3a19a4f1fb7e108afd14f43393a79d,3c8f79fff6b73fd5a3eaebdc20b37078
+52,52,-0.7444922063270066,539e71c4caa0d99daa99307259cce47a,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+53,53,0.16988450181070822,84640f156cd8cf32ecf2fa3aa9904dfc,0a13a288ab79c3e522be11f1763e9fb8,3c8f79fff6b73fd5a3eaebdc20b37078
+54,54,-0.43226601330437225,539e71c4caa0d99daa99307259cce47a,e338631de2b9905f20cc62bd7764ce49,3c8f79fff6b73fd5a3eaebdc20b37078
+55,55,-0.43226601330437225,539e71c4caa0d99daa99307259cce47a,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+56,56,-0.43226601330437225,539e71c4caa0d99daa99307259cce47a,e338631de2b9905f20cc62bd7764ce49,3c8f79fff6b73fd5a3eaebdc20b37078
+57,57,0.013771405299391045,84640f156cd8cf32ecf2fa3aa9904dfc,08b6dc106726445b6bacb324193b6670,ee93c727c612367ffb4d9881f8948d68
+58,58,-0.45456788423456046,84640f156cd8cf32ecf2fa3aa9904dfc,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+59,59,0.32599759832202535,84640f156cd8cf32ecf2fa3aa9904dfc,577f2100c2b03c70ba37174fcc43261f,78469ccb0d9544e55354632ff51ed422
+60,60,-0.6775865935364421,84640f156cd8cf32ecf2fa3aa9904dfc,7ee018fa21054e085847b83adcde16ea,84fd569b1b2309788c39aced8f88d084
+61,61,-0.2761529167930551,527a325f46248058e3c5df9fb548f05f,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+62,62,-0.5214734970251249,1c3ca9815f059a8c26c6d1fae6071105,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+63,63,-0.49917162609493676,c605b44dc95c980270fac46dafb45cce,536e6a0637e2923835b9c85dc9398745,979ec370acfcbeb6a79e131db9ac4c62
+64,64,-0.5883791098156894,bf9100bc60ba7ae103ce1297f9e7187b,7ee018fa21054e085847b83adcde16ea,84fd569b1b2309788c39aced8f88d084
+65,65,3.782787592501191,fd5c8b8aa48f0f39108b280da9c3bb99,8dffa1b0f662ed25cf44b4c0daff9207,78469ccb0d9544e55354632ff51ed422
+66,66,-0.12003982028173796,bf9100bc60ba7ae103ce1297f9e7187b,adfeed89343845820ec6e4afc2bcde15,b106244c2b3aa33fb9f1a9193badc19c
+67,67,-0.2761529167930551,ab5678265d1b6b884ddc30cac35dc0b7,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+68,68,1.1288649518087994,84c5238bc7dfedff64df3e71e39456d3,577f2100c2b03c70ba37174fcc43261f,78469ccb0d9544e55354632ff51ed422
+69,69,-0.6775865935364421,84640f156cd8cf32ecf2fa3aa9904dfc,5aa6086753a8f50f472a7879ddf6e57e,3c8f79fff6b73fd5a3eaebdc20b37078
+70,70,-0.9006053028383237,84640f156cd8cf32ecf2fa3aa9904dfc,e338631de2b9905f20cc62bd7764ce49,3c8f79fff6b73fd5a3eaebdc20b37078
+71,71,-0.6775865935364421,84640f156cd8cf32ecf2fa3aa9904dfc,a472426cf770b94bf774a0efb89080bd,84fd569b1b2309788c39aced8f88d084
+72,72,-0.7444922063270066,84640f156cd8cf32ecf2fa3aa9904dfc,7ee018fa21054e085847b83adcde16ea,84fd569b1b2309788c39aced8f88d084
+73,73,-0.2761529167930551,1c3ca9815f059a8c26c6d1fae6071105,a472426cf770b94bf774a0efb89080bd,84fd569b1b2309788c39aced8f88d084
+74,74,-0.3430585295836196,bf9100bc60ba7ae103ce1297f9e7187b,e338631de2b9905f20cc62bd7764ce49,3c8f79fff6b73fd5a3eaebdc20b37078
+75,75,-0.43226601330437225,bf9100bc60ba7ae103ce1297f9e7187b,5e92ce840abf12dc17408ca42b189836,84fd569b1b2309788c39aced8f88d084
+76,76,-0.6106809807458776,539e71c4caa0d99daa99307259cce47a,647fb6b80c46798ce5a1596eeea761b8,3c8f79fff6b73fd5a3eaebdc20b37078
+77,77,-1.056718399349641,539e71c4caa0d99daa99307259cce47a,647fb6b80c46798ce5a1596eeea761b8,3c8f79fff6b73fd5a3eaebdc20b37078
+78,78,-0.5883791098156894,539e71c4caa0d99daa99307259cce47a,1c3a19a4f1fb7e108afd14f43393a79d,3c8f79fff6b73fd5a3eaebdc20b37078
+79,79,-0.9006053028383237,539e71c4caa0d99daa99307259cce47a,b70d45665d14cc40b4cfd7839aee9dee,3c8f79fff6b73fd5a3eaebdc20b37078

obp/dataset/obd/bts/men/item_context.csv

@@ -0,0 +1,35 @@
+,item_id,item_feature_0,item_feature_1,item_feature_2,item_feature_3
+0,0,-0.6771831139635117,c82d13885d8bf7a3b8b9fa6f0842ba60,088abf8a8657959e46ac19af8da80d15,8ea65bc866b36a8f00ae913e0c3acc29
+1,1,-0.7202996418188664,77490d05a721c6d93edf580642ffd8bd,088abf8a8657959e46ac19af8da80d15,8ea65bc866b36a8f00ae913e0c3acc29
+2,2,0.7456623052631924,77490d05a721c6d93edf580642ffd8bd,bfcce809dad48aadc7fcbe714f9eabd7,7a0c97ee71eb7985bd0a6271ce57cec5
+3,3,-0.6987413778911891,135f410ec21307919cd92df77f1e2a36,ff2de7df709624e5b79199b850382ea0,68f8b5168b2a322db725a6cd6f5c900b
+4,4,1.6511093902256406,61a525de9976c0f3fa29d400caf26c56,ee987234ffe4f3d901846ac3f7417738,7a0c97ee71eb7985bd0a6271ce57cec5
+5,5,0.14203091528822703,61a525de9976c0f3fa29d400caf26c56,bb7caf7f0c11f7827fb23b331777b871,8ea65bc866b36a8f00ae913e0c3acc29
+6,6,1.6511093902256406,c82d13885d8bf7a3b8b9fa6f0842ba60,818dfe387422471f09a34db693a78212,7a0c97ee71eb7985bd0a6271ce57cec5
+7,7,2.8583721701755715,61a525de9976c0f3fa29d400caf26c56,bfcce809dad48aadc7fcbe714f9eabd7,7a0c97ee71eb7985bd0a6271ce57cec5
+8,8,1.349293695238158,61a525de9976c0f3fa29d400caf26c56,7daaf8717f83289266063b6cc1728087,7a0c97ee71eb7985bd0a6271ce57cec5
+9,9,1.1983858477444165,135f410ec21307919cd92df77f1e2a36,bfcce809dad48aadc7fcbe714f9eabd7,7a0c97ee71eb7985bd0a6271ce57cec5
+10,10,1.5864345984426087,135f410ec21307919cd92df77f1e2a36,bfcce809dad48aadc7fcbe714f9eabd7,7a0c97ee71eb7985bd0a6271ce57cec5
+11,11,0.44384661027570976,c82d13885d8bf7a3b8b9fa6f0842ba60,24ea3b3a472c51dd6299ebdfb220a55f,0c3b42b13b5a49fcb746da9f60e63717
+12,12,1.1983858477444165,c82d13885d8bf7a3b8b9fa6f0842ba60,0e077f97ef2dcda0dc404f873fc5f96c,7a0c97ee71eb7985bd0a6271ce57cec5
+13,13,0.6163127216971285,135f410ec21307919cd92df77f1e2a36,0e077f97ef2dcda0dc404f873fc5f96c,7a0c97ee71eb7985bd0a6271ce57cec5
+14,14,-1.000557072878672,135f410ec21307919cd92df77f1e2a36,865945b5265169a2176a6e5f084ab2eb,8ea65bc866b36a8f00ae913e0c3acc29
+15,15,-0.37536741897602904,c82d13885d8bf7a3b8b9fa6f0842ba60,786ff5d72b02d1e68a43508d9579977d,68f8b5168b2a322db725a6cd6f5c900b
+16,16,-0.5909500582528024,c82d13885d8bf7a3b8b9fa6f0842ba60,32338184693488f3a469822fd0a08387,68f8b5168b2a322db725a6cd6f5c900b
+17,17,-0.6987413778911891,135f410ec21307919cd92df77f1e2a36,32338184693488f3a469822fd0a08387,68f8b5168b2a322db725a6cd6f5c900b
+18,18,-0.9143240171679625,77490d05a721c6d93edf580642ffd8bd,32338184693488f3a469822fd0a08387,68f8b5168b2a322db725a6cd6f5c900b
+19,19,-0.7634161696742211,77490d05a721c6d93edf580642ffd8bd,32338184693488f3a469822fd0a08387,68f8b5168b2a322db725a6cd6f5c900b
+20,20,-0.6125083221804798,77490d05a721c6d93edf580642ffd8bd,32338184693488f3a469822fd0a08387,68f8b5168b2a322db725a6cd6f5c900b
+21,21,-0.6987413778911891,135f410ec21307919cd92df77f1e2a36,ff2de7df709624e5b79199b850382ea0,68f8b5168b2a322db725a6cd6f5c900b
+22,22,-0.6987413778911891,17ef71cb22e550d31e5eaa4d629c4abd,088abf8a8657959e46ac19af8da80d15,8ea65bc866b36a8f00ae913e0c3acc29
+23,23,-0.5693917943251251,e1b1451d555c82a01874347dbecdfeae,01b306b40a448bff555c06d5d72c0171,7a0c97ee71eb7985bd0a6271ce57cec5
+24,24,0.4222883463480324,f15de9aa508214df06454736b488717c,7daaf8717f83289266063b6cc1728087,7a0c97ee71eb7985bd0a6271ce57cec5
+25,25,-0.4616004746867384,135f410ec21307919cd92df77f1e2a36,088abf8a8657959e46ac19af8da80d15,8ea65bc866b36a8f00ae913e0c3acc29
+26,26,0.8965701527569339,77490d05a721c6d93edf580642ffd8bd,746facf4548f3da6d628b8e35bf9e6ec,7a0c97ee71eb7985bd0a6271ce57cec5
+27,27,-0.8496492253849305,17ef71cb22e550d31e5eaa4d629c4abd,32338184693488f3a469822fd0a08387,68f8b5168b2a322db725a6cd6f5c900b
+28,28,-1.0652318646617038,17ef71cb22e550d31e5eaa4d629c4abd,a46137fea33ac48f0809591a76630ea5,68f8b5168b2a322db725a6cd6f5c900b
+29,29,-0.8496492253849305,17ef71cb22e550d31e5eaa4d629c4abd,008dc8758000efaf5b318227fcb71f8d,8ea65bc866b36a8f00ae913e0c3acc29
+30,30,-0.9143240171679625,17ef71cb22e550d31e5eaa4d629c4abd,865945b5265169a2176a6e5f084ab2eb,8ea65bc866b36a8f00ae913e0c3acc29
+31,31,-0.4616004746867384,e1b1451d555c82a01874347dbecdfeae,008dc8758000efaf5b318227fcb71f8d,8ea65bc866b36a8f00ae913e0c3acc29
+32,32,-0.5262752664697704,f15de9aa508214df06454736b488717c,a46137fea33ac48f0809591a76630ea5,68f8b5168b2a322db725a6cd6f5c900b
+33,33,-0.6125083221804798,f15de9aa508214df06454736b488717c,088abf8a8657959e46ac19af8da80d15,8ea65bc866b36a8f00ae913e0c3acc29