added four corner states

PeaBrane · Mar 18, 2023 · f48eb28 · f48eb28
1 parent f9b4c7a
commit f48eb28
Show file tree

Hide file tree

Showing 12 changed files with 9,280 additions and 903 deletions.
diff --git a/data/arizona_boulders.csv b/data/arizona_boulders.csv
diff --git a/data/arizona_boulders_processed.csv b/data/arizona_boulders_processed.csv
diff --git a/data/colorado_boulders.csv b/data/colorado_boulders.csv
diff --git a/data/colorado_boulders_processed.csv b/data/colorado_boulders_processed.csv
diff --git a/data/jtree_boulders_processed.csv b/data/jtree_boulders_processed.csv
diff --git a/data/nevada_boulders.csv b/data/nevada_boulders.csv
diff --git a/data/nevada_boulders_processed.csv b/data/nevada_boulders_processed.csv
diff --git a/data/new_mexico_boulders.csv b/data/new_mexico_boulders.csv
diff --git a/data/new_mexico_boulders_processed.csv b/data/new_mexico_boulders_processed.csv
diff --git a/data/utah_boulders.csv b/data/utah_boulders.csv
diff --git a/data/utah_boulders_processed.csv b/data/utah_boulders_processed.csv
diff --git a/script.py b/script.py
@@ -1,6 +1,7 @@
 import os
 import pandas as pd
 import numpy as np
+from time import sleep
 
 from urllib import request
 from bs4 import BeautifulSoup
@@ -9,19 +10,29 @@
 from rich.progress import track
 
 
+def get_vote(url):
+    response = request.urlopen(url)
+    soup = BeautifulSoup(response.read(), 'html.parser')
+
+    results = soup.body.find_all(string=re.compile('.*{0}.*'.format('Avg')), recursive=True)[0]
+    start_index = results.find('from') + 5
+    end_index = results.find('votes') - 5
+
+    vote = int(results[start_index:end_index])
+    return vote
+
+
 def get_votes(df: pd.DataFrame):
-
     votes = []
     for index in track(range(len(df))):
         url = df['URL'].iloc[index]
-        response = request.urlopen(url)
-        soup = BeautifulSoup(response.read(), 'html.parser')
-
-        results = soup.body.find_all(string=re.compile('.*{0}.*'.format('Avg')), recursive=True)[0]
-        start_index = results.find('from') + 5
-        end_index = results.find('votes') - 5
+
+        try:
+            vote = get_vote(url)
+        except:
+            print('retrying url...')
+            sleep(3)
 
-        vote = int(results[start_index:end_index])
         votes.append(vote)
 
     return np.array(votes)
@@ -31,7 +42,6 @@ def build_df(df: pd.DataFrame,
              votes: np.array,
              boulder=True, 
              tol=1e-7):
-
     latitudes, longitudes = df['Area Latitude'].to_numpy(), df['Area Longitude'].to_numpy()
     coords = np.stack([latitudes, longitudes], axis=-1)
     distances = coords[np.newaxis, ...] - coords[:, np.newaxis]
@@ -76,8 +86,7 @@ def process_problems(filename: pd.DataFrame,
         vote_threshold (int, optional): problems whose vote count below this threshold will be ignored. Defaults to 2.
         vote_split (int, optional): problems whose vote count above this threshold will be treated as popular problems. Defaults to 20.
         boulder (bool, optional): whether the problem is a boulder of roped problem. Defaults to True.
-    """
-
+    """   
     df = pd.read_csv(filename)
     votes = get_votes(df)
 
@@ -96,7 +105,7 @@ def process_problems(filename: pd.DataFrame,
     # df_processed.loc[popular_mask].to_csv(filename[:-4] + '_processed_popular.csv', index=False)
 
 
-filenames = ['jtree_boulders.csv']
+filenames = ['colorado_boulders.csv', 'arizona_boulders.csv', 'new_mexico_boulders.csv']
 filenames = [os.path.join('data', filename) for filename in filenames]
 for filename in filenames:
     process_problems(filename)