Skip to content

Commit

Permalink
Use MeshDB backend (#102)
Browse files Browse the repository at this point in the history 
* Implement meshdb-connected database client

* Edit env example to remove unused secrets

* Use auth token to communicate with MeshDB

* Empty commit to test builds

* Remove "cached" from variable name

* Update to new schema
  • Loading branch information
@Andrew-Dickinson
Andrew-Dickinson authored Mar 23, 2024
1 parent b9251a2 commit 8ebb262
Show file tree
Hide file tree
Showing 10 changed files with 408 additions and 267 deletions.
41 changes: 15 additions & 26 deletions .env-example
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,21 @@
# Sheets database

SPREADSHEET_ID =
SPREADSHEET_ID_TEST =
MAPS_API =

# from credentials.json file provided by Google Sheets API
GOOGLE_SHEETS_TYPE =
GOOGLE_SHEETS_PROJECT_ID =
GOOGLE_SHEETS_PRIVATE_KEY_ID =
GOOGLE_SHEETS_PRIVATE_KEY =
GOOGLE_SHEETS_CLIEND_EMAIL =
GOOGLE_SHEETS_CLIENT_ID =
GOOGLE_SHEETS_AUTH_URI =
GOOGLE_SHEETS_TOKEN_URI =
GOOGLE_SHEETS_AUTH_PROVIDER_X509_CERT_URL =
GOOGLE_SHEETS_CLIENT_509_CERT_URL =


# Mesh-wide secrets
OMNI_PASS =

# nycmesh-tool config
# see https://github.com/byxorna/nycmesh-tool
NYCMESH_TOOL_AUTH_TOKEN =
LBE_USERNAME=
LBE_PASSWORD=
OMNI_PASS=
UISP_AUTH_TOKEN=

# Slack secrets
SLACK_APP_TOKEN=
SLACK_BOT_TOKEN=

LBE_PASSWORD=
LBE_USERNAME=
ON_MESH=false
# Config
ON_MESH=false

# Mesh DB
MESHDB_API_ENDPOINT_BASE=http://meshdb.meshsvc-grand.mesh.nycmesh.net/api/v1/
MESHDB_AUTH_TOKEN=

# CI / CD creds
DOCKER_USERNAME=andybaumgar
DOCKER_PASSWORD=desolate-smatter-abash-intromit
4 changes: 3 additions & 1 deletion mesh-database-client/mesh_database_client/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
from .database import DatabaseClient
from .database import DatabaseClient
from .meshdb_client import MeshDBDatabaseClient
from .spreadsheet_client import SpreadsheetDatabaseClient
255 changes: 27 additions & 228 deletions mesh-database-client/mesh_database_client/database.py
View file
Original file line number Diff line number Diff line change
@@ -1,238 +1,37 @@
from __future__ import print_function

import os
import os.path

import pandas as pd
from dotenv import load_dotenv
from google.auth.transport.requests import Request
from google.oauth2 import service_account
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from numpy import sqrt

load_dotenv()


class DatabaseClient:
def __init__(self, spreadsheet_id=None, include_active=False):
self.spreadsheet_id = spreadsheet_id
if self.spreadsheet_id is None:
self.spreadsheet_id = os.environ.get("SPREADSHEET_ID")

SCOPES = ["https://www.googleapis.com/auth/spreadsheets.readonly"]
# created in Google Cloud admin
SERVICE_ACCOUNT_FILE = "credentials.json"

credentials_dict = self._get_sheets_credentials_from_env()
credentials = service_account.Credentials.from_service_account_info(
credentials_dict
)
self.service = build("sheets", "v4", credentials=credentials)

# Call the Sheets API
self.sheet = self.service.spreadsheets()

self.signup_df = self.get_signup_df()
self.links_df = self.get_links_df()

if include_active:
self.active_node_df = self.get_active_node_df()
self.active_link_df = self.get_active_link_df()

def _get_sheets_credentials_from_env(self):
try:
credentials_mapping = {
"type": "GOOGLE_SHEETS_TYPE",
"project_id": "GOOGLE_SHEETS_PROJECT_ID",
"private_key_id": "GOOGLE_SHEETS_PRIVATE_KEY_ID",
"private_key": "GOOGLE_SHEETS_PRIVATE_KEY",
"client_email": "GOOGLE_SHEETS_CLIEND_EMAIL",
"client_id": "GOOGLE_SHEETS_CLIENT_ID",
"auth_uri": "GOOGLE_SHEETS_AUTH_URI",
"token_uri": "GOOGLE_SHEETS_TOKEN_URI",
"auth_provider_x509_cert_url": "GOOGLE_SHEETS_AUTH_PROVIDER_X509_CERT_URL",
"client_x509_cert_url": "GOOGLE_SHEETS_CLIENT_509_CERT_URL",
}

credentials = {}
for key, value in credentials_mapping.items():
credentials[key] = os.environ.get(value)

credentials["private_key"] = credentials["private_key"].replace("\\n", "\n")

return credentials
except Exception as e:
print(e)
raise ValueError("Problem parsing Google Sheets credentials")

def get_range_as_df(self, range):
result = (
self.sheet.values()
.get(spreadsheetId=self.spreadsheet_id, range=range)
.execute()
)
values = result.get("values", [])

df = pd.DataFrame(values[1:], columns=values[0])
return df

def get_signup_df(self):
df = self.get_range_as_df("Form Responses 1!A:AP")

# force columns to be specific type
df["NN"] = pd.to_numeric(df["NN"], errors="coerce").fillna(0).astype(int)
df["ID"] = pd.to_numeric(df["ID"], errors="coerce").fillna(0).astype(int)

df["Latitude"] = pd.to_numeric(df["Latitude"], errors="coerce")
df["Longitude"] = pd.to_numeric(df["Longitude"], errors="coerce")

df["installDate"] = pd.to_datetime(df["installDate"], errors="coerce")

df.drop(df.tail(1).index, inplace=True)

return df

def get_links_df(self):
df = self.get_range_as_df("Links!A:I")

# force columns to be specific type
df["to"] = pd.to_numeric(df["to"], errors="coerce").fillna(0).astype(int)
df["from"] = pd.to_numeric(df["from"], errors="coerce").fillna(0).astype(int)

return df

def get_active_node_df(self):
df = self.signup_df.copy()
df = df.sort_values(by=["installDate"])
df = df[df["Status"].isin(["Installed", "NN assigned"])]
df = df[df["NN"] != 0]
df = df.drop_duplicates(subset="NN", keep="first")

return df

def get_active_link_df(self):
df = self.links_df.copy()

columns_list = list(df.columns)
columns_list[6] = "to_nn"
columns_list[7] = "from_nn"

df.columns = columns_list

df["to"] = df["to_nn"]
df["from"] = df["from_nn"]

df["to"] = pd.to_numeric(df["to"], errors="coerce").fillna(0).astype(int)
df["from"] = pd.to_numeric(df["from"], errors="coerce").fillna(0).astype(int)

df = df[~df["status"].isin(["dead", "planned"])]

df = df[(df["to"] != 0) & (df["from"] != 0)]

# enure only active nodes are in links df
nns = self.active_node_df["NN"]
df = df[df["from"].isin(nns) & df["to"].isin(nns)]

return df

def name_to_nn(self, name):
signup_df = self.signup_df
name_df = signup_df[signup_df["Name"].str.contains(name, case=False)]

if name_df.empty:
return None

entry = name_df.iloc[0]
if (nn := entry["NN"]) == 0:
return None
return nn
"""
Given a member's name, fuzzy match to search for an Active install
and return that install's NN, or None if not found
"""
raise NotImplementedError()

def email_to_nn(self, email):
signup_df = self.signup_df
email_df = signup_df.query(
f'Email.str.contains("{email}") and NN > 0', engine="python"
)

if email_df.empty:
return None

entry = email_df.iloc[0]
if (nn := entry["NN"]) == 0:
return None
return nn

# def address_to_nn(self, address):
# signup_df = self.signup_df

# geocode_result = self.gmaps.geocode(address)
# location = geocode_result[0]['geometry']['location']
# lat = location['lat']
# lng = location['lng']

# deg_to_feet = 288200

# lat_diff = abs(signup_df['Latitude']-lat)*deg_to_feet
# lng_diff = abs(signup_df['Longitude']-lng)*deg_to_feet
# distance = sqrt(lat_diff**2 + lng_diff**2)
# signup_df['distance'] = distance
# min_distance = distance.min()

# # check if closest signup request is further than 200ft
# if min_distance > 200:
# return None

# min_index = distance.idxmin()
# closest = signup_df.iloc[min_index]

# if closest['NN'] == 0:
# return None

# return closest['NN']
"""
Given a member's email, search for an Active install
and return that install's NN, or None if not found
"""
raise NotImplementedError()

def nn_to_linked_nn(self, nn):
links_df = self.links_df

from_df = links_df[links_df["from"] == nn]
to_df = links_df[links_df["to"] == nn]

from_nns = from_df["to"].tolist()
to_nns = to_df["from"].tolist()

connected_nodes = from_nns + to_nns

return connected_nodes
"""
Given an NN, return a list of the NNs of all directly adjacent nodes
using the Links table or None if the NN is not found
"""
raise NotImplementedError()

def get_nn(self, input_number):
if input_number is None:
return None

input_number = int(input_number)

id_rows = self.signup_df[self.signup_df["ID"] == input_number]
for index, row in id_rows.iterrows():
if row["Status"] == "NN assigned":
return input_number
elif row["Status"] == "Installed":
return row["NN"]

nn_rows = self.signup_df[self.signup_df["NN"] == input_number]
for index, row in nn_rows.iterrows():
if row["Status"] == "Installed":
return input_number

return None
"""
Given an input number which might be an NN or install number,
search for an Active install and return that install's NN,
or None if not found
"""
raise NotImplementedError()

def nn_to_location(self, nn):
if not isinstance(nn, int):
raise ValueError("nn must be an integer")
row = self.signup_df[self.signup_df["NN"] == nn].iloc[0]
return {"Latitude": row.Latitude, "Longitude": row.Longitude}


if __name__ == "__main__":
database_client = DatabaseClient(include_active=True)
name = database_client.name_to_nn("test")
print(name)
"""
Given an NN, return the lat/lon of the underlying building as a dict:
{"Latitude": ..., "Longitude": ...}
or None if the NN is not found
"""
raise NotImplementedError()
8 changes: 8 additions & 0 deletions mesh-database-client/mesh_database_client/endpoints.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import os

MESHDB_API_ENDPOINT_BASE = os.environ["MESHDB_API_ENDPOINT_BASE"]


MEMBER_LOOKUP_ENDPOINT = os.path.join(MESHDB_API_ENDPOINT_BASE, "members/lookup/")
INSTALL_LOOKUP_ENDPOINT = os.path.join(MESHDB_API_ENDPOINT_BASE, "installs/lookup/")
BUILDINGS_ENDPOINT = os.path.join(MESHDB_API_ENDPOINT_BASE, "buildings/")
Loading

0 comments on commit 8ebb262

Please sign in to comment.