add the Rock the Vote connector (move-coop#225)

This commit adds the `RockTheVote` connector that wraps around the
Rock the Vote's Rocky API. This commit only implements endpoints
for requesting registrant reports, which is a report to list out
registration records for a given partner.

docs/rockthevote.rst

@@ -0,0 +1,56 @@
+Rock the Vote
+=============
+
+`Rock the Vote <https://www.rockthevote.org/>`_ is an online registration tool.
+
+The Parsons Connector makes use of Rock the Vote's Rocky API. In order to authenticate with the
+API, users will need to specify the ID and the API key of the RTV partner organization for the
+data.
+
+**********
+QuickStart
+**********
+
+To use the RockTheVote class you can either store the partner ID and API key as an
+environmental variable (RTV_PARTNER_ID and RTV_PARTNER_API_KEY, respectively), or you can
+pass them in as arguments to the class.
+
+.. code-block:: python
+
+   from parsons import RockTheVote
+
+   rtv = RockTheVote()  # If specified as environment variables, no need to pass them in
+
+   rtv = RockTheVote(partner_id='123', partner_api_key='supersecretkey') # Pass credentials directly
+
+To fetch a list of registrations submitted for the partner ID, use the `run_registration_report`
+method. It is possible to filter the results by providing a parameter to specify a start date
+for the registrations.
+
+.. code-block:: python
+
+   from parsons import RockTheVote
+
+   rtv = RockTheVote()
+
+   registrants = rtv.run_registration_report(since='2020-01-01')
+
+The `run_registration_report` will block as the report is being generated and downloaded from the
+Rocky API. For larger reports, this can take a long time. If you have other things you want to do
+while the report is running, you can break up the creation of the report from the fetching of the
+data.
+
+.. code-block:: python
+
+   from parsons import RockTheVote
+
+   rtv = RockTheVote()
+
+   report_id = rtv.create_registration_report(since='2020-01-01')
+
+   # Do some other stuff here
+
+   registrants = rtv.get_registration_report(report_id)
+
+.. autoclass :: parsons.rockthevote.rtv.RockTheVote
+   :inherited-members:

parsons/__init__.py

@@ -40,6 +40,7 @@
     from parsons.bill_com.bill_com import BillCom
     from parsons.newmode.newmode import Newmode
     from parsons.databases.mysql.mysql import MySQL
+    from parsons.rockthevote.rtv import RockTheVote
 
     __all__ = [
         'VAN',
@@ -73,7 +74,8 @@
         'Freshdesk',
         'BillCom',
         'Newmode',
-        'MySQL'
+        'MySQL',
+        'RockTheVote',
     ]
 
 # Define the default logging config for Parsons and its submodules. For now the

parsons/etl/etl.py

@@ -935,3 +935,16 @@ def sort(self, columns=None, reverse=False):
         self.table = petl.sort(self.table, key=columns, reverse=reverse)
 
         return self
+
+    def set_header(self, new_header):
+        """
+        Replace the header row of the table.
+
+        `Args:`
+            new_header: list
+                List of new header column names
+        `Returns:`
+            `Parsons Table` and also updates self
+        """
+        self.table = petl.setheader(self.table, new_header)
+        return self

parsons/rockthevote/rtv.py

@@ -0,0 +1,213 @@
+import datetime
+import logging
+import petl
+import re
+import requests
+import time
+
+from dateutil.parser import parse as parse_date
+
+from parsons import Table
+from parsons.utilities import check_env
+from parsons.utilities.api_connector import APIConnector
+
+logger = logging.getLogger(__name__)
+
+DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S UTC'
+"""Datetime format for sending date's to the API."""
+
+REQUEST_HEADERS = {
+    # For some reason, RTV's firewall REALLY doesn't like the
+    # user-agent string that Python's request library gives by default,
+    # though it seems fine with the curl user agent
+    # For more info on user agents, see:
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+    'user-agent': 'curl/7.54.0'
+}
+"""Standard request header for sending requests to the API."""
+
+STATUS_URL_PARSE_REGEX = re.compile(r'(\d+)$')
+"""Regex for parsing the report ID from the status URL."""
+
+
+class RTVFailure(Exception):
+    """Exception raised when there is an error with the connector."""
+
+
+class RockTheVote:
+    """
+    `Args:`
+        partner_id: str
+            The RockTheVote partner ID for the RTV account
+        partner_api_key: str
+            The API Key for the partner
+    `Returns`:
+        RockTheVote class
+    """
+
+    def __init__(self, partner_id=None, partner_api_key=None):
+        self.partner_id = check_env.check('RTV_PARTNER_ID', partner_id)
+        self.partner_api_key = check_env.check('RTV_PARTNER_API_KEY', partner_api_key)
+
+        self.client = APIConnector('https://vr.rockthevote.com/api/v4', headers=REQUEST_HEADERS)
+
+    def create_registration_report(self, before=None, since=None):
+        """
+        Create a new registration report.
+
+        `Args:`
+            before: str
+                Date before which to return registrations for
+            since: str
+                Date for filtering registrations
+        `Returns:`
+            int
+                The ID of the created report.
+        """
+        report_url = f'{self.client.uri}/registrant_reports.json'
+        # Create the report for the new data
+        report_parameters = {
+            'partner_id': self.partner_id,
+            'partner_API_key': self.partner_api_key,
+        }
+
+        if since:
+            since_date = parse_date(since)
+            report_parameters['since'] = since_date.strftime(DATETIME_FORMAT)
+        if before:
+            before_date = parse_date(before)
+            report_parameters['before'] = before_date.strftime(DATETIME_FORMAT)
+
+        # The report parameters get passed into the request as JSON in the body
+        # of the request.
+        response = self.client.request(report_url, 'post', json=report_parameters)
+        if response.status_code != requests.codes.ok:
+            print(response.text)
+            raise RTVFailure("Couldn't create RTV registrations report")
+
+        response_json = response.json()
+        # The RTV API says the response should include the report_id, but I have not found
+        # that to be the case
+        report_id = response_json.get('report_id')
+        if report_id:
+            return report_id
+
+        # If the response didn't include the report_id, then we will parse it out of the URL.
+        status_url = response_json.get('status_url')
+        url_match = STATUS_URL_PARSE_REGEX.search(status_url)
+        if url_match:
+            report_id = url_match.group(1)
+
+        return report_id
+
+    def get_registration_report(self, report_id, block=False, poll_interval_seconds=60,
+                                report_timeout_seconds=3600):
+        """
+        Get data from an existing registration report.
+
+        `Args:`
+            report_id: int
+                The ID of the report to get data from
+            block: bool
+                Whether or not to block execution until the report is complete
+            poll_interval_seconds: int
+                If blocking, how long to pause between attempts to check if the report is done
+            report_timeout_seconds: int
+                If blocking, how long to wait for the report before timing out
+        `Returns:`
+            Parsons Table
+                Parsons table with the report data.
+        """
+        credentials = {
+            'partner_id': self.partner_id,
+            'partner_API_key': self.partner_api_key,
+        }
+        status_url = f'{self.client.uri}/registrant_reports/{report_id}'
+        download_url = None
+
+        # Let's figure out at what time should we just give up because we waited
+        # too long
+        end_time = datetime.datetime.now() + datetime.timedelta(seconds=report_timeout_seconds)
+
+        # If we have a download URL, we can move on and just download the
+        # report. Otherwise, as long as we haven't run out of time, we will
+        # check the status.
+        while not download_url and datetime.datetime.now() < end_time:
+            logger.debug(f'Registrations report not ready yet, sleeping %s seconds',
+                         poll_interval_seconds)
+
+            # Check the status again via the status endpoint
+            status_response = self.client.request(status_url, 'get', params=credentials)
+
+            # Check to make sure the call got a valid response
+            if status_response.status_code == requests.codes.ok:
+                status_json = status_response.json()
+
+                # Grab the download_url from the response.
+                download_url = status_json.get('download_url')
+
+                if not download_url and not block:
+                    return None
+            else:
+                raise RTVFailure("Couldn't get report status")
+
+            if not download_url:
+                # We just got the status, so we should wait a minute before
+                # we check it again.
+                time.sleep(poll_interval_seconds)
+
+        # If we never got a valid download_url, then we timed out waiting for
+        # the report to generate. We will log an error and exit.
+        if not download_url:
+            raise RTVFailure('Timed out waiting for report')
+
+        # Download the report data
+        download_response = self.client.request(download_url, 'get', params=credentials)
+
+        # Check to make sure the call got a valid response
+        if download_response.status_code == requests.codes.ok:
+            report_data = download_response.text
+
+            # Load the report data into a Parsons Table
+            table = Table.from_csv_string(report_data)
+
+            # Transform the data from the report's CSV format to something more
+            # Pythonic (snake case)
+            normalized_column_names = [
+                re.sub(r'\s', '_', name).lower()
+                for name in table.columns
+            ]
+            normalized_column_names = [
+                re.sub(r'[^A-Za-z\d_]', '', name)
+                for name in normalized_column_names
+            ]
+            table.table = petl.setheader(table.table, normalized_column_names)
+            return table
+        else:
+            raise RTVFailure('Unable to download report data')
+
+    def run_registration_report(self, before=None, since=None, poll_interval_seconds=60,
+                                report_timeout_seconds=3600):
+        """
+        Run a new registration report.
+
+        This method will block until the report has finished generating, or until the specified
+        timeout is reached.
+
+        `Args:`
+            before: str
+                Date before which to return registrations for
+            since: str
+                Date for filtering registrations
+            poll_interval_seconds: int
+                If blocking, how long to pause between attempts to check if the report is done
+            report_timeout_seconds: int
+                If blocking, how long to wait for the report before timing out
+        `Returns:`
+            int
+                The ID of the created report.
+        """
+        report_id = self.create_registration_report(before=before, since=since)
+        return self.get_registration_report(report_id, block=True,
+                                            poll_interval_seconds=poll_interval_seconds,
+                                            report_timeout_seconds=report_timeout_seconds)

requirements.txt

@@ -28,6 +28,7 @@ suds-py3==1.3.4.0
 newmode==0.1.6
 mysql-connector-python==8.0.18
 braintree==4.0.0
+python-dateutil==2.8.1
 
 # Testing Requirements
 requests-mock==1.5.2

test/test_etl.py

@@ -537,7 +537,7 @@ def test_first(self):
 
     def test_get_item(self):
         # Test indexing on table
-        
+
         # Test a valid column
         tbl = Table(self.lst)
         lst = [1, 4, 7, 10, 13]
@@ -714,7 +714,7 @@ def test_get_column_max_with(self):
         # Doesn't break for non-strings
         self.assertEqual(tbl.get_column_max_width('b'), 5)
 
-        # Evaluates based on byte length rather than char length 
+        # Evaluates based on byte length rather than char length
         self.assertEqual(tbl.get_column_max_width('c'), 33)
 
     def test_sort(self):
@@ -732,4 +732,18 @@ def test_sort(self):
         # Test reverse sort
         unsorted_tbl = Table([['a', 'b'],[3, 1],[2, 2],[1, 3]])
         sorted_tbl = unsorted_tbl.sort(reverse=True)
-        self.assertEqual(sorted_tbl[0], {'a': 3, 'b': 1})
+        self.assertEqual(sorted_tbl[0], {'a': 3, 'b': 1})
+
+    def test_set_header(self):
+
+        # Rename columns
+        tbl = Table([['one', 'two'], [1, 2], [3, 4]])
+        new_tbl = tbl.set_header(['oneone', 'twotwo'])
+
+        self.assertEqual(new_tbl[0], {'oneone': 1, 'twotwo': 2})
+
+        # Change number of columns
+        tbl = Table([['one', 'two'], [1, 2], [3, 4]])
+        new_tbl = tbl.set_header(['one'])
+
+        self.assertEqual(new_tbl[0], {'one': 1})

test/test_rockthevote/sample.csv

@@ -0,0 +1,2 @@
+Status,Tracking Source,Tracking ID,Open Tracking ID,State API Submission Result,Language,Date of birth,Email address,US citizen?,Salutation,First name,Middle name,Last name,Name suffix,Home address,Home unit,Home city,Home County,Home state,Home zip code,Has mailing address?,Mailing address,Mailing unit,Mailing city,Mailing County,Mailing state,Mailing zip code,Party,Race,Phone,Phone type,Opt-in to RTV email?,Opt-in to RTV sms?,Opt-in to Partner email?,Opt-in to Partner SMS/robocall,Survey question 1,Survey answer 1,Survey question 2,Survey answer 2,Volunteer for RTV,Volunteer for partner,Ineligible reason,Pre-Registered,Started registration,Finish with State,Built via API,Has State License,Has SSN,VR Application Submission Modifications,VR Application Submission Errors,VR Application Status,VR Application Status Details,VR Application Status Imported DateTime,Submitted Via State API,Submitted Signature to State API
+Step 1,ABCD,"",,,English,"",[email protected],Yes,,Carol,,King,,,,Laurel,,MD,20708,No,,,,,,,,,,,No,No,Yes,No,,,,,No,No,,No,1982-02-10 00:56:02 -0400,No,No,No,No,"","",,,,No,