VAN: Upload Saved Lists (move-coop#106)

* Set up client, methods.

* Update requirements.

* Saved list.

* Add tests.

* Fix tests.

* Struggling with the unittests...

* Better testing and EveryAction DB functionality.

* Fix typos.

* Make the code cleaner.

* Fix scores method too.

* Final cleanup.

parsons/ngpvan/saved_lists.py

@@ -1,7 +1,10 @@
 """NGPVAN Saved List Endpoints"""
 
 from parsons.etl.table import Table
+from parsons.utilities import cloud_storage
 import logging
+import uuid
+from suds.client import Client
 
 logger = logging.getLogger(__name__)
 
@@ -64,6 +67,81 @@ def download_saved_list(self, saved_list_id):
         else:
             return Table.from_csv(job['downloadUrl'])
 
+    def upload_saved_list(self, tbl, list_name, folder_id, url_type, id_type='vanid', replace=False,
+                          **url_kwargs):
+        """
+        Upload a saved list. Invalid or unmatched person id records will be ignored. Your api user
+        must be shared on the target folder.
+
+        `Args:`
+            tbl: parsons.Table
+                A parsons table object containing one column of person ids.
+            list_name: str
+                The saved list name.
+            folder_id: int
+                The folder id where the list will be stored.
+            url_post_type: str
+                The cloud file storage to use to post the file. Currently only ``S3``.
+            id_type: str
+                The primary key type. The options, beyond ``vanid`` are specific to your
+                instance of VAN.
+            replace: boolean
+                Replace saved list if already exists.
+            **url_kwargs: kwargs
+                Arguments to configure your cloud storage url type.
+                    * S3 requires ``bucket`` argument and, if not stored as env variables
+                      ``aws_access_key`` and ``aws_secret_access_key``.
+        `Returns:`
+            dict
+                Upload results information included the number of matched and saved
+                records in your list.
+        """
+
+        # Move to cloud storage
+        file_name = str(uuid.uuid1())
+        url = cloud_storage.post_file(tbl, url_type, file_path=file_name + '.zip', **url_kwargs)
+        logger.info(f'Table uploaded to {url_type}.')
+
+        # Create XML
+        xml = self.connection.soap_client.factory.create('CreateAndStoreSavedListMetaData')
+        xml.SavedList._Name = list_name
+        xml.DestinationFolder._ID = folder_id
+        xml.SourceFile.FileName = file_name + '.csv'
+        xml.SourceFile.FileUrl = url
+        xml.SourceFile.FileCompression = 'zip'
+        xml.Options.OverwriteExistingList = replace
+
+        # Describe file
+        file_desc = self.connection.soap_client.factory.create('SeparatedFileFormatDescription')
+        file_desc._name = 'csv'
+        file_desc.HasHeaderRow = True
+
+        # Only support single column for now
+        col = self.connection.soap_client.factory.create('Column')
+        col.Name = id_type
+        col.RefersTo._Path = f"Person[@PersonIDType=\'{id_type}\']"
+        col._Index = '0'
+
+        # VAN errors for this method are not particularly useful or helpful. For that reason, we
+        # will check that the folder exists and if the list already exists.
+        logger.info('Validating folder id and list name.')
+        if folder_id not in [x['folderId'] for x in self.get_folders()]:
+            raise ValueError("Folder does not exist or is not shared with API user.")
+
+        if not replace:
+            if list_name in [x['name'] for x in self.get_saved_lists(folder_id)]:
+                raise ValueError("Saved list already exists. Set to replace argument to True or "
+                                 "change list name.")
+
+        # Assemble request
+        file_desc.Columns.Column.append(col)
+        xml.SourceFile.Format = file_desc
+
+        r = Client.dict(self.connection.soap_client.service.CreateAndStoreSavedList(xml))
+        if r:
+            logger.info(f"Uploaded {r['ListSize']} records to {r['_Name']} saved list.")
+        return r
+
 
 class Folders(object):
 

parsons/ngpvan/scores.py

@@ -1,7 +1,7 @@
 """NGPVAN Score Endpoints"""
 
 from parsons.etl.table import Table
-from parsons.utilities import cloud_storage, files
+from parsons.utilities import cloud_storage
 import uuid
 import logging
 import petl
@@ -170,20 +170,19 @@ def upload_scores(self, tbl, config, url_type, id_type='vanid', email=None, auto
         """
 
         # Move to cloud storage
-        file_name = str(uuid.uuid1()) + '.zip'
-        public_url = cloud_storage.post_file(tbl, url_type, file_path=file_name, **url_kwargs)
-        csv_name = files.extract_file_name(file_name, include_suffix=False) + '.csv'
+        file_name = str(uuid.uuid1())
+        url = cloud_storage.post_file(tbl, url_type, file_path=file_name + '.zip', **url_kwargs)
         logger.info(f'Table uploaded to {url_type}.')
 
         # Generate shell request
         json = {"description": 'A description',
                 "file": {
                     "columnDelimiter": 'csv',
                     "columns": [{'name': c} for c in tbl.columns],
-                    "fileName": csv_name,
+                    "fileName": file_name + '.csv',
                     "hasHeader": "True",
                     "hasQuotes": "False",
-                    "sourceUrl": public_url},
+                    "sourceUrl": url},
                 "actions": []
                 }
 

parsons/ngpvan/van_connector.py

@@ -1,10 +1,12 @@
+from suds.client import Client
 import logging
 from parsons.utilities import check_env
 from parsons.utilities.api_connector import APIConnector
 
 logger = logging.getLogger(__name__)
 
 URI = 'https://api.securevan.com/v4/'
+SOAP_URI = 'https://api.securevan.com/Services/V3/ListService.asmx?WSDL'
 
 
 class VANConnector(object):
@@ -27,6 +29,34 @@ def __init__(self, api_key=None, auth_name='default', db=None):
         self.auth = (self.auth_name, self.api_key + '|' + str(self.db_code))
         self.api = APIConnector(self.uri, auth=self.auth, data_key='items')
 
+        # We will not create the SOAP client unless we need to as this triggers checking for
+        # valid credentials. As not all API keys are provisioned for SOAP, this keeps it from
+        # raising a permission exception when creating the class.
+        self._soap_client = None
+
+    @property
+    def soap_client(self):
+
+        if not self._soap_client:
+
+            # Create the SOAP client
+            soap_auth = {'Header': {'DatabaseMode': self.soap_client_db(), 'APIKey': self.api_key}}
+            self._soap_client = Client(SOAP_URI, soapheaders=soap_auth)
+
+        return self._soap_client
+
+    def soap_client_db(self):
+        """
+        Parse the REST database name to the accepted SOAP format
+        """
+
+        if self.db == 'MyVoters':
+            return 'MyVoterFile'
+        if self.db == 'EveryAction':
+            return 'MyCampaign'
+        else:
+            return self.db
+
     def get_request(self, endpoint, **kwargs):
 
         r = self.api.get_request(self.uri + endpoint, **kwargs)

parsons/utilities/cloud_storage.py

@@ -1,4 +1,3 @@
-
 """
 This utility method is a generalizable method for moving files to an
 online file storage class. It is used by methods that require access

requirements.txt

@@ -26,6 +26,7 @@ docutils<0.15,>=0.10 # Botocore and Sphinx have conflicting needs.
 urllib3<1.25,>=1.21.1 # Requests 2.20.0 requirement
 simplejson==3.16.0
 twilio==6.30.0
+suds-py3==1.3.4.0
 
 # Testing Requirements
 requests-mock==1.5.2

test/test_van/test_ngpvan.py

@@ -1,5 +1,5 @@
-import unittest
 import os
+import unittest
 import requests_mock
 from parsons.ngpvan.van import VAN
 from parsons.etl.table import Table
@@ -48,170 +48,6 @@ def test_get_canvass_responses_result_codes(self, m):
         m.get(self.van.connection.uri + 'canvassResponses/resultCodes', json=json)
         assert_matching_tables(Table(json), self.van.get_canvass_responses_result_codes())
 
-    @requests_mock.Mocker()
-    def test_get_saved_lists(self, m):
-
-        json = {'count': 1, 'items': [
-            {"savedListId": 517612,
-             "listCount": 974656,
-             "name": "LikelyParents(16andunder)_DWID_S... - MN",
-             "doorCount": 520709,
-             "description": "null"
-             }
-        ], 'nextPageLink': None}
-
-        m.get(self.van.connection.uri + 'savedLists', json=json)
-
-        expected = ['savedListId', 'listCount', 'name', 'doorCount','description']
-
-        self.assertTrue(validate_list(expected, self.van.get_saved_lists()))
-
-    @requests_mock.Mocker()
-    def test_get_saved_list(self, m):
-
-        saved_list_id = 517612
-
-        json = {"savedListId": 517612,
-                "listCount": 974656,
-                "name": "LikelyParents(16andunder)_DWID_S... - MN",
-                "doorCount": 520709,
-                "description": "null"
-                }
-
-        m.get(self.van.connection.uri + f'savedLists/{saved_list_id}', json=json)
-
-        expected = ['savedListId', 'listCount', 'name', 'doorCount', 'description']
-
-        self.assertEqual(self.van.get_saved_list(saved_list_id), json)
-
-    @requests_mock.Mocker()
-    def test_get_folders(self, m):
-
-        json = {u'count': 2,
-                u'items': [
-                    {
-                        u'folderId': 5046,
-                        u'name': u'#2018_MN_active_universe'
-                    },
-                    {u'folderId': 2168,
-                     u'name': u'API Generated Lists'
-                     }
-                ], u'nextPageLink': None}
-
-        m.get(self.van.connection.uri + 'folders', json=json)
-
-        expected = ['folderId', 'name']
-
-        self.assertTrue(validate_list(expected, self.van.get_folders()))
-
-    @requests_mock.Mocker()
-    def test_get_folder(self, m):
-
-        folder_id = 5046
-
-        json = {"folderId": 5046, "name": "#2018_MN_active_universe"}
-
-        m.get(self.van.connection.uri + f'folders/{folder_id}', json=json)
-
-        self.assertEqual(json, self.van.get_folder(folder_id))
-
-    @requests_mock.Mocker()
-    def test_export_job_types(self, m):
-
-        json = {u'count': 1, u'items':
-                [{u'exportJobTypeId': 4, u'name': u'SavedListExport'}],
-                u'nextPageLink': None}
-
-        m.get(self.van.connection.uri + 'exportJobTypes', json=json)
-
-        expected = ['exportJobTypeId', 'name']
-
-        self.assertTrue(validate_list(expected, self.van.get_export_job_types()))
-
-    @requests_mock.Mocker()
-    def test_export_job_create(self, m):
-
-        saved_list_id = 517612
-
-        json = {"status": "Completed",
-                "errorCode": "null",
-                "exportJobGuid": "bf4d1297-1c77-3fb2-03bd-f0acda122d37",
-                "activistCodes": "null",
-                "canvassFileRequestId": 448,
-                "dateExpired": "2018-09-08T16:04:00Z",
-                "surveyQuestions": "null",
-                "webhookUrl": "https://www.nothing.com/",
-                "downloadUrl": "https://ngpvan.blob.core.windows.net/canvass-files-savedlistexport/bf4d1297-1c77-3fb2-03bd-f0acda122d37_2018-09-08T13:03:27.7191831-04:00.csv",  # noqa: E501
-                "savedListId": 517612,
-                "districtFields": "null",
-                "canvassFileRequestGuid": "bf4d1297-1c77-3fb2-03bd-f0acda122d37",
-                "customFields": "null",
-                "type": 4,
-                "exportJobId": 448}
-
-        m.post(self.van.connection.uri + 'exportJobs', json=json, status_code=201)
-
-        expected = [
-            'status',
-            'errorCode',
-            'exportJobGuid',
-            'activistCodes',
-            'canvassFileRequestId',
-            'dateExpired',
-            'surveyQuestions',
-            'webhookUrl',
-            'downloadUrl',
-            'savedListId',
-            'districtFields',
-            'canvassFileRequestGuid',
-            'customFields',
-            'type',
-            'exportJobId']
-
-        self.assertEqual(json,self.van.export_job_create(saved_list_id))
-
-    @requests_mock.Mocker()
-    def test_get_export_job(self, m):
-
-        export_job_id = 448
-
-        json = {"status": "Completed",
-                "errorCode": "null",
-                "exportJobGuid": "bf4d1297-1c77-3fb2-03bd-f0acda122d37",
-                "activistCodes": "null",
-                "canvassFileRequestId": 448,
-                "dateExpired": "2018-09-08T16:04:00Z",
-                "surveyQuestions": "null",
-                "webhookUrl": "https://www.nothing.com/",
-                "downloadUrl": "https://ngpvan.blob.core.windows.net/canvass-files-savedlistexport/bf4d1297-1c77-3fb2-03bd-f0acda122d37_2018-09-08T13:03:27.7191831-04:00.csv",  # noqa: E501
-                "savedListId": 517612,
-                "districtFields": "null",
-                "canvassFileRequestGuid": "bf4d1297-1c77-3fb2-03bd-f0acda122d37",
-                "customFields": "null",
-                "type": 4,
-                "exportJobId": 448}
-
-        expected = [
-            'status',
-            'errorCode',
-            'exportJobGuid',
-            'activistCodes',
-            'canvassFileRequestId',
-            'dateExpired',
-            'surveyQuestions',
-            'webhookUrl',
-            'downloadUrl',
-            'savedListId',
-            'districtFields',
-            'canvassFileRequestGuid',
-            'customFields',
-            'type',
-            'exportJobId']
-
-        m.get(self.van.connection.uri + f'exportJobs/{export_job_id}', json=json)
-
-        self.assertEqual(json, self.van.get_export_job(export_job_id))
-
     @requests_mock.Mocker()
     def test_get_survey_questions(self, m):