Moving common testing tools to gcp-repo-tools.

* Updating travis.yml for new GAE tool.
* Updating tox.ini to use new tools.
* Updating imports in various tests to use new tools.
* Updating root conftest.py to provide configuration.
* Updating secrets.tar with client-secrets, needed for incoming bigquery tests.

.gitignore

@@ -12,6 +12,7 @@ managed_vms/django_tutorial/static/*
 lib
 testing/resources/test-env.sh
 testing/resources/service-account.json
+testing/resources/client-secrets.json
 secrets.tar
 .cache
 junit.xml

.travis.yml

@@ -1,8 +1,8 @@
 sudo: false
 language: python
 services:
-  - memcached
-  - mysql
+- memcached
+- mysql
 branches:
   only:
   - master
@@ -13,15 +13,15 @@ env:
   global:
   - PATH=${PATH}:${HOME}/gcloud/google-cloud-sdk/bin
   - GOOGLE_APPLICATION_CREDENTIALS=${TRAVIS_BUILD_DIR}/testing/resources/service-account.json
+  - GOOGLE_CLIENT_SECRETS=${TRAVIS_BUILD_DIR}/testing/resources/client-secrets.json
   - GAE_PYTHONPATH=${HOME}/.cache/google_appengine
-  - secure: YIowCOMJ97rTcehKVT6Gi3u0Etm8s9+TBRGsNPJLgSF2zZdsh9IHcIc+tMDUMR3lpOe8y2a060RuODQcRsW1W1LIHej+ZE/gv6vATT6qNA3eKfKmZ9AyrpBO0fTOHlHrGBuU9ktBPR+iqvnq8MLWjnUozPFMJbuNBFITU7JP8jc=
+  - GAE_ROOT=${HOME}/.cache/
+  - secure: Orp9Et2TIwCG/Hf59aa0NUDF1pNcwcS4TFulXX175918cFREOzf/cNZNg+Ui585ZRFjbifZdc858tVuCVd8XlxQPXQgp7bwB7nXs3lby3LYg4+HD83Gaz7KOWxRLWVor6IVn8OxeCzwl6fJkdmffsTTO9csC4yZ7izHr+u7hiO4=
 before_install:
 - openssl aes-256-cbc -k "$secrets_password" -in secrets.tar.enc -out secrets.tar -d
 - tar xvf secrets.tar
 install:
 - pip install tox coverage
-- pip install -e git+https://github.com/GoogleCloudPlatform/python-repo-tools#egg=python-repo-tools
-- gcp-python-repo-tools download-appengine-sdk `dirname "${GAE_PYTHONPATH}"`
 script:
 - source ${TRAVIS_BUILD_DIR}/testing/resources/test-env.sh
 - tox

appengine/bigquery/main_test.py

@@ -14,7 +14,7 @@
 
 import re
 
-from apiclient.http import HttpMock
+from googleapiclient.http import HttpMock
 import main
 import mock
 import pytest
@@ -23,7 +23,7 @@
 
 @pytest.fixture
 def app(cloud_config, testbed):
-    main.PROJECTID = cloud_config.GCLOUD_PROJECT
+    main.PROJECTID = cloud_config.project
     return webtest.TestApp(main.app)
 
 

appengine/conftest.py

@@ -1,35 +1,27 @@
-import pytest
-import testing.appengine
-
-
-def pytest_configure(config):
-    testing.appengine.setup_sdk_imports()
-
-
-def pytest_runtest_call(item):
-    testing.appengine.import_appengine_config()
-
-
-@pytest.yield_fixture
-def testbed():
-    testbed = testing.appengine.setup_testbed()
-    yield testbed
-    testbed.deactivate()
-
-
-@pytest.fixture
-def login(testbed):
-    def _login(email='[email protected]', id='123', is_admin=False):
-        testbed.setup_env(
-            user_email=email,
-            user_id=id,
-            user_is_admin='1' if is_admin else '0',
-            overwrite=True)
-    return _login
-
-
-@pytest.fixture
-def run_tasks(testbed):
-    def _run_tasks(app):
-        testing.appengine.run_tasks(testbed, app)
-    return _run_tasks
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Import py.test hooks and fixtures for App Engine
+from gcp.testing.appengine import (
+    login,
+    pytest_configure,
+    pytest_runtest_call,
+    run_tasks,
+    testbed)
+
+(login)
+(pytest_configure)
+(pytest_runtest_call)
+(run_tasks)
+(testbed)

appengine/mailgun/main_test.py

@@ -12,12 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from googleapiclient.http import HttpMockSequence
+import httplib2
 import main
+import mock
 import pytest
-from testing import Http2Mock
 import webtest
 
 
+class HttpMockSequenceWithCredentials(HttpMockSequence):
+    def add_credentials(self, *args):
+        pass
+
+
 @pytest.fixture
 def app():
     return webtest.TestApp(main.app)
@@ -29,27 +36,31 @@ def test_get(app):
 
 
 def test_post(app):
-    http = Http2Mock(responses=[{}])
+    http = HttpMockSequenceWithCredentials([
+        ({'status': '200'}, '')])
+    patch_http = mock.patch.object(httplib2, 'Http', lambda: http)
 
-    with http:
+    with patch_http:
         response = app.post('/', {
             'recipient': '[email protected]',
             'submit': 'Send simple email'})
 
         assert response.status_int == 200
 
-    http = Http2Mock(responses=[{}])
+    http = HttpMockSequenceWithCredentials([
+        ({'status': '200'}, '')])
 
-    with http:
+    with patch_http:
         response = app.post('/', {
             'recipient': '[email protected]',
             'submit': 'Send complex email'})
 
         assert response.status_int == 200
 
-    http = Http2Mock(responses=[{'status': 500, 'body': 'Test error'}])
+    http = HttpMockSequenceWithCredentials([
+        ({'status': '500'}, 'Test error')])
 
-    with http, pytest.raises(Exception):
+    with patch_http, pytest.raises(Exception):
         app.post('/', {
             'recipient': '[email protected]',
             'submit': 'Send simple email'})

appengine/storage/main_test.py

@@ -18,7 +18,7 @@
 
 
 def test_get(cloud_config):
-    main.BUCKET_NAME = cloud_config.GCLOUD_PROJECT
+    main.BUCKET_NAME = cloud_config.project
     app = webtest.TestApp(main.app)
 
     response = app.get('/')

bigquery/api/async_query_test.py

@@ -22,7 +22,7 @@ def test_async_query(cloud_config, capsys):
         'GROUP BY corpus;')
 
     main(
-        project_id=cloud_config.GCLOUD_PROJECT,
+        project_id=cloud_config.project,
         query_string=query,
         batch=False,
         num_retries=5,

bigquery/api/export_data_to_cloud_storage_test.py

@@ -12,47 +12,47 @@
 # limitations under the License.
 
 from export_data_to_cloud_storage import main
-from testing import mark_flaky
+from gcp.testing.flaky import flaky
 
 DATASET_ID = 'test_dataset'
 TABLE_ID = 'test_table'
 
 
-@mark_flaky
+@flaky
 def test_export_table_csv(cloud_config):
     cloud_storage_output_uri = \
-        'gs://{}/output.csv'.format(cloud_config.CLOUD_STORAGE_BUCKET)
+        'gs://{}/output.csv'.format(cloud_config.storage_bucket)
     main(
         cloud_storage_output_uri,
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID,
         num_retries=5,
         interval=1,
         export_format="CSV")
 
 
-@mark_flaky
+@flaky
 def test_export_table_json(cloud_config):
     cloud_storage_output_uri = \
-        'gs://{}/output.json'.format(cloud_config.CLOUD_STORAGE_BUCKET)
+        'gs://{}/output.json'.format(cloud_config.storage_bucket)
     main(
         cloud_storage_output_uri,
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID,
         num_retries=5,
         interval=1,
         export_format="NEWLINE_DELIMITED_JSON")
 
 
-@mark_flaky
+@flaky
 def test_export_table_avro(cloud_config):
     cloud_storage_output_uri = \
-        'gs://{}/output.avro'.format(cloud_config.CLOUD_STORAGE_BUCKET)
+        'gs://{}/output.avro'.format(cloud_config.storage_bucket)
     main(
         cloud_storage_output_uri,
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID,
         num_retries=5,

bigquery/api/getting_started_test.py

@@ -17,7 +17,7 @@
 
 
 def test_main(cloud_config, capsys):
-    main(cloud_config.GCLOUD_PROJECT)
+    main(cloud_config.project)
 
     out, _ = capsys.readouterr()
 

bigquery/api/list_datasets_projects_test.py

@@ -17,7 +17,7 @@
 
 
 def test_main(cloud_config, capsys):
-    main(cloud_config.GCLOUD_PROJECT)
+    main(cloud_config.project)
 
     out, _ = capsys.readouterr()
 

bigquery/api/load_data_by_post_test.py

@@ -13,22 +13,22 @@
 
 import re
 
+from gcp.testing.flaky import flaky
 from load_data_by_post import load_data
-from testing import mark_flaky
 
 DATASET_ID = 'ephemeral_test_dataset'
 TABLE_ID = 'load_data_by_post'
 
 
-@mark_flaky
+@flaky
 def test_load_csv_data(cloud_config, resource, capsys):
     schema_path = resource('schema.json')
     data_path = resource('data.csv')
 
     load_data(
         schema_path,
         data_path,
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID
     )
@@ -39,15 +39,15 @@ def test_load_csv_data(cloud_config, resource, capsys):
         r'Waiting for job to finish.*Job complete.', re.DOTALL), out)
 
 
-@mark_flaky
+@flaky
 def test_load_json_data(cloud_config, resource, capsys):
     schema_path = resource('schema.json')
     data_path = resource('data.json')
 
     load_data(
         schema_path,
         data_path,
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID
     )

bigquery/api/load_data_from_csv_test.py

@@ -11,22 +11,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+from gcp.testing.flaky import flaky
 from load_data_from_csv import main
-from testing import mark_flaky
 
 DATASET_ID = 'test_dataset'
 TABLE_ID = 'test_import_table'
 
 
-@mark_flaky
+@flaky
 def test_load_table(cloud_config, resource):
     cloud_storage_input_uri = 'gs://{}/data.csv'.format(
-        cloud_config.CLOUD_STORAGE_BUCKET)
+        cloud_config.storage_bucket)
     schema_file = resource('schema.json')
 
     main(
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID,
         schema_file=schema_file,

bigquery/api/streaming_test.py

@@ -27,7 +27,7 @@ def test_stream_row_to_bigquery(cloud_config, resource, capsys):
     streaming.get_rows = lambda: rows
 
     streaming.main(
-        cloud_config.GCLOUD_PROJECT,
+        cloud_config.project,
         DATASET_ID,
         TABLE_ID,
         num_retries=5)

bigquery/api/sync_query_test.py

@@ -22,7 +22,7 @@ def test_sync_query(cloud_config, capsys):
         'GROUP BY corpus;')
 
     main(
-        project_id=cloud_config.GCLOUD_PROJECT,
+        project_id=cloud_config.project,
         query=query,
         timeout=30,
         num_retries=5)

blog/introduction_to_data_models_in_cloud_datastore/blog_test.py

@@ -12,9 +12,9 @@
 # limitations under the License.
 
 from blog import main
-from testing import mark_flaky
+from gcp.testing.flaky import flaky
 
 
-@mark_flaky
+@flaky
 def test_main(cloud_config):
-    main(cloud_config.GCLOUD_PROJECT)
+    main(cloud_config.project)

blog/introduction_to_data_models_in_cloud_datastore/wiki_test.py

@@ -11,10 +11,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from testing import mark_flaky
+from gcp.testing.flaky import flaky
 from wiki import main
 
 
-@mark_flaky
+@flaky
 def test_main(cloud_config):
-    main(cloud_config.GCLOUD_PROJECT)
+    main(cloud_config.project)

cloud_logging/api/list_logs_test.py

@@ -17,6 +17,6 @@
 
 
 def test_main(cloud_config, capsys):
-    list_logs.main(cloud_config.GCLOUD_PROJECT)
+    list_logs.main(cloud_config.project)
     out, _ = capsys.readouterr()
     assert re.search(re.compile(r'.*', re.S), out)