Initial Python3 support

Notable Changes:
  -Convert all str types to byte types at cython layer (ascii encoding)
  -Use six for packages that have different names in Python2/Python3
  -By default, unit tests are compiled/run in Python2.7 and Python3.4
  -Ensure MACOSX_BUILD_TARGET is at least 10.7

setup.py

@@ -33,9 +33,11 @@
 import os.path
 import shutil
 import sys
+import sysconfig
 
 from distutils import core as _core
 from distutils import extension as _extension
+import pkg_resources
 import setuptools
 from setuptools.command import egg_info
 
@@ -110,6 +112,16 @@
   DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
 
 
+# By default, Python3 distutils enforces compatibility of
+# c plugins (.so files) with the OSX version Python3 was built with.
+# For Python3.4, this is OSX 10.6, but we need Thread Local Support (__thread)
+if 'darwin' in sys.platform and PY3:
+  mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
+  if mac_target and (pkg_resources.parse_version(mac_target) <
+                     pkg_resources.parse_version('10.7.0')):
+    os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
+
+
 def cython_extensions(module_names, extra_sources, include_dirs,
                       libraries, define_macros, build_with_cython=False):
   # Set compiler directives linetrace argument only if we care about tracing;

src/python/grpcio/commands.py

@@ -191,7 +191,7 @@ def run(self):
       except subprocess.CalledProcessError as e:
         sys.stderr.write(
             'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
-                command, e.message, e.output))
+                command, str(e), e.output))
 
     # Generated proto directories dont include __init__.py, but
     # these are needed for python package resolution

src/python/grpcio/grpc/_channel.py

@@ -85,7 +85,7 @@ def _deadline(timeout):
 
 
 def _unknown_code_details(unknown_cygrpc_code, details):
-  return b'Server sent unknown code {} and details "{}"'.format(
+  return 'Server sent unknown code {} and details "{}"'.format(
       unknown_cygrpc_code, details)
 
 
@@ -142,7 +142,7 @@ def _handle_event(event, state, response_deserializer):
         response = _common.deserialize(
             serialized_response, response_deserializer)
         if response is None:
-          details = b'Exception deserializing response!'
+          details = 'Exception deserializing response!'
           _abort(state, grpc.StatusCode.INTERNAL, details)
         else:
           state.response = response
@@ -186,7 +186,7 @@ def consume_request_iterator():
         if state.code is None and not state.cancelled:
           if serialized_request is None:
             call.cancel()
-            details = b'Exception serializing request!'
+            details = 'Exception serializing request!'
             _abort(state, grpc.StatusCode.INTERNAL, details)
             return
           else:
@@ -230,7 +230,7 @@ def cancel(self):
       if self._state.code is None:
         self._call.cancel()
         self._state.cancelled = True
-        _abort(self._state, grpc.StatusCode.CANCELLED, b'Cancelled!')
+        _abort(self._state, grpc.StatusCode.CANCELLED, 'Cancelled!')
         self._state.condition.notify_all()
       return False
 
@@ -402,7 +402,7 @@ def _start_unary_request(request, timeout, request_serializer):
   if serialized_request is None:
     state = _RPCState(
         (), _EMPTY_METADATA, _EMPTY_METADATA, grpc.StatusCode.INTERNAL,
-        b'Exception serializing request!')
+        'Exception serializing request!')
     rendezvous = _Rendezvous(state, None, None, deadline)
     return deadline, deadline_timespec, None, rendezvous
   else:

src/python/grpcio/grpc/_common.py

@@ -97,3 +97,16 @@ def serialize(message, serializer):
 def deserialize(serialized_message, deserializer):
   return _transform(serialized_message, deserializer,
                     'Exception deserializing message!')
+
+
+def _encode(s):
+  if isinstance(s, bytes):
+    return s
+  else:
+    return s.encode('ascii')
+
+
+def fully_qualified_method(group, method):
+  group = _encode(group)
+  method = _encode(method)
+  return b'/' + group + b'/' + method

src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi

@@ -55,6 +55,7 @@ cdef class Call:
   def cancel(
       self, grpc_status_code error_code=GRPC_STATUS__DO_NOT_USE,
       details=None):
+    details = str_to_bytes(details)
     if not self.is_valid:
       raise ValueError("invalid call object cannot be used from Python")
     if (details is None) != (error_code == GRPC_STATUS__DO_NOT_USE):
@@ -63,12 +64,6 @@ cdef class Call:
     cdef grpc_call_error result
     cdef char *c_details = NULL
     if error_code != GRPC_STATUS__DO_NOT_USE:
-      if isinstance(details, bytes):
-        pass
-      elif isinstance(details, basestring):
-        details = details.encode()
-      else:
-        raise TypeError("expected details to be str or bytes")
       self.references.append(details)
       c_details = details
       with nogil:

src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi

@@ -34,18 +34,13 @@ cdef class Channel:
 
   def __cinit__(self, target, ChannelArgs arguments=None,
                 ChannelCredentials channel_credentials=None):
+    target = str_to_bytes(target)
     cdef grpc_channel_args *c_arguments = NULL
     cdef char *c_target = NULL
     self.c_channel = NULL
     self.references = []
     if arguments is not None:
       c_arguments = &arguments.c_args
-    if isinstance(target, bytes):
-      pass
-    elif isinstance(target, basestring):
-      target = target.encode()
-    else:
-      raise TypeError("expected target to be str or bytes")
     c_target = target
     if channel_credentials is None:
       with nogil:
@@ -62,25 +57,14 @@ cdef class Channel:
   def create_call(self, Call parent, int flags,
                   CompletionQueue queue not None,
                   method, host, Timespec deadline not None):
+    method = str_to_bytes(method)
+    host = str_to_bytes(host)
     if queue.is_shutting_down:
       raise ValueError("queue must not be shutting down or shutdown")
-    if isinstance(method, bytes):
-      pass
-    elif isinstance(method, basestring):
-      method = method.encode()
-    else:
-      raise TypeError("expected method to be str or bytes")
     cdef char *method_c_string = method
     cdef char *host_c_string = NULL
-    if host is None:
-      pass
-    elif isinstance(host, bytes):
+    if host is not None:
       host_c_string = host
-    elif isinstance(host, basestring):
-      host = host.encode()
-      host_c_string = host
-    else:
-      raise TypeError("expected host to be str, bytes, or None")
     cdef Call operation_call = Call()
     operation_call.references = [self, method, host, queue]
     cdef grpc_call *parent_call = NULL

src/python/grpcio/grpc/_cython/_cygrpc/credentials.pxd.pxi

@@ -54,7 +54,7 @@ cdef class ServerCredentials:
 cdef class CredentialsMetadataPlugin:
 
   cdef object plugin_callback
-  cdef str plugin_name
+  cdef bytes plugin_name
 
   cdef grpc_metadata_credentials_plugin make_c_plugin(self)
 

src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi

@@ -82,7 +82,7 @@ cdef class ServerCredentials:
 
 cdef class CredentialsMetadataPlugin:
 
-  def __cinit__(self, object plugin_callback, str name):
+  def __cinit__(self, object plugin_callback, name):
     """
     Args:
       plugin_callback (callable): Callback accepting a service URL (str/bytes)
@@ -93,6 +93,7 @@ cdef class CredentialsMetadataPlugin:
         successful).
       name (str): Plugin name.
     """
+    name = str_to_bytes(name)
     if not callable(plugin_callback):
       raise ValueError('expected callable plugin_callback')
     self.plugin_callback = plugin_callback
@@ -129,7 +130,8 @@ cdef void plugin_get_metadata(
     grpc_credentials_plugin_metadata_cb cb, void *user_data) with gil:
   def python_callback(
       Metadata metadata, grpc_status_code status,
-      const char *error_details):
+      error_details):
+    error_details = str_to_bytes(error_details)
     cb(user_data, metadata.c_metadata_array.metadata,
        metadata.c_metadata_array.count, status, error_details)
   cdef CredentialsMetadataPlugin self = <CredentialsMetadataPlugin>state
@@ -148,14 +150,7 @@ def channel_credentials_google_default():
 
 def channel_credentials_ssl(pem_root_certificates,
                             SslPemKeyCertPair ssl_pem_key_cert_pair):
-  if pem_root_certificates is None:
-    pass
-  elif isinstance(pem_root_certificates, bytes):
-    pass
-  elif isinstance(pem_root_certificates, basestring):
-    pem_root_certificates = pem_root_certificates.encode()
-  else:
-    raise TypeError("expected str or bytes for pem_root_certificates")
+  pem_root_certificates = str_to_bytes(pem_root_certificates)
   cdef ChannelCredentials credentials = ChannelCredentials()
   cdef const char *c_pem_root_certificates = NULL
   if pem_root_certificates is not None:
@@ -207,12 +202,7 @@ def call_credentials_google_compute_engine():
 
 def call_credentials_service_account_jwt_access(
     json_key, Timespec token_lifetime not None):
-  if isinstance(json_key, bytes):
-    pass
-  elif isinstance(json_key, basestring):
-    json_key = json_key.encode()
-  else:
-    raise TypeError("expected json_key to be str or bytes")
+  json_key = str_to_bytes(json_key)
   cdef CallCredentials credentials = CallCredentials()
   cdef char *json_key_c_string = json_key
   with nogil:
@@ -223,12 +213,7 @@ def call_credentials_service_account_jwt_access(
   return credentials
 
 def call_credentials_google_refresh_token(json_refresh_token):
-  if isinstance(json_refresh_token, bytes):
-    pass
-  elif isinstance(json_refresh_token, basestring):
-    json_refresh_token = json_refresh_token.encode()
-  else:
-    raise TypeError("expected json_refresh_token to be str or bytes")
+  json_refresh_token = str_to_bytes(json_refresh_token)
   cdef CallCredentials credentials = CallCredentials()
   cdef char *json_refresh_token_c_string = json_refresh_token
   with nogil:
@@ -238,18 +223,8 @@ def call_credentials_google_refresh_token(json_refresh_token):
   return credentials
 
 def call_credentials_google_iam(authorization_token, authority_selector):
-  if isinstance(authorization_token, bytes):
-    pass
-  elif isinstance(authorization_token, basestring):
-    authorization_token = authorization_token.encode()
-  else:
-    raise TypeError("expected authorization_token to be str or bytes")
-  if isinstance(authority_selector, bytes):
-    pass
-  elif isinstance(authority_selector, basestring):
-    authority_selector = authority_selector.encode()
-  else:
-    raise TypeError("expected authority_selector to be str or bytes")
+  authorization_token = str_to_bytes(authorization_token)
+  authority_selector = str_to_bytes(authority_selector)
   cdef CallCredentials credentials = CallCredentials()
   cdef char *authorization_token_c_string = authorization_token
   cdef char *authority_selector_c_string = authority_selector
@@ -272,16 +247,10 @@ def call_credentials_metadata_plugin(CredentialsMetadataPlugin plugin):
 
 def server_credentials_ssl(pem_root_certs, pem_key_cert_pairs,
                            bint force_client_auth):
+  pem_root_certs = str_to_bytes(pem_root_certs)
   cdef char *c_pem_root_certs = NULL
-  if pem_root_certs is None:
-    pass
-  elif isinstance(pem_root_certs, bytes):
-    c_pem_root_certs = pem_root_certs
-  elif isinstance(pem_root_certs, basestring):
-    pem_root_certs = pem_root_certs.encode()
+  if pem_root_certs is not None: 
     c_pem_root_certs = pem_root_certs
-  else:
-    raise TypeError("expected pem_root_certs to be str or bytes")
   pem_key_cert_pairs = list(pem_key_cert_pairs)
   for pair in pem_key_cert_pairs:
     if not isinstance(pair, SslPemKeyCertPair):

src/python/grpcio/grpc/_cython/_cygrpc/grpc_string.pyx.pxi

@@ -0,0 +1,39 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+# This function will ascii encode unicode string inputs if neccesary.
+# In Python3, unicode strings are the default str type.
+cdef bytes str_to_bytes(object s):
+  if s is None or isinstance(s, bytes):
+    return s
+  elif isinstance(s, unicode):
+    return s.encode('ascii')
+  else:
+    raise TypeError('Expected bytes, str, or unicode, not {}'.format(type(s)))