Some more tests; cleanup; more documentation.

py/vtdb/db_object.py

@@ -1,9 +1,11 @@
-"""Module containing base classes and helper methods for database objects.
-
-The base classes represent different sharding schemes like
-unsharded, range-sharded and custom-sharded tables.
-This abstracts sharding details and provides methods
-for common database access patterns.
+"""Module containing the base class for database classes and decorator for db method.
+
+The base class DBObjectBase is the base class for all other database base classes.
+It has methods for common database operations like select, insert, update and delete.
+This module also contains the definition for ShardRouting which is used for determining
+the routing of a query during cursor creation.
+The module also has the db_class_method decorator and db_wrapper method which are
+used for cursor creation and calling the database method.
 """
 import functools
 import logging
@@ -25,25 +27,66 @@ class ShardRouting(object):
   keyspace: keyspace where the table resides.
   sharding_key: sharding key of the table.
   keyrange: keyrange for the query.
+  entity_column_name: the name of the lookup based entity used for routing.
   entity_id_sharding_key_map: this map is used for in clause queries.
   shard_name: this is used to route queries for custom sharded keyspaces.
   """
-
-  keyspace = None
-  sharding_key = None
-  keyrange = None
-  entity_column_name = None
-  entity_id_sharding_key_map = None
-  shard_name = None # For custom sharding
-
   def __init__(self, keyspace):
+    # keyspace of the table.
     self.keyspace = keyspace
+    # sharding_key, entity_column_name and entity_id_sharding_key
+    # are used primarily for routing range-sharded keyspace queries.
+    self.sharding_key = None
+    self.entity_column_name = None
+    self.entity_id_sharding_key_map = None
+    self.keyrange = None
+    self.shard_name = None
 
 
 def _is_iterable_container(x):
   return hasattr(x, '__iter__')
 
 
+def create_cursor_from_params(vtgate_conn, tablet_type, is_dml, table_class):
+  """This method creates the cursor from the required params.
+
+  This is mainly used for creating lookup cursor during create_shard_routing,
+  as there is no real cursor available.
+
+  Args:
+    vtgate_conn: connection to vtgate server.
+    tablet_type: tablet type for the cursor.
+    is_dml: indicates writable cursor or not.
+    table_class: table for which the cursor is being created.
+
+  Returns:
+    cursor
+  """
+  cursor = table_class.create_vtgate_cursor(vtgate_conn, tablet_type, is_dml)
+  return cursor
+
+
+def create_cursor_from_old_cursor(old_cursor, table_class):
+  """This method creates the cursor from an existing cursor.
+
+  This is mainly used for creating lookup cursor during db operations on
+  other database tables.
+
+  Args:
+    old_cursor: existing cursor from which important params are evaluated.
+    table_class: table for which the cursor is being created.
+
+  Returns:
+    cursor
+  """
+  cursor = table_class.create_vtgate_cursor(old_cursor._conn,
+                                            old_cursor.tablet_type,
+                                            old_cursor.is_writable())
+  return cursor
+
+
+
+
 def db_wrapper(method):
   """Decorator that is used to create the appropriate cursor
   for the table and call the database method with it.

py/vtdb/db_object_custom_sharded.py

@@ -1,9 +1,8 @@
-"""Module containing base classes and helper methods for database objects.
+"""Module containing base class for tables in custom sharded keyspace.
 
-The base classes represent different sharding schemes like
-unsharded, range-sharded and custom-sharded tables.
-This abstracts sharding details and provides methods
-for common database access patterns.
+Vitess sharding scheme is range-sharded. Vitess supports routing for
+other sharding schemes by allowing explicit shard_name addressing.
+This implementation is not fully complete as yet.
 """
 import logging
 

py/vtdb/db_object_lookup.py

@@ -1,9 +1,9 @@
-"""Module containing base classes and helper methods for database objects.
+"""Module containing base class for lookup database tables.
 
-The base classes represent different sharding schemes like
-unsharded, range-sharded and custom-sharded tables.
-This abstracts sharding details and provides methods
-for common database access patterns.
+LookupDBObject defines the base class for lookup tables and defines
+relevant methods. LookupDBObject inherits from DBObjectUnsharded and
+extends the functionality for getting, creating, updating and deleting
+the lookup relationship.
 """
 import functools
 import logging
@@ -37,8 +37,9 @@ def create(class_, cursor, **bind_vars):
   def update(class_, cursor, sharding_key_column_name, sharding_key,
              entity_id_column, new_entity_id):
     where_column_value_pairs = [(sharding_key_column_name, sharding_key),]
+    update_columns = {entity_id_column:new_entity_id}
     return class_.update_columns(cursor, where_column_value_pairs,
-                                 entity_id_column=new_entity_id)
+                                 **update_columns)
 
   @classmethod
   def delete(class_, cursor, sharding_key_column_name, sharding_key):

py/vtdb/db_object_range_sharded.py

@@ -1,9 +1,12 @@
-"""Module containing base classes and helper methods for database objects.
-
-The base classes represent different sharding schemes like
-unsharded, range-sharded and custom-sharded tables.
-This abstracts sharding details and provides methods
-for common database access patterns.
+"""Module containing base classes for range-sharded database objects.
+
+There are two base classes for tables that live in range-sharded keyspace -
+1. DBObjectRangeSharded -  This should be used for tables that only reference lookup entities
+but don't create or manage them. Please see examples in test/clientlib_tests/db_class_sharded.py.
+2. DBObjectEntityRangeSharded - This inherits from DBObjectRangeSharded and is used for tables
+and also create new lookup relationships.
+This module also contains helper methods for cursor creation for accessing lookup tables
+and methods for dml and select for the above mentioned base classes.
 """
 import functools
 import logging
@@ -17,23 +20,19 @@
 from vtdb import sql_builder
 from vtdb import vtgate_cursor
 
+
+# This creates a 64 binary packed string for keyspace_id.
+# This is used for cursor creation so that keyspace_id can
+# be passed as rpc param for vtgate.
 pack_keyspace_id = struct.Struct('!Q').pack
 
+
+# This unpacks the keyspace_id so that it can be used
+# in bind variables.
 def unpack_keyspace_id(kid):
  return struct.Struct('!Q').unpack(kid)[0]
 
 
-def create_cursor_from_params(vtgate_conn, tablet_type, is_dml, table_class):
-  cursor = table_class.create_vtgate_cursor(vtgate_conn, tablet_type, is_dml)
-  return cursor
-
-
-def create_cursor_from_old_cursor(old_cursor, table_class):
-  cursor = table_class.create_vtgate_cursor(old_cursor._conn,
-                                            old_cursor.tablet_type,
-                                            old_cursor.is_writable())
-  return cursor
-
 
 
 class DBObjectRangeSharded(db_object.DBObjectBase):
@@ -95,20 +94,23 @@ def create_shard_routing(class_, *pargs,  **kargs):
     routing = db_object.ShardRouting(class_.keyspace)
     entity_id_map = None
 
-
     entity_id_map = kargs.get("entity_id_map", None)
     if entity_id_map is None:
-      keyrange = kargs.get("keyrange", None)
-      if keyrange is not None:
-        routing.keyrange = keyrange
+      kr = None
+      key_range = kargs.get("keyrange", None)
+      if isinstance(key_range, keyrange.KeyRange):
+        kr = key_range
+      else:
+        kr = keyrange.KeyRange(key_range)
+      if kr is not None:
+        routing.keyrange = kr
       # Both entity_id_map and keyrange have been evaluated. Return.
-      return routing, kargs
+      return routing
 
     # entity_id_map is not None
     if len(entity_id_map) != 1:
       dbexceptions.ProgrammingError("Invalid entity_id_map '%s'" % entity_id_map)
 
-    #logging.info("In create_shard_routing entity_id_map %s" % (entity_id_map))
     entity_id_col = entity_id_map.keys()[0]
     entity_id = entity_id_map[entity_id_col]
 
@@ -130,7 +132,8 @@ def create_shard_routing(class_, *pargs,  **kargs):
 
   @classmethod
   def create_vtgate_cursor(class_, vtgate_conn, tablet_type, is_dml, **cursor_kargs):
-    cursor_method = functools.partial(create_cursor_from_params, vtgate_conn, tablet_type, False)
+    cursor_method = functools.partial(db_object.create_cursor_from_params,
+                                      vtgate_conn, tablet_type, False)
     routing = class_.create_shard_routing(cursor_method, **cursor_kargs)
     if is_dml:
       if routing.sharding_key is None or db_object._is_iterable_container(routing.sharding_key):
@@ -225,7 +228,11 @@ def select_by_ids(class_, cursor, where_column_value_pairs,
     if cursor.routing.sharding_key is not None:
       # If the in-clause is based on sharding key
       entity_col_name = class_.sharding_key_column_name
-      for sk in list(cursor.routing.sharding_key):
+      if db_object._is_iterable_container(cursor.routing.sharding_key):
+        for sk in list(cursor.routing.sharding_key):
+          entity_id_keyspace_id_map[sk] = pack_keyspace_id(class_.sharding_key_to_keyspace_id(sk))
+      else:
+        sk = cursor.routing.sharding_key
         entity_id_keyspace_id_map[sk] = pack_keyspace_id(class_.sharding_key_to_keyspace_id(sk))
     elif cursor.routing.entity_id_sharding_key_map is not None:
       # If the in-clause is based on entity column
@@ -305,11 +312,12 @@ def update_columns(class_, cursor, where_column_value_pairs,
     rowcount = cursor.execute(query, bind_vars)
 
     # If the entity_id column is being updated, update lookup map.
-    for entity_col in class_.entity_id_lookup_map.keys():
-      if entity_col in update_columns:
-        class_.update_sharding_key_entity_id_lookup(cursor, sharding_key,
-                                                    entity_col,
-                                                    update_columns[entity_col])
+    if class_.entity_id_lookup_map is not None:
+      for entity_col in class_.entity_id_lookup_map.keys():
+        if entity_col in update_columns:
+          class_.update_sharding_key_entity_id_lookup(cursor, sharding_key,
+                                                      entity_col,
+                                                      update_columns[entity_col])
 
     return rowcount
 
@@ -372,7 +380,6 @@ def delete_sharding_key_entity_id_lookup(class_, cursor_method,
   def update_sharding_key_entity_id_lookup(class_, cursor_method,
                                            sharding_key, entity_id_column,
                                            new_entity_id):
-
     sharding_key_lookup_column = class_.get_lookup_column_name(class_.sharding_key_column_name)
     entity_id_lookup_column = class_.get_lookup_column_name(entity_id_column)
     lookup_class = class_.entity_id_lookup_map[entity_id_column]
@@ -468,7 +475,8 @@ def update_columns(class_, cursor, where_column_value_pairs,
     rowcount = cursor.execute(query, bind_vars)
 
     # If the entity_id column is being updated, update lookup map.
-    lookup_cursor_method = functools.partial(create_cursor_from_old_cursor, cursor)
+    lookup_cursor_method = functools.partial(
+        db_object.create_cursor_from_old_cursor, cursor)
     for entity_col in class_.entity_id_lookup_map.keys():
       if entity_col in update_columns:
         class_.update_sharding_key_entity_id_lookup(lookup_cursor_method,
@@ -498,7 +506,8 @@ def delete_by_columns(class_, cursor, where_column_value_pairs,
     rowcount = cursor.rowcount
 
     #delete the lookup map.
-    lookup_cursor_method = functools.partial(create_cursor_from_old_cursor, cursor)
+    lookup_cursor_method = functools.partial(
+        db_object.create_cursor_from_old_cursor, cursor)
     class_.delete_sharding_key_entity_id_lookup(lookup_cursor_method, sharding_key)
 
     return rowcount

py/vtdb/db_object_unsharded.py

@@ -1,9 +1,8 @@
-"""Module containing base classes and helper methods for database objects.
+"""Module containing base class for tables in unsharded keyspace.
 
-The base classes represent different sharding schemes like
-unsharded, range-sharded and custom-sharded tables.
-This abstracts sharding details and provides methods
-for common database access patterns.
+DBObjectUnsharded inherits from DBObjectBase, the implementation
+for the common database operations is defined in DBObjectBase.
+DBObjectUnsharded defines the cursor creation methods for the same.
 """
 import functools
 import logging
@@ -20,8 +19,8 @@
 class DBObjectUnsharded(db_object.DBObjectBase):
   """Base class for unsharded db classes.
 
-  This provides default implementation of routing helper methods, cursor
-  creation and common database access operations.
+  This provides default implementation of routing helper methods and cursor
+  creation. The common database access operations are defined in the base class.
   """
   keyspace = None
   sharding = shard_constants.UNSHARDED