Now tests work consistently in py2 and 3

It's a nice way of saying that there is still one failing, consistently.

Author: Byron

gitdb/const.py

@@ -1,5 +1,4 @@
-from gitdb.utils.encoding import force_bytes
-
-NULL_BYTE = force_bytes("\0")
+BYTE_SPACE = b' '
+NULL_BYTE = b'\0'
 NULL_HEX_SHA = "0" * 40
 NULL_BIN_SHA = NULL_BYTE * 20

gitdb/db/base.py

@@ -9,6 +9,7 @@
     hex_to_bin
 )
 
+from gitdb.utils.encoding import force_text
 from gitdb.exc import (
     BadObject,
     AmbiguousObjectName
@@ -122,8 +123,6 @@ def db_path(self, rela_path):
         """
         :return: the given relative path relative to our database root, allowing
             to pontentially access datafiles"""
-        from gitdb.utils.encoding import force_text
-
         return join(self._root_path, force_text(rela_path))
     #} END interface
 
@@ -234,12 +233,12 @@ def update_cache(self, force=False):
 
     def partial_to_complete_sha_hex(self, partial_hexsha):
         """
-        :return: 20 byte binary sha1 from the given less-than-40 byte hexsha
+        :return: 20 byte binary sha1 from the given less-than-40 byte hexsha (bytes or str)
         :param partial_hexsha: hexsha with less than 40 byte
         :raise AmbiguousObjectName: """
         databases = list()
         _databases_recursive(self, databases)
-
+        partial_hexsha = force_text(partial_hexsha)
         len_partial_hexsha = len(partial_hexsha)
         if len_partial_hexsha % 2 != 0:
             partial_binsha = hex_to_bin(partial_hexsha + "0")

gitdb/db/loose.py

@@ -109,7 +109,7 @@ def readable_db_object_path(self, hexsha):
 
     def partial_to_complete_sha_hex(self, partial_hexsha):
         """:return: 20 byte binary sha1 string which matches the given name uniquely
-        :param name: hexadecimal partial name
+        :param name: hexadecimal partial name (bytes or ascii string)
         :raise AmbiguousObjectName:
         :raise BadObject: """
         candidate = None

gitdb/fun.py

@@ -14,7 +14,9 @@
 from itertools import islice
 from functools import reduce
 
-from gitdb.utils.compat import izip, buffer, xrange
+from gitdb.const import NULL_BYTE, BYTE_SPACE
+from gitdb.utils.encoding import force_text
+from gitdb.utils.compat import izip, buffer, xrange, PY3
 from gitdb.typ import (
     str_blob_type,
     str_commit_type,
@@ -30,12 +32,12 @@
 delta_types = (OFS_DELTA, REF_DELTA)
 
 type_id_to_type_map =   {
-    0 : "",             # EXT 1
+    0 : b'',             # EXT 1
     1 : str_commit_type,
     2 : str_tree_type,
     3 : str_blob_type,
     4 : str_tag_type,
-    5 : "",             # EXT 2
+    5 : b'',             # EXT 2
     OFS_DELTA : "OFS_DELTA",    # OFFSET DELTA
     REF_DELTA : "REF_DELTA"     # REFERENCE DELTA
 }
@@ -394,11 +396,9 @@ def loose_object_header_info(m):
     :return: tuple(type_string, uncompressed_size_in_bytes) the type string of the
         object as well as its uncompressed size in bytes.
     :param m: memory map from which to read the compressed object data"""
-    from gitdb.const import NULL_BYTE
-
     decompress_size = 8192      # is used in cgit as well
     hdr = decompressobj().decompress(m, decompress_size)
-    type_name, size = hdr[:hdr.find(NULL_BYTE)].split(" ".encode("ascii"))
+    type_name, size = hdr[:hdr.find(NULL_BYTE)].split(BYTE_SPACE)
 
     return type_name, int(size)
 
@@ -413,12 +413,21 @@ def pack_object_header_info(data):
     type_id = (c >> 4) & 7          # numeric type
     size = c & 15                   # starting size
     s = 4                           # starting bit-shift size
-    while c & 0x80:
-        c = byte_ord(data[i])
-        i += 1
-        size += (c & 0x7f) << s
-        s += 7
-    # END character loop
+    if PY3:
+        while c & 0x80:
+            c = data[i]
+            i += 1
+            size += (c & 0x7f) << s
+            s += 7
+        # END character loop
+    else:
+        while c & 0x80:
+            c = ord(data[i])
+            i += 1
+            size += (c & 0x7f) << s
+            s += 7
+        # END character loop
+    # end performance at expense of maintenance ...
     return (type_id, size, i)
 
 def create_pack_object_header(obj_type, obj_size):
@@ -429,16 +438,29 @@ def create_pack_object_header(obj_type, obj_size):
     :param obj_type: pack type_id of the object
     :param obj_size: uncompressed size in bytes of the following object stream"""
     c = 0       # 1 byte
-    hdr = str() # output string
-
-    c = (obj_type << 4) | (obj_size & 0xf)
-    obj_size >>= 4
-    while obj_size:
-        hdr += chr(c | 0x80)
-        c = obj_size & 0x7f
-        obj_size >>= 7
-    #END until size is consumed
-    hdr += chr(c)
+    if PY3:
+        hdr = bytearray() # output string
+
+        c = (obj_type << 4) | (obj_size & 0xf)
+        obj_size >>= 4
+        while obj_size:
+            hdr.append(c | 0x80)
+            c = obj_size & 0x7f
+            obj_size >>= 7
+        #END until size is consumed
+        hdr.append(c)
+    else:
+        hdr = bytes() # output string
+
+        c = (obj_type << 4) | (obj_size & 0xf)
+        obj_size >>= 4
+        while obj_size:
+            hdr += chr(c | 0x80)
+            c = obj_size & 0x7f
+            obj_size >>= 7
+        #END until size is consumed
+        hdr += chr(c)
+    # end handle interpreter
     return hdr
 
 def msb_size(data, offset=0):
@@ -449,24 +471,36 @@ def msb_size(data, offset=0):
     i = 0
     l = len(data)
     hit_msb = False
-    while i < l:
-        c = byte_ord(data[i+offset])
-        size |= (c & 0x7f) << i*7
-        i += 1
-        if not c & 0x80:
-            hit_msb = True
-            break
-        # END check msb bit
-    # END while in range
+    if PY3:
+        while i < l:
+            c = data[i+offset]
+            size |= (c & 0x7f) << i*7
+            i += 1
+            if not c & 0x80:
+                hit_msb = True
+                break
+            # END check msb bit
+        # END while in range
+    else:
+        while i < l:
+            c = ord(data[i+offset])
+            size |= (c & 0x7f) << i*7
+            i += 1
+            if not c & 0x80:
+                hit_msb = True
+                break
+            # END check msb bit
+        # END while in range
+    # end performance ...
     if not hit_msb:
         raise AssertionError("Could not find terminating MSB byte in data stream")
     return i+offset, size
 
 def loose_object_header(type, size):
     """
-    :return: string representing the loose object header, which is immediately
+    :return: bytes representing the loose object header, which is immediately
         followed by the content stream of size 'size'"""
-    return "%s %i\0" % (type, size)
+    return ('%s %i\0' % (force_text(type), size)).encode('ascii')
 
 def write_object(type, size, read, write, chunk_size=chunk_size):
     """
@@ -611,48 +645,93 @@ def apply_delta_data(src_buf, src_buf_size, delta_buf, delta_buf_size, write):
     **Note:** transcribed to python from the similar routine in patch-delta.c"""
     i = 0
     db = delta_buf
-    while i < delta_buf_size:
-        c = ord(db[i])
-        i += 1
-        if c & 0x80:
-            cp_off, cp_size = 0, 0
-            if (c & 0x01):
-                cp_off = ord(db[i])
-                i += 1
-            if (c & 0x02):
-                cp_off |= (ord(db[i]) << 8)
-                i += 1
-            if (c & 0x04):
-                cp_off |= (ord(db[i]) << 16)
-                i += 1
-            if (c & 0x08):
-                cp_off |= (ord(db[i]) << 24)
-                i += 1
-            if (c & 0x10):
-                cp_size = ord(db[i])
-                i += 1
-            if (c & 0x20):
-                cp_size |= (ord(db[i]) << 8)
-                i += 1
-            if (c & 0x40):
-                cp_size |= (ord(db[i]) << 16)
-                i += 1
-
-            if not cp_size:
-                cp_size = 0x10000
-
-            rbound = cp_off + cp_size
-            if (rbound < cp_size or
-                rbound > src_buf_size):
-                break
-            write(buffer(src_buf, cp_off, cp_size))
-        elif c:
-            write(db[i:i+c])
-            i += c
-        else:
-            raise ValueError("unexpected delta opcode 0")
-        # END handle command byte
-    # END while processing delta data
+    if PY3:
+        while i < delta_buf_size:
+            c = db[i]
+            i += 1
+            if c & 0x80:
+                cp_off, cp_size = 0, 0
+                if (c & 0x01):
+                    cp_off = db[i]
+                    i += 1
+                if (c & 0x02):
+                    cp_off |= (db[i] << 8)
+                    i += 1
+                if (c & 0x04):
+                    cp_off |= (db[i] << 16)
+                    i += 1
+                if (c & 0x08):
+                    cp_off |= (db[i] << 24)
+                    i += 1
+                if (c & 0x10):
+                    cp_size = db[i]
+                    i += 1
+                if (c & 0x20):
+                    cp_size |= (db[i] << 8)
+                    i += 1
+                if (c & 0x40):
+                    cp_size |= (db[i] << 16)
+                    i += 1
+
+                if not cp_size:
+                    cp_size = 0x10000
+
+                rbound = cp_off + cp_size
+                if (rbound < cp_size or
+                    rbound > src_buf_size):
+                    break
+                write(buffer(src_buf, cp_off, cp_size))
+            elif c:
+                write(db[i:i+c])
+                i += c
+            else:
+                raise ValueError("unexpected delta opcode 0")
+            # END handle command byte
+        # END while processing delta data
+    else:
+        while i < delta_buf_size:
+            c = ord(db[i])
+            i += 1
+            if c & 0x80:
+                cp_off, cp_size = 0, 0
+                if (c & 0x01):
+                    cp_off = ord(db[i])
+                    i += 1
+                if (c & 0x02):
+                    cp_off |= (ord(db[i]) << 8)
+                    i += 1
+                if (c & 0x04):
+                    cp_off |= (ord(db[i]) << 16)
+                    i += 1
+                if (c & 0x08):
+                    cp_off |= (ord(db[i]) << 24)
+                    i += 1
+                if (c & 0x10):
+                    cp_size = ord(db[i])
+                    i += 1
+                if (c & 0x20):
+                    cp_size |= (ord(db[i]) << 8)
+                    i += 1
+                if (c & 0x40):
+                    cp_size |= (ord(db[i]) << 16)
+                    i += 1
+
+                if not cp_size:
+                    cp_size = 0x10000
+
+                rbound = cp_off + cp_size
+                if (rbound < cp_size or
+                    rbound > src_buf_size):
+                    break
+                write(buffer(src_buf, cp_off, cp_size))
+            elif c:
+                write(db[i:i+c])
+                i += c
+            else:
+                raise ValueError("unexpected delta opcode 0")
+            # END handle command byte
+        # END while processing delta data
+    # end save byte_ord call and prevent performance regression in py2
 
     # yes, lets use the exact same error message that git uses :)
     assert i == delta_buf_size, "delta replay has gone wild"