Merge "Port FileLikeIter to Python 3"

swift/common/utils.py

@@ -454,6 +454,8 @@ class FileLikeIter(object):
     def __init__(self, iterable):
         """
         Wraps an iterable to behave as a file-like object.
+
+        The iterable must yield bytes strings.
         """
         self.iterator = iter(iterable)
         self.buf = None
@@ -474,10 +476,11 @@ def next(self):
             return rv
         else:
             return next(self.iterator)
+    __next__ = next
 
     def read(self, size=-1):
         """
-        read([size]) -> read at most size bytes, returned as a string.
+        read([size]) -> read at most size bytes, returned as a bytes string.
 
         If the size argument is negative or omitted, read until EOF is reached.
         Notice that when in non-blocking mode, less data than what was
@@ -486,43 +489,43 @@ def read(self, size=-1):
         if self.closed:
             raise ValueError('I/O operation on closed file')
         if size < 0:
-            return ''.join(self)
+            return b''.join(self)
         elif not size:
-            chunk = ''
+            chunk = b''
         elif self.buf:
             chunk = self.buf
             self.buf = None
         else:
             try:
                 chunk = next(self.iterator)
             except StopIteration:
-                return ''
+                return b''
         if len(chunk) > size:
             self.buf = chunk[size:]
             chunk = chunk[:size]
         return chunk
 
     def readline(self, size=-1):
         """
-        readline([size]) -> next line from the file, as a string.
+        readline([size]) -> next line from the file, as a bytes string.
 
         Retain newline.  A non-negative size argument limits the maximum
         number of bytes to return (an incomplete line may be returned then).
         Return an empty string at EOF.
         """
         if self.closed:
             raise ValueError('I/O operation on closed file')
-        data = ''
-        while '\n' not in data and (size < 0 or len(data) < size):
+        data = b''
+        while b'\n' not in data and (size < 0 or len(data) < size):
             if size < 0:
                 chunk = self.read(1024)
             else:
                 chunk = self.read(size - len(data))
             if not chunk:
                 break
             data += chunk
-        if '\n' in data:
-            data, sep, rest = data.partition('\n')
+        if b'\n' in data:
+            data, sep, rest = data.partition(b'\n')
             data += sep
             if self.buf:
                 self.buf = rest + self.buf
@@ -532,7 +535,7 @@ def readline(self, size=-1):
 
     def readlines(self, sizehint=-1):
         """
-        readlines([size]) -> list of strings, each a line from the file.
+        readlines([size]) -> list of bytes strings, each a line from the file.
 
         Call readline() repeatedly and return a list of the lines so read.
         The optional size argument, if given, is an approximate bound on the
@@ -3370,7 +3373,7 @@ def read(self, length=None):
         if not length:
             length = self.read_chunk_size
         if self.no_more_data_for_this_file:
-            return ''
+            return b''
 
         # read enough data to know whether we're going to run
         # into a boundary in next [length] bytes
@@ -3396,22 +3399,22 @@ def read(self, length=None):
         # if it does, just return data up to the boundary
         else:
             ret, self.input_buffer = self.input_buffer.split(self.boundary, 1)
-            self.no_more_files = self.input_buffer.startswith('--')
+            self.no_more_files = self.input_buffer.startswith(b'--')
             self.no_more_data_for_this_file = True
             self.input_buffer = self.input_buffer[2:]
         return ret
 
     def readline(self):
         if self.no_more_data_for_this_file:
-            return ''
+            return b''
         boundary_pos = newline_pos = -1
         while newline_pos < 0 and boundary_pos < 0:
             try:
                 chunk = self.wsgi_input.read(self.read_chunk_size)
             except (IOError, ValueError) as e:
                 raise swift.common.exceptions.ChunkReadError(str(e))
             self.input_buffer += chunk
-            newline_pos = self.input_buffer.find('\r\n')
+            newline_pos = self.input_buffer.find(b'\r\n')
             boundary_pos = self.input_buffer.find(self.boundary)
             if not chunk:
                 self.no_more_files = True
@@ -3420,7 +3423,7 @@ def readline(self):
         if newline_pos >= 0 and \
                 (boundary_pos < 0 or newline_pos < boundary_pos):
             # Use self.read to ensure any logic there happens...
-            ret = ''
+            ret = b''
             to_read = newline_pos + 2
             while to_read > 0:
                 chunk = self.read(to_read)

test/unit/common/test_utils.py

@@ -3493,14 +3493,14 @@ def test_get_swift_info_with_unmatched_disallowed_sections(self):
 class TestFileLikeIter(unittest.TestCase):
 
     def test_iter_file_iter(self):
-        in_iter = ['abc', 'de', 'fghijk', 'l']
+        in_iter = [b'abc', b'de', b'fghijk', b'l']
         chunks = []
         for chunk in utils.FileLikeIter(in_iter):
             chunks.append(chunk)
         self.assertEqual(chunks, in_iter)
 
     def test_next(self):
-        in_iter = ['abc', 'de', 'fghijk', 'l']
+        in_iter = [b'abc', b'de', b'fghijk', b'l']
         chunks = []
         iter_file = utils.FileLikeIter(in_iter)
         while True:
@@ -3512,12 +3512,12 @@ def test_next(self):
         self.assertEqual(chunks, in_iter)
 
     def test_read(self):
-        in_iter = ['abc', 'de', 'fghijk', 'l']
+        in_iter = [b'abc', b'de', b'fghijk', b'l']
         iter_file = utils.FileLikeIter(in_iter)
-        self.assertEqual(iter_file.read(), ''.join(in_iter))
+        self.assertEqual(iter_file.read(), b''.join(in_iter))
 
     def test_read_with_size(self):
-        in_iter = ['abc', 'de', 'fghijk', 'l']
+        in_iter = [b'abc', b'de', b'fghijk', b'l']
         chunks = []
         iter_file = utils.FileLikeIter(in_iter)
         while True:
@@ -3526,14 +3526,15 @@ def test_read_with_size(self):
                 break
             self.assertTrue(len(chunk) <= 2)
             chunks.append(chunk)
-        self.assertEqual(''.join(chunks), ''.join(in_iter))
+        self.assertEqual(b''.join(chunks), b''.join(in_iter))
 
     def test_read_with_size_zero(self):
         # makes little sense, but file supports it, so...
-        self.assertEqual(utils.FileLikeIter('abc').read(0), '')
+        self.assertEqual(utils.FileLikeIter(b'abc').read(0), b'')
 
     def test_readline(self):
-        in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
+        in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
+                   b'trailing.']
         lines = []
         iter_file = utils.FileLikeIter(in_iter)
         while True:
@@ -3543,22 +3544,23 @@ def test_readline(self):
             lines.append(line)
         self.assertEqual(
             lines,
-            [v if v == 'trailing.' else v + '\n'
-             for v in ''.join(in_iter).split('\n')])
+            [v if v == b'trailing.' else v + b'\n'
+             for v in b''.join(in_iter).split(b'\n')])
 
     def test_readline2(self):
         self.assertEqual(
-            utils.FileLikeIter(['abc', 'def\n']).readline(4),
-            'abcd')
+            utils.FileLikeIter([b'abc', b'def\n']).readline(4),
+            b'abcd')
 
     def test_readline3(self):
         self.assertEqual(
-            utils.FileLikeIter(['a' * 1111, 'bc\ndef']).readline(),
-            ('a' * 1111) + 'bc\n')
+            utils.FileLikeIter([b'a' * 1111, b'bc\ndef']).readline(),
+            (b'a' * 1111) + b'bc\n')
 
     def test_readline_with_size(self):
 
-        in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
+        in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
+                   b'trailing.']
         lines = []
         iter_file = utils.FileLikeIter(in_iter)
         while True:
@@ -3568,19 +3570,21 @@ def test_readline_with_size(self):
             lines.append(line)
         self.assertEqual(
             lines,
-            ['ab', 'c\n', 'd\n', 'ef', 'g\n', 'h\n', 'ij', '\n', '\n', 'k\n',
-             'tr', 'ai', 'li', 'ng', '.'])
+            [b'ab', b'c\n', b'd\n', b'ef', b'g\n', b'h\n', b'ij', b'\n', b'\n',
+             b'k\n', b'tr', b'ai', b'li', b'ng', b'.'])
 
     def test_readlines(self):
-        in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
+        in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
+                   b'trailing.']
         lines = utils.FileLikeIter(in_iter).readlines()
         self.assertEqual(
             lines,
-            [v if v == 'trailing.' else v + '\n'
-             for v in ''.join(in_iter).split('\n')])
+            [v if v == b'trailing.' else v + b'\n'
+             for v in b''.join(in_iter).split(b'\n')])
 
     def test_readlines_with_size(self):
-        in_iter = ['abc\n', 'd', '\nef', 'g\nh', '\nij\n\nk\n', 'trailing.']
+        in_iter = [b'abc\n', b'd', b'\nef', b'g\nh', b'\nij\n\nk\n',
+                   b'trailing.']
         iter_file = utils.FileLikeIter(in_iter)
         lists_of_lines = []
         while True:
@@ -3590,12 +3594,13 @@ def test_readlines_with_size(self):
             lists_of_lines.append(lines)
         self.assertEqual(
             lists_of_lines,
-            [['ab'], ['c\n'], ['d\n'], ['ef'], ['g\n'], ['h\n'], ['ij'],
-             ['\n', '\n'], ['k\n'], ['tr'], ['ai'], ['li'], ['ng'], ['.']])
+            [[b'ab'], [b'c\n'], [b'd\n'], [b'ef'], [b'g\n'], [b'h\n'], [b'ij'],
+             [b'\n', b'\n'], [b'k\n'], [b'tr'], [b'ai'], [b'li'], [b'ng'],
+             [b'.']])
 
     def test_close(self):
-        iter_file = utils.FileLikeIter('abcdef')
-        self.assertEqual(next(iter_file), 'a')
+        iter_file = utils.FileLikeIter([b'a', b'b', b'c'])
+        self.assertEqual(next(iter_file), b'a')
         iter_file.close()
         self.assertTrue(iter_file.closed)
         self.assertRaises(ValueError, iter_file.next)