Merge branch 'master' into 184-int

Conflicts:
	paramiko/config.py
	tests/test_util.py

.travis.yml

@@ -4,6 +4,7 @@ python:
   - "2.7"
   - "3.2"
   - "3.3"
+  - "3.4"
 install:
   # Self-install for setup.py-driven deps
   - pip install -e .
@@ -17,16 +18,16 @@ script:
   # Run 'docs' first since its objects.inv is referred to by 'www'.
   # Also force warnings to be errors since most of them tend to be actual
   # problems.
-  - invoke docs -o -W
-  - invoke www -o -W
+  # Finally, skip them under Python 3.2 due to sphinx shenanigans
+  - "[[ $TRAVIS_PYTHON_VERSION != 3.2 ]] && invoke docs -o -W || true"
+  - "[[ $TRAVIS_PYTHON_VERSION != 3.2 ]] && invoke www -o -W || true"
 notifications:
   irc:
     channels: "irc.freenode.org#paramiko"
     template:
       - "%{repository}@%{branch}: %{message} (%{build_url})"
     on_success: change
     on_failure: change
-    use_notice: true
   email: false
 after_success:
   - coveralls

README

@@ -35,7 +35,7 @@ Requirements
 ------------
 
   - Python 2.6 or better <http://www.python.org/> - this includes Python
-    3.3 and higher as well.
+    3.2 and higher as well.
   - pycrypto 2.1 or better <https://www.dlitz.net/software/pycrypto/>
   - ecdsa 0.9 or better <https://pypi.python.org/pypi/ecdsa>
 

dev-requirements.txt

@@ -1,9 +1,9 @@
 # Older junk
 tox>=1.4,<1.5
 # For newer tasks like building Sphinx docs.
-# NOTE: Requires Python >=2.6
-invoke>=0.7.0
+invoke>=0.7.0,<0.8
 invocations>=0.5.0
 sphinx>=1.1.3
-alabaster>=0.4.0
+alabaster>=0.6.1
 releases>=0.5.2
+wheel==0.23.0

paramiko/__init__.py

@@ -17,14 +17,13 @@
 # 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
 
 import sys
+from paramiko._version import __version__, __version_info__
 
 if sys.version_info < (2, 6):
     raise RuntimeError('You need Python 2.6+ for this module.')
 
 
 __author__ = "Jeff Forcier <[email protected]>"
-__version__ = "1.13.0"
-__version_info__ = tuple([ int(d) for d in __version__.split(".") ])
 __license__ = "GNU Lesser General Public License (LGPL)"
 
 

paramiko/_version.py

@@ -0,0 +1,2 @@
+__version_info__ = (1, 15, 0)
+__version__ = '.'.join(map(str, __version_info__))

paramiko/channel.py

@@ -280,7 +280,7 @@ def exit_status_ready(self):
     def recv_exit_status(self):
         """
         Return the exit status from the process on the server.  This is
-        mostly useful for retrieving the reults of an `exec_command`.
+        mostly useful for retrieving the results of an `exec_command`.
         If the command hasn't finished yet, this method will wait until
         it does, or until the channel is closed.  If no exit status is
         provided by the server, -1 is returned.
@@ -330,7 +330,7 @@ def request_x11(self, screen_number=0, auth_protocol=None, auth_cookie=None,
         If you omit the auth_cookie, a new secure random 128-bit value will be
         generated, used, and returned.  You will need to use this value to
         verify incoming x11 requests and replace them with the actual local
-        x11 cookie (which requires some knoweldge of the x11 protocol).
+        x11 cookie (which requires some knowledge of the x11 protocol).
         
         If a handler is passed in, the handler is called from another thread
         whenever a new x11 connection arrives.  The default handler queues up
@@ -339,7 +339,7 @@ def request_x11(self, screen_number=0, auth_protocol=None, auth_cookie=None,
         
             handler(channel: Channel, (address: str, port: int))
         
-        :param int screen_number: the x11 screen number (0, 10, etc)
+        :param int screen_number: the x11 screen number (0, 10, etc.)
         :param str auth_protocol:
             the name of the X11 authentication method used; if none is given,
             ``"MIT-MAGIC-COOKIE-1"`` is used
@@ -744,10 +744,10 @@ def sendall(self, s):
         :raises socket.timeout:
             if sending stalled for longer than the timeout set by `settimeout`.
         :raises socket.error:
-            if an error occured before the entire string was sent.
+            if an error occurred before the entire string was sent.
         
         .. note::
-            If the channel is closed while only part of the data hase been
+            If the channel is closed while only part of the data has been
             sent, there is no way to determine how much data (if any) was sent.
             This is irritating, but identically follows Python's API.
         """
@@ -771,7 +771,7 @@ def sendall_stderr(self, s):
         :raises socket.timeout:
             if sending stalled for longer than the timeout set by `settimeout`.
         :raises socket.error:
-            if an error occured before the entire string was sent.
+            if an error occurred before the entire string was sent.
             
         .. versionadded:: 1.1
         """
@@ -812,7 +812,7 @@ def makefile_stderr(self, *params):
     def fileno(self):
         """
         Returns an OS-level file descriptor which can be used for polling, but
-        but not for reading or writing.  This is primaily to allow Python's
+        but not for reading or writing.  This is primarily to allow Python's
         ``select`` module to work.
 
         The first time ``fileno`` is called on a channel, a pipe is created to

paramiko/config.py

@@ -57,8 +57,8 @@ def parse(self, file_obj):
 
         host = {"host": ['*'], "config": {}}
         for line in file_obj:
-            line = line.rstrip('\n').lstrip()
-            if (line == '') or (line[0] == '#'):
+            line = line.rstrip('\r\n').lstrip()
+            if not line or line.startswith('#'):
                 continue
 
             match = re.match(self.SETTINGS_REGEX, line)
@@ -107,8 +107,10 @@ def lookup(self, hostname):
 
         :param str hostname: the hostname to lookup
         """
-        matches = [config for config in self._config if
-                   self._allowed(hostname, config['host'])]
+        matches = [
+            config for config in self._config
+            if self._allowed(config['host'], hostname)
+        ]
 
         ret = {}
         for match in matches:
@@ -124,7 +126,7 @@ def lookup(self, hostname):
         ret = self._expand_variables(ret, hostname)
         return ret
 
-    def _allowed(self, hostname, hosts):
+    def _allowed(self, hosts, hostname):
         match = False
         for host in hosts:
             if host.startswith('!') and fnmatch.fnmatch(hostname, host[1:]):
@@ -196,10 +198,12 @@ def _expand_variables(self, config, hostname):
                 for find, replace in replacements[k]:
                     if isinstance(config[k], list):
                         for item in range(len(config[k])):
-                            config[k][item] = config[k][item].\
-                                replace(find, str(replace))
+                            if find in config[k][item]:
+                                config[k][item] = config[k][item].\
+                                    replace(find, str(replace))
                     else:
-                        config[k] = config[k].replace(find, str(replace))
+                        if find in config[k]:
+                            config[k] = config[k].replace(find, str(replace))
         return config
 
     def _get_hosts(self, host):

paramiko/file.py

@@ -124,9 +124,15 @@ def read(self, size=None):
         file first).  If the ``size`` argument is negative or omitted, read all
         the remaining data in the file.
 
+        .. note::
+            ``'b'`` mode flag is ignored (``self.FLAG_BINARY`` in
+            ``self._flags``), because SSH treats all files as binary, since we
+            have no idea what encoding the file is in, or even if the file is
+            text data.
+
         :param int size: maximum number of bytes to read
         :return:
-            data read from the file (as a `str`), or an empty string if EOF was
+            data read from the file (as bytes), or an empty string if EOF was
             encountered immediately
         """
         if self._closed:
@@ -148,12 +154,12 @@ def read(self, size=None):
                 result += new_data
                 self._realpos += len(new_data)
                 self._pos += len(new_data)
-            return result if self._flags & self.FLAG_BINARY else u(result)
+            return result 
         if size <= len(self._rbuffer):
             result = self._rbuffer[:size]
             self._rbuffer = self._rbuffer[size:]
             self._pos += len(result)
-            return result if self._flags & self.FLAG_BINARY else u(result)
+            return result 
         while len(self._rbuffer) < size:
             read_size = size - len(self._rbuffer)
             if self._flags & self.FLAG_BUFFERED:
@@ -169,7 +175,7 @@ def read(self, size=None):
         result = self._rbuffer[:size]
         self._rbuffer = self._rbuffer[size:]
         self._pos += len(result)
-        return result if self._flags & self.FLAG_BINARY else u(result)
+        return result 
 
     def readline(self, size=None):
         """
@@ -186,8 +192,12 @@ def readline(self, size=None):
 
         :param int size: maximum length of returned string.
         :return:
-            next line of the file (`str`), or an empty string if the end of the
+            next line of the file, or an empty string if the end of the
             file has been reached.
+
+            If the file was opened in binary (``'b'``) mode: bytes are returned
+            Else: the encoding of the file is assumed to be UTF-8 and character
+            strings (`str`) are returned
         """
         # it's almost silly how complex this function is.
         if self._closed:
@@ -277,7 +287,8 @@ def seek(self, offset, whence=0):
         Set the file's current position, like stdio's ``fseek``.  Not all file
         objects support seeking.
 
-        .. note:: If a file is opened in append mode (``'a'`` or ``'a+'``), any seek
+        .. note::
+            If a file is opened in append mode (``'a'`` or ``'a+'``), any seek
             operations will be undone at the next write (as the file position
             will move back to the end of the file).
         

paramiko/hostkeys.py

@@ -179,7 +179,7 @@ def keys(self):
         entries = []
         for e in self._entries:
             for h in e.hostnames:
-                if h.startswith('|1|') and constant_time_bytes_eq(self.hash_host(hostname, h), h) or h == hostname:
+                if h.startswith('|1|') and not hostname.startswith('|1|') and constant_time_bytes_eq(self.hash_host(hostname, h), h) or h == hostname:
                     entries.append(e)
         if len(entries) == 0:
             return None

paramiko/py3compat.py

@@ -39,6 +39,8 @@ def b(s, encoding='utf8'):  # NOQA
             return s
         elif isinstance(s, unicode):
             return s.encode(encoding)
+        elif isinstance(s, buffer):
+            return s
         else:
             raise TypeError("Expected unicode or bytes, got %r" % s)
 
@@ -49,6 +51,8 @@ def u(s, encoding='utf8'):  # NOQA
             return s.decode(encoding)
         elif isinstance(s, unicode):
             return s
+        elif isinstance(s, buffer):
+            return s.decode(encoding)
         else:
             raise TypeError("Expected unicode or bytes, got %r" % s)
 

paramiko/sftp_client.py

@@ -196,6 +196,71 @@ def listdir_attr(self, path='.'):
         self._request(CMD_CLOSE, handle)
         return filelist
 
+    def listdir_iter(self, path='.', read_aheads=50):
+        """
+        Generator version of `.listdir_attr`.
+
+        See the API docs for `.listdir_attr` for overall details.
+
+        This function adds one more kwarg on top of `.listdir_attr`:
+        ``read_aheads``, an integer controlling how many
+        ``SSH_FXP_READDIR`` requests are made to the server. The default of 50
+        should suffice for most file listings as each request/response cycle
+        may contain multiple files (dependant on server implementation.)
+
+        .. versionadded:: 1.15
+        """
+        path = self._adjust_cwd(path)
+        self._log(DEBUG, 'listdir(%r)' % path)
+        t, msg = self._request(CMD_OPENDIR, path)
+
+        if t != CMD_HANDLE:
+            raise SFTPError('Expected handle')
+
+        handle = msg.get_string()
+
+        nums = list()
+        while True:
+            try:
+                # Send out a bunch of readdir requests so that we can read the
+                # responses later on Section 6.7 of the SSH file transfer RFC
+                # explains this
+                # http://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt
+                for i in range(read_aheads):
+                    num = self._async_request(type(None), CMD_READDIR, handle)
+                    nums.append(num)
+
+
+                # For each of our sent requests
+                # Read and parse the corresponding packets
+                # If we're at the end of our queued requests, then fire off
+                # some more requests
+                # Exit the loop when we've reached the end of the directory
+                # handle
+                for num in nums:
+                    t, pkt_data = self._read_packet()
+                    msg = Message(pkt_data)
+                    new_num = msg.get_int()
+                    if num == new_num:
+                        if t == CMD_STATUS:
+                            self._convert_status(msg)
+                    count = msg.get_int()
+                    for i in range(count):
+                        filename = msg.get_text()
+                        longname = msg.get_text()
+                        attr = SFTPAttributes._from_msg(
+                            msg, filename, longname)
+                        if (filename != '.') and (filename != '..'):
+                            yield attr
+
+                # If we've hit the end of our queued requests, reset nums.
+                nums = list()
+
+            except EOFError:
+                self._request(CMD_CLOSE, handle)
+                return
+
+
     def open(self, filename, mode='r', bufsize=-1):
         """
         Open a file on the remote server.  The arguments are the same as for
@@ -534,9 +599,7 @@ def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True):
             an `.SFTPAttributes` object containing attributes about the given
             file.
 
-        .. versionadded:: 1.4
-        .. versionchanged:: 1.7.4
-            Began returning rich attribute objects.
+        .. versionadded:: 1.10
         """
         with self.file(remotepath, 'wb') as fr:
             fr.set_pipelined(True)
@@ -566,7 +629,9 @@ def put(self, localpath, remotepath, callback=None, confirm=True):
         The SFTP operations use pipelining for speed.
 
         :param str localpath: the local file to copy
-        :param str remotepath: the destination path on the SFTP server
+        :param str remotepath: the destination path on the SFTP server. Note
+            that the filename should be included. Only specifying a directory
+            may result in an error.
         :param callable callback:
             optional callback function (form: ``func(int, int)``) that accepts
             the bytes transferred so far and the total bytes to be transferred
@@ -584,7 +649,7 @@ def put(self, localpath, remotepath, callback=None, confirm=True):
         """
         file_size = os.stat(localpath).st_size
         with open(localpath, 'rb') as fl:
-            return self.putfo(fl, remotepath, os.stat(localpath).st_size, callback, confirm)
+            return self.putfo(fl, remotepath, file_size, callback, confirm)
 
     def getfo(self, remotepath, fl, callback=None):
         """
@@ -601,9 +666,7 @@ def getfo(self, remotepath, fl, callback=None):
             the bytes transferred so far and the total bytes to be transferred
         :return: the `number <int>` of bytes written to the opened file object
 
-        .. versionadded:: 1.4
-        .. versionchanged:: 1.7.4
-            Added the ``callable`` param.
+        .. versionadded:: 1.10
         """
         with self.open(remotepath, 'rb') as fr:
             file_size = self.stat(remotepath).st_size

paramiko/transport.py

@@ -164,6 +164,8 @@ def __init__(self, sock):
         :param socket sock:
             a socket or socket-like object to create the session over.
         """
+        self.active = False
+
         if isinstance(sock, string_types):
             # convert "host:port" into (host, port)
             hl = sock.split(':', 1)
@@ -219,7 +221,6 @@ def __init__(self, sock):
         self.H = None
         self.K = None
 
-        self.active = False
         self.initial_kex_done = False
         self.in_kex = False
         self.authenticated = False