diff --git a/.gitignore b/.gitignore index e1ea2a6096..60eb6c6731 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ nosetests.xml junit-report.xml pylint.txt toy.py +tox.ini violations.pyflakes.txt cover/ build/ diff --git a/AUTHORS.rst b/AUTHORS.rst index 2010caecc9..307dbb3098 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -124,3 +124,7 @@ Patches and Suggestions - Wilfred Hughes @dontYetKnow - Dmitry Medvinsky - Bryce Boe @bboe +- Colin Dunklau @cdunklau +- Hugo Osvaldo Barrera @hobarrera +- Łukasz Langa @llanga +- Dave Shawley diff --git a/README.rst b/README.rst index 0b848355e1..3d03641528 100644 --- a/README.rst +++ b/README.rst @@ -72,6 +72,11 @@ Or, if you absolutely must: But, you really shouldn't do that. +Documentation +------------- + +Documentation is available at http://docs.python-requests.org/. + Contribute ---------- diff --git a/docs/api.rst b/docs/api.rst index cf68ca4077..08cb1b8d1f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -48,13 +48,11 @@ Request Sessions Exceptions ~~~~~~~~~~ -.. module:: requests - -.. autoexception:: RequestException -.. autoexception:: ConnectionError -.. autoexception:: HTTPError -.. autoexception:: URLRequired -.. autoexception:: TooManyRedirects +.. autoexception:: requests.exceptions.RequestException +.. autoexception:: requests.exceptions.ConnectionError +.. autoexception:: requests.exceptions.HTTPError +.. autoexception:: requests.exceptions.URLRequired +.. autoexception:: requests.exceptions.TooManyRedirects Status Code Lookup @@ -76,18 +74,17 @@ Status Code Lookup Cookies ~~~~~~~ -.. autofunction:: dict_from_cookiejar -.. autofunction:: cookiejar_from_dict -.. autofunction:: add_dict_to_cookiejar +.. autofunction:: requests.utils.dict_from_cookiejar +.. autofunction:: requests.utils.cookiejar_from_dict +.. autofunction:: requests.utils.add_dict_to_cookiejar Encodings ~~~~~~~~~ -.. autofunction:: get_encodings_from_content -.. autofunction:: get_encoding_from_headers -.. autofunction:: get_unicode_from_response -.. autofunction:: decode_gzip +.. autofunction:: requests.utils.get_encodings_from_content +.. autofunction:: requests.utils.get_encoding_from_headers +.. autofunction:: requests.utils.get_unicode_from_response Classes diff --git a/docs/community/support.rst b/docs/community/support.rst index 318691f0d3..a187a332a9 100644 --- a/docs/community/support.rst +++ b/docs/community/support.rst @@ -3,7 +3,7 @@ Support ======= -If you have a questions or issues about Requests, there are several options: +If you have questions or issues about Requests, there are several options: Send a Tweet ------------ diff --git a/docs/user/authentication.rst b/docs/user/authentication.rst index d25f373c51..66bc3ca243 100644 --- a/docs/user/authentication.rst +++ b/docs/user/authentication.rst @@ -49,7 +49,7 @@ OAuth 1 Authentication A common form of authentication for several web APIs is OAuth. The ``requests-oauthlib`` library allows Requests users to easily make OAuth authenticated requests:: - >>> import request + >>> import requests >>> from requests_oauthlib import OAuth1 >>> url = 'https://api.twitter.com/1.1/account/verify_credentials.json' diff --git a/requests/__init__.py b/requests/__init__.py index 1ea4aff4e0..2dd194b02d 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -48,6 +48,12 @@ __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2013 Kenneth Reitz' +# Attempt to enable urllib3's SNI support, if possible +try: + from requests.packages.urllib3.contrib import pyopenssl + pyopenssl.inject_into_urllib3() +except ImportError: + pass from . import utils from .models import Request, Response, PreparedRequest diff --git a/requests/models.py b/requests/models.py index 49c6364c94..55716307fb 100644 --- a/requests/models.py +++ b/requests/models.py @@ -18,6 +18,7 @@ from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header from .packages.urllib3.filepost import encode_multipart_formdata +from .packages.urllib3.util import parse_url from .exceptions import HTTPError, RequestException, MissingSchema, InvalidURL from .utils import ( guess_filename, get_auth_from_url, requote_uri, @@ -284,19 +285,28 @@ def prepare_url(self, url, params): pass # Support for unicode domain names and paths. - scheme, netloc, path, _params, query, fragment = urlparse(url) + scheme, auth, host, port, path, query, fragment = parse_url(url) if not scheme: raise MissingSchema("Invalid URL %r: No schema supplied" % url) - if not netloc: - raise InvalidURL("Invalid URL %t: No netloc supplied" % url) + if not host: + raise InvalidURL("Invalid URL %t: No host supplied" % url) + # Only want to apply IDNA to the hostname try: - netloc = netloc.encode('idna').decode('utf-8') + host = host.encode('idna').decode('utf-8') except UnicodeError: raise InvalidURL('URL has an invalid label.') + # Carefully reconstruct the network location + netloc = auth or '' + if netloc: + netloc += '@' + netloc += host + if port: + netloc += ':' + str(port) + # Bare domains aren't valid URLs. if not path: path = '/' @@ -308,8 +318,6 @@ def prepare_url(self, url, params): netloc = netloc.encode('utf-8') if isinstance(path, str): path = path.encode('utf-8') - if isinstance(_params, str): - _params = _params.encode('utf-8') if isinstance(query, str): query = query.encode('utf-8') if isinstance(fragment, str): @@ -322,7 +330,7 @@ def prepare_url(self, url, params): else: query = enc_params - url = requote_uri(urlunparse([scheme, netloc, path, _params, query, fragment])) + url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) self.url = url def prepare_headers(self, headers): @@ -646,7 +654,7 @@ def json(self, **kwargs): def links(self): """Returns the parsed header links of the response, if any.""" - header = self.headers['link'] + header = self.headers.get('link') # l = MultiDict() l = {} diff --git a/requests/sessions.py b/requests/sessions.py index 185d5df73a..77df0e82dd 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -11,14 +11,13 @@ import os from datetime import datetime -from .compat import cookielib +from .compat import cookielib, OrderedDict, urljoin, urlparse from .cookies import cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar from .models import Request, PreparedRequest from .hooks import default_hooks, dispatch_hook from .utils import from_key_val_list, default_headers from .exceptions import TooManyRedirects, InvalidSchema -from .compat import urlparse, urljoin from .adapters import HTTPAdapter from .utils import requote_uri, get_environ_proxies, get_netrc_auth @@ -223,9 +222,9 @@ def __init__(self): self.cookies = cookiejar_from_dict({}) # Default connection adapters. - self.adapters = {} - self.mount('http://', HTTPAdapter()) + self.adapters = OrderedDict() self.mount('https://', HTTPAdapter()) + self.mount('http://', HTTPAdapter()) def __enter__(self): return self @@ -490,8 +489,13 @@ def close(self): v.close() def mount(self, prefix, adapter): - """Registers a connection adapter to a prefix.""" + """Registers a connection adapter to a prefix. + + Adapters are sorted in descending order by key length.""" self.adapters[prefix] = adapter + keys_to_move = [k for k in self.adapters if len(k) < len(prefix)] + for key in keys_to_move: + self.adapters[key] = self.adapters.pop(key) def __getstate__(self): return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) diff --git a/requests/structures.py b/requests/structures.py index 05f5ac15ce..8d02ea67b6 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -9,6 +9,7 @@ """ import os +import collections from itertools import islice @@ -33,43 +34,79 @@ def read(self, n): return "".join(islice(self.i, None, n)) -class CaseInsensitiveDict(dict): - """Case-insensitive Dictionary +class CaseInsensitiveDict(collections.MutableMapping): + """ + A case-insensitive ``dict``-like object. + + Implements all methods and operations of + ``collections.MutableMapping`` as well as dict's ``copy``. Also + provides ``lower_items``. + + All keys are expected to be strings. The structure remembers the + case of the last key to be set, and ``iter(instance)``, + ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` + will contain case-sensitive keys. However, querying and contains + testing is case insensitive: + + cid = CaseInsensitiveDict() + cid['Accept'] = 'application/json' + cid['aCCEPT'] == 'application/json' # True + list(cid) == ['Accept'] # True For example, ``headers['content-encoding']`` will return the - value of a ``'Content-Encoding'`` response header.""" + value of a ``'Content-Encoding'`` response header, regardless + of how the header name was originally stored. - @property - def lower_keys(self): - if not hasattr(self, '_lower_keys') or not self._lower_keys: - self._lower_keys = dict((k.lower(), k) for k in list(self.keys())) - return self._lower_keys + If the constructor, ``.update``, or equality comparison + operations are given keys that have equal ``.lower()``s, the + behavior is undefined. - def _clear_lower_keys(self): - if hasattr(self, '_lower_keys'): - self._lower_keys.clear() + """ + def __init__(self, data=None, **kwargs): + self._store = dict() + if data is None: + data = {} + self.update(data, **kwargs) def __setitem__(self, key, value): - dict.__setitem__(self, key, value) - self._clear_lower_keys() + # Use the lowercased key for lookups, but store the actual + # key alongside the value. + self._store[key.lower()] = (key, value) - def __delitem__(self, key): - dict.__delitem__(self, self.lower_keys.get(key.lower(), key)) - self._lower_keys.clear() + def __getitem__(self, key): + return self._store[key.lower()][1] - def __contains__(self, key): - return key.lower() in self.lower_keys + def __delitem__(self, key): + del self._store[key.lower()] - def __getitem__(self, key): - # We allow fall-through here, so values default to None - if key in self: - return dict.__getitem__(self, self.lower_keys[key.lower()]) + def __iter__(self): + return (casedkey for casedkey, mappedvalue in self._store.values()) - def get(self, key, default=None): - if key in self: - return self[key] + def __len__(self): + return len(self._store) + + def lower_items(self): + """Like iteritems(), but with all lowercase keys.""" + return ( + (lowerkey, keyval[1]) + for (lowerkey, keyval) + in self._store.items() + ) + + def __eq__(self, other): + if isinstance(other, collections.Mapping): + other = CaseInsensitiveDict(other) else: - return default + return NotImplemented + # Compare insensitively + return dict(self.lower_items()) == dict(other.lower_items()) + + # Copy is required + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, dict(self.items())) class LookupDict(dict): diff --git a/requests/utils.py b/requests/utils.py index 68f3e625aa..d690559dd2 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -23,6 +23,7 @@ from .compat import parse_http_list as _parse_list_header from .compat import quote, urlparse, bytes, str, OrderedDict, urlunparse from .cookies import RequestsCookieJar, cookiejar_from_dict +from .structures import CaseInsensitiveDict _hush_pyflakes = (RequestsCookieJar,) @@ -449,11 +450,11 @@ def default_user_agent(): def default_headers(): - return { + return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), 'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')), 'Accept': '*/*' - } + }) def parse_header_links(value): diff --git a/setup.py b/setup.py index 64e305bcc5..2b93ba9efd 100755 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'requests.packages.charade', 'requests.packages.urllib3', 'requests.packages.urllib3.packages', + 'requests.packages.urllib3.contrib', 'requests.packages.urllib3.packages.ssl_match_hostname' ] @@ -51,9 +52,7 @@ 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - # 'Programming Language :: Python :: 3.0', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', + ), ) diff --git a/test_requests.py b/test_requests.py index 9339476162..60e4498434 100644 --- a/test_requests.py +++ b/test_requests.py @@ -11,8 +11,10 @@ import requests from requests.auth import HTTPDigestAuth +from requests.adapters import HTTPAdapter from requests.compat import str, cookielib from requests.cookies import cookiejar_from_dict +from requests.structures import CaseInsensitiveDict try: import StringIO @@ -458,6 +460,220 @@ def test_session_pickling(self): r = s.send(r.prepare()) self.assertEqual(r.status_code, 200) + def test_fixes_1329(self): + """ + Ensure that header updates are done case-insensitively. + """ + s = requests.Session() + s.headers.update({'ACCEPT': 'BOGUS'}) + s.headers.update({'accept': 'application/json'}) + r = s.get(httpbin('get')) + headers = r.request.headers + # ASCII encode because of key comparison changes in py3 + self.assertEqual( + headers['accept'.encode('ascii')], + 'application/json' + ) + self.assertEqual( + headers['Accept'.encode('ascii')], + 'application/json' + ) + self.assertEqual( + headers['ACCEPT'.encode('ascii')], + 'application/json' + ) + + def test_transport_adapter_ordering(self): + s = requests.Session() + order = ['https://', 'http://'] + self.assertEqual(order, list(s.adapters)) + s.mount('http://git', HTTPAdapter()) + s.mount('http://github', HTTPAdapter()) + s.mount('http://github.com', HTTPAdapter()) + s.mount('http://github.com/about/', HTTPAdapter()) + order = [ + 'http://github.com/about/', + 'http://github.com', + 'http://github', + 'http://git', + 'https://', + 'http://', + ] + self.assertEqual(order, list(s.adapters)) + s.mount('http://gittip', HTTPAdapter()) + s.mount('http://gittip.com', HTTPAdapter()) + s.mount('http://gittip.com/about/', HTTPAdapter()) + order = [ + 'http://github.com/about/', + 'http://gittip.com/about/', + 'http://github.com', + 'http://gittip.com', + 'http://github', + 'http://gittip', + 'http://git', + 'https://', + 'http://', + ] + self.assertEqual(order, list(s.adapters)) + s2 = requests.Session() + s2.adapters = {'http://': HTTPAdapter()} + s2.mount('https://', HTTPAdapter()) + self.assertTrue('http://' in s2.adapters) + self.assertTrue('https://' in s2.adapters) + + def test_long_authinfo_in_url(self): + url = 'http://{0}:{1}@{2}:9000/path?query#frag'.format( + 'E8A3BE87-9E3F-4620-8858-95478E385B5B', + 'EA770032-DA4D-4D84-8CE9-29C6D910BF1E', + 'exactly-------------sixty-----------three------------characters', + ) + r = requests.Request('GET', url).prepare() + self.assertEqual(r.url, url) + + +class TestCaseInsensitiveDict(unittest.TestCase): + + def test_mapping_init(self): + cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'}) + self.assertEqual(len(cid), 2) + self.assertTrue('foo' in cid) + self.assertTrue('bar' in cid) + + def test_iterable_init(self): + cid = CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]) + self.assertEqual(len(cid), 2) + self.assertTrue('foo' in cid) + self.assertTrue('bar' in cid) + + def test_kwargs_init(self): + cid = CaseInsensitiveDict(FOO='foo', BAr='bar') + self.assertEqual(len(cid), 2) + self.assertTrue('foo' in cid) + self.assertTrue('bar' in cid) + + def test_docstring_example(self): + cid = CaseInsensitiveDict() + cid['Accept'] = 'application/json' + self.assertEqual(cid['aCCEPT'], 'application/json') + self.assertEqual(list(cid), ['Accept']) + + def test_len(self): + cid = CaseInsensitiveDict({'a': 'a', 'b': 'b'}) + cid['A'] = 'a' + self.assertEqual(len(cid), 2) + + def test_getitem(self): + cid = CaseInsensitiveDict({'Spam': 'blueval'}) + self.assertEqual(cid['spam'], 'blueval') + self.assertEqual(cid['SPAM'], 'blueval') + + def test_fixes_649(self): + """__setitem__ should behave case-insensitively.""" + cid = CaseInsensitiveDict() + cid['spam'] = 'oneval' + cid['Spam'] = 'twoval' + cid['sPAM'] = 'redval' + cid['SPAM'] = 'blueval' + self.assertEqual(cid['spam'], 'blueval') + self.assertEqual(cid['SPAM'], 'blueval') + self.assertEqual(list(cid.keys()), ['SPAM']) + + def test_delitem(self): + cid = CaseInsensitiveDict() + cid['Spam'] = 'someval' + del cid['sPam'] + self.assertFalse('spam' in cid) + self.assertEqual(len(cid), 0) + + def test_contains(self): + cid = CaseInsensitiveDict() + cid['Spam'] = 'someval' + self.assertTrue('Spam' in cid) + self.assertTrue('spam' in cid) + self.assertTrue('SPAM' in cid) + self.assertTrue('sPam' in cid) + self.assertFalse('notspam' in cid) + + def test_get(self): + cid = CaseInsensitiveDict() + cid['spam'] = 'oneval' + cid['SPAM'] = 'blueval' + self.assertEqual(cid.get('spam'), 'blueval') + self.assertEqual(cid.get('SPAM'), 'blueval') + self.assertEqual(cid.get('sPam'), 'blueval') + self.assertEqual(cid.get('notspam', 'default'), 'default') + + def test_update(self): + cid = CaseInsensitiveDict() + cid['spam'] = 'blueval' + cid.update({'sPam': 'notblueval'}) + self.assertEqual(cid['spam'], 'notblueval') + cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'}) + cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'}) + self.assertEqual(len(cid), 2) + self.assertEqual(cid['foo'], 'anotherfoo') + self.assertEqual(cid['bar'], 'anotherbar') + + def test_update_retains_unchanged(self): + cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'}) + cid.update({'foo': 'newfoo'}) + self.assertEquals(cid['bar'], 'bar') + + def test_iter(self): + cid = CaseInsensitiveDict({'Spam': 'spam', 'Eggs': 'eggs'}) + keys = frozenset(['Spam', 'Eggs']) + self.assertEqual(frozenset(iter(cid)), keys) + + def test_equality(self): + cid = CaseInsensitiveDict({'SPAM': 'blueval', 'Eggs': 'redval'}) + othercid = CaseInsensitiveDict({'spam': 'blueval', 'eggs': 'redval'}) + self.assertEqual(cid, othercid) + del othercid['spam'] + self.assertNotEqual(cid, othercid) + self.assertEqual(cid, {'spam': 'blueval', 'eggs': 'redval'}) + + def test_setdefault(self): + cid = CaseInsensitiveDict({'Spam': 'blueval'}) + self.assertEqual( + cid.setdefault('spam', 'notblueval'), + 'blueval' + ) + self.assertEqual( + cid.setdefault('notspam', 'notblueval'), + 'notblueval' + ) + + def test_lower_items(self): + cid = CaseInsensitiveDict({ + 'Accept': 'application/json', + 'user-Agent': 'requests', + }) + keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items()) + lowerkeyset = frozenset(['accept', 'user-agent']) + self.assertEqual(keyset, lowerkeyset) + + def test_preserve_key_case(self): + cid = CaseInsensitiveDict({ + 'Accept': 'application/json', + 'user-Agent': 'requests', + }) + keyset = frozenset(['Accept', 'user-Agent']) + self.assertEqual(frozenset(i[0] for i in cid.items()), keyset) + self.assertEqual(frozenset(cid.keys()), keyset) + self.assertEqual(frozenset(cid), keyset) + + def test_preserve_last_key_case(self): + cid = CaseInsensitiveDict({ + 'Accept': 'application/json', + 'user-Agent': 'requests', + }) + cid.update({'ACCEPT': 'application/json'}) + cid['USER-AGENT'] = 'requests' + keyset = frozenset(['ACCEPT', 'USER-AGENT']) + self.assertEqual(frozenset(i[0] for i in cid.items()), keyset) + self.assertEqual(frozenset(cid.keys()), keyset) + self.assertEqual(frozenset(cid), keyset) + if __name__ == '__main__': unittest.main()