Merge branch 'trunk' into 336-security-docs

twisted · May 1, 2024 · 9c97fc5 · 9c97fc5
2 parents 1160133 + eb301b5
commit 9c97fc5
Show file tree

Hide file tree

Showing 10 changed files with 475 additions and 100 deletions.
diff --git a/README.rst b/README.rst
@@ -1,10 +1,26 @@
 treq: High-level Twisted HTTP Client API
 ========================================
 
-|pypi|_
-|calver|_
-|coverage|_
-|documentation|_
+.. |pypi| image:: https://img.shields.io/pypi/v/treq.svg
+    :alt: PyPI
+    :target: https://pypi.org/project/treq/
+
+.. |calver| image:: https://img.shields.io/badge/calver-YY.MM.MICRO-22bfda.svg
+    :alt: calver: YY.MM.MICRO
+    :target: https://calver.org/
+
+.. |coverage| image:: https://coveralls.io/repos/github/twisted/treq/badge.svg
+    :alt: Coverage
+    :target: https://coveralls.io/github/twisted/treq
+
+.. |documentation| image:: https://readthedocs.org/projects/treq/badge/
+    :alt: Documentation
+    :target: https://treq.readthedocs.org
+
+|pypi|
+|calver|
+|coverage|
+|documentation|
 
 ``treq`` is an HTTP library inspired by
 `requests <https://requests.readthedocs.io/>`_ but written on top of
@@ -18,15 +34,16 @@ using Twisted.
 
     >>> import treq
 
-    >>> def done(response):
+    >>> async def main(reactor):
+    ...     response = await treq.get("https://github.com")
     ...     print(response.code)
-    ...     reactor.stop()
+    ...     body = await response.text()
+    ...     print("<!DOCTYPE html>" in body)
 
-    >>> treq.get("https://github.com").addCallback(done)
-
-    >>> from twisted.internet import reactor
-    >>> reactor.run()
+    >>> from twisted.internet.task import react
+    >>> react(main)
     200
+    True
 
 For more info `read the docs <https://treq.readthedocs.org>`_.
 
@@ -35,7 +52,7 @@ Contributing
 
 ``treq`` development is hosted on `GitHub <https://github.com/twisted/treq>`_.
 
-We welcome contributions: feel to fork and send contributions over.
+We welcome contributions: feel free to fork and send contributions over.
 See `CONTRIBUTING.rst <https://github.com/twisted/treq/blob/master/CONTRIBUTING.rst>`_ for more info.
 
 Code of Conduct
@@ -50,18 +67,3 @@ Copyright and License
 See `LICENSE <./LICENSE>`_ for legal details and copyright notices.
 
 
-.. _pypi: https://pypi.org/project/treq/
-.. |pypi| image:: https://img.shields.io/pypi/v/treq.svg
-    :alt: PyPI
-
-.. _calver: https://calver.org/
-.. |calver| image:: https://img.shields.io/badge/calver-YY.MM.MICRO-22bfda.svg
-    :alt: calver: YY.MM.MICRO
-
-.. _coverage: https://coveralls.io/github/twisted/treq
-.. |coverage| image:: https://coveralls.io/repos/github/twisted/treq/badge.svg
-    :alt: Coverage
-
-.. _documentation: https://treq.readthedocs.org
-.. |documentation| image:: https://readthedocs.org/projects/treq/badge/
-    :alt: Documentation
diff --git a/changelog.d/382.misc.rst b/changelog.d/382.misc.rst
diff --git a/changelog.d/384.feature.rst b/changelog.d/384.feature.rst
@@ -0,0 +1 @@
+The new :mod:`treq.cookies` module provides helper functions for working with `http.cookiejar.Cookie` and `CookieJar` objects.
diff --git a/docs/api.rst b/docs/api.rst
@@ -89,6 +89,15 @@ Authentication
 
 .. autoexception:: UnknownAuthConfig
 
+Cookies
+-------
+
+.. module:: treq.cookies
+
+.. autofunction:: scoped_cookie
+
+.. autofunction:: search
+
 Test Helpers
 ------------
 

diff --git a/docs/examples/using_cookies.py b/docs/examples/using_cookies.py
@@ -4,19 +4,16 @@
 import treq
 
 
-def main(reactor, *args):
-    d = treq.get('https://httpbin.org/cookies/set?hello=world')
+async def main(reactor):
+    resp = await treq.get("https://httpbin.org/cookies/set?hello=world")
 
-    def _get_jar(resp):
-        jar = resp.cookies()
+    jar = resp.cookies()
+    [cookie] = treq.cookies.search(jar, domain="httpbin.org", name="hello")
+    print("The server set our hello cookie to: {}".format(cookie.value))
 
-        print('The server set our hello cookie to: {}'.format(jar['hello']))
+    await treq.get("https://httpbin.org/cookies", cookies=jar).addCallback(
+        print_response
+    )
 
-        return treq.get('https://httpbin.org/cookies', cookies=jar)
 
-    d.addCallback(_get_jar)
-    d.addCallback(print_response)
-
-    return d
-
-react(main, [])
+react(main)
diff --git a/pyproject.toml b/pyproject.toml
@@ -14,6 +14,9 @@ directory = "changelog.d"
 title_format = "{version} ({project_date})"
 issue_format = "`#{issue} <https://github.com/twisted/treq/issues/{issue}>`__"
 
+[tool.ruff]
+line-length = 88
+
 [tool.mypy]
 namespace_packages = true
 plugins = "mypy_zope:plugin"

diff --git a/src/treq/client.py b/src/treq/client.py
@@ -2,29 +2,53 @@
 import mimetypes
 import uuid
 from collections import abc
-from http.cookiejar import Cookie, CookieJar
+from http.cookiejar import CookieJar
 from json import dumps as json_dumps
-from typing import (Any, Callable, Iterable, Iterator, List, Mapping,
-                    Optional, Tuple, Union)
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 from urllib.parse import quote_plus
 from urllib.parse import urlencode as _urlencode
 
 from hyperlink import DecodedURL, EncodedURL
 from requests.cookies import merge_cookies
+from treq.cookies import scoped_cookie
 from twisted.internet.defer import Deferred
 from twisted.internet.interfaces import IProtocol
 from twisted.python.components import proxyForInterface, registerAdapter
 from twisted.python.filepath import FilePath
-from twisted.web.client import (BrowserLikeRedirectAgent, ContentDecoderAgent,
-                                CookieAgent, FileBodyProducer, GzipDecoder,
-                                IAgent, RedirectAgent)
+from twisted.web.client import (
+    BrowserLikeRedirectAgent,
+    ContentDecoderAgent,
+    CookieAgent,
+    FileBodyProducer,
+    GzipDecoder,
+    IAgent,
+    RedirectAgent,
+)
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import IBodyProducer, IResponse
 
 from treq import multipart
-from treq._types import (_CookiesType, _DataType, _FilesType, _FileValue,
-                         _HeadersType, _ITreqReactor, _JSONType, _ParamsType,
-                         _URLType)
+from treq._types import (
+    _CookiesType,
+    _DataType,
+    _FilesType,
+    _FileValue,
+    _HeadersType,
+    _ITreqReactor,
+    _JSONType,
+    _ParamsType,
+    _URLType,
+)
 from treq.auth import add_auth
 from treq.response import _Response
 
@@ -55,39 +79,7 @@ def _scoped_cookiejar_from_dict(
     if cookie_dict is None:
         return cookie_jar
     for k, v in cookie_dict.items():
-        secure = url_object.scheme == "https"
-        port_specified = not (
-            (url_object.scheme == "https" and url_object.port == 443)
-            or (url_object.scheme == "http" and url_object.port == 80)
-        )
-        port = str(url_object.port) if port_specified else None
-        domain = url_object.host
-        netscape_domain = domain if "." in domain else domain + ".local"
-
-        cookie_jar.set_cookie(
-            Cookie(
-                # Scoping
-                domain=netscape_domain,
-                port=port,
-                secure=secure,
-                port_specified=port_specified,
-                # Contents
-                name=k,
-                value=v,
-                # Constant/always-the-same stuff
-                version=0,
-                path="/",
-                expires=None,
-                discard=False,
-                comment=None,
-                comment_url=None,
-                rfc2109=False,
-                path_specified=False,
-                domain_specified=False,
-                domain_initial_dot=False,
-                rest={},
-            )
-        )
+        cookie_jar.set_cookie(scoped_cookie(url_object, k, v))
     return cookie_jar
 
 
@@ -254,8 +246,8 @@ def request(
         if not isinstance(cookies, CookieJar):
             cookies = _scoped_cookiejar_from_dict(parsed_url, cookies)
 
-        cookies = merge_cookies(self._cookiejar, cookies)
-        wrapped_agent: IAgent = CookieAgent(self._agent, cookies)
+        merge_cookies(self._cookiejar, cookies)
+        wrapped_agent: IAgent = CookieAgent(self._agent, self._cookiejar)
 
         if allow_redirects:
             if browser_like_redirects:
@@ -289,7 +281,7 @@ def gotResult(result):
         if not unbuffered:
             d.addCallback(_BufferedResponse)
 
-        return d.addCallback(_Response, cookies)
+        return d.addCallback(_Response, self._cookiejar)
 
     def _request_headers(
         self, headers: Optional[_HeadersType], stacklevel: int

diff --git a/src/treq/cookies.py b/src/treq/cookies.py
@@ -0,0 +1,99 @@
+"""
+Convenience helpers for :mod:`http.cookiejar`
+"""
+
+from typing import Union, Iterable, Optional
+from http.cookiejar import Cookie, CookieJar
+
+from hyperlink import EncodedURL
+
+
+def scoped_cookie(origin: Union[str, EncodedURL], name: str, value: str) -> Cookie:
+    """
+    Create a cookie scoped to a given URL's origin.
+
+    You can insert the result directly into a `CookieJar`, like::
+
+        jar = CookieJar()
+        jar.set_cookie(scoped_cookie("https://example.tld", "flavor", "chocolate"))
+
+        await treq.get("https://domain.example", cookies=jar)
+
+    :param origin:
+        A URL that specifies the domain and port number of the cookie.
+
+        If the protocol is HTTP*S* the cookie is marked ``Secure``, meaning
+        it will not be attached to HTTP requests. Otherwise the cookie will be
+        attached to both HTTP and HTTPS requests
+
+    :param name: Name of the cookie.
+
+    :param value: Value of the cookie.
+
+    .. note::
+
+        This does not scope the cookies to any particular path, only the
+        host, port, and scheme of the given URL.
+    """
+    if isinstance(origin, EncodedURL):
+        url_object = origin
+    else:
+        url_object = EncodedURL.from_text(origin)
+
+    secure = url_object.scheme == "https"
+    port_specified = not (
+        (url_object.scheme == "https" and url_object.port == 443)
+        or (url_object.scheme == "http" and url_object.port == 80)
+    )
+    port = str(url_object.port) if port_specified else None
+    domain = url_object.host
+    netscape_domain = domain if "." in domain else domain + ".local"
+    return Cookie(
+        # Scoping
+        domain=netscape_domain,
+        port=port,
+        secure=secure,
+        port_specified=port_specified,
+        # Contents
+        name=name,
+        value=value,
+        # Constant/always-the-same stuff
+        version=0,
+        path="/",
+        expires=None,
+        discard=False,
+        comment=None,
+        comment_url=None,
+        rfc2109=False,
+        path_specified=False,
+        domain_specified=False,
+        domain_initial_dot=False,
+        rest={},
+    )
+
+
+def search(
+    jar: CookieJar, *, domain: str, name: Optional[str] = None
+) -> Iterable[Cookie]:
+    """
+    Raid the cookie jar for matching cookies.
+
+    This is O(n) on the number of cookies in the jar.
+
+    :param jar: The `CookieJar` (or subclass thereof) to search.
+
+    :param domain:
+        Domain, as in the URL, to match. ``.local`` is appended to
+        a bare hostname. Subdomains are not matched (i.e., searching
+        for ``foo.bar.tld`` won't return a cookie set for ``bar.tld``).
+
+    :param name: Cookie name to match (exactly)
+    """
+    netscape_domain = domain if "." in domain else domain + ".local"
+
+    for c in jar:
+        if c.domain != netscape_domain:
+            continue
+        if name is not None and c.name != name:
+            continue
+        yield c
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		The new :mod:`treq.cookies` module provides helper functions for working with `http.cookiejar.Cookie` and `CookieJar` objects.