diff options
| -rw-r--r-- | .travis.yml | 10 | ||||
| -rw-r--r-- | 3.0-HISTORY.rst | 77 | ||||
| -rw-r--r-- | AUTHORS.rst | 6 | ||||
| -rw-r--r-- | HISTORY.rst | 5 | ||||
| -rw-r--r-- | README.rst | 2 | ||||
| -rw-r--r-- | docs/api.rst | 3 | ||||
| -rw-r--r-- | docs/community/faq.rst | 3 | ||||
| -rw-r--r-- | docs/dev/todo.rst | 1 | ||||
| -rw-r--r-- | docs/index.rst | 2 | ||||
| -rw-r--r-- | docs/user/quickstart.rst | 6 | ||||
| -rw-r--r-- | requests/__init__.py | 8 | ||||
| -rw-r--r-- | requests/__version__.py | 4 | ||||
| -rw-r--r-- | requests/adapters.py | 148 | ||||
| -rw-r--r-- | requests/api.py | 8 | ||||
| -rw-r--r-- | requests/auth.py | 37 | ||||
| -rw-r--r-- | requests/compat.py | 8 | ||||
| -rw-r--r-- | requests/cookies.py | 31 | ||||
| -rw-r--r-- | requests/exceptions.py | 12 | ||||
| -rw-r--r-- | requests/hooks.py | 4 | ||||
| -rw-r--r-- | requests/models.py | 182 | ||||
| -rw-r--r-- | requests/packages.py | 14 | ||||
| -rw-r--r-- | requests/sessions.py | 202 | ||||
| -rw-r--r-- | requests/structures.py | 4 | ||||
| -rw-r--r-- | requests/utils.py | 76 | ||||
| -rwxr-xr-x | setup.py | 3 | ||||
| -rw-r--r-- | tests/test_lowlevel.py | 17 | ||||
| -rw-r--r-- | tests/test_packages.py | 13 | ||||
| -rw-r--r-- | tests/test_requests.py | 705 | ||||
| -rw-r--r-- | tests/test_utils.py | 2 | ||||
| -rw-r--r-- | tox.ini | 2 |
30 files changed, 1167 insertions, 428 deletions
diff --git a/.travis.yml b/.travis.yml index aae4b560..1968ae52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ sudo: false language: python python: - # - "2.6" - "2.7" - "3.4" - "3.5" @@ -9,20 +8,21 @@ python: - "3.7-dev" # - "pypy" -- appears to hang # - "pypy3" +matrix: + allow_failures: + - python: 3.7-dev # command to install dependencies install: "make" # command to run tests script: - - | - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6" ]] ; then make test-readme; fi + - make test-readme - make ci cache: pip jobs: include: - stage: test script: - - | - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6" ]] ; then make test-readme; fi + - make test-readme - make ci - stage: coverage python: 3.6 diff --git a/3.0-HISTORY.rst b/3.0-HISTORY.rst new file mode 100644 index 00000000..a65aad1f --- /dev/null +++ b/3.0-HISTORY.rst @@ -0,0 +1,77 @@ +3.0.0 (2017-xx-xx) +++++++++++++++++++ + +- Support for Python 2.6 has been dropped. + + - The ``OrderedDict`` import no longer exists in compat.py because it is part + of ``collections`` in Python 2.7 and newer. + +- Simplified logic for determining Content-Length and Transfer-Encoding. + Requests will now avoid setting both headers on the same request, and + raise an exception if this is done manually by a user. + +- Remove the HTTPProxyAuth class in favor of supporting proxy auth via + the proxies parameter. + +- Relax how Requests strips bodies from redirects. 3.0.0 only supports body + removal on 301/302 POST redirects and all 303 redirects. + +- Remove support for non-string/bytes parameters in ``_basic_auth_str``. + +- Prevent ``Session.merge_environment`` from erroneously setting the + ``verify`` parameter to ``None`` instead of ``True``. + +- Streaming responses with ``Response.iter_lines`` or ``Response.iter_content`` + now requires an encoding to be set if one isn't provided by the server. + +- Exception raised during read timeout for ``Response.iter_content`` and + ``Response.iter_lines`` changed from ``ConnectionError`` to more + specific ``ReadTimeout``. + +- Raise exception if multiple locations are returned during a redirect. + +- Update ConnectionPool connections when TLS/SSL settings change. + +- Remove simplejson import and only use standard json module. + +- Strip surrounding whitespace from urls. + +- MissingSchema and InvalidSchema renamed to MissingScheme and InvalidScheme + respectively. + +- Change merge order for environment settings to avoid excluding Session-level + settings. + +- Encode redirect URIs as latin-1 before performing redirects in Python 3 to + avoid mangling during the requoting process. + +- Remove the ``__bool__`` and ``__nonzero__`` methods from a ``Response`` + object. + + This has been a planned feature for over a year. The behaviour is surprising + to most people and breaks most of the assumptions that people have about + Response objects. This resolves issue `#2002`_ + +- Skip over empty chunks in iterators. Empty chunks could prematurely signal + the end of a request body's transmission, skipping them allows all of the + data through. See `#2631`_ for more details. + +- Rename the ``req`` argument from ``Session.resolve_redirects`` method + to ``request``. + +- Rename the ``resp`` argument from ``Session.resolve_redirects`` to + ``response``. + +- New ``PreparedRequest.send`` method. Now, you can + ``Request().prepare().send()``. + +- All porcelain API functions (e.g. ``requests.get``, etc) now accept an + optional ``session`` parameter. If provided, the session given will be used + for the request, in place of one being created for you. + +- URLs are now automatically stripped of leading/trailing whitespace. + +- ``Response.raise_for_status()`` now returns the response object for good responses + +.. _#2002: https://github.com/kennethreitz/requests/issues/2002 +.. _#2631: https://github.com/kennethreitz/requests/issues/2631 diff --git a/AUTHORS.rst b/AUTHORS.rst index 907687d4..eb580a4d 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -125,7 +125,7 @@ Patches and Suggestions - Bryce Boe <bbzbryce@gmail.com> (`@bboe <https://github.com/bboe>`_) - Colin Dunklau <colin.dunklau@gmail.com> (`@cdunklau <https://github.com/cdunklau>`_) - Bob Carroll <bob.carroll@alum.rit.edu> (`@rcarz <https://github.com/rcarz>`_) -- Hugo Osvaldo Barrera <hugo@osvaldobarrera.com.ar> (`@hobarrera <https://github.com/hobarrera>`_) +- Hugo Osvaldo Barrera <hugo@barrera.io> (`@hobarrera <https://github.com/hobarrera>`_) - Łukasz Langa <lukasz@langa.pl> - Dave Shawley <daveshawley@gmail.com> - James Clarke (`@jam <https://github.com/jam>`_) @@ -157,16 +157,19 @@ Patches and Suggestions - Muhammad Yasoob Ullah Khalid <yasoob.khld@gmail.com> (`@yasoob <https://github.com/yasoob>`_) - Paul van der Linden (`@pvanderlinden <https://github.com/pvanderlinden>`_) - Colin Dickson (`@colindickson <https://github.com/colindickson>`_) +- Sabari Kumar Murugesan (`@neosab <https://github.com/neosab>`_) - Smiley Barry (`@smiley <https://github.com/smiley>`_) - Shagun Sodhani (`@shagunsodhani <https://github.com/shagunsodhani>`_) - Robin Linderborg (`@vienno <https://github.com/vienno>`_) - Brian Samek (`@bsamek <https://github.com/bsamek>`_) - Dmitry Dygalo (`@Stranger6667 <https://github.com/Stranger6667>`_) +- Tomáš Heger (`@geckon <https://github.com/geckon>`_) - piotrjurkiewicz - Jesse Shapiro <jesse@jesseshapiro.net> (`@haikuginger <https://github.com/haikuginger>`_) - Nate Prewitt <nate.prewitt@gmail.com> (`@nateprewitt <https://github.com/nateprewitt>`_) - Maik Himstedt - Michael Hunsinger +- Jeremy Cline <jcline@redhat.com> (`@jeremycline <https://github.com/jeremycline>`_) - Brian Bamsch <bbamsch32@gmail.com> (`@bbamsch <https://github.com/bbamsch>`_) - Om Prakash Kumar <omprakash070@gmail.com> (`@iamprakashom <https://github.com/iamprakashom>`_) - Philipp Konrad <gardiac2002@gmail.com> (`@gardiac2002 <https://github.com/gardiac2002>`_) @@ -183,6 +186,7 @@ Patches and Suggestions - Ed Morley (`@edmorley <https://github.com/edmorley>`_) - Matt Liu <liumatt@gmail.com> (`@mlcrazy <https://github.com/mlcrazy>`_) - Taylor Hoff <primdevs@protonmail.com> (`@PrimordialHelios <https://github.com/PrimordialHelios>`_) +- Hugo van Kemenade (`@hugovk <https://github.com/hugovk>`_) - Arthur Vigil (`@ahvigil <https://github.com/ahvigil>`_) - Nehal J Wani (`@nehaljwani <https://github.com/nehaljwani>`_) - Demetrios Bairaktaris (`@DemetriosBairaktaris <https://github.com/demetriosbairaktaris>`_) diff --git a/HISTORY.rst b/HISTORY.rst index 61f7e232..fc71d0b8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -241,6 +241,11 @@ Or, even better:: - Updated bundled idna to v2.5. - Updated bundled certifi to 2017.4.17. +- Altered how ``SessionRedirectMixin.resolve_redirects`` and ``Session.send`` + process redirect history. Developers who subclass ``resolve_redirects`` will + find a different ``.history`` attribute - the first element now contains the + original response, and the last element now contains the active response. + 2.13.0 (2017-01-24) +++++++++++++++++++ @@ -77,7 +77,7 @@ Requests is ready for today's web. - ``.netrc`` Support - Chunked Requests -Requests officially supports Python 2.6–2.7 & 3.4–3.6, and runs great on PyPy. +Requests officially supports Python 2.7 & 3.4–3.7, and runs great on PyPy. Installation ------------ diff --git a/docs/api.rst b/docs/api.rst index ef84bf60..cecb0fe9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -74,7 +74,6 @@ Authentication .. autoclass:: requests.auth.AuthBase .. autoclass:: requests.auth.HTTPBasicAuth -.. autoclass:: requests.auth.HTTPProxyAuth .. autoclass:: requests.auth.HTTPDigestAuth @@ -242,7 +241,7 @@ API Changes } # In requests 1.x, this was legal, in requests 2.x, - # this raises requests.exceptions.MissingSchema + # this raises requests.exceptions.MissingScheme requests.get("http://example.org", proxies=proxies) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 7f5a0e1e..e1c8e9ff 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -56,7 +56,6 @@ Python 3 Support? Yes! Here's a list of Python platforms that are officially supported: -* Python 2.6 * Python 2.7 * Python 3.4 * Python 3.5 @@ -70,7 +69,7 @@ These errors occur when :ref:`SSL certificate verification <verification>` fails to match the certificate the server responds with to the hostname Requests thinks it's contacting. If you're certain the server's SSL setup is correct (for example, because you can visit the site with your browser) and -you're using Python 2.6 or 2.7, a possible explanation is that you need +you're using Python 2.7, a possible explanation is that you need Server-Name-Indication. `Server-Name-Indication`_, or SNI, is an official extension to SSL where the diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 1766a28a..b1a3f7eb 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -51,7 +51,6 @@ Runtime Environments Requests currently supports the following versions of Python: -- Python 2.6 - Python 2.7 - Python 3.4 - Python 3.5 diff --git a/docs/index.rst b/docs/index.rst index bd9aded2..5ffe739c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -108,7 +108,7 @@ Requests is ready for today's web. - Chunked Requests - ``.netrc`` Support -Requests officially supports Python 2.6–2.7 & 3.4–3.7, and runs great on PyPy. +Requests officially supports Python 2.7 & 3.4–3.7, and runs great on PyPy. The User Guide diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 032e70f8..d393bf05 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -384,10 +384,14 @@ But, since our ``status_code`` for ``r`` was ``200``, when we call ``raise_for_status()`` we get:: >>> r.raise_for_status() - None + <Response [200]> All is well. +.. note:: ``raise_for_status`` returns the response object for a successful response. This eases chaining in trivial cases, where we want bad codes to raise an exception, but use the response otherwise: + + >>> value = requests.get('http://httpbin.org/ip').raise_for_status().json()['origin'] + Response Headers ---------------- diff --git a/requests/__init__.py b/requests/__init__.py index 6fa855df..bd9c8bb3 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -110,7 +110,6 @@ from .__version__ import __build__, __author__, __author_email__, __license__ from .__version__ import __copyright__, __cake__ from . import utils -from . import packages from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options from .sessions import session, Session @@ -123,12 +122,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/requests/__version__.py b/requests/__version__.py index dc33eef6..5347c7cc 100644 --- a/requests/__version__.py +++ b/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.18.4' -__build__ = 0x021804 +__version__ = '3.0.0' +__build__ = 0x030000 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/requests/adapters.py b/requests/adapters.py index a4b02842..fe0f9049 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -35,14 +35,14 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidScheme, InvalidProxyURL) from .auth import _basic_auth_str try: from urllib3.contrib.socks import SOCKSProxyManager except ImportError: def SOCKSProxyManager(*args, **kwargs): - raise InvalidSchema("Missing dependencies for SOCKS support.") + raise InvalidScheme("Missing dependencies for SOCKS support.") DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 @@ -50,6 +50,67 @@ DEFAULT_RETRIES = 0 DEFAULT_POOL_TIMEOUT = None +def _pool_kwargs(verify, cert): + """Create a dictionary of keyword arguments to pass to a + :class:`PoolManager <urllib3.poolmanager.PoolManager>` with the + necessary SSL configuration. + + :param verify: Whether we should actually verify the certificate; + optionally a path to a CA certificate bundle or + directory of CA certificates. + :param cert: The path to the client certificate and key, if any. + This can either be the path to the certificate and + key concatenated in a single file, or as a tuple of + (cert_file, key_file). + """ + pool_kwargs = {} + if verify: + + cert_loc = None + + # Allow self-specified cert location. + if verify is not True: + cert_loc = verify + + if not cert_loc: + cert_loc = DEFAULT_CA_BUNDLE_PATH + + if not cert_loc or not os.path.exists(cert_loc): + raise IOError("Could not find a suitable TLS CA certificate bundle, " + "invalid path: {0}".format(cert_loc)) + + pool_kwargs['cert_reqs'] = 'CERT_REQUIRED' + + if not os.path.isdir(cert_loc): + pool_kwargs['ca_certs'] = cert_loc + pool_kwargs['ca_cert_dir'] = None + else: + pool_kwargs['ca_cert_dir'] = cert_loc + pool_kwargs['ca_certs'] = None + else: + pool_kwargs['cert_reqs'] = 'CERT_NONE' + pool_kwargs['ca_certs'] = None + pool_kwargs['ca_cert_dir'] = None + + if cert: + if not isinstance(cert, basestring): + pool_kwargs['cert_file'] = cert[0] + pool_kwargs['key_file'] = cert[1] + else: + pool_kwargs['cert_file'] = cert + pool_kwargs['key_file'] = None + + cert_file = pool_kwargs['cert_file'] + key_file = pool_kwargs['key_file'] + if cert_file and not os.path.exists(cert_file): + raise IOError("Could not find the TLS certificate file, " + "invalid path: {0}".format(cert_file)) + if key_file and not os.path.exists(key_file): + raise IOError("Could not find the TLS key file, " + "invalid path: {0}".format(key_file)) + return pool_kwargs + + class BaseAdapter(object): """The Base Transport Adapter""" @@ -127,8 +188,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -199,58 +259,6 @@ class HTTPAdapter(BaseAdapter): return manager - def cert_verify(self, conn, url, verify, cert): - """Verify a SSL certificate. This method should not be called from user - code, and is only exposed for use when subclassing the - :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. - - :param conn: The urllib3 connection object associated with the cert. - :param url: The requested URL. - :param verify: Either a boolean, in which case it controls whether we verify - the server's TLS certificate, or a string, in which case it must be a path - to a CA bundle to use - :param cert: The SSL certificate to verify. - """ - if url.lower().startswith('https') and verify: - - cert_loc = None - - # Allow self-specified cert location. - if verify is not True: - cert_loc = verify - - if not cert_loc: - cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) - - if not cert_loc or not os.path.exists(cert_loc): - raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) - - conn.cert_reqs = 'CERT_REQUIRED' - - if not os.path.isdir(cert_loc): - conn.ca_certs = cert_loc - else: - conn.ca_cert_dir = cert_loc - else: - conn.cert_reqs = 'CERT_NONE' - conn.ca_certs = None - conn.ca_cert_dir = None - - if cert: - if not isinstance(cert, basestring): - conn.cert_file = cert[0] - conn.key_file = cert[1] - else: - conn.cert_file = cert - conn.key_file = None - if conn.cert_file and not os.path.exists(conn.cert_file): - raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) - if conn.key_file and not os.path.exists(conn.key_file): - raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) - def build_response(self, req, resp): """Builds a :class:`Response <requests.Response>` object from a urllib3 response. This should not be called from user code, and is only exposed @@ -288,7 +296,7 @@ class HTTPAdapter(BaseAdapter): return response - def get_connection(self, url, proxies=None): + def get_connection(self, url, proxies=None, verify=None, cert=None): """Returns a urllib3 connection for the given URL. This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. @@ -297,6 +305,7 @@ class HTTPAdapter(BaseAdapter): :param proxies: (optional) A Requests-style dictionary of proxies used on this request. :rtype: urllib3.ConnectionPool """ + pool_kwargs = _pool_kwargs(verify, cert) proxy = select_proxy(url, proxies) if proxy: @@ -306,12 +315,12 @@ class HTTPAdapter(BaseAdapter): raise InvalidProxyURL("Please check proxy URL. It is malformed" " and could be missing the host.") proxy_manager = self.proxy_manager_for(proxy) - conn = proxy_manager.connection_from_url(url) + conn = proxy_manager.connection_from_url(url, pool_kwargs=pool_kwargs) else: # Only scheme should be lower case parsed = urlparse(url) url = parsed.geturl() - conn = self.poolmanager.connection_from_url(url) + conn = self.poolmanager.connection_from_url(url, pool_kwargs=pool_kwargs) return conn @@ -406,10 +415,8 @@ class HTTPAdapter(BaseAdapter): :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ + conn = self.get_connection(request.url, proxies, verify, cert) - conn = self.get_connection(request.url, proxies) - - self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) @@ -442,7 +449,8 @@ class HTTPAdapter(BaseAdapter): preload_content=False, decode_content=False, retries=self.max_retries, - timeout=timeout + timeout=timeout, + enforce_content_length=True ) # Send the request. @@ -463,7 +471,10 @@ class HTTPAdapter(BaseAdapter): low_conn.endheaders() for i in request.body: - low_conn.send(hex(len(i))[2:].encode('utf-8')) + chunk_size = len(i) + if chunk_size == 0: + continue + low_conn.send(hex(chunk_size)[2:].encode('utf-8')) low_conn.send(b'\r\n') low_conn.send(i) low_conn.send(b'\r\n') @@ -471,11 +482,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For Python 3.3+ versions, this is the default r = low_conn.getresponse() resp = HTTPResponse.from_httplib( @@ -483,7 +493,9 @@ class HTTPAdapter(BaseAdapter): pool=conn, connection=low_conn, preload_content=False, - decode_content=False + decode_content=False, + enforce_content_length=True, + request_method=request.method ) except: # If we hit any problems here, clean up the connection. diff --git a/requests/api.py b/requests/api.py index bc2115c1..b02834b1 100644 --- a/requests/api.py +++ b/requests/api.py @@ -13,11 +13,12 @@ This module implements the Requests API. from . import sessions -def request(method, url, **kwargs): +def request(method, url, session=None, **kwargs): """Constructs and sends a :class:`Request <Request>`. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. + :param session: :class:`Session` object to use for this request. If none is given, one will be provided. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. @@ -54,7 +55,10 @@ def request(method, url, **kwargs): # By using the 'with' statement we are sure the session is closed, thus we # avoid leaving sockets open which can trigger a ResourceWarning in some # cases, and look like a memory leak in others. - with sessions.Session() as session: + + session = sessions.Session() if session is None else session + + with session: return session.request(method=method, url=url, **kwargs) diff --git a/requests/auth.py b/requests/auth.py index 1a182dff..e3cbcffd 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -12,7 +12,6 @@ import re import time import hashlib import threading -import warnings from base64 import b64encode @@ -28,33 +27,13 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data' def _basic_auth_str(username, password): """Returns a Basic Auth string.""" - # "I want us to put a big-ol' comment on top of it that - # says that this behaviour is dumb but we need to preserve - # it because people are relying on it." - # - Lukasa - # - # These are here solely to maintain backwards compatibility - # for things like ints. This will be removed in 3.0.0. if not isinstance(username, basestring): - warnings.warn( - "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " - "a string or bytes object in the near future to avoid " - "problems.".format(username), - category=DeprecationWarning, - ) - username = str(username) + raise TypeError('username must be of type str or bytes, ' + 'instead it was %s' % type(username)) if not isinstance(password, basestring): - warnings.warn( - "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " - "a string or bytes object in the near future to avoid " - "problems.".format(password), - category=DeprecationWarning, - ) - password = str(password) - # -- End Removal -- + raise TypeError('password must be of type str or bytes, ' + 'instead it was %s' % type(password)) if isinstance(username, str): username = username.encode('latin1') @@ -97,14 +76,6 @@ class HTTPBasicAuth(AuthBase): return r -class HTTPProxyAuth(HTTPBasicAuth): - """Attaches HTTP Proxy Authentication to a given Request object.""" - - def __call__(self, r): - r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password) - return r - - class HTTPDigestAuth(AuthBase): """Attaches HTTP Digest Authentication to the given Request object.""" diff --git a/requests/compat.py b/requests/compat.py index f417cfd8..55da272a 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -25,11 +25,6 @@ is_py2 = (_ver[0] == 2) #: Python 3.x? is_py3 = (_ver[0] == 3) -try: - import simplejson as json -except ImportError: - import json - # --------- # Specifics # --------- @@ -44,8 +39,6 @@ if is_py2: from Cookie import Morsel from StringIO import StringIO - from urllib3.packages.ordered_dict import OrderedDict - builtin_str = str bytes = str str = unicode @@ -59,7 +52,6 @@ elif is_py3: from http import cookiejar as cookielib from http.cookies import Morsel from io import StringIO - from collections import OrderedDict builtin_str = str str = str diff --git a/requests/cookies.py b/requests/cookies.py index ab3c88b9..d4abfd0e 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -414,7 +414,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def copy(self): """Return a copy of this RequestsCookieJar.""" - new_cj = RequestsCookieJar() + new_cj = RequestsCookieJar(self._policy) new_cj.update(self) return new_cj @@ -440,20 +440,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: diff --git a/requests/exceptions.py b/requests/exceptions.py index a80cad80..1c61bf87 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -69,12 +69,12 @@ class TooManyRedirects(RequestException): """Too many redirects.""" -class MissingSchema(RequestException, ValueError): - """The URL schema (e.g. http or https) is missing.""" +class MissingScheme(RequestException, ValueError): + """The URL scheme (e.g. http or https) is missing.""" -class InvalidSchema(RequestException, ValueError): - """See defaults.py for valid schemas.""" +class InvalidScheme(RequestException, ValueError): + """See defaults.py for valid schemes.""" class InvalidURL(RequestException, ValueError): @@ -108,6 +108,10 @@ class RetryError(RequestException): class UnrewindableBodyError(RequestException): """Requests encountered an error when trying to rewind a body""" + +class InvalidBodyError(RequestException, ValueError): + """An invalid request body was specified""" + # Warnings diff --git a/requests/hooks.py b/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/requests/models.py b/requests/models.py index ce4e284e..c3391ad2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -9,6 +9,7 @@ This module contains the primary objects that power Requests. import collections import datetime +import codecs import sys # Import encoding now, to avoid implicit import later. @@ -26,20 +27,23 @@ from io import UnsupportedOperation from .hooks import default_hooks from .structures import CaseInsensitiveDict +import requests from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .exceptions import ( - HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError) + HTTPError, MissingScheme, InvalidURL, ChunkedEncodingError, + ContentDecodingError, ConnectionError, StreamConsumedError, + InvalidHeader, InvalidBodyError, ReadTimeout) from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( guess_filename, get_auth_from_url, requote_uri, stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, check_header_validity) + iter_slices, guess_json_utf, super_len, check_header_validity, + is_stream) from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, is_py2, chardet, builtin_str, basestring) -from .compat import json as complexjson +import json as complexjson from .status_codes import codes #: The set of HTTP status codes that indicate an automatically @@ -331,8 +335,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_method(self, method): """Prepares the given HTTP method.""" self.method = method - if self.method is not None: - self.method = to_native_string(self.method.upper()) + if self.method is None: + raise ValueError('Request method cannot be "None"') + self.method = to_native_string(self.method.upper()) @staticmethod def _get_idna_encoded_host(host): @@ -356,8 +361,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): else: url = unicode(url) if is_py2 else str(url) - # Remove leading whitespaces from url - url = url.lstrip() + # Ignore any leading and trailing whitespace characters. + url = url.strip() # Don't do any URL preparation for non-HTTP schemes like `mailto`, # `data` etc to work around exceptions from `url_parse`, which @@ -373,10 +378,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): raise InvalidURL(*e.args) if not scheme: - error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") + error = ("Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?") error = error.format(to_native_string(url, 'utf8')) - raise MissingSchema(error) + raise MissingScheme(error) if not host: raise InvalidURL("Invalid URL %r: No host supplied" % url) @@ -459,17 +464,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if not isinstance(body, bytes): body = body.encode('utf-8') - is_stream = all([ - hasattr(data, '__iter__'), - not isinstance(data, (basestring, list, tuple, collections.Mapping)) - ]) - - try: - length = super_len(data) - except (TypeError, AttributeError, UnsupportedOperation): - length = None - - if is_stream: + if is_stream(data): body = data if getattr(body, 'tell', None) is not None: @@ -486,10 +481,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if files: raise NotImplementedError('Streamed bodies and files are mutually exclusive.') - if length: - self.headers['Content-Length'] = builtin_str(length) - else: - self.headers['Transfer-Encoding'] = 'chunked' else: # Multi-part file uploads. if files: @@ -502,27 +493,40 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): else: content_type = 'application/x-www-form-urlencoded' - self.prepare_content_length(body) - # Add content-type if it wasn't explicitly provided. if content_type and ('content-type' not in self.headers): self.headers['Content-Type'] = content_type + self.prepare_content_length(body) self.body = body def prepare_content_length(self, body): - """Prepare Content-Length header based on request method and body""" + """Prepares Content-Length header. + + If the length of the body of the request can be computed, Content-Length + is set using ``super_len``. If user has manually set either a + Transfer-Encoding or Content-Length header when it should not be set + (they should be mutually exclusive) an InvalidHeader + error will be raised. + """ if body is not None: length = super_len(body) + if length: - # If length exists, set it. Otherwise, we fallback - # to Transfer-Encoding: chunked. self.headers['Content-Length'] = builtin_str(length) + elif is_stream(body): + self.headers['Transfer-Encoding'] = 'chunked' + else: + raise InvalidBodyError('Non-null body must have length or be streamable.') elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None: # Set Content-Length to 0 for methods that can have a body # but don't provide one. (i.e. not GET or HEAD) self.headers['Content-Length'] = '0' + if 'Transfer-Encoding' in self.headers and 'Content-Length' in self.headers: + raise InvalidHeader('Conflicting Headers: Both Transfer-Encoding and ' + 'Content-Length are set.') + def prepare_auth(self, auth, url=''): """Prepares the given HTTP auth data.""" @@ -574,6 +578,14 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): for event in hooks: self.register_hook(event, hooks[event]) + def send(self, session=None, **send_kwargs): + """Sends the PreparedRequest to the given Session. + If none is provided, one is created for you.""" + session = requests.Session() if session is None else session + + with session: + return session.send(self, **send_kwargs) + class Response(object): """The :class:`Response <Response>` object, which contains a @@ -606,7 +618,8 @@ class Response(object): #: Final URL location of Response. self.url = None - #: Encoding to decode with when accessing r.text. + #: Encoding to decode with when accessing r.text or + #: r.iter_content(decode_unicode=True) self.encoding = None #: A list of :class:`Response <Response>` objects from @@ -644,10 +657,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): @@ -660,26 +670,6 @@ class Response(object): def __repr__(self): return '<Response [%s]>' % (self.status_code) - def __bool__(self): - """Returns True if :attr:`status_code` is less than 400. - - This attribute checks if the status code of the response is between - 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This - is **not** a check to see if the response code is ``200 OK``. - """ - return self.ok - - def __nonzero__(self): - """Returns True if :attr:`status_code` is less than 400. - - This attribute checks if the status code of the response is between - 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This - is **not** a check to see if the response code is ``200 OK``. - """ - return self.ok - def __iter__(self): """Allows you to use a response as an iterator.""" return self.iter_content(128) @@ -734,8 +724,8 @@ class Response(object): chunks are received. If stream=False, data is returned as a single chunk. - If decode_unicode is True, content will be decoded using the best - available encoding based on the response. + If using decode_unicode, the encoding must be set to a valid encoding + enumeration before invoking iter_content. """ def generate(): @@ -745,11 +735,14 @@ class Response(object): for chunk in self.raw.stream(chunk_size, decode_content=True): yield chunk except ProtocolError as e: - raise ChunkedEncodingError(e) + if self.headers.get('Transfer-Encoding') == 'chunked': + raise ChunkedEncodingError(e) + else: + raise ConnectionError(e) except DecodeError as e: raise ContentDecodingError(e) except ReadTimeoutError as e: - raise ConnectionError(e) + raise ReadTimeout(e) else: # Standard file-like object. while True: @@ -772,6 +765,16 @@ class Response(object): chunks = reused_chunks if self._content_consumed else stream_chunks if decode_unicode: + if self.encoding is None: + raise TypeError( + 'encoding must be set before consuming streaming ' + 'responses' + ) + + # check encoding value here, don't wait for the generator to be + # consumed before raising an exception + codecs.lookup(self.encoding) + chunks = stream_decode_response_unicode(chunks, self) return chunks @@ -783,23 +786,67 @@ class Response(object): .. note:: This method is not reentrant safe. """ + carriage_return = u'\r' if decode_unicode else b'\r' + line_feed = u'\n' if decode_unicode else b'\n' pending = None - - for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode): - + last_chunk_ends_with_cr = False + + for chunk in self.iter_content(chunk_size=chunk_size, + decode_unicode=decode_unicode): + # Skip any null responses: if there is pending data it is necessarily an + # incomplete chunk, so if we don't have more data we don't want to bother + # trying to get it. Unconsumed pending data will be yielded anyway in the + # end of the loop if the stream ends. + if not chunk: + continue + + # Consume any pending data if pending is not None: chunk = pending + chunk + pending = None + # Either split on a line, or split on a specified delimiter if delimiter: lines = chunk.split(delimiter) else: + # Python splitlines() supports the universal newline (PEP 278). + # That means, '\r', '\n', and '\r\n' are all treated as end of + # line. If the last chunk ends with '\r', and the current chunk + # starts with '\n', they should be merged and treated as only + # *one* new line separator '\r\n' by splitlines(). + # This rule only applies when splitlines() is used. + + # The last chunk ends with '\r', so the '\n' at chunk[0] + # is just the second half of a '\r\n' pair rather than a + # new line break. Just skip it. + skip_first_char = last_chunk_ends_with_cr and chunk.startswith(line_feed) + last_chunk_ends_with_cr = chunk.endswith(carriage_return) + if skip_first_char: + chunk = chunk[1:] + # it's possible that after stripping the '\n' then chunk becomes empty + if not chunk: + continue lines = chunk.splitlines() - if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: + # Calling `.split(delimiter)` will always end with whatever text + # remains beyond the delimiter, or '' if the delimiter is the end + # of the text. On the other hand, `.splitlines()` doesn't include + # a '' if the text ends in a line delimiter. + # + # For example: + # + # 'abc\ndef\n'.split('\n') ~> ['abc', 'def', ''] + # 'abc\ndef\n'.splitlines() ~> ['abc', 'def'] + # + # So if we have a specified delimiter, we always pop the final + # item and prepend it to the next chunk. + # + # If we're using `splitlines()`, we only do this if the chunk + # ended midway through a line. + incomplete_line = lines[-1] and lines[-1][-1] == chunk[-1] + if delimiter or incomplete_line: pending = lines.pop() - else: - pending = None for line in lines: yield line @@ -910,7 +957,8 @@ class Response(object): return l def raise_for_status(self): - """Raises stored :class:`HTTPError`, if one occurred.""" + """Raises stored :class:`HTTPError`, if one occurred. + Otherwise, returns the response object (self).""" http_error_msg = '' if isinstance(self.reason, bytes): @@ -934,6 +982,8 @@ class Response(object): if http_error_msg: raise HTTPError(http_error_msg, response=self) + return self + def close(self): """Releases the connection back to the pool. Once this method has been called the underlying ``raw`` object must not be accessed again. diff --git a/requests/packages.py b/requests/packages.py deleted file mode 100644 index 7232fe0f..00000000 --- a/requests/packages.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -# This code exists for backwards compatibility reasons. -# I don't like it either. Just look the other way. :) - -for package in ('urllib3', 'idna', 'chardet'): - locals()[package] = __import__(package) - # This traversal is apparently necessary such that the identities are - # preserved (requests.packages.urllib3.* is urllib3.*) - for mod in list(sys.modules): - if mod == package or mod.startswith(package + '.'): - sys.modules['requests.packages.' + mod] = sys.modules[mod] - -# Kinda cool, though, right? diff --git a/requests/sessions.py b/requests/sessions.py index 4ffed46a..66ed53ea 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -10,26 +10,28 @@ requests (cookies, auth, proxies). import os import sys import time -from collections import Mapping +from collections import Mapping, OrderedDict from datetime import timedelta from .auth import _basic_auth_str -from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse +from .compat import cookielib, urljoin, urlparse, is_py3, str from .cookies import ( - cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) + cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, + merge_cookies, _copy_cookie_jar) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string from .utils import to_key_val_list, default_headers from .exceptions import ( - TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) + TooManyRedirects, InvalidScheme, ChunkedEncodingError, + ConnectionError, ContentDecodingError, InvalidHeader) from .structures import CaseInsensitiveDict from .adapters import HTTPAdapter from .utils import ( requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, - get_auth_from_url, rewind_body + get_auth_from_url, is_valid_location, rewind_body ) from .status_codes import codes @@ -50,7 +52,7 @@ else: def merge_setting(request_setting, session_setting, dict_class=OrderedDict): """Determines appropriate setting for a given request, taking into account the explicit setting on that request, and the setting in the session. If a - setting is a dictionary, they will be merged together using `dict_class` + setting is a dictionary, they will be merged together using `dict_class`. """ if session_setting is None: @@ -95,7 +97,7 @@ def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): class SessionRedirectMixin(object): - def get_redirect_target(self, resp): + def get_redirect_target(self, response): """Receives a Response. Returns a redirect URI or ``None``""" # Due to the nature of how requests processes redirects this method will # be called at least once upon the original response and at least twice @@ -103,8 +105,13 @@ class SessionRedirectMixin(object): # If a custom mixin is used to handle this logic, it may be advantageous # to cache the redirect location onto the response object as a private # attribute. - if resp.is_redirect: - location = resp.headers['location'] + if response.is_redirect: + if not is_valid_location(response): + raise InvalidHeader('Response contains multiple Location headers. ' + 'Unable to perform redirect.') + + location = response.headers['location'] + # Currently the underlying http module on py3 decode headers # in latin1, but empirical evidence suggests that latin1 is very # rarely used with non-ASCII characters in HTTP headers. @@ -116,60 +123,61 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None - def resolve_redirects(self, resp, req, stream=False, timeout=None, - verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): - """Receives a Response. Returns a generator of Responses or Requests.""" + def resolve_redirects(self, response, request, stream=False, timeout=None, + verify=True, cert=None, proxies=None, + yield_requests=False, **adapter_kwargs): + """Given a Response, yields Responses until 'Location' header-based + redirection ceases, or the Session.max_redirects limit has been + reached. + """ - hist = [] # keep track of history + history = [response] # keep track of history; seed it with the original response - url = self.get_redirect_target(resp) - previous_fragment = urlparse(req.url).fragment - while url: - prepared_request = req.copy() + location_url = self.get_redirect_target(response) + previous_fragment = urlparse(request.url).fragment - # Update history and keep track of redirects. - # resp.history must ignore the original request in this loop - hist.append(resp) - resp.history = hist[1:] + while location_url: + prepared_request = request.copy() try: - resp.content # Consume socket so it can be released - except (ChunkedEncodingError, ContentDecodingError, RuntimeError): - resp.raw.read(decode_content=False) + response.content # Consume socket so it can be released + except (ChunkedEncodingError, ConnectionError, ContentDecodingError, RuntimeError): + response.raw.read(decode_content=False) - if len(resp.history) >= self.max_redirects: - raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) + if len(response.history) >= self.max_redirects: + raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=response) # Release the connection back into the pool. - resp.close() + response.close() # Handle redirection without scheme (see: RFC 1808 Section 4) - if url.startswith('//'): - parsed_rurl = urlparse(resp.url) - url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) + if location_url.startswith('//'): + parsed_rurl = urlparse(response.url) + location_url = '%s:%s' % (to_native_string(parsed_rurl.scheme), location_url) # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) - parsed = urlparse(url) + parsed = urlparse(location_url) if parsed.fragment == '' and previous_fragment: parsed = parsed._replace(fragment=previous_fragment) elif parsed.fragment: previous_fragment = parsed.fragment - url = parsed.geturl() + location_url = parsed.geturl() # Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. if not parsed.netloc: - url = urljoin(resp.url, requote_uri(url)) + location_url = urljoin(response.url, requote_uri(location_url)) else: - url = requote_uri(url) + location_url = requote_uri(location_url) - prepared_request.url = to_native_string(url) + prepared_request.url = to_native_string(location_url) - self.rebuild_method(prepared_request, resp) + method_changed = self.rebuild_method(prepared_request, response) - # https://github.com/requests/requests/issues/1084 - if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): + # https://github.com/kennethreitz/requests/issues/2590 + # If method is changed to GET we need to remove body and associated headers. + if method_changed and prepared_request.method == 'GET': # https://github.com/requests/requests/issues/3490 purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding') for header in purged_headers: @@ -185,13 +193,13 @@ class SessionRedirectMixin(object): # Extract any cookies sent on the response to the cookiejar # in the new request. Because we've mutated our copied prepared # request, use the old one that we haven't yet touched. - extract_cookies_to_jar(prepared_request._cookies, req, resp.raw) + extract_cookies_to_jar(prepared_request._cookies, request, response.raw) merge_cookies(prepared_request._cookies, self.cookies) prepared_request.prepare_cookies(prepared_request._cookies) # Rebuild auth and proxy information. proxies = self.rebuild_proxies(prepared_request, proxies) - self.rebuild_auth(prepared_request, resp) + self.rebuild_auth(prepared_request, response) # A failed tell() sets `_body_position` to `object()`. This non-None # value ensures `rewindable` will be True, allowing us to raise an @@ -206,14 +214,14 @@ class SessionRedirectMixin(object): rewind_body(prepared_request) # Override the original request. - req = prepared_request + request = prepared_request if yield_requests: - yield req + yield request else: - resp = self.send( - req, + response = self.send( + request, stream=stream, timeout=timeout, verify=verify, @@ -222,16 +230,21 @@ class SessionRedirectMixin(object): allow_redirects=False, **adapter_kwargs ) + # copy our history tracker into the response + response.history = history[:] + # append the new response to the history tracker for the next iteration + history.append(response) - extract_cookies_to_jar(self.cookies, prepared_request, resp.raw) + extract_cookies_to_jar(self.cookies, prepared_request, response.raw) # extract redirect url, if any, for the next loop - url = self.get_redirect_target(resp) - yield resp + location_url = self.get_redirect_target(response) + yield response def rebuild_auth(self, prepared_request, response): """When being redirected we may want to strip authentication from the - request to avoid leaking credentials. This method intelligently removes + request to avoid leaking credentials. This method intelligently + removes and reapplies authentication where possible to avoid credential loss. """ headers = prepared_request.headers @@ -254,11 +267,11 @@ class SessionRedirectMixin(object): return def rebuild_proxies(self, prepared_request, proxies): - """This method re-evaluates the proxy configuration by considering the - environment variables. If we are redirected to a URL covered by - NO_PROXY, we strip the proxy configuration. Otherwise, we set missing - proxy keys for this URL (in case they were stripped by a previous - redirect). + """This method re-evaluates the proxy configuration by + considering the environment variables. If we are redirected to a + URL covered by NO_PROXY, we strip the proxy configuration. + Otherwise, we set missing proxy keys for this URL (in case they + were stripped by a previous redirect). This method also replaces the Proxy-Authorization header where necessary. @@ -297,24 +310,26 @@ class SessionRedirectMixin(object): def rebuild_method(self, prepared_request, response): """When being redirected we may want to change the method of the request based on certain specs or browser behavior. + + :rtype bool: + :return: boolean expressing if the method changed during rebuild. """ - method = prepared_request.method + method = original_method = prepared_request.method # http://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' - # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if response.status_code == codes.found and method != 'HEAD': - method = 'GET' - - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if response.status_code == codes.moved and method == 'POST': + # If a POST is responded to with a 301 or 302, turn it into a GET. This has + # become a common pattern in browsers and was introduced into later versions + # of HTTP RFCs. While some browsers transform other methods to GET, little of + # that has been standardized. For that reason, we're using curl as a model + # which only supports POST->GET. + if response.status_code in (codes.found, codes.moved) and method == 'POST': method = 'GET' prepared_request.method = method + return method != original_method class Session(SessionRedirectMixin): @@ -410,7 +425,7 @@ class Session(SessionRedirectMixin): :class:`Session`. :param request: :class:`Request` instance to prepare with this - session's settings. + Session's settings. :rtype: requests.PreparedRequest """ cookies = request.cookies or {} @@ -420,8 +435,8 @@ class Session(SessionRedirectMixin): cookies = cookiejar_from_dict(cookies) # Merge with session cookies - merged_cookies = merge_cookies( - merge_cookies(RequestsCookieJar(), self.cookies), cookies) + session_cookies = _copy_cookie_jar(self.cookies) + merged_cookies = merge_cookies(session_cookies, cookies) # Set environment's basic authentication if not explicitly set. auth = request.auth @@ -447,7 +462,7 @@ class Session(SessionRedirectMixin): params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, json=None): - """Constructs a :class:`Request <Request>`, prepares it and sends it. + """Constructs a :class:`Request <Request>`, prepares it, and sends it. Returns :class:`Response <Response>` object. :param method: method for the new :class:`Request` object. @@ -608,7 +623,8 @@ class Session(SessionRedirectMixin): if isinstance(request, Request): raise ValueError('You can only send PreparedRequests.') - # Set up variables needed for resolve_redirects and dispatching of hooks + # Set up variables needed for resolve_redirects and dispatching of + # hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') hooks = request.hooks @@ -626,7 +642,7 @@ class Session(SessionRedirectMixin): elapsed = preferred_clock() - start r.elapsed = timedelta(seconds=elapsed) - # Response manipulation hooks + # Response manipulation hooks. r = dispatch_hook('response', hooks, r, **kwargs) # Persist cookies @@ -641,16 +657,12 @@ class Session(SessionRedirectMixin): # Redirect resolving generator. gen = self.resolve_redirects(r, request, **kwargs) - # Resolve redirects if allowed. + # Resolve redirects, if allowed. history = [resp for resp in gen] if allow_redirects else [] - # Shuffle things around if there's history. + # If there is a history, replace ``r`` with the last response if history: - # Insert the first (original) request at the start - history.insert(0, r) - # Get the last request made r = history.pop() - r.history = history # If redirects aren't being followed, store the response on the Request for Response.next(). if not allow_redirects: @@ -670,25 +682,37 @@ class Session(SessionRedirectMixin): :rtype: dict """ + # Merge all the kwargs except for proxies. + stream = merge_setting(stream, self.stream) + verify = merge_setting(verify, self.verify) + cert = merge_setting(cert, self.cert) # Gather clues from the surrounding environment. + # We do this after merging the Session values to make sure we don't + # accidentally exclude them. if self.trust_env: - # Set environment's proxies. - no_proxy = proxies.get('no_proxy') if proxies is not None else None - env_proxies = get_environ_proxies(url, no_proxy=no_proxy) - for (k, v) in env_proxies.items(): - proxies.setdefault(k, v) - # Look for requests environment configuration and be compatible # with cURL. if verify is True or verify is None: verify = (os.environ.get('REQUESTS_CA_BUNDLE') or - os.environ.get('CURL_CA_BUNDLE')) + os.environ.get('CURL_CA_BUNDLE') or + verify) - # Merge all the kwargs. - proxies = merge_setting(proxies, self.proxies) - stream = merge_setting(stream, self.stream) - verify = merge_setting(verify, self.verify) - cert = merge_setting(cert, self.cert) + # Now we handle proxies. + # Proxies need to be built up backwards. This is because None values + # can delete proxy information, which can then be re-added by a more + # specific layer. So we begin by getting the environment's proxies, + # then add the Session, then add the request. + no_proxy = proxies.get('no_proxy') if proxies is not None else None + if no_proxy is None: + no_proxy = self.proxies.get('no_proxy') + + env_proxies = {} + + if self.trust_env: + env_proxies = get_environ_proxies(url, no_proxy=no_proxy) or {} + + new_proxies = merge_setting(self.proxies, env_proxies) + proxies = merge_setting(proxies, new_proxies) return {'verify': verify, 'proxies': proxies, 'stream': stream, 'cert': cert} @@ -705,10 +729,10 @@ class Session(SessionRedirectMixin): return adapter # Nothing matches :-/ - raise InvalidSchema("No connection adapters were found for '%s'" % url) + raise InvalidScheme("No connection adapters were found for '%s'" % url) def close(self): - """Closes all adapters and as such the session""" + """Closes all adapters and, as such, the Session.""" for v in self.adapters.values(): v.close() @@ -724,7 +748,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): diff --git a/requests/structures.py b/requests/structures.py index 05d2b3f5..cb5104a6 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -9,8 +9,6 @@ Data structures that power Requests. import collections -from .compat import OrderedDict - class CaseInsensitiveDict(collections.MutableMapping): """A case-insensitive ``dict``-like object. @@ -40,7 +38,7 @@ class CaseInsensitiveDict(collections.MutableMapping): """ def __init__(self, data=None, **kwargs): - self._store = OrderedDict() + self._store = collections.OrderedDict() if data is None: data = {} self.update(data, **kwargs) diff --git a/requests/utils.py b/requests/utils.py index 3f50d485..c718a783 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -27,8 +27,8 @@ from . import certs from ._internal_utils import to_native_string from .compat import parse_http_list as _parse_list_header from .compat import ( - quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, - proxy_bypass, urlunparse, basestring, integer_types, is_py3, + quote, urlparse, bytes, str, unquote, getproxies, + proxy_bypass, urlunparse, basestring, integer_types, is_py2, is_py3, proxy_bypass_environment, getproxies_environment) from .cookies import cookiejar_from_dict from .structures import CaseInsensitiveDict @@ -277,7 +277,7 @@ def from_key_val_list(value): if isinstance(value, (str, bytes, bool, int)): raise ValueError('cannot encode objects that are not 2-tuples') - return OrderedDict(value) + return collections.OrderedDict(value) def to_key_val_list(value): @@ -495,11 +495,6 @@ def get_encoding_from_headers(headers): def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" - if r.encoding is None: - for item in iterator: - yield item - return - decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') for chunk in iterator: rv = decoder.decode(chunk) @@ -567,7 +562,26 @@ def unquote_unreserved(uri): :rtype: str """ - parts = uri.split('%') + # This convert function is used to optionally convert the output of `chr`. + # In Python 3, `chr` returns a unicode string, while in Python 2 it returns + # a bytestring. Here we deal with that by optionally converting. + def convert(is_bytes, c): + if is_py2 and not is_bytes: + return c.decode('ascii') + elif is_py3 and is_bytes: + return c.encode('ascii') + else: + return c + + # Handle both bytestrings and unicode strings. + is_bytes = isinstance(uri, bytes) + splitchar = u'%' + base = u'' + if is_bytes: + splitchar = splitchar.encode('ascii') + base = base.encode('ascii') + + parts = uri.split(splitchar) for i in range(1, len(parts)): h = parts[i][0:2] if len(h) == 2 and h.isalnum(): @@ -577,12 +591,12 @@ def unquote_unreserved(uri): raise InvalidURL("Invalid percent-escape sequence: '%s'" % h) if c in UNRESERVED_SET: - parts[i] = c + parts[i][2:] + parts[i] = convert(is_bytes, c) + parts[i][2:] else: - parts[i] = '%' + parts[i] + parts[i] = splitchar + parts[i] else: - parts[i] = '%' + parts[i] - return ''.join(parts) + parts[i] = splitchar + parts[i] + return base.join(parts) def requote_uri(uri): @@ -732,22 +746,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): - try: - bypass = proxy_bypass(parsed.hostname) - except (TypeError, socket.gaierror): - bypass = False - - if bypass: - return True - - return False + return bool(proxy_bypass(parsed.hostname)) def get_environ_proxies(url, no_proxy=None): @@ -846,6 +846,19 @@ def parse_header_links(value): return links +def is_valid_location(response): + """Verify that multiple Location headers weren't + returned from the last response. + """ + headers = getattr(response.raw, 'headers', None) + if headers is not None: + getlist = getattr(headers, 'getlist', None) + if getlist is not None: + return len(getlist('location')) <= 1 + # If response.raw isn't urllib3-like we can't reliably check this + return True + + # Null bytes; no need to recreate these on each call to guess_json_utf _null = '\x00'.encode('ascii') # encoding to ASCII for Python 3 _null2 = _null * 2 @@ -973,3 +986,10 @@ def rewind_body(prepared_request): "body for redirect.") else: raise UnrewindableBodyError("Unable to rewind request body for redirect.") + + +def is_stream(data): + """Given data, determines if it should be sent as a stream.""" + is_iterable = getattr(data, '__iter__', False) + is_io_type = not isinstance(data, (basestring, list, tuple, collections.Mapping)) + return is_iterable and is_io_type @@ -83,7 +83,6 @@ setup( 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', @@ -97,6 +96,6 @@ setup( extras_require={ 'security': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], 'socks': ['PySocks>=1.5.6, !=1.5.7'], - 'socks:sys_platform == "win32" and (python_version == "2.7" or python_version == "2.6")': ['win_inet_pton'], + 'socks:sys_platform == "win32" and python_version == "2.7"': ['win_inet_pton'], }, ) diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index 6d6268cd..6c6b0863 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -23,6 +23,23 @@ def test_chunked_upload(): assert r.status_code == 200 assert r.request.headers['Transfer-Encoding'] == 'chunked' +def test_incorrect_content_length(): + """Test ConnectionError raised for incomplete responses""" + close_server = threading.Event() + server = Server.text_response_server( + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 50\r\n\r\n" + + "Hello World." + ) + with server as (host, port): + url = 'http://{0}:{1}/'.format(host, port) + r = requests.Request('GET', url).prepare() + s = requests.Session() + with pytest.raises(requests.exceptions.ConnectionError) as e: + resp = s.send(r) + assert "12 bytes read, 38 more expected" in str(e) + close_server.set() # release server block + def test_digestauth_401_count_reset_on_redirect(): """Ensure we correctly reset num_401_calls after a successful digest auth, diff --git a/tests/test_packages.py b/tests/test_packages.py deleted file mode 100644 index b55cb68c..00000000 --- a/tests/test_packages.py +++ /dev/null @@ -1,13 +0,0 @@ -import requests - - -def test_can_access_urllib3_attribute(): - requests.packages.urllib3 - - -def test_can_access_idna_attribute(): - requests.packages.idna - - -def test_can_access_chardet_attribute(): - requests.packages.chardet diff --git a/tests/test_requests.py b/tests/test_requests.py index b3747474..3c61cdfa 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -13,27 +13,45 @@ import warnings import io import requests import pytest +import pytest_httpbin from requests.adapters import HTTPAdapter from requests.auth import HTTPDigestAuth, _basic_auth_str from requests.compat import ( Morsel, cookielib, getproxies, str, urlparse, - builtin_str, OrderedDict) + builtin_str) from requests.cookies import ( cookiejar_from_dict, morsel_to_cookie) from requests.exceptions import ( - ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, - MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError, SSLError, InvalidProxyURL) + ConnectionError, ConnectTimeout, InvalidScheme, InvalidURL, + MissingScheme, ReadTimeout, Timeout, RetryError, TooManyRedirects, + ProxyError, InvalidHeader, UnrewindableBodyError, InvalidBodyError, + SSLError, InvalidProxyURL) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin from requests.models import urlencode from requests.hooks import default_hooks +from requests.utils import DEFAULT_CA_BUNDLE_PATH from .compat import StringIO, u from .utils import override_environ from urllib3.util import Timeout as Urllib3Timeout +class SendRecordingAdapter(HTTPAdapter): + """ + A basic subclass of the HTTPAdapter that records the arguments used to + ``send``. + """ + def __init__(self, *args, **kwargs): + super(SendRecordingAdapter, self).__init__(*args, **kwargs) + + self.send_calls = [] + + def send(self, *args, **kwargs): + self.send_calls.append((args, kwargs)) + return super(SendRecordingAdapter, self).send(*args, **kwargs) + + # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) TARPIT = 'http://10.255.255.1' @@ -64,15 +82,13 @@ class TestRequests: requests.put requests.patch requests.post - # Not really an entry point, but people rely on it. - from requests.packages.urllib3.poolmanager import PoolManager @pytest.mark.parametrize( 'exception, url', ( - (MissingSchema, 'hiwpefhipowhefopw'), - (InvalidSchema, 'localhost:3128'), - (InvalidSchema, 'localhost.localdomain:3128/'), - (InvalidSchema, '10.122.1.1:3128/'), + (MissingScheme, 'hiwpefhipowhefopw'), + (InvalidScheme, 'localhost:3128'), + (InvalidScheme, 'localhost.localdomain:3128/'), + (InvalidScheme, '10.122.1.1:3128/'), (InvalidURL, 'http://'), )) def test_invalid_url(self, exception, url): @@ -80,7 +96,7 @@ class TestRequests: requests.get(url) def test_basic_building(self): - req = requests.Request() + req = requests.Request(method='GET') req.url = 'http://kennethreitz.org/' req.data = {'life': '42'} @@ -126,7 +142,8 @@ class TestRequests: assert request.url == expected def test_params_original_order_is_preserved_by_default(self): - param_ordered_dict = OrderedDict((('z', 1), ('a', 1), ('k', 1), ('d', 1))) + param_ordered_dict = collections.OrderedDict( + (('z', 1), ('a', 1), ('k', 1), ('d', 1))) session = requests.Session() request = requests.Request('GET', 'http://example.com/', params=param_ordered_dict) prep = session.prepare_request(request) @@ -211,49 +228,107 @@ class TestRequests: else: pytest.fail('Expected custom max number of redirects to be respected but was not') - def test_http_301_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '301')) - assert r.status_code == 200 - assert r.request.method == 'GET' - assert r.history[0].status_code == 301 - assert r.history[0].is_redirect + @pytest.mark.parametrize( + 'method, body, expected', ( + ('GET', None, 'GET'), + ('HEAD', None, 'HEAD'), + ('POST', 'test', 'GET'), + ('PUT', 'put test', 'PUT'), + ('PATCH', 'patch test', 'PATCH'), + ('DELETE', '', 'DELETE') + ) + ) + def test_http_301_for_redirectable_methods(self, httpbin, method, body, expected): + """Tests all methods except OPTIONS for expected redirect behaviour. - def test_http_301_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '301'), allow_redirects=True) - print(r.content) - assert r.status_code == 200 - assert r.request.method == 'HEAD' + OPTIONS responses can behave differently depending on the server, so + we don't have anything uniform to test except how httpbin responds + to them. For that reason they aren't included here. + """ + params = {'url': '/%s' % expected.lower(), 'status_code': '301'} + r = requests.request(method, httpbin('redirect-to'), data=body, params=params) + + assert r.request.url == httpbin(expected.lower()) + assert r.request.method == expected assert r.history[0].status_code == 301 assert r.history[0].is_redirect - def test_http_302_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '302')) - assert r.status_code == 200 - assert r.request.method == 'GET' - assert r.history[0].status_code == 302 - assert r.history[0].is_redirect + if expected in ('GET', 'HEAD'): + assert r.request.body is None + else: + assert r.json()['data'] == body - def test_http_302_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '302'), allow_redirects=True) - assert r.status_code == 200 - assert r.request.method == 'HEAD' + @pytest.mark.parametrize( + 'method, body, expected', ( + ('GET', None, 'GET'), + ('HEAD', None, 'HEAD'), + ('POST', 'test', 'GET'), + ('PUT', 'put test', 'PUT'), + ('PATCH', 'patch test', 'PATCH'), + ('DELETE', '', 'DELETE') + ) + ) + def test_http_302_for_redirectable_methods(self, httpbin, method, body, expected): + """Tests all methods except OPTIONS for expected redirect behaviour. + + OPTIONS responses can behave differently depending on the server, so + we don't have anything uniform to test except how httpbin responds + to them. For that reason they aren't included here. + """ + params = {'url': '/%s' % expected.lower()} + r = requests.request(method, httpbin('redirect-to'), data=body, params=params) + + assert r.request.url == httpbin(expected.lower()) + assert r.request.method == expected assert r.history[0].status_code == 302 assert r.history[0].is_redirect - def test_http_303_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '303')) - assert r.status_code == 200 - assert r.request.method == 'GET' - assert r.history[0].status_code == 303 - assert r.history[0].is_redirect + if expected in ('GET', 'HEAD'): + assert r.request.body is None + else: + assert r.json()['data'] == body - def test_http_303_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '303'), allow_redirects=True) - assert r.status_code == 200 - assert r.request.method == 'HEAD' + @pytest.mark.parametrize( + 'method, body, expected', ( + ('GET', None, 'GET'), + ('HEAD', None, 'HEAD'), + ('POST', 'test', 'GET'), + ('PUT', 'put test', 'GET'), + ('PATCH', 'patch test', 'GET'), + ('DELETE', '', 'GET') + ) + ) + def test_http_303_for_redirectable_methods(self, httpbin, method, body, expected): + """Tests all methods except OPTIONS for expected redirect behaviour. + + OPTIONS responses can behave differently depending on the server, so + we don't have anything uniform to test except how httpbin responds + to them. For that reason they aren't included here. + """ + params = {'url': '/%s' % expected.lower(), 'status_code': '303'} + r = requests.request(method, httpbin('redirect-to'), data=body, params=params) + + assert r.request.url == httpbin(expected.lower()) + assert r.request.method == expected assert r.history[0].status_code == 303 assert r.history[0].is_redirect + assert r.request.body is None + + def test_multiple_location_headers(self, httpbin): + headers = [('Location', 'http://example.com'), + ('Location', 'https://example.com/1')] + params = '&'.join(['%s=%s' % (k, v) for k, v in headers]) + ses = requests.Session() + req = requests.Request('GET', httpbin('response-headers?%s' % params)) + prep = ses.prepare_request(req) + resp = ses.send(prep) + # change response to redirect + resp.status_code = 302 + with pytest.raises(InvalidHeader): + # next triggers yield on generator + next(ses.resolve_redirects(resp, prep)) + def test_header_and_body_removal_on_redirect(self, httpbin): purged_headers = ('Content-Length', 'Content-Type') ses = requests.Session() @@ -417,6 +492,35 @@ class TestRequests: assert cookies['foo'] == 'bar' assert cookies['cookie'] == 'tasty' + @pytest.mark.parametrize( + 'jar', ( + requests.cookies.RequestsCookieJar(), + cookielib.CookieJar() + )) + def test_custom_cookie_policy_persistence(self, httpbin, jar): + """Verify a custom CookiePolicy is propagated on each session request.""" + + class TestCookiePolicy(cookielib.DefaultCookiePolicy): + """Policy to restrict all cookies from localhost (127.0.0.1).""" + def __init__(self): + cookielib.DefaultCookiePolicy.__init__(self, blocked_domains=['127.0.0.1']) + + # Establish session with jar and set some cookies. + s = requests.Session() + s.cookies = jar + s.get(httpbin('cookies/set?k1=v1&k2=v2')) + assert len(s.cookies) == 2 + + # Set different policy. + s.cookies.set_policy(TestCookiePolicy()) + assert isinstance(s.cookies._policy, TestCookiePolicy) + + # No cookies were sent to our blocked domain and none were set. + resp = s.get(httpbin('cookies/set?k3=v3')) + assert 'Cookie' not in resp.request.headers + assert len(s.cookies) == 2 + assert 'k3' not in s.cookies + def test_requests_in_history_are_not_overridden(self, httpbin): resp = requests.get(httpbin('redirect/3')) urls = [r.url for r in resp.history] @@ -442,11 +546,11 @@ class TestRequests: def test_headers_preserve_order(self, httpbin): """Preserve order when headers provided as OrderedDict.""" ses = requests.Session() - ses.headers = OrderedDict() + ses.headers = collections.OrderedDict() ses.headers['Accept-Encoding'] = 'identity' ses.headers['First'] = '1' ses.headers['Second'] = '2' - headers = OrderedDict([('Third', '3'), ('Fourth', '4')]) + headers = collections.OrderedDict([('Third', '3'), ('Fourth', '4')]) headers['Fifth'] = '5' headers['Second'] = '222' req = requests.Request('GET', httpbin('get'), headers=headers) @@ -494,8 +598,6 @@ class TestRequests: 'username, password', ( ('user', 'pass'), (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8')), - (42, 42), - (None, None), )) def test_set_basicauth(self, httpbin, username, password): auth = (username, password) @@ -506,6 +608,18 @@ class TestRequests: assert p.headers['Authorization'] == _basic_auth_str(username, password) + @pytest.mark.parametrize( + 'username, password', ( + ('user', 1234), + (None, 'test'), + )) + def test_non_str_basicauth(self, username, password): + """Ensure we only allow string or bytes values for basicauth""" + with pytest.raises(TypeError) as e: + requests.auth._basic_auth_str(username, password) + + assert 'must be of type str or bytes' in str(e) + def test_basicauth_encodes_byte_strings(self): """Ensure b'test' formats as the byte string "test" rather than the unicode string "b'test'" in Python 3. @@ -768,6 +882,10 @@ class TestRequests: r = requests.get(httpbin('status', '500')) assert not r.ok + def test_raise_for_status_returns_self(self, httpbin): + r = requests.get(httpbin('status', '200')) + assert r.raise_for_status() is r + def test_decompress_gzip(self, httpbin): r = requests.get(httpbin('gzip')) r.content.decode('ascii') @@ -844,7 +962,7 @@ class TestRequests: def test_urlencoded_get_query_multivalued_param(self, httpbin): - r = requests.get(httpbin('get'), params=dict(test=['foo', 'baz'])) + r = requests.get(httpbin('get'), params={'test': ['foo', 'baz']}) assert r.status_code == 200 assert r.url == httpbin('get?test=foo&test=baz') @@ -855,8 +973,8 @@ class TestRequests: files={'file': ('test_requests.py', open(__file__, 'rb'))}) assert r.status_code == 200 - @pytest.mark.parametrize( - 'data', ( + @pytest.mark.parametrize('data', + ( {'stuff': u('ëlïxr')}, {'stuff': u('ëlïxr').encode('utf-8')}, {'stuff': 'elixr'}, @@ -1190,9 +1308,24 @@ class TestRequests: r = requests.Response() r.raw = io.BytesIO(b'the content') r.encoding = 'ascii' + chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) + @pytest.mark.parametrize( + 'encoding, exception', ( + (None, TypeError), + ('invalid encoding', LookupError), + )) + def test_decode_unicode_encoding(self, encoding, exception): + # raise an exception if encoding isn't set + r = requests.Response() + r.raw = io.BytesIO(b'the content') + r.encoding = encoding + + with pytest.raises(exception): + chunks = r.iter_content(decode_unicode=True) + def test_response_reason_unicode(self): # check for unicode HTTP status r = requests.Response() @@ -1245,6 +1378,109 @@ class TestRequests: assert r.request.url == pr.request.url assert r.request.headers == pr.request.headers + + def test_response_lines(self): + """ + iter_lines should be able to handle data dribbling in which delimiters + might not be lined up ideally. + """ + mock_chunks = [ + b'This \r\n', + b'', + b'is\r', + b'\n', + b'a', + b' ', + b'', + b'', + b'test.', + b'\r', + b'\n', + b'end.', + ] + mock_data = b''.join(mock_chunks) + unicode_mock_data = mock_data.decode('utf-8') + + def mock_iter_content(*args, **kwargs): + if kwargs.get("decode_unicode"): + return (e.decode('utf-8') for e in mock_chunks) + return (e for e in mock_chunks) + + r = requests.Response() + r._content_consumed = True + r.iter_content = mock_iter_content + + # decode_unicode=None, output raw bytes + assert list(r.iter_lines(delimiter=b'\r\n')) == mock_data.split(b'\r\n') + + # decode_unicode=True, output unicode strings + assert list(r.iter_lines(decode_unicode=True, delimiter=u'\r\n')) == unicode_mock_data.split(u'\r\n') + + # When delimiter is None, we should yield the same result as splitlines() + # which supports the universal newline. + # '\r', '\n', and '\r\n' are all treated as one line break. + + # decode_unicode=None, output raw bytes + result = list(r.iter_lines()) + assert result == mock_data.splitlines() + + # decode_unicode=True, output unicode strings + result = list(r.iter_lines(decode_unicode=True)) + assert result == unicode_mock_data.splitlines() + + # If we change all the line breaks to `\r`, we should be okay. + # decode_unicode=None, output raw bytes + mock_chunks = [chunk.replace(b'\n', b'\r') for chunk in mock_chunks] + mock_data = b''.join(mock_chunks) + assert list(r.iter_lines()) == mock_data.splitlines() + + # decode_unicode=True, output unicode strings + unicode_mock_data = mock_data.decode('utf-8') + assert list(r.iter_lines(decode_unicode=True)) == unicode_mock_data.splitlines() + + + @pytest.mark.parametrize( + 'content, expected_no_delimiter, expected_delimiter', ( + ([b''], [], []), + ([b'line\n'], [u'line'], [u'line\n']), + ([b'line', b'\n'], [u'line'], [u'line\n']), + ([b'line\r\n'], [u'line'], [u'line', u'']), + # Empty chunk in the end of stream, same behavior as the previous + ([b'line\r\n', b''], [u'line'], [u'line', u'']), + ([b'line', b'\r\n'], [u'line'], [u'line', u'']), + ([b'a\r', b'\nb\r'], [u'a', u'b'], [u'a', u'b\r']), + ([b'a\r', b'\n', b'\nb'], [u'a', u'', u'b'], [u'a', u'\nb']), + ([b'a\n', b'\nb'], [u'a', u'', u'b'], [u'a\n\nb']), + ([b'a\r\n', b'\rb\n'], [u'a', u'', u'b'], [u'a', u'\rb\n']), + ([b'a\nb', b'c'], [u'a', u'bc'], [u'a\nbc']), + ([b'a\n', b'\rb', b'\r\nc'], [u'a', u'', u'b', u'c'], [u'a\n\rb', u'c']), + ([b'a\r\nb', b'', b'c'], [u'a', u'bc'], [u'a', u'bc']) # Empty chunk with pending data + )) + def test_response_lines_parametrized(self, content, expected_no_delimiter, expected_delimiter): + """ + Test a lot of potential chunk splits to ensure consistency of + iter_lines(delimiter=x), as well as the legacy behavior of + iter_lines() without delimiter + https://github.com/kennethreitz/requests/pull/2431#issuecomment-72333964 + """ + mock_chunks = content + def mock_iter_content(*args, **kwargs): + if kwargs.get("decode_unicode"): + return (e.decode('utf-8') for e in mock_chunks) + return (e for e in mock_chunks) + + r = requests.Response() + r._content_consumed = True + r.iter_content = mock_iter_content + + # decode_unicode=True, output unicode strings + assert list(r.iter_lines(decode_unicode=True)) == expected_no_delimiter + assert list(r.iter_lines(decode_unicode=True, delimiter='\r\n')) == expected_delimiter + + # decode_unicode=None, output raw bytes + assert list(r.iter_lines()) == [line.encode('utf-8') for line in expected_no_delimiter] + assert list(r.iter_lines(delimiter=b'\r\n')) == [line.encode('utf-8') for line in expected_delimiter] + def test_prepared_request_is_pickleable(self, httpbin): p = requests.Request('GET', httpbin('get')).prepare() @@ -1558,9 +1794,10 @@ class TestRequests: def test_manual_redirect_with_partial_body_read(self, httpbin): s = requests.Session() - r1 = s.get(httpbin('redirect/2'), allow_redirects=False, stream=True) + req = requests.Request('GET', httpbin('redirect/2')).prepare() + r1 = s.send(req, allow_redirects=False, stream=True) assert r1.is_redirect - rg = s.resolve_redirects(r1, r1.request, stream=True) + rg = s.resolve_redirects(r1, req, stream=True) # read only the first eight bytes of the response body, # then follow the redirect @@ -1730,11 +1967,12 @@ class TestRequests: prep = r.prepare() assert 'stuff=elixr' == prep.body - def test_response_iter_lines(self, httpbin): + @pytest.mark.parametrize('decode_unicode', (True, False)) + def test_response_iter_lines(self, httpbin, decode_unicode): r = requests.get(httpbin('stream/4'), stream=True) assert r.status_code == 200 - - it = r.iter_lines() + r.encoding = 'utf-8' + it = r.iter_lines(decode_unicode=decode_unicode) next(it) assert len(list(it)) == 3 @@ -1762,6 +2000,59 @@ class TestRequests: next(r.iter_lines()) assert len(list(r.iter_lines())) == 3 + def test_environment_comes_after_session(self, httpbin): + """The Session arguments should come before environment arguments.""" + # We get proxies from the environment and verify from the argument. + s = requests.Session() + a = SendRecordingAdapter() + s.mount('http://', a) + + # Both of these arguments are safe fallbacks that we can easily + # detect, but which will allow the request to succeed. + s.verify = False + s.proxies = {'http': None} + + old_proxy = os.environ.get('HTTP_PROXY') + old_bundle = os.environ.get('REQUESTS_CA_BUNDLE') + + try: + os.environ['HTTP_PROXY'] = '10.10.10.10:3128' + os.environ['REQUESTS_CA_BUNDLE'] = '/path/to/nowhere' + + s.get(httpbin('get'), timeout=5) + finally: + if old_proxy is not None: + os.environ['HTTP_PROXY'] = old_proxy + else: + del os.environ['HTTP_PROXY'] + + if old_bundle is not None: + os.environ['REQUESTS_CA_BUNDLE'] = old_bundle + else: + del os.environ['REQUESTS_CA_BUNDLE'] + + call = a.send_calls[0] + assert call[1]['verify'] == False + + proxies = call[1]['proxies'] + with pytest.raises(KeyError): + proxies['http'] + + @pytest.fixture(autouse=True) + def test_merge_environment_settings_verify(self, monkeypatch): + """Assert CA environment settings are merged as expected when missing""" + session = requests.Session() + monkeypatch.delenv('CURL_CA_BUNDLE', raising=False) + monkeypatch.delenv('REQUESTS_CA_BUNDLE', raising=False) + + assert session.trust_env is True + assert session.verify is True + assert 'REQUESTS_CA_BUNDLE' not in os.environ + assert 'CURL_CA_BUNDLE' not in os.environ + merged_settings = session.merge_environment_settings( + 'http://example.com', {}, False, True, None) + assert merged_settings['verify'] is True + def test_session_close_proxy_clear(self, mocker): proxies = { 'one': mocker.Mock(), @@ -1804,6 +2095,33 @@ class TestRequests: resp.close() assert resp.raw.closed + def test_updating_ca_cert(self, httpbin_secure): + """Assert that requests use the latest configured CA certificates.""" + session = requests.session() + session.verify = pytest_httpbin.certs.where() + session.get(httpbin_secure('/')) + session.verify = True + with pytest.raises(requests.exceptions.SSLError) as e: + session.get(httpbin_secure('/')) + assert 'certificate verify failed' in str(e) + + def test_updating_client_cert(self, httpbin_secure): + """Assert that requests use the latest configured client certificates.""" + ca_file = pytest_httpbin.certs.where() + cert_dir = os.path.dirname(ca_file) + # All we need is a valid certificate and key to make a request. httpbin_secure + # won't check the signature or subject name, so it's okay that these happen to + # be the server's certificate and key. + cert = os.path.join(cert_dir, 'cert.pem') + key = os.path.join(cert_dir, 'key.pem') + session = requests.session() + session.verify = ca_file + resp = session.get(httpbin_secure('/')) + resp_with_cert = session.get(httpbin_secure('/'), cert=(cert, key)) + assert resp_with_cert.raw._pool.cert_file == cert + assert resp_with_cert.raw._pool.key_file == key + assert resp.raw._pool is not resp_with_cert.raw._pool + def test_empty_stream_with_auth_does_not_set_content_length_header(self, httpbin): """Ensure that a byte stream with size 0 will not set both a Content-Length and Transfer-Encoding header. @@ -1839,6 +2157,61 @@ class TestRequests: assert 'Transfer-Encoding' in prepared_request.headers assert 'Content-Length' not in prepared_request.headers + def test_chunked_upload_with_manually_set_content_length_header_raises_error(self, httpbin): + """Ensure that if a user manually sets a content length header, when + the data is chunked, that an InvalidHeader error is raised. + """ + data = (i for i in [b'a', b'b', b'c']) + url = httpbin('post') + with pytest.raises(InvalidHeader): + r = requests.post(url, data=data, headers={'Content-Length': 'foo'}) + + def test_content_length_with_manually_set_transfer_encoding_raises_error(self, httpbin): + """Ensure that if a user manually sets a Transfer-Encoding header when + data is not chunked that an InvalidHeader error is raised. + """ + data = 'test data' + url = httpbin('post') + with pytest.raises(InvalidHeader): + r = requests.post(url, data=data, headers={'Transfer-Encoding': 'chunked'}) + + def test_null_body_does_not_raise_error(self, httpbin): + url = httpbin('post') + try: + requests.post(url, data=None) + except InvalidHeader: + pytest.fail('InvalidHeader error raised unexpectedly.') + + @pytest.mark.parametrize( + 'body, expected', ( + (None, ('Content-Length', '0')), + ('test_data', ('Content-Length', '9')), + (io.BytesIO(b'test_data'), ('Content-Length', '9')), + (StringIO.StringIO(''), ('Transfer-Encoding', 'chunked')) + )) + def test_prepare_content_length(self, httpbin, body, expected): + """Test prepare_content_length creates expected header.""" + prep = requests.PreparedRequest() + prep.headers = {} + prep.method = 'POST' + + # Ensure Content-Length is set appropriately. + key, value = expected + prep.prepare_content_length(body) + assert prep.headers[key] == value + + def test_prepare_content_length_with_bad_body(self, httpbin): + """Test prepare_content_length raises exception with unsendable body.""" + # Initialize minimum required PreparedRequest. + prep = requests.PreparedRequest() + prep.headers = {} + prep.method = 'POST' + + with pytest.raises(InvalidBodyError) as e: + # Send object that isn't iterable and has no accessible content. + prep.prepare_content_length(object()) + assert "Non-null body must have length or be streamable." in str(e) + def test_custom_redirect_mixin(self, httpbin): """Tests a custom mixin to overwrite ``get_redirect_target``. @@ -2024,6 +2397,18 @@ class TestCaseInsensitiveDict: cid['changed'] = True assert cid != cid_copy + def test_url_surrounding_whitespace(self, httpbin): + """Test case with URLs surrounded by whitespace characters.""" + get_url = httpbin('get') + # All surrounding whitespaces are supposed to be ignored: + assert requests.get(get_url + ' ').status_code == 200 + assert requests.get(' ' + get_url).status_code == 200 + assert requests.get(get_url + ' \t ').status_code == 200 + assert requests.get(' \t' + get_url).status_code == 200 + assert requests.get(get_url + '\n').status_code == 200 + # The whitespaces can't be in the middle of the URL though: + assert requests.get(get_url + ' abc').status_code == 404 + class TestMorselToCookieExpires: """Tests for morsel_to_cookie when morsel contains expires.""" @@ -2166,6 +2551,7 @@ class RedirectSession(SessionRedirectMixin): self.max_redirects = 30 self.cookies = {} self.trust_env = False + self.location = '/' def send(self, *args, **kwargs): self.calls.append(SendCall(args, kwargs)) @@ -2180,7 +2566,7 @@ class RedirectSession(SessionRedirectMixin): except IndexError: r.status_code = 200 - r.headers = CaseInsensitiveDict({'Location': '/'}) + r.headers = CaseInsensitiveDict({'Location': self.location}) r.raw = self._build_raw() r.request = request return r @@ -2295,6 +2681,16 @@ def test_prepared_copy(kwargs): assert getattr(p, attr) == getattr(copy, attr) +def test_prepare_requires_a_request_method(): + req = requests.Request() + with pytest.raises(ValueError): + req.prepare() + + prepped = PreparedRequest() + with pytest.raises(ValueError): + prepped.prepare() + + def test_urllib3_retries(httpbin): from urllib3.util import Retry s = requests.Session() @@ -2446,3 +2842,200 @@ class TestPreparingURLs(object): r = requests.Request('GET', url=input, params=params) p = r.prepare() assert p.url == expected + + +class TestGetConnection(object): + """ + Tests for the :meth:`requests.adapters.HTTPAdapter.get_connection` that assert + the connections are correctly configured. + """ + @pytest.mark.parametrize( + 'proxies, verify, cert, expected', + ( + ( + {}, + True, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + False, + None, + { + 'cert_reqs': 'CERT_NONE', + 'ca_certs': None, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + __file__, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': __file__, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + os.path.dirname(__file__), + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': None, + 'ca_cert_dir': os.path.dirname(__file__), + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + True, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + True, + __file__, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': None, + }, + ), + ( + {}, + True, + (__file__, __file__), + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': __file__, + }, + ), + ( + {}, + True, + (__file__, __file__), + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': __file__, + }, + ), + ( + {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + True, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + os.path.dirname(__file__), + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': None, + 'ca_cert_dir': os.path.dirname(__file__), + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + __file__, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': __file__, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + True, + __file__, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': None, + }, + ), + ( + {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + True, + (__file__, __file__), + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': __file__, + }, + ), + ) + ) + def test_get_https_connection(self, proxies, verify, cert, expected): + """Assert connections are configured correctly.""" + adapter = requests.adapters.HTTPAdapter() + connection = adapter.get_connection( + 'https://example.com', proxies=proxies, verify=verify, cert=cert) + actual_config = {} + for key, value in connection.__dict__.items(): + if key in expected: + actual_config[key] = value + assert actual_config == expected + + @pytest.mark.parametrize( + 'verify, cert', + ( + ('a/path/that/does/not/exist', None), + (True, 'a/path/that/does/not/exist'), + (True, (__file__, 'a/path/that/does/not/exist')), + (True, ('a/path/that/does/not/exist', __file__)), + ) + ) + def test_cert_files_missing(self, verify, cert): + """ + Assert an IOError is raised when one of the certificate files or + directories can't be found. + """ + adapter = requests.adapters.HTTPAdapter() + with pytest.raises(IOError) as excinfo: + adapter.get_connection('https://example.com', verify=verify, cert=cert) + excinfo.match('invalid path: a/path/that/does/not/exist') diff --git a/tests/test_utils.py b/tests/test_utils.py index f39cd67b..54b83335 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -660,7 +660,7 @@ def test_add_dict_to_cookiejar(cookiejar): cookiedict = {'test': 'cookies', 'good': 'cookies'} cj = add_dict_to_cookiejar(cookiejar, cookiedict) - cookies = dict((cookie.name, cookie.value) for cookie in cj) + cookies = {cookie.name: cookie.value for cookie in cj} assert cookiedict == cookies @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py34,py35,py36 +envlist = py27,py34,py35,py36 [testenv] |