jpayne@7: """ jpayne@7: requests.adapters jpayne@7: ~~~~~~~~~~~~~~~~~ jpayne@7: jpayne@7: This module contains the transport adapters that Requests uses to define jpayne@7: and maintain connections. jpayne@7: """ jpayne@7: jpayne@7: import os.path jpayne@7: import socket # noqa: F401 jpayne@7: jpayne@7: from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError jpayne@7: from urllib3.exceptions import HTTPError as _HTTPError jpayne@7: from urllib3.exceptions import InvalidHeader as _InvalidHeader jpayne@7: from urllib3.exceptions import ( jpayne@7: LocationValueError, jpayne@7: MaxRetryError, jpayne@7: NewConnectionError, jpayne@7: ProtocolError, jpayne@7: ) jpayne@7: from urllib3.exceptions import ProxyError as _ProxyError jpayne@7: from urllib3.exceptions import ReadTimeoutError, ResponseError jpayne@7: from urllib3.exceptions import SSLError as _SSLError jpayne@7: from urllib3.poolmanager import PoolManager, proxy_from_url jpayne@7: from urllib3.util import Timeout as TimeoutSauce jpayne@7: from urllib3.util import parse_url jpayne@7: from urllib3.util.retry import Retry jpayne@7: jpayne@7: from .auth import _basic_auth_str jpayne@7: from .compat import basestring, urlparse jpayne@7: from .cookies import extract_cookies_to_jar jpayne@7: from .exceptions import ( jpayne@7: ConnectionError, jpayne@7: ConnectTimeout, jpayne@7: InvalidHeader, jpayne@7: InvalidProxyURL, jpayne@7: InvalidSchema, jpayne@7: InvalidURL, jpayne@7: ProxyError, jpayne@7: ReadTimeout, jpayne@7: RetryError, jpayne@7: SSLError, jpayne@7: ) jpayne@7: from .models import Response jpayne@7: from .structures import CaseInsensitiveDict jpayne@7: from .utils import ( jpayne@7: DEFAULT_CA_BUNDLE_PATH, jpayne@7: extract_zipped_paths, jpayne@7: get_auth_from_url, jpayne@7: get_encoding_from_headers, jpayne@7: prepend_scheme_if_needed, jpayne@7: select_proxy, jpayne@7: urldefragauth, jpayne@7: ) jpayne@7: jpayne@7: try: jpayne@7: from urllib3.contrib.socks import SOCKSProxyManager jpayne@7: except ImportError: jpayne@7: jpayne@7: def SOCKSProxyManager(*args, **kwargs): jpayne@7: raise InvalidSchema("Missing dependencies for SOCKS support.") jpayne@7: jpayne@7: jpayne@7: DEFAULT_POOLBLOCK = False jpayne@7: DEFAULT_POOLSIZE = 10 jpayne@7: DEFAULT_RETRIES = 0 jpayne@7: DEFAULT_POOL_TIMEOUT = None jpayne@7: jpayne@7: jpayne@7: class BaseAdapter: jpayne@7: """The Base Transport Adapter""" jpayne@7: jpayne@7: def __init__(self): jpayne@7: super().__init__() jpayne@7: jpayne@7: def send( jpayne@7: self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None jpayne@7: ): jpayne@7: """Sends PreparedRequest object. Returns Response object. jpayne@7: jpayne@7: :param request: The :class:`PreparedRequest ` being sent. jpayne@7: :param stream: (optional) Whether to stream the request content. jpayne@7: :param timeout: (optional) How long to wait for the server to send jpayne@7: data before giving up, as a float, or a :ref:`(connect timeout, jpayne@7: read timeout) ` tuple. jpayne@7: :type timeout: float or tuple jpayne@7: :param verify: (optional) Either a boolean, in which case it controls whether we verify jpayne@7: the server's TLS certificate, or a string, in which case it must be a path jpayne@7: to a CA bundle to use jpayne@7: :param cert: (optional) Any user-provided SSL certificate to be trusted. jpayne@7: :param proxies: (optional) The proxies dictionary to apply to the request. jpayne@7: """ jpayne@7: raise NotImplementedError jpayne@7: jpayne@7: def close(self): jpayne@7: """Cleans up adapter specific items.""" jpayne@7: raise NotImplementedError jpayne@7: jpayne@7: jpayne@7: class HTTPAdapter(BaseAdapter): jpayne@7: """The built-in HTTP Adapter for urllib3. jpayne@7: jpayne@7: Provides a general-case interface for Requests sessions to contact HTTP and jpayne@7: HTTPS urls by implementing the Transport Adapter interface. This class will jpayne@7: usually be created by the :class:`Session ` class under the jpayne@7: covers. jpayne@7: jpayne@7: :param pool_connections: The number of urllib3 connection pools to cache. jpayne@7: :param pool_maxsize: The maximum number of connections to save in the pool. jpayne@7: :param max_retries: The maximum number of retries each connection jpayne@7: should attempt. Note, this applies only to failed DNS lookups, socket jpayne@7: connections and connection timeouts, never to requests where data has jpayne@7: made it to the server. By default, Requests does not retry failed jpayne@7: connections. If you need granular control over the conditions under jpayne@7: which we retry a request, import urllib3's ``Retry`` class and pass jpayne@7: that instead. jpayne@7: :param pool_block: Whether the connection pool should block for connections. jpayne@7: jpayne@7: Usage:: jpayne@7: jpayne@7: >>> import requests jpayne@7: >>> s = requests.Session() jpayne@7: >>> a = requests.adapters.HTTPAdapter(max_retries=3) jpayne@7: >>> s.mount('http://', a) jpayne@7: """ jpayne@7: jpayne@7: __attrs__ = [ jpayne@7: "max_retries", jpayne@7: "config", jpayne@7: "_pool_connections", jpayne@7: "_pool_maxsize", jpayne@7: "_pool_block", jpayne@7: ] jpayne@7: jpayne@7: def __init__( jpayne@7: self, jpayne@7: pool_connections=DEFAULT_POOLSIZE, jpayne@7: pool_maxsize=DEFAULT_POOLSIZE, jpayne@7: max_retries=DEFAULT_RETRIES, jpayne@7: pool_block=DEFAULT_POOLBLOCK, jpayne@7: ): jpayne@7: if max_retries == DEFAULT_RETRIES: jpayne@7: self.max_retries = Retry(0, read=False) jpayne@7: else: jpayne@7: self.max_retries = Retry.from_int(max_retries) jpayne@7: self.config = {} jpayne@7: self.proxy_manager = {} jpayne@7: jpayne@7: super().__init__() jpayne@7: jpayne@7: self._pool_connections = pool_connections jpayne@7: self._pool_maxsize = pool_maxsize jpayne@7: self._pool_block = pool_block jpayne@7: jpayne@7: self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) jpayne@7: jpayne@7: def __getstate__(self): jpayne@7: return {attr: getattr(self, attr, None) for attr in self.__attrs__} jpayne@7: jpayne@7: def __setstate__(self, state): jpayne@7: # Can't handle by adding 'proxy_manager' to self.__attrs__ because jpayne@7: # self.poolmanager uses a lambda function, which isn't pickleable. jpayne@7: self.proxy_manager = {} jpayne@7: self.config = {} jpayne@7: jpayne@7: for attr, value in state.items(): jpayne@7: setattr(self, attr, value) jpayne@7: jpayne@7: self.init_poolmanager( jpayne@7: self._pool_connections, self._pool_maxsize, block=self._pool_block jpayne@7: ) jpayne@7: jpayne@7: def init_poolmanager( jpayne@7: self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs jpayne@7: ): jpayne@7: """Initializes a urllib3 PoolManager. jpayne@7: jpayne@7: This method should not be called from user code, and is only jpayne@7: exposed for use when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param connections: The number of urllib3 connection pools to cache. jpayne@7: :param maxsize: The maximum number of connections to save in the pool. jpayne@7: :param block: Block when no free connections are available. jpayne@7: :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. jpayne@7: """ jpayne@7: # save these values for pickling jpayne@7: self._pool_connections = connections jpayne@7: self._pool_maxsize = maxsize jpayne@7: self._pool_block = block jpayne@7: jpayne@7: self.poolmanager = PoolManager( jpayne@7: num_pools=connections, jpayne@7: maxsize=maxsize, jpayne@7: block=block, jpayne@7: **pool_kwargs, jpayne@7: ) jpayne@7: jpayne@7: def proxy_manager_for(self, proxy, **proxy_kwargs): jpayne@7: """Return urllib3 ProxyManager for the given proxy. jpayne@7: jpayne@7: This method should not be called from user code, and is only jpayne@7: exposed for use when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param proxy: The proxy to return a urllib3 ProxyManager for. jpayne@7: :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. jpayne@7: :returns: ProxyManager jpayne@7: :rtype: urllib3.ProxyManager jpayne@7: """ jpayne@7: if proxy in self.proxy_manager: jpayne@7: manager = self.proxy_manager[proxy] jpayne@7: elif proxy.lower().startswith("socks"): jpayne@7: username, password = get_auth_from_url(proxy) jpayne@7: manager = self.proxy_manager[proxy] = SOCKSProxyManager( jpayne@7: proxy, jpayne@7: username=username, jpayne@7: password=password, jpayne@7: num_pools=self._pool_connections, jpayne@7: maxsize=self._pool_maxsize, jpayne@7: block=self._pool_block, jpayne@7: **proxy_kwargs, jpayne@7: ) jpayne@7: else: jpayne@7: proxy_headers = self.proxy_headers(proxy) jpayne@7: manager = self.proxy_manager[proxy] = proxy_from_url( jpayne@7: proxy, jpayne@7: proxy_headers=proxy_headers, jpayne@7: num_pools=self._pool_connections, jpayne@7: maxsize=self._pool_maxsize, jpayne@7: block=self._pool_block, jpayne@7: **proxy_kwargs, jpayne@7: ) jpayne@7: jpayne@7: return manager jpayne@7: jpayne@7: def cert_verify(self, conn, url, verify, cert): jpayne@7: """Verify a SSL certificate. This method should not be called from user jpayne@7: code, and is only exposed for use when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param conn: The urllib3 connection object associated with the cert. jpayne@7: :param url: The requested URL. jpayne@7: :param verify: Either a boolean, in which case it controls whether we verify jpayne@7: the server's TLS certificate, or a string, in which case it must be a path jpayne@7: to a CA bundle to use jpayne@7: :param cert: The SSL certificate to verify. jpayne@7: """ jpayne@7: if url.lower().startswith("https") and verify: jpayne@7: jpayne@7: cert_loc = None jpayne@7: jpayne@7: # Allow self-specified cert location. jpayne@7: if verify is not True: jpayne@7: cert_loc = verify jpayne@7: jpayne@7: if not cert_loc: jpayne@7: cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) jpayne@7: jpayne@7: if not cert_loc or not os.path.exists(cert_loc): jpayne@7: raise OSError( jpayne@7: f"Could not find a suitable TLS CA certificate bundle, " jpayne@7: f"invalid path: {cert_loc}" jpayne@7: ) jpayne@7: jpayne@7: conn.cert_reqs = "CERT_REQUIRED" jpayne@7: jpayne@7: if not os.path.isdir(cert_loc): jpayne@7: conn.ca_certs = cert_loc jpayne@7: else: jpayne@7: conn.ca_cert_dir = cert_loc jpayne@7: else: jpayne@7: conn.cert_reqs = "CERT_NONE" jpayne@7: conn.ca_certs = None jpayne@7: conn.ca_cert_dir = None jpayne@7: jpayne@7: if cert: jpayne@7: if not isinstance(cert, basestring): jpayne@7: conn.cert_file = cert[0] jpayne@7: conn.key_file = cert[1] jpayne@7: else: jpayne@7: conn.cert_file = cert jpayne@7: conn.key_file = None jpayne@7: if conn.cert_file and not os.path.exists(conn.cert_file): jpayne@7: raise OSError( jpayne@7: f"Could not find the TLS certificate file, " jpayne@7: f"invalid path: {conn.cert_file}" jpayne@7: ) jpayne@7: if conn.key_file and not os.path.exists(conn.key_file): jpayne@7: raise OSError( jpayne@7: f"Could not find the TLS key file, invalid path: {conn.key_file}" jpayne@7: ) jpayne@7: jpayne@7: def build_response(self, req, resp): jpayne@7: """Builds a :class:`Response ` object from a urllib3 jpayne@7: response. This should not be called from user code, and is only exposed jpayne@7: for use when subclassing the jpayne@7: :class:`HTTPAdapter ` jpayne@7: jpayne@7: :param req: The :class:`PreparedRequest ` used to generate the response. jpayne@7: :param resp: The urllib3 response object. jpayne@7: :rtype: requests.Response jpayne@7: """ jpayne@7: response = Response() jpayne@7: jpayne@7: # Fallback to None if there's no status_code, for whatever reason. jpayne@7: response.status_code = getattr(resp, "status", None) jpayne@7: jpayne@7: # Make headers case-insensitive. jpayne@7: response.headers = CaseInsensitiveDict(getattr(resp, "headers", {})) jpayne@7: jpayne@7: # Set encoding. jpayne@7: response.encoding = get_encoding_from_headers(response.headers) jpayne@7: response.raw = resp jpayne@7: response.reason = response.raw.reason jpayne@7: jpayne@7: if isinstance(req.url, bytes): jpayne@7: response.url = req.url.decode("utf-8") jpayne@7: else: jpayne@7: response.url = req.url jpayne@7: jpayne@7: # Add new cookies from the server. jpayne@7: extract_cookies_to_jar(response.cookies, req, resp) jpayne@7: jpayne@7: # Give the Response some context. jpayne@7: response.request = req jpayne@7: response.connection = self jpayne@7: jpayne@7: return response jpayne@7: jpayne@7: def get_connection(self, url, proxies=None): jpayne@7: """Returns a urllib3 connection for the given URL. This should not be jpayne@7: called from user code, and is only exposed for use when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param url: The URL to connect to. jpayne@7: :param proxies: (optional) A Requests-style dictionary of proxies used on this request. jpayne@7: :rtype: urllib3.ConnectionPool jpayne@7: """ jpayne@7: proxy = select_proxy(url, proxies) jpayne@7: jpayne@7: if proxy: jpayne@7: proxy = prepend_scheme_if_needed(proxy, "http") jpayne@7: proxy_url = parse_url(proxy) jpayne@7: if not proxy_url.host: jpayne@7: raise InvalidProxyURL( jpayne@7: "Please check proxy URL. It is malformed " jpayne@7: "and could be missing the host." jpayne@7: ) jpayne@7: proxy_manager = self.proxy_manager_for(proxy) jpayne@7: conn = proxy_manager.connection_from_url(url) jpayne@7: else: jpayne@7: # Only scheme should be lower case jpayne@7: parsed = urlparse(url) jpayne@7: url = parsed.geturl() jpayne@7: conn = self.poolmanager.connection_from_url(url) jpayne@7: jpayne@7: return conn jpayne@7: jpayne@7: def close(self): jpayne@7: """Disposes of any internal state. jpayne@7: jpayne@7: Currently, this closes the PoolManager and any active ProxyManager, jpayne@7: which closes any pooled connections. jpayne@7: """ jpayne@7: self.poolmanager.clear() jpayne@7: for proxy in self.proxy_manager.values(): jpayne@7: proxy.clear() jpayne@7: jpayne@7: def request_url(self, request, proxies): jpayne@7: """Obtain the url to use when making the final request. jpayne@7: jpayne@7: If the message is being sent through a HTTP proxy, the full URL has to jpayne@7: be used. Otherwise, we should only use the path portion of the URL. jpayne@7: jpayne@7: This should not be called from user code, and is only exposed for use jpayne@7: when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param request: The :class:`PreparedRequest ` being sent. jpayne@7: :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. jpayne@7: :rtype: str jpayne@7: """ jpayne@7: proxy = select_proxy(request.url, proxies) jpayne@7: scheme = urlparse(request.url).scheme jpayne@7: jpayne@7: is_proxied_http_request = proxy and scheme != "https" jpayne@7: using_socks_proxy = False jpayne@7: if proxy: jpayne@7: proxy_scheme = urlparse(proxy).scheme.lower() jpayne@7: using_socks_proxy = proxy_scheme.startswith("socks") jpayne@7: jpayne@7: url = request.path_url jpayne@7: if is_proxied_http_request and not using_socks_proxy: jpayne@7: url = urldefragauth(request.url) jpayne@7: jpayne@7: return url jpayne@7: jpayne@7: def add_headers(self, request, **kwargs): jpayne@7: """Add any headers needed by the connection. As of v2.0 this does jpayne@7: nothing by default, but is left for overriding by users that subclass jpayne@7: the :class:`HTTPAdapter `. jpayne@7: jpayne@7: This should not be called from user code, and is only exposed for use jpayne@7: when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param request: The :class:`PreparedRequest ` to add headers to. jpayne@7: :param kwargs: The keyword arguments from the call to send(). jpayne@7: """ jpayne@7: pass jpayne@7: jpayne@7: def proxy_headers(self, proxy): jpayne@7: """Returns a dictionary of the headers to add to any request sent jpayne@7: through a proxy. This works with urllib3 magic to ensure that they are jpayne@7: correctly sent to the proxy, rather than in a tunnelled request if jpayne@7: CONNECT is being used. jpayne@7: jpayne@7: This should not be called from user code, and is only exposed for use jpayne@7: when subclassing the jpayne@7: :class:`HTTPAdapter `. jpayne@7: jpayne@7: :param proxy: The url of the proxy being used for this request. jpayne@7: :rtype: dict jpayne@7: """ jpayne@7: headers = {} jpayne@7: username, password = get_auth_from_url(proxy) jpayne@7: jpayne@7: if username: jpayne@7: headers["Proxy-Authorization"] = _basic_auth_str(username, password) jpayne@7: jpayne@7: return headers jpayne@7: jpayne@7: def send( jpayne@7: self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None jpayne@7: ): jpayne@7: """Sends PreparedRequest object. Returns Response object. jpayne@7: jpayne@7: :param request: The :class:`PreparedRequest ` being sent. jpayne@7: :param stream: (optional) Whether to stream the request content. jpayne@7: :param timeout: (optional) How long to wait for the server to send jpayne@7: data before giving up, as a float, or a :ref:`(connect timeout, jpayne@7: read timeout) ` tuple. jpayne@7: :type timeout: float or tuple or urllib3 Timeout object jpayne@7: :param verify: (optional) Either a boolean, in which case it controls whether jpayne@7: we verify the server's TLS certificate, or a string, in which case it jpayne@7: must be a path to a CA bundle to use jpayne@7: :param cert: (optional) Any user-provided SSL certificate to be trusted. jpayne@7: :param proxies: (optional) The proxies dictionary to apply to the request. jpayne@7: :rtype: requests.Response jpayne@7: """ jpayne@7: jpayne@7: try: jpayne@7: conn = self.get_connection(request.url, proxies) jpayne@7: except LocationValueError as e: jpayne@7: raise InvalidURL(e, request=request) jpayne@7: jpayne@7: self.cert_verify(conn, request.url, verify, cert) jpayne@7: url = self.request_url(request, proxies) jpayne@7: self.add_headers( jpayne@7: request, jpayne@7: stream=stream, jpayne@7: timeout=timeout, jpayne@7: verify=verify, jpayne@7: cert=cert, jpayne@7: proxies=proxies, jpayne@7: ) jpayne@7: jpayne@7: chunked = not (request.body is None or "Content-Length" in request.headers) jpayne@7: jpayne@7: if isinstance(timeout, tuple): jpayne@7: try: jpayne@7: connect, read = timeout jpayne@7: timeout = TimeoutSauce(connect=connect, read=read) jpayne@7: except ValueError: jpayne@7: raise ValueError( jpayne@7: f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " jpayne@7: f"or a single float to set both timeouts to the same value." jpayne@7: ) jpayne@7: elif isinstance(timeout, TimeoutSauce): jpayne@7: pass jpayne@7: else: jpayne@7: timeout = TimeoutSauce(connect=timeout, read=timeout) jpayne@7: jpayne@7: try: jpayne@7: resp = conn.urlopen( jpayne@7: method=request.method, jpayne@7: url=url, jpayne@7: body=request.body, jpayne@7: headers=request.headers, jpayne@7: redirect=False, jpayne@7: assert_same_host=False, jpayne@7: preload_content=False, jpayne@7: decode_content=False, jpayne@7: retries=self.max_retries, jpayne@7: timeout=timeout, jpayne@7: chunked=chunked, jpayne@7: ) jpayne@7: jpayne@7: except (ProtocolError, OSError) as err: jpayne@7: raise ConnectionError(err, request=request) jpayne@7: jpayne@7: except MaxRetryError as e: jpayne@7: if isinstance(e.reason, ConnectTimeoutError): jpayne@7: # TODO: Remove this in 3.0.0: see #2811 jpayne@7: if not isinstance(e.reason, NewConnectionError): jpayne@7: raise ConnectTimeout(e, request=request) jpayne@7: jpayne@7: if isinstance(e.reason, ResponseError): jpayne@7: raise RetryError(e, request=request) jpayne@7: jpayne@7: if isinstance(e.reason, _ProxyError): jpayne@7: raise ProxyError(e, request=request) jpayne@7: jpayne@7: if isinstance(e.reason, _SSLError): jpayne@7: # This branch is for urllib3 v1.22 and later. jpayne@7: raise SSLError(e, request=request) jpayne@7: jpayne@7: raise ConnectionError(e, request=request) jpayne@7: jpayne@7: except ClosedPoolError as e: jpayne@7: raise ConnectionError(e, request=request) jpayne@7: jpayne@7: except _ProxyError as e: jpayne@7: raise ProxyError(e) jpayne@7: jpayne@7: except (_SSLError, _HTTPError) as e: jpayne@7: if isinstance(e, _SSLError): jpayne@7: # This branch is for urllib3 versions earlier than v1.22 jpayne@7: raise SSLError(e, request=request) jpayne@7: elif isinstance(e, ReadTimeoutError): jpayne@7: raise ReadTimeout(e, request=request) jpayne@7: elif isinstance(e, _InvalidHeader): jpayne@7: raise InvalidHeader(e, request=request) jpayne@7: else: jpayne@7: raise jpayne@7: jpayne@7: return self.build_response(request, resp)