Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/site-packages/urllib3/connectionpool.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 from __future__ import annotations
jpayne@69 2
jpayne@69 3 import errno
jpayne@69 4 import logging
jpayne@69 5 import queue
jpayne@69 6 import sys
jpayne@69 7 import typing
jpayne@69 8 import warnings
jpayne@69 9 import weakref
jpayne@69 10 from socket import timeout as SocketTimeout
jpayne@69 11 from types import TracebackType
jpayne@69 12
jpayne@69 13 from ._base_connection import _TYPE_BODY
jpayne@69 14 from ._collections import HTTPHeaderDict
jpayne@69 15 from ._request_methods import RequestMethods
jpayne@69 16 from .connection import (
jpayne@69 17 BaseSSLError,
jpayne@69 18 BrokenPipeError,
jpayne@69 19 DummyConnection,
jpayne@69 20 HTTPConnection,
jpayne@69 21 HTTPException,
jpayne@69 22 HTTPSConnection,
jpayne@69 23 ProxyConfig,
jpayne@69 24 _wrap_proxy_error,
jpayne@69 25 )
jpayne@69 26 from .connection import port_by_scheme as port_by_scheme
jpayne@69 27 from .exceptions import (
jpayne@69 28 ClosedPoolError,
jpayne@69 29 EmptyPoolError,
jpayne@69 30 FullPoolError,
jpayne@69 31 HostChangedError,
jpayne@69 32 InsecureRequestWarning,
jpayne@69 33 LocationValueError,
jpayne@69 34 MaxRetryError,
jpayne@69 35 NewConnectionError,
jpayne@69 36 ProtocolError,
jpayne@69 37 ProxyError,
jpayne@69 38 ReadTimeoutError,
jpayne@69 39 SSLError,
jpayne@69 40 TimeoutError,
jpayne@69 41 )
jpayne@69 42 from .response import BaseHTTPResponse
jpayne@69 43 from .util.connection import is_connection_dropped
jpayne@69 44 from .util.proxy import connection_requires_http_tunnel
jpayne@69 45 from .util.request import _TYPE_BODY_POSITION, set_file_position
jpayne@69 46 from .util.retry import Retry
jpayne@69 47 from .util.ssl_match_hostname import CertificateError
jpayne@69 48 from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout
jpayne@69 49 from .util.url import Url, _encode_target
jpayne@69 50 from .util.url import _normalize_host as normalize_host
jpayne@69 51 from .util.url import parse_url
jpayne@69 52 from .util.util import to_str
jpayne@69 53
jpayne@69 54 if typing.TYPE_CHECKING:
jpayne@69 55 import ssl
jpayne@69 56
jpayne@69 57 from typing_extensions import Self
jpayne@69 58
jpayne@69 59 from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
jpayne@69 60
jpayne@69 61 log = logging.getLogger(__name__)
jpayne@69 62
jpayne@69 63 _TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
jpayne@69 64
jpayne@69 65
jpayne@69 66 # Pool objects
jpayne@69 67 class ConnectionPool:
jpayne@69 68 """
jpayne@69 69 Base class for all connection pools, such as
jpayne@69 70 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
jpayne@69 71
jpayne@69 72 .. note::
jpayne@69 73 ConnectionPool.urlopen() does not normalize or percent-encode target URIs
jpayne@69 74 which is useful if your target server doesn't support percent-encoded
jpayne@69 75 target URIs.
jpayne@69 76 """
jpayne@69 77
jpayne@69 78 scheme: str | None = None
jpayne@69 79 QueueCls = queue.LifoQueue
jpayne@69 80
jpayne@69 81 def __init__(self, host: str, port: int | None = None) -> None:
jpayne@69 82 if not host:
jpayne@69 83 raise LocationValueError("No host specified.")
jpayne@69 84
jpayne@69 85 self.host = _normalize_host(host, scheme=self.scheme)
jpayne@69 86 self.port = port
jpayne@69 87
jpayne@69 88 # This property uses 'normalize_host()' (not '_normalize_host()')
jpayne@69 89 # to avoid removing square braces around IPv6 addresses.
jpayne@69 90 # This value is sent to `HTTPConnection.set_tunnel()` if called
jpayne@69 91 # because square braces are required for HTTP CONNECT tunneling.
jpayne@69 92 self._tunnel_host = normalize_host(host, scheme=self.scheme).lower()
jpayne@69 93
jpayne@69 94 def __str__(self) -> str:
jpayne@69 95 return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
jpayne@69 96
jpayne@69 97 def __enter__(self) -> Self:
jpayne@69 98 return self
jpayne@69 99
jpayne@69 100 def __exit__(
jpayne@69 101 self,
jpayne@69 102 exc_type: type[BaseException] | None,
jpayne@69 103 exc_val: BaseException | None,
jpayne@69 104 exc_tb: TracebackType | None,
jpayne@69 105 ) -> typing.Literal[False]:
jpayne@69 106 self.close()
jpayne@69 107 # Return False to re-raise any potential exceptions
jpayne@69 108 return False
jpayne@69 109
jpayne@69 110 def close(self) -> None:
jpayne@69 111 """
jpayne@69 112 Close all pooled connections and disable the pool.
jpayne@69 113 """
jpayne@69 114
jpayne@69 115
jpayne@69 116 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
jpayne@69 117 _blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
jpayne@69 118
jpayne@69 119
jpayne@69 120 class HTTPConnectionPool(ConnectionPool, RequestMethods):
jpayne@69 121 """
jpayne@69 122 Thread-safe connection pool for one host.
jpayne@69 123
jpayne@69 124 :param host:
jpayne@69 125 Host used for this HTTP Connection (e.g. "localhost"), passed into
jpayne@69 126 :class:`http.client.HTTPConnection`.
jpayne@69 127
jpayne@69 128 :param port:
jpayne@69 129 Port used for this HTTP Connection (None is equivalent to 80), passed
jpayne@69 130 into :class:`http.client.HTTPConnection`.
jpayne@69 131
jpayne@69 132 :param timeout:
jpayne@69 133 Socket timeout in seconds for each individual connection. This can
jpayne@69 134 be a float or integer, which sets the timeout for the HTTP request,
jpayne@69 135 or an instance of :class:`urllib3.util.Timeout` which gives you more
jpayne@69 136 fine-grained control over request timeouts. After the constructor has
jpayne@69 137 been parsed, this is always a `urllib3.util.Timeout` object.
jpayne@69 138
jpayne@69 139 :param maxsize:
jpayne@69 140 Number of connections to save that can be reused. More than 1 is useful
jpayne@69 141 in multithreaded situations. If ``block`` is set to False, more
jpayne@69 142 connections will be created but they will not be saved once they've
jpayne@69 143 been used.
jpayne@69 144
jpayne@69 145 :param block:
jpayne@69 146 If set to True, no more than ``maxsize`` connections will be used at
jpayne@69 147 a time. When no free connections are available, the call will block
jpayne@69 148 until a connection has been released. This is a useful side effect for
jpayne@69 149 particular multithreaded situations where one does not want to use more
jpayne@69 150 than maxsize connections per host to prevent flooding.
jpayne@69 151
jpayne@69 152 :param headers:
jpayne@69 153 Headers to include with all requests, unless other headers are given
jpayne@69 154 explicitly.
jpayne@69 155
jpayne@69 156 :param retries:
jpayne@69 157 Retry configuration to use by default with requests in this pool.
jpayne@69 158
jpayne@69 159 :param _proxy:
jpayne@69 160 Parsed proxy URL, should not be used directly, instead, see
jpayne@69 161 :class:`urllib3.ProxyManager`
jpayne@69 162
jpayne@69 163 :param _proxy_headers:
jpayne@69 164 A dictionary with proxy headers, should not be used directly,
jpayne@69 165 instead, see :class:`urllib3.ProxyManager`
jpayne@69 166
jpayne@69 167 :param \\**conn_kw:
jpayne@69 168 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
jpayne@69 169 :class:`urllib3.connection.HTTPSConnection` instances.
jpayne@69 170 """
jpayne@69 171
jpayne@69 172 scheme = "http"
jpayne@69 173 ConnectionCls: (
jpayne@69 174 type[BaseHTTPConnection] | type[BaseHTTPSConnection]
jpayne@69 175 ) = HTTPConnection
jpayne@69 176
jpayne@69 177 def __init__(
jpayne@69 178 self,
jpayne@69 179 host: str,
jpayne@69 180 port: int | None = None,
jpayne@69 181 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
jpayne@69 182 maxsize: int = 1,
jpayne@69 183 block: bool = False,
jpayne@69 184 headers: typing.Mapping[str, str] | None = None,
jpayne@69 185 retries: Retry | bool | int | None = None,
jpayne@69 186 _proxy: Url | None = None,
jpayne@69 187 _proxy_headers: typing.Mapping[str, str] | None = None,
jpayne@69 188 _proxy_config: ProxyConfig | None = None,
jpayne@69 189 **conn_kw: typing.Any,
jpayne@69 190 ):
jpayne@69 191 ConnectionPool.__init__(self, host, port)
jpayne@69 192 RequestMethods.__init__(self, headers)
jpayne@69 193
jpayne@69 194 if not isinstance(timeout, Timeout):
jpayne@69 195 timeout = Timeout.from_float(timeout)
jpayne@69 196
jpayne@69 197 if retries is None:
jpayne@69 198 retries = Retry.DEFAULT
jpayne@69 199
jpayne@69 200 self.timeout = timeout
jpayne@69 201 self.retries = retries
jpayne@69 202
jpayne@69 203 self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize)
jpayne@69 204 self.block = block
jpayne@69 205
jpayne@69 206 self.proxy = _proxy
jpayne@69 207 self.proxy_headers = _proxy_headers or {}
jpayne@69 208 self.proxy_config = _proxy_config
jpayne@69 209
jpayne@69 210 # Fill the queue up so that doing get() on it will block properly
jpayne@69 211 for _ in range(maxsize):
jpayne@69 212 self.pool.put(None)
jpayne@69 213
jpayne@69 214 # These are mostly for testing and debugging purposes.
jpayne@69 215 self.num_connections = 0
jpayne@69 216 self.num_requests = 0
jpayne@69 217 self.conn_kw = conn_kw
jpayne@69 218
jpayne@69 219 if self.proxy:
jpayne@69 220 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
jpayne@69 221 # We cannot know if the user has added default socket options, so we cannot replace the
jpayne@69 222 # list.
jpayne@69 223 self.conn_kw.setdefault("socket_options", [])
jpayne@69 224
jpayne@69 225 self.conn_kw["proxy"] = self.proxy
jpayne@69 226 self.conn_kw["proxy_config"] = self.proxy_config
jpayne@69 227
jpayne@69 228 # Do not pass 'self' as callback to 'finalize'.
jpayne@69 229 # Then the 'finalize' would keep an endless living (leak) to self.
jpayne@69 230 # By just passing a reference to the pool allows the garbage collector
jpayne@69 231 # to free self if nobody else has a reference to it.
jpayne@69 232 pool = self.pool
jpayne@69 233
jpayne@69 234 # Close all the HTTPConnections in the pool before the
jpayne@69 235 # HTTPConnectionPool object is garbage collected.
jpayne@69 236 weakref.finalize(self, _close_pool_connections, pool)
jpayne@69 237
jpayne@69 238 def _new_conn(self) -> BaseHTTPConnection:
jpayne@69 239 """
jpayne@69 240 Return a fresh :class:`HTTPConnection`.
jpayne@69 241 """
jpayne@69 242 self.num_connections += 1
jpayne@69 243 log.debug(
jpayne@69 244 "Starting new HTTP connection (%d): %s:%s",
jpayne@69 245 self.num_connections,
jpayne@69 246 self.host,
jpayne@69 247 self.port or "80",
jpayne@69 248 )
jpayne@69 249
jpayne@69 250 conn = self.ConnectionCls(
jpayne@69 251 host=self.host,
jpayne@69 252 port=self.port,
jpayne@69 253 timeout=self.timeout.connect_timeout,
jpayne@69 254 **self.conn_kw,
jpayne@69 255 )
jpayne@69 256 return conn
jpayne@69 257
jpayne@69 258 def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:
jpayne@69 259 """
jpayne@69 260 Get a connection. Will return a pooled connection if one is available.
jpayne@69 261
jpayne@69 262 If no connections are available and :prop:`.block` is ``False``, then a
jpayne@69 263 fresh connection is returned.
jpayne@69 264
jpayne@69 265 :param timeout:
jpayne@69 266 Seconds to wait before giving up and raising
jpayne@69 267 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
jpayne@69 268 :prop:`.block` is ``True``.
jpayne@69 269 """
jpayne@69 270 conn = None
jpayne@69 271
jpayne@69 272 if self.pool is None:
jpayne@69 273 raise ClosedPoolError(self, "Pool is closed.")
jpayne@69 274
jpayne@69 275 try:
jpayne@69 276 conn = self.pool.get(block=self.block, timeout=timeout)
jpayne@69 277
jpayne@69 278 except AttributeError: # self.pool is None
jpayne@69 279 raise ClosedPoolError(self, "Pool is closed.") from None # Defensive:
jpayne@69 280
jpayne@69 281 except queue.Empty:
jpayne@69 282 if self.block:
jpayne@69 283 raise EmptyPoolError(
jpayne@69 284 self,
jpayne@69 285 "Pool is empty and a new connection can't be opened due to blocking mode.",
jpayne@69 286 ) from None
jpayne@69 287 pass # Oh well, we'll create a new connection then
jpayne@69 288
jpayne@69 289 # If this is a persistent connection, check if it got disconnected
jpayne@69 290 if conn and is_connection_dropped(conn):
jpayne@69 291 log.debug("Resetting dropped connection: %s", self.host)
jpayne@69 292 conn.close()
jpayne@69 293
jpayne@69 294 return conn or self._new_conn()
jpayne@69 295
jpayne@69 296 def _put_conn(self, conn: BaseHTTPConnection | None) -> None:
jpayne@69 297 """
jpayne@69 298 Put a connection back into the pool.
jpayne@69 299
jpayne@69 300 :param conn:
jpayne@69 301 Connection object for the current host and port as returned by
jpayne@69 302 :meth:`._new_conn` or :meth:`._get_conn`.
jpayne@69 303
jpayne@69 304 If the pool is already full, the connection is closed and discarded
jpayne@69 305 because we exceeded maxsize. If connections are discarded frequently,
jpayne@69 306 then maxsize should be increased.
jpayne@69 307
jpayne@69 308 If the pool is closed, then the connection will be closed and discarded.
jpayne@69 309 """
jpayne@69 310 if self.pool is not None:
jpayne@69 311 try:
jpayne@69 312 self.pool.put(conn, block=False)
jpayne@69 313 return # Everything is dandy, done.
jpayne@69 314 except AttributeError:
jpayne@69 315 # self.pool is None.
jpayne@69 316 pass
jpayne@69 317 except queue.Full:
jpayne@69 318 # Connection never got put back into the pool, close it.
jpayne@69 319 if conn:
jpayne@69 320 conn.close()
jpayne@69 321
jpayne@69 322 if self.block:
jpayne@69 323 # This should never happen if you got the conn from self._get_conn
jpayne@69 324 raise FullPoolError(
jpayne@69 325 self,
jpayne@69 326 "Pool reached maximum size and no more connections are allowed.",
jpayne@69 327 ) from None
jpayne@69 328
jpayne@69 329 log.warning(
jpayne@69 330 "Connection pool is full, discarding connection: %s. Connection pool size: %s",
jpayne@69 331 self.host,
jpayne@69 332 self.pool.qsize(),
jpayne@69 333 )
jpayne@69 334
jpayne@69 335 # Connection never got put back into the pool, close it.
jpayne@69 336 if conn:
jpayne@69 337 conn.close()
jpayne@69 338
jpayne@69 339 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
jpayne@69 340 """
jpayne@69 341 Called right before a request is made, after the socket is created.
jpayne@69 342 """
jpayne@69 343
jpayne@69 344 def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:
jpayne@69 345 # Nothing to do for HTTP connections.
jpayne@69 346 pass
jpayne@69 347
jpayne@69 348 def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
jpayne@69 349 """Helper that always returns a :class:`urllib3.util.Timeout`"""
jpayne@69 350 if timeout is _DEFAULT_TIMEOUT:
jpayne@69 351 return self.timeout.clone()
jpayne@69 352
jpayne@69 353 if isinstance(timeout, Timeout):
jpayne@69 354 return timeout.clone()
jpayne@69 355 else:
jpayne@69 356 # User passed us an int/float. This is for backwards compatibility,
jpayne@69 357 # can be removed later
jpayne@69 358 return Timeout.from_float(timeout)
jpayne@69 359
jpayne@69 360 def _raise_timeout(
jpayne@69 361 self,
jpayne@69 362 err: BaseSSLError | OSError | SocketTimeout,
jpayne@69 363 url: str,
jpayne@69 364 timeout_value: _TYPE_TIMEOUT | None,
jpayne@69 365 ) -> None:
jpayne@69 366 """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
jpayne@69 367
jpayne@69 368 if isinstance(err, SocketTimeout):
jpayne@69 369 raise ReadTimeoutError(
jpayne@69 370 self, url, f"Read timed out. (read timeout={timeout_value})"
jpayne@69 371 ) from err
jpayne@69 372
jpayne@69 373 # See the above comment about EAGAIN in Python 3.
jpayne@69 374 if hasattr(err, "errno") and err.errno in _blocking_errnos:
jpayne@69 375 raise ReadTimeoutError(
jpayne@69 376 self, url, f"Read timed out. (read timeout={timeout_value})"
jpayne@69 377 ) from err
jpayne@69 378
jpayne@69 379 def _make_request(
jpayne@69 380 self,
jpayne@69 381 conn: BaseHTTPConnection,
jpayne@69 382 method: str,
jpayne@69 383 url: str,
jpayne@69 384 body: _TYPE_BODY | None = None,
jpayne@69 385 headers: typing.Mapping[str, str] | None = None,
jpayne@69 386 retries: Retry | None = None,
jpayne@69 387 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
jpayne@69 388 chunked: bool = False,
jpayne@69 389 response_conn: BaseHTTPConnection | None = None,
jpayne@69 390 preload_content: bool = True,
jpayne@69 391 decode_content: bool = True,
jpayne@69 392 enforce_content_length: bool = True,
jpayne@69 393 ) -> BaseHTTPResponse:
jpayne@69 394 """
jpayne@69 395 Perform a request on a given urllib connection object taken from our
jpayne@69 396 pool.
jpayne@69 397
jpayne@69 398 :param conn:
jpayne@69 399 a connection from one of our connection pools
jpayne@69 400
jpayne@69 401 :param method:
jpayne@69 402 HTTP request method (such as GET, POST, PUT, etc.)
jpayne@69 403
jpayne@69 404 :param url:
jpayne@69 405 The URL to perform the request on.
jpayne@69 406
jpayne@69 407 :param body:
jpayne@69 408 Data to send in the request body, either :class:`str`, :class:`bytes`,
jpayne@69 409 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
jpayne@69 410
jpayne@69 411 :param headers:
jpayne@69 412 Dictionary of custom headers to send, such as User-Agent,
jpayne@69 413 If-None-Match, etc. If None, pool headers are used. If provided,
jpayne@69 414 these headers completely replace any pool-specific headers.
jpayne@69 415
jpayne@69 416 :param retries:
jpayne@69 417 Configure the number of retries to allow before raising a
jpayne@69 418 :class:`~urllib3.exceptions.MaxRetryError` exception.
jpayne@69 419
jpayne@69 420 Pass ``None`` to retry until you receive a response. Pass a
jpayne@69 421 :class:`~urllib3.util.retry.Retry` object for fine-grained control
jpayne@69 422 over different types of retries.
jpayne@69 423 Pass an integer number to retry connection errors that many times,
jpayne@69 424 but no other types of errors. Pass zero to never retry.
jpayne@69 425
jpayne@69 426 If ``False``, then retries are disabled and any exception is raised
jpayne@69 427 immediately. Also, instead of raising a MaxRetryError on redirects,
jpayne@69 428 the redirect response will be returned.
jpayne@69 429
jpayne@69 430 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
jpayne@69 431
jpayne@69 432 :param timeout:
jpayne@69 433 If specified, overrides the default timeout for this one
jpayne@69 434 request. It may be a float (in seconds) or an instance of
jpayne@69 435 :class:`urllib3.util.Timeout`.
jpayne@69 436
jpayne@69 437 :param chunked:
jpayne@69 438 If True, urllib3 will send the body using chunked transfer
jpayne@69 439 encoding. Otherwise, urllib3 will send the body using the standard
jpayne@69 440 content-length form. Defaults to False.
jpayne@69 441
jpayne@69 442 :param response_conn:
jpayne@69 443 Set this to ``None`` if you will handle releasing the connection or
jpayne@69 444 set the connection to have the response release it.
jpayne@69 445
jpayne@69 446 :param preload_content:
jpayne@69 447 If True, the response's body will be preloaded during construction.
jpayne@69 448
jpayne@69 449 :param decode_content:
jpayne@69 450 If True, will attempt to decode the body based on the
jpayne@69 451 'content-encoding' header.
jpayne@69 452
jpayne@69 453 :param enforce_content_length:
jpayne@69 454 Enforce content length checking. Body returned by server must match
jpayne@69 455 value of Content-Length header, if present. Otherwise, raise error.
jpayne@69 456 """
jpayne@69 457 self.num_requests += 1
jpayne@69 458
jpayne@69 459 timeout_obj = self._get_timeout(timeout)
jpayne@69 460 timeout_obj.start_connect()
jpayne@69 461 conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
jpayne@69 462
jpayne@69 463 try:
jpayne@69 464 # Trigger any extra validation we need to do.
jpayne@69 465 try:
jpayne@69 466 self._validate_conn(conn)
jpayne@69 467 except (SocketTimeout, BaseSSLError) as e:
jpayne@69 468 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
jpayne@69 469 raise
jpayne@69 470
jpayne@69 471 # _validate_conn() starts the connection to an HTTPS proxy
jpayne@69 472 # so we need to wrap errors with 'ProxyError' here too.
jpayne@69 473 except (
jpayne@69 474 OSError,
jpayne@69 475 NewConnectionError,
jpayne@69 476 TimeoutError,
jpayne@69 477 BaseSSLError,
jpayne@69 478 CertificateError,
jpayne@69 479 SSLError,
jpayne@69 480 ) as e:
jpayne@69 481 new_e: Exception = e
jpayne@69 482 if isinstance(e, (BaseSSLError, CertificateError)):
jpayne@69 483 new_e = SSLError(e)
jpayne@69 484 # If the connection didn't successfully connect to it's proxy
jpayne@69 485 # then there
jpayne@69 486 if isinstance(
jpayne@69 487 new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
jpayne@69 488 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
jpayne@69 489 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
jpayne@69 490 raise new_e
jpayne@69 491
jpayne@69 492 # conn.request() calls http.client.*.request, not the method in
jpayne@69 493 # urllib3.request. It also calls makefile (recv) on the socket.
jpayne@69 494 try:
jpayne@69 495 conn.request(
jpayne@69 496 method,
jpayne@69 497 url,
jpayne@69 498 body=body,
jpayne@69 499 headers=headers,
jpayne@69 500 chunked=chunked,
jpayne@69 501 preload_content=preload_content,
jpayne@69 502 decode_content=decode_content,
jpayne@69 503 enforce_content_length=enforce_content_length,
jpayne@69 504 )
jpayne@69 505
jpayne@69 506 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
jpayne@69 507 # legitimately able to close the connection after sending a valid response.
jpayne@69 508 # With this behaviour, the received response is still readable.
jpayne@69 509 except BrokenPipeError:
jpayne@69 510 pass
jpayne@69 511 except OSError as e:
jpayne@69 512 # MacOS/Linux
jpayne@69 513 # EPROTOTYPE and ECONNRESET are needed on macOS
jpayne@69 514 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
jpayne@69 515 # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
jpayne@69 516 if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
jpayne@69 517 raise
jpayne@69 518
jpayne@69 519 # Reset the timeout for the recv() on the socket
jpayne@69 520 read_timeout = timeout_obj.read_timeout
jpayne@69 521
jpayne@69 522 if not conn.is_closed:
jpayne@69 523 # In Python 3 socket.py will catch EAGAIN and return None when you
jpayne@69 524 # try and read into the file pointer created by http.client, which
jpayne@69 525 # instead raises a BadStatusLine exception. Instead of catching
jpayne@69 526 # the exception and assuming all BadStatusLine exceptions are read
jpayne@69 527 # timeouts, check for a zero timeout before making the request.
jpayne@69 528 if read_timeout == 0:
jpayne@69 529 raise ReadTimeoutError(
jpayne@69 530 self, url, f"Read timed out. (read timeout={read_timeout})"
jpayne@69 531 )
jpayne@69 532 conn.timeout = read_timeout
jpayne@69 533
jpayne@69 534 # Receive the response from the server
jpayne@69 535 try:
jpayne@69 536 response = conn.getresponse()
jpayne@69 537 except (BaseSSLError, OSError) as e:
jpayne@69 538 self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
jpayne@69 539 raise
jpayne@69 540
jpayne@69 541 # Set properties that are used by the pooling layer.
jpayne@69 542 response.retries = retries
jpayne@69 543 response._connection = response_conn # type: ignore[attr-defined]
jpayne@69 544 response._pool = self # type: ignore[attr-defined]
jpayne@69 545
jpayne@69 546 log.debug(
jpayne@69 547 '%s://%s:%s "%s %s HTTP/%s" %s %s',
jpayne@69 548 self.scheme,
jpayne@69 549 self.host,
jpayne@69 550 self.port,
jpayne@69 551 method,
jpayne@69 552 url,
jpayne@69 553 response.version,
jpayne@69 554 response.status,
jpayne@69 555 response.length_remaining,
jpayne@69 556 )
jpayne@69 557
jpayne@69 558 return response
jpayne@69 559
jpayne@69 560 def close(self) -> None:
jpayne@69 561 """
jpayne@69 562 Close all pooled connections and disable the pool.
jpayne@69 563 """
jpayne@69 564 if self.pool is None:
jpayne@69 565 return
jpayne@69 566 # Disable access to the pool
jpayne@69 567 old_pool, self.pool = self.pool, None
jpayne@69 568
jpayne@69 569 # Close all the HTTPConnections in the pool.
jpayne@69 570 _close_pool_connections(old_pool)
jpayne@69 571
jpayne@69 572 def is_same_host(self, url: str) -> bool:
jpayne@69 573 """
jpayne@69 574 Check if the given ``url`` is a member of the same host as this
jpayne@69 575 connection pool.
jpayne@69 576 """
jpayne@69 577 if url.startswith("/"):
jpayne@69 578 return True
jpayne@69 579
jpayne@69 580 # TODO: Add optional support for socket.gethostbyname checking.
jpayne@69 581 scheme, _, host, port, *_ = parse_url(url)
jpayne@69 582 scheme = scheme or "http"
jpayne@69 583 if host is not None:
jpayne@69 584 host = _normalize_host(host, scheme=scheme)
jpayne@69 585
jpayne@69 586 # Use explicit default port for comparison when none is given
jpayne@69 587 if self.port and not port:
jpayne@69 588 port = port_by_scheme.get(scheme)
jpayne@69 589 elif not self.port and port == port_by_scheme.get(scheme):
jpayne@69 590 port = None
jpayne@69 591
jpayne@69 592 return (scheme, host, port) == (self.scheme, self.host, self.port)
jpayne@69 593
jpayne@69 594 def urlopen( # type: ignore[override]
jpayne@69 595 self,
jpayne@69 596 method: str,
jpayne@69 597 url: str,
jpayne@69 598 body: _TYPE_BODY | None = None,
jpayne@69 599 headers: typing.Mapping[str, str] | None = None,
jpayne@69 600 retries: Retry | bool | int | None = None,
jpayne@69 601 redirect: bool = True,
jpayne@69 602 assert_same_host: bool = True,
jpayne@69 603 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
jpayne@69 604 pool_timeout: int | None = None,
jpayne@69 605 release_conn: bool | None = None,
jpayne@69 606 chunked: bool = False,
jpayne@69 607 body_pos: _TYPE_BODY_POSITION | None = None,
jpayne@69 608 preload_content: bool = True,
jpayne@69 609 decode_content: bool = True,
jpayne@69 610 **response_kw: typing.Any,
jpayne@69 611 ) -> BaseHTTPResponse:
jpayne@69 612 """
jpayne@69 613 Get a connection from the pool and perform an HTTP request. This is the
jpayne@69 614 lowest level call for making a request, so you'll need to specify all
jpayne@69 615 the raw details.
jpayne@69 616
jpayne@69 617 .. note::
jpayne@69 618
jpayne@69 619 More commonly, it's appropriate to use a convenience method
jpayne@69 620 such as :meth:`request`.
jpayne@69 621
jpayne@69 622 .. note::
jpayne@69 623
jpayne@69 624 `release_conn` will only behave as expected if
jpayne@69 625 `preload_content=False` because we want to make
jpayne@69 626 `preload_content=False` the default behaviour someday soon without
jpayne@69 627 breaking backwards compatibility.
jpayne@69 628
jpayne@69 629 :param method:
jpayne@69 630 HTTP request method (such as GET, POST, PUT, etc.)
jpayne@69 631
jpayne@69 632 :param url:
jpayne@69 633 The URL to perform the request on.
jpayne@69 634
jpayne@69 635 :param body:
jpayne@69 636 Data to send in the request body, either :class:`str`, :class:`bytes`,
jpayne@69 637 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
jpayne@69 638
jpayne@69 639 :param headers:
jpayne@69 640 Dictionary of custom headers to send, such as User-Agent,
jpayne@69 641 If-None-Match, etc. If None, pool headers are used. If provided,
jpayne@69 642 these headers completely replace any pool-specific headers.
jpayne@69 643
jpayne@69 644 :param retries:
jpayne@69 645 Configure the number of retries to allow before raising a
jpayne@69 646 :class:`~urllib3.exceptions.MaxRetryError` exception.
jpayne@69 647
jpayne@69 648 If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
jpayne@69 649 :class:`~urllib3.util.retry.Retry` object for fine-grained control
jpayne@69 650 over different types of retries.
jpayne@69 651 Pass an integer number to retry connection errors that many times,
jpayne@69 652 but no other types of errors. Pass zero to never retry.
jpayne@69 653
jpayne@69 654 If ``False``, then retries are disabled and any exception is raised
jpayne@69 655 immediately. Also, instead of raising a MaxRetryError on redirects,
jpayne@69 656 the redirect response will be returned.
jpayne@69 657
jpayne@69 658 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
jpayne@69 659
jpayne@69 660 :param redirect:
jpayne@69 661 If True, automatically handle redirects (status codes 301, 302,
jpayne@69 662 303, 307, 308). Each redirect counts as a retry. Disabling retries
jpayne@69 663 will disable redirect, too.
jpayne@69 664
jpayne@69 665 :param assert_same_host:
jpayne@69 666 If ``True``, will make sure that the host of the pool requests is
jpayne@69 667 consistent else will raise HostChangedError. When ``False``, you can
jpayne@69 668 use the pool on an HTTP proxy and request foreign hosts.
jpayne@69 669
jpayne@69 670 :param timeout:
jpayne@69 671 If specified, overrides the default timeout for this one
jpayne@69 672 request. It may be a float (in seconds) or an instance of
jpayne@69 673 :class:`urllib3.util.Timeout`.
jpayne@69 674
jpayne@69 675 :param pool_timeout:
jpayne@69 676 If set and the pool is set to block=True, then this method will
jpayne@69 677 block for ``pool_timeout`` seconds and raise EmptyPoolError if no
jpayne@69 678 connection is available within the time period.
jpayne@69 679
jpayne@69 680 :param bool preload_content:
jpayne@69 681 If True, the response's body will be preloaded into memory.
jpayne@69 682
jpayne@69 683 :param bool decode_content:
jpayne@69 684 If True, will attempt to decode the body based on the
jpayne@69 685 'content-encoding' header.
jpayne@69 686
jpayne@69 687 :param release_conn:
jpayne@69 688 If False, then the urlopen call will not release the connection
jpayne@69 689 back into the pool once a response is received (but will release if
jpayne@69 690 you read the entire contents of the response such as when
jpayne@69 691 `preload_content=True`). This is useful if you're not preloading
jpayne@69 692 the response's content immediately. You will need to call
jpayne@69 693 ``r.release_conn()`` on the response ``r`` to return the connection
jpayne@69 694 back into the pool. If None, it takes the value of ``preload_content``
jpayne@69 695 which defaults to ``True``.
jpayne@69 696
jpayne@69 697 :param bool chunked:
jpayne@69 698 If True, urllib3 will send the body using chunked transfer
jpayne@69 699 encoding. Otherwise, urllib3 will send the body using the standard
jpayne@69 700 content-length form. Defaults to False.
jpayne@69 701
jpayne@69 702 :param int body_pos:
jpayne@69 703 Position to seek to in file-like body in the event of a retry or
jpayne@69 704 redirect. Typically this won't need to be set because urllib3 will
jpayne@69 705 auto-populate the value when needed.
jpayne@69 706 """
jpayne@69 707 parsed_url = parse_url(url)
jpayne@69 708 destination_scheme = parsed_url.scheme
jpayne@69 709
jpayne@69 710 if headers is None:
jpayne@69 711 headers = self.headers
jpayne@69 712
jpayne@69 713 if not isinstance(retries, Retry):
jpayne@69 714 retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
jpayne@69 715
jpayne@69 716 if release_conn is None:
jpayne@69 717 release_conn = preload_content
jpayne@69 718
jpayne@69 719 # Check host
jpayne@69 720 if assert_same_host and not self.is_same_host(url):
jpayne@69 721 raise HostChangedError(self, url, retries)
jpayne@69 722
jpayne@69 723 # Ensure that the URL we're connecting to is properly encoded
jpayne@69 724 if url.startswith("/"):
jpayne@69 725 url = to_str(_encode_target(url))
jpayne@69 726 else:
jpayne@69 727 url = to_str(parsed_url.url)
jpayne@69 728
jpayne@69 729 conn = None
jpayne@69 730
jpayne@69 731 # Track whether `conn` needs to be released before
jpayne@69 732 # returning/raising/recursing. Update this variable if necessary, and
jpayne@69 733 # leave `release_conn` constant throughout the function. That way, if
jpayne@69 734 # the function recurses, the original value of `release_conn` will be
jpayne@69 735 # passed down into the recursive call, and its value will be respected.
jpayne@69 736 #
jpayne@69 737 # See issue #651 [1] for details.
jpayne@69 738 #
jpayne@69 739 # [1] <https://github.com/urllib3/urllib3/issues/651>
jpayne@69 740 release_this_conn = release_conn
jpayne@69 741
jpayne@69 742 http_tunnel_required = connection_requires_http_tunnel(
jpayne@69 743 self.proxy, self.proxy_config, destination_scheme
jpayne@69 744 )
jpayne@69 745
jpayne@69 746 # Merge the proxy headers. Only done when not using HTTP CONNECT. We
jpayne@69 747 # have to copy the headers dict so we can safely change it without those
jpayne@69 748 # changes being reflected in anyone else's copy.
jpayne@69 749 if not http_tunnel_required:
jpayne@69 750 headers = headers.copy() # type: ignore[attr-defined]
jpayne@69 751 headers.update(self.proxy_headers) # type: ignore[union-attr]
jpayne@69 752
jpayne@69 753 # Must keep the exception bound to a separate variable or else Python 3
jpayne@69 754 # complains about UnboundLocalError.
jpayne@69 755 err = None
jpayne@69 756
jpayne@69 757 # Keep track of whether we cleanly exited the except block. This
jpayne@69 758 # ensures we do proper cleanup in finally.
jpayne@69 759 clean_exit = False
jpayne@69 760
jpayne@69 761 # Rewind body position, if needed. Record current position
jpayne@69 762 # for future rewinds in the event of a redirect/retry.
jpayne@69 763 body_pos = set_file_position(body, body_pos)
jpayne@69 764
jpayne@69 765 try:
jpayne@69 766 # Request a connection from the queue.
jpayne@69 767 timeout_obj = self._get_timeout(timeout)
jpayne@69 768 conn = self._get_conn(timeout=pool_timeout)
jpayne@69 769
jpayne@69 770 conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
jpayne@69 771
jpayne@69 772 # Is this a closed/new connection that requires CONNECT tunnelling?
jpayne@69 773 if self.proxy is not None and http_tunnel_required and conn.is_closed:
jpayne@69 774 try:
jpayne@69 775 self._prepare_proxy(conn)
jpayne@69 776 except (BaseSSLError, OSError, SocketTimeout) as e:
jpayne@69 777 self._raise_timeout(
jpayne@69 778 err=e, url=self.proxy.url, timeout_value=conn.timeout
jpayne@69 779 )
jpayne@69 780 raise
jpayne@69 781
jpayne@69 782 # If we're going to release the connection in ``finally:``, then
jpayne@69 783 # the response doesn't need to know about the connection. Otherwise
jpayne@69 784 # it will also try to release it and we'll have a double-release
jpayne@69 785 # mess.
jpayne@69 786 response_conn = conn if not release_conn else None
jpayne@69 787
jpayne@69 788 # Make the request on the HTTPConnection object
jpayne@69 789 response = self._make_request(
jpayne@69 790 conn,
jpayne@69 791 method,
jpayne@69 792 url,
jpayne@69 793 timeout=timeout_obj,
jpayne@69 794 body=body,
jpayne@69 795 headers=headers,
jpayne@69 796 chunked=chunked,
jpayne@69 797 retries=retries,
jpayne@69 798 response_conn=response_conn,
jpayne@69 799 preload_content=preload_content,
jpayne@69 800 decode_content=decode_content,
jpayne@69 801 **response_kw,
jpayne@69 802 )
jpayne@69 803
jpayne@69 804 # Everything went great!
jpayne@69 805 clean_exit = True
jpayne@69 806
jpayne@69 807 except EmptyPoolError:
jpayne@69 808 # Didn't get a connection from the pool, no need to clean up
jpayne@69 809 clean_exit = True
jpayne@69 810 release_this_conn = False
jpayne@69 811 raise
jpayne@69 812
jpayne@69 813 except (
jpayne@69 814 TimeoutError,
jpayne@69 815 HTTPException,
jpayne@69 816 OSError,
jpayne@69 817 ProtocolError,
jpayne@69 818 BaseSSLError,
jpayne@69 819 SSLError,
jpayne@69 820 CertificateError,
jpayne@69 821 ProxyError,
jpayne@69 822 ) as e:
jpayne@69 823 # Discard the connection for these exceptions. It will be
jpayne@69 824 # replaced during the next _get_conn() call.
jpayne@69 825 clean_exit = False
jpayne@69 826 new_e: Exception = e
jpayne@69 827 if isinstance(e, (BaseSSLError, CertificateError)):
jpayne@69 828 new_e = SSLError(e)
jpayne@69 829 if isinstance(
jpayne@69 830 new_e,
jpayne@69 831 (
jpayne@69 832 OSError,
jpayne@69 833 NewConnectionError,
jpayne@69 834 TimeoutError,
jpayne@69 835 SSLError,
jpayne@69 836 HTTPException,
jpayne@69 837 ),
jpayne@69 838 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
jpayne@69 839 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
jpayne@69 840 elif isinstance(new_e, (OSError, HTTPException)):
jpayne@69 841 new_e = ProtocolError("Connection aborted.", new_e)
jpayne@69 842
jpayne@69 843 retries = retries.increment(
jpayne@69 844 method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
jpayne@69 845 )
jpayne@69 846 retries.sleep()
jpayne@69 847
jpayne@69 848 # Keep track of the error for the retry warning.
jpayne@69 849 err = e
jpayne@69 850
jpayne@69 851 finally:
jpayne@69 852 if not clean_exit:
jpayne@69 853 # We hit some kind of exception, handled or otherwise. We need
jpayne@69 854 # to throw the connection away unless explicitly told not to.
jpayne@69 855 # Close the connection, set the variable to None, and make sure
jpayne@69 856 # we put the None back in the pool to avoid leaking it.
jpayne@69 857 if conn:
jpayne@69 858 conn.close()
jpayne@69 859 conn = None
jpayne@69 860 release_this_conn = True
jpayne@69 861
jpayne@69 862 if release_this_conn:
jpayne@69 863 # Put the connection back to be reused. If the connection is
jpayne@69 864 # expired then it will be None, which will get replaced with a
jpayne@69 865 # fresh connection during _get_conn.
jpayne@69 866 self._put_conn(conn)
jpayne@69 867
jpayne@69 868 if not conn:
jpayne@69 869 # Try again
jpayne@69 870 log.warning(
jpayne@69 871 "Retrying (%r) after connection broken by '%r': %s", retries, err, url
jpayne@69 872 )
jpayne@69 873 return self.urlopen(
jpayne@69 874 method,
jpayne@69 875 url,
jpayne@69 876 body,
jpayne@69 877 headers,
jpayne@69 878 retries,
jpayne@69 879 redirect,
jpayne@69 880 assert_same_host,
jpayne@69 881 timeout=timeout,
jpayne@69 882 pool_timeout=pool_timeout,
jpayne@69 883 release_conn=release_conn,
jpayne@69 884 chunked=chunked,
jpayne@69 885 body_pos=body_pos,
jpayne@69 886 preload_content=preload_content,
jpayne@69 887 decode_content=decode_content,
jpayne@69 888 **response_kw,
jpayne@69 889 )
jpayne@69 890
jpayne@69 891 # Handle redirect?
jpayne@69 892 redirect_location = redirect and response.get_redirect_location()
jpayne@69 893 if redirect_location:
jpayne@69 894 if response.status == 303:
jpayne@69 895 # Change the method according to RFC 9110, Section 15.4.4.
jpayne@69 896 method = "GET"
jpayne@69 897 # And lose the body not to transfer anything sensitive.
jpayne@69 898 body = None
jpayne@69 899 headers = HTTPHeaderDict(headers)._prepare_for_method_change()
jpayne@69 900
jpayne@69 901 try:
jpayne@69 902 retries = retries.increment(method, url, response=response, _pool=self)
jpayne@69 903 except MaxRetryError:
jpayne@69 904 if retries.raise_on_redirect:
jpayne@69 905 response.drain_conn()
jpayne@69 906 raise
jpayne@69 907 return response
jpayne@69 908
jpayne@69 909 response.drain_conn()
jpayne@69 910 retries.sleep_for_retry(response)
jpayne@69 911 log.debug("Redirecting %s -> %s", url, redirect_location)
jpayne@69 912 return self.urlopen(
jpayne@69 913 method,
jpayne@69 914 redirect_location,
jpayne@69 915 body,
jpayne@69 916 headers,
jpayne@69 917 retries=retries,
jpayne@69 918 redirect=redirect,
jpayne@69 919 assert_same_host=assert_same_host,
jpayne@69 920 timeout=timeout,
jpayne@69 921 pool_timeout=pool_timeout,
jpayne@69 922 release_conn=release_conn,
jpayne@69 923 chunked=chunked,
jpayne@69 924 body_pos=body_pos,
jpayne@69 925 preload_content=preload_content,
jpayne@69 926 decode_content=decode_content,
jpayne@69 927 **response_kw,
jpayne@69 928 )
jpayne@69 929
jpayne@69 930 # Check if we should retry the HTTP response.
jpayne@69 931 has_retry_after = bool(response.headers.get("Retry-After"))
jpayne@69 932 if retries.is_retry(method, response.status, has_retry_after):
jpayne@69 933 try:
jpayne@69 934 retries = retries.increment(method, url, response=response, _pool=self)
jpayne@69 935 except MaxRetryError:
jpayne@69 936 if retries.raise_on_status:
jpayne@69 937 response.drain_conn()
jpayne@69 938 raise
jpayne@69 939 return response
jpayne@69 940
jpayne@69 941 response.drain_conn()
jpayne@69 942 retries.sleep(response)
jpayne@69 943 log.debug("Retry: %s", url)
jpayne@69 944 return self.urlopen(
jpayne@69 945 method,
jpayne@69 946 url,
jpayne@69 947 body,
jpayne@69 948 headers,
jpayne@69 949 retries=retries,
jpayne@69 950 redirect=redirect,
jpayne@69 951 assert_same_host=assert_same_host,
jpayne@69 952 timeout=timeout,
jpayne@69 953 pool_timeout=pool_timeout,
jpayne@69 954 release_conn=release_conn,
jpayne@69 955 chunked=chunked,
jpayne@69 956 body_pos=body_pos,
jpayne@69 957 preload_content=preload_content,
jpayne@69 958 decode_content=decode_content,
jpayne@69 959 **response_kw,
jpayne@69 960 )
jpayne@69 961
jpayne@69 962 return response
jpayne@69 963
jpayne@69 964
jpayne@69 965 class HTTPSConnectionPool(HTTPConnectionPool):
jpayne@69 966 """
jpayne@69 967 Same as :class:`.HTTPConnectionPool`, but HTTPS.
jpayne@69 968
jpayne@69 969 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
jpayne@69 970 ``assert_hostname`` and ``host`` in this order to verify connections.
jpayne@69 971 If ``assert_hostname`` is False, no verification is done.
jpayne@69 972
jpayne@69 973 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
jpayne@69 974 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
jpayne@69 975 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
jpayne@69 976 the connection socket into an SSL socket.
jpayne@69 977 """
jpayne@69 978
jpayne@69 979 scheme = "https"
jpayne@69 980 ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection
jpayne@69 981
jpayne@69 982 def __init__(
jpayne@69 983 self,
jpayne@69 984 host: str,
jpayne@69 985 port: int | None = None,
jpayne@69 986 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
jpayne@69 987 maxsize: int = 1,
jpayne@69 988 block: bool = False,
jpayne@69 989 headers: typing.Mapping[str, str] | None = None,
jpayne@69 990 retries: Retry | bool | int | None = None,
jpayne@69 991 _proxy: Url | None = None,
jpayne@69 992 _proxy_headers: typing.Mapping[str, str] | None = None,
jpayne@69 993 key_file: str | None = None,
jpayne@69 994 cert_file: str | None = None,
jpayne@69 995 cert_reqs: int | str | None = None,
jpayne@69 996 key_password: str | None = None,
jpayne@69 997 ca_certs: str | None = None,
jpayne@69 998 ssl_version: int | str | None = None,
jpayne@69 999 ssl_minimum_version: ssl.TLSVersion | None = None,
jpayne@69 1000 ssl_maximum_version: ssl.TLSVersion | None = None,
jpayne@69 1001 assert_hostname: str | typing.Literal[False] | None = None,
jpayne@69 1002 assert_fingerprint: str | None = None,
jpayne@69 1003 ca_cert_dir: str | None = None,
jpayne@69 1004 **conn_kw: typing.Any,
jpayne@69 1005 ) -> None:
jpayne@69 1006 super().__init__(
jpayne@69 1007 host,
jpayne@69 1008 port,
jpayne@69 1009 timeout,
jpayne@69 1010 maxsize,
jpayne@69 1011 block,
jpayne@69 1012 headers,
jpayne@69 1013 retries,
jpayne@69 1014 _proxy,
jpayne@69 1015 _proxy_headers,
jpayne@69 1016 **conn_kw,
jpayne@69 1017 )
jpayne@69 1018
jpayne@69 1019 self.key_file = key_file
jpayne@69 1020 self.cert_file = cert_file
jpayne@69 1021 self.cert_reqs = cert_reqs
jpayne@69 1022 self.key_password = key_password
jpayne@69 1023 self.ca_certs = ca_certs
jpayne@69 1024 self.ca_cert_dir = ca_cert_dir
jpayne@69 1025 self.ssl_version = ssl_version
jpayne@69 1026 self.ssl_minimum_version = ssl_minimum_version
jpayne@69 1027 self.ssl_maximum_version = ssl_maximum_version
jpayne@69 1028 self.assert_hostname = assert_hostname
jpayne@69 1029 self.assert_fingerprint = assert_fingerprint
jpayne@69 1030
jpayne@69 1031 def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override]
jpayne@69 1032 """Establishes a tunnel connection through HTTP CONNECT."""
jpayne@69 1033 if self.proxy and self.proxy.scheme == "https":
jpayne@69 1034 tunnel_scheme = "https"
jpayne@69 1035 else:
jpayne@69 1036 tunnel_scheme = "http"
jpayne@69 1037
jpayne@69 1038 conn.set_tunnel(
jpayne@69 1039 scheme=tunnel_scheme,
jpayne@69 1040 host=self._tunnel_host,
jpayne@69 1041 port=self.port,
jpayne@69 1042 headers=self.proxy_headers,
jpayne@69 1043 )
jpayne@69 1044 conn.connect()
jpayne@69 1045
jpayne@69 1046 def _new_conn(self) -> BaseHTTPSConnection:
jpayne@69 1047 """
jpayne@69 1048 Return a fresh :class:`urllib3.connection.HTTPConnection`.
jpayne@69 1049 """
jpayne@69 1050 self.num_connections += 1
jpayne@69 1051 log.debug(
jpayne@69 1052 "Starting new HTTPS connection (%d): %s:%s",
jpayne@69 1053 self.num_connections,
jpayne@69 1054 self.host,
jpayne@69 1055 self.port or "443",
jpayne@69 1056 )
jpayne@69 1057
jpayne@69 1058 if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap]
jpayne@69 1059 raise ImportError(
jpayne@69 1060 "Can't connect to HTTPS URL because the SSL module is not available."
jpayne@69 1061 )
jpayne@69 1062
jpayne@69 1063 actual_host: str = self.host
jpayne@69 1064 actual_port = self.port
jpayne@69 1065 if self.proxy is not None and self.proxy.host is not None:
jpayne@69 1066 actual_host = self.proxy.host
jpayne@69 1067 actual_port = self.proxy.port
jpayne@69 1068
jpayne@69 1069 return self.ConnectionCls(
jpayne@69 1070 host=actual_host,
jpayne@69 1071 port=actual_port,
jpayne@69 1072 timeout=self.timeout.connect_timeout,
jpayne@69 1073 cert_file=self.cert_file,
jpayne@69 1074 key_file=self.key_file,
jpayne@69 1075 key_password=self.key_password,
jpayne@69 1076 cert_reqs=self.cert_reqs,
jpayne@69 1077 ca_certs=self.ca_certs,
jpayne@69 1078 ca_cert_dir=self.ca_cert_dir,
jpayne@69 1079 assert_hostname=self.assert_hostname,
jpayne@69 1080 assert_fingerprint=self.assert_fingerprint,
jpayne@69 1081 ssl_version=self.ssl_version,
jpayne@69 1082 ssl_minimum_version=self.ssl_minimum_version,
jpayne@69 1083 ssl_maximum_version=self.ssl_maximum_version,
jpayne@69 1084 **self.conn_kw,
jpayne@69 1085 )
jpayne@69 1086
jpayne@69 1087 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
jpayne@69 1088 """
jpayne@69 1089 Called right before a request is made, after the socket is created.
jpayne@69 1090 """
jpayne@69 1091 super()._validate_conn(conn)
jpayne@69 1092
jpayne@69 1093 # Force connect early to allow us to validate the connection.
jpayne@69 1094 if conn.is_closed:
jpayne@69 1095 conn.connect()
jpayne@69 1096
jpayne@69 1097 # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
jpayne@69 1098 if not conn.is_verified and not conn.proxy_is_verified:
jpayne@69 1099 warnings.warn(
jpayne@69 1100 (
jpayne@69 1101 f"Unverified HTTPS request is being made to host '{conn.host}'. "
jpayne@69 1102 "Adding certificate verification is strongly advised. See: "
jpayne@69 1103 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
jpayne@69 1104 "#tls-warnings"
jpayne@69 1105 ),
jpayne@69 1106 InsecureRequestWarning,
jpayne@69 1107 )
jpayne@69 1108
jpayne@69 1109
jpayne@69 1110 def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:
jpayne@69 1111 """
jpayne@69 1112 Given a url, return an :class:`.ConnectionPool` instance of its host.
jpayne@69 1113
jpayne@69 1114 This is a shortcut for not having to parse out the scheme, host, and port
jpayne@69 1115 of the url before creating an :class:`.ConnectionPool` instance.
jpayne@69 1116
jpayne@69 1117 :param url:
jpayne@69 1118 Absolute URL string that must include the scheme. Port is optional.
jpayne@69 1119
jpayne@69 1120 :param \\**kw:
jpayne@69 1121 Passes additional parameters to the constructor of the appropriate
jpayne@69 1122 :class:`.ConnectionPool`. Useful for specifying things like
jpayne@69 1123 timeout, maxsize, headers, etc.
jpayne@69 1124
jpayne@69 1125 Example::
jpayne@69 1126
jpayne@69 1127 >>> conn = connection_from_url('http://google.com/')
jpayne@69 1128 >>> r = conn.request('GET', '/')
jpayne@69 1129 """
jpayne@69 1130 scheme, _, host, port, *_ = parse_url(url)
jpayne@69 1131 scheme = scheme or "http"
jpayne@69 1132 port = port or port_by_scheme.get(scheme, 80)
jpayne@69 1133 if scheme == "https":
jpayne@69 1134 return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
jpayne@69 1135 else:
jpayne@69 1136 return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
jpayne@69 1137
jpayne@69 1138
jpayne@69 1139 @typing.overload
jpayne@69 1140 def _normalize_host(host: None, scheme: str | None) -> None:
jpayne@69 1141 ...
jpayne@69 1142
jpayne@69 1143
jpayne@69 1144 @typing.overload
jpayne@69 1145 def _normalize_host(host: str, scheme: str | None) -> str:
jpayne@69 1146 ...
jpayne@69 1147
jpayne@69 1148
jpayne@69 1149 def _normalize_host(host: str | None, scheme: str | None) -> str | None:
jpayne@69 1150 """
jpayne@69 1151 Normalize hosts for comparisons and use with sockets.
jpayne@69 1152 """
jpayne@69 1153
jpayne@69 1154 host = normalize_host(host, scheme)
jpayne@69 1155
jpayne@69 1156 # httplib doesn't like it when we include brackets in IPv6 addresses
jpayne@69 1157 # Specifically, if we include brackets but also pass the port then
jpayne@69 1158 # httplib crazily doubles up the square brackets on the Host header.
jpayne@69 1159 # Instead, we need to make sure we never pass ``None`` as the port.
jpayne@69 1160 # However, for backward compatibility reasons we can't actually
jpayne@69 1161 # *assert* that. See http://bugs.python.org/issue28539
jpayne@69 1162 if host and host.startswith("[") and host.endswith("]"):
jpayne@69 1163 host = host[1:-1]
jpayne@69 1164 return host
jpayne@69 1165
jpayne@69 1166
jpayne@69 1167 def _url_from_pool(
jpayne@69 1168 pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None
jpayne@69 1169 ) -> str:
jpayne@69 1170 """Returns the URL from a given connection pool. This is mainly used for testing and logging."""
jpayne@69 1171 return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url
jpayne@69 1172
jpayne@69 1173
jpayne@69 1174 def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:
jpayne@69 1175 """Drains a queue of connections and closes each one."""
jpayne@69 1176 try:
jpayne@69 1177 while True:
jpayne@69 1178 conn = pool.get(block=False)
jpayne@69 1179 if conn:
jpayne@69 1180 conn.close()
jpayne@69 1181 except queue.Empty:
jpayne@69 1182 pass # Done.