Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/site-packages/urllib3/connectionpool.py @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 from __future__ import annotations
jpayne@68 2
jpayne@68 3 import errno
jpayne@68 4 import logging
jpayne@68 5 import queue
jpayne@68 6 import sys
jpayne@68 7 import typing
jpayne@68 8 import warnings
jpayne@68 9 import weakref
jpayne@68 10 from socket import timeout as SocketTimeout
jpayne@68 11 from types import TracebackType
jpayne@68 12
jpayne@68 13 from ._base_connection import _TYPE_BODY
jpayne@68 14 from ._collections import HTTPHeaderDict
jpayne@68 15 from ._request_methods import RequestMethods
jpayne@68 16 from .connection import (
jpayne@68 17 BaseSSLError,
jpayne@68 18 BrokenPipeError,
jpayne@68 19 DummyConnection,
jpayne@68 20 HTTPConnection,
jpayne@68 21 HTTPException,
jpayne@68 22 HTTPSConnection,
jpayne@68 23 ProxyConfig,
jpayne@68 24 _wrap_proxy_error,
jpayne@68 25 )
jpayne@68 26 from .connection import port_by_scheme as port_by_scheme
jpayne@68 27 from .exceptions import (
jpayne@68 28 ClosedPoolError,
jpayne@68 29 EmptyPoolError,
jpayne@68 30 FullPoolError,
jpayne@68 31 HostChangedError,
jpayne@68 32 InsecureRequestWarning,
jpayne@68 33 LocationValueError,
jpayne@68 34 MaxRetryError,
jpayne@68 35 NewConnectionError,
jpayne@68 36 ProtocolError,
jpayne@68 37 ProxyError,
jpayne@68 38 ReadTimeoutError,
jpayne@68 39 SSLError,
jpayne@68 40 TimeoutError,
jpayne@68 41 )
jpayne@68 42 from .response import BaseHTTPResponse
jpayne@68 43 from .util.connection import is_connection_dropped
jpayne@68 44 from .util.proxy import connection_requires_http_tunnel
jpayne@68 45 from .util.request import _TYPE_BODY_POSITION, set_file_position
jpayne@68 46 from .util.retry import Retry
jpayne@68 47 from .util.ssl_match_hostname import CertificateError
jpayne@68 48 from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_DEFAULT, Timeout
jpayne@68 49 from .util.url import Url, _encode_target
jpayne@68 50 from .util.url import _normalize_host as normalize_host
jpayne@68 51 from .util.url import parse_url
jpayne@68 52 from .util.util import to_str
jpayne@68 53
jpayne@68 54 if typing.TYPE_CHECKING:
jpayne@68 55 import ssl
jpayne@68 56
jpayne@68 57 from typing_extensions import Self
jpayne@68 58
jpayne@68 59 from ._base_connection import BaseHTTPConnection, BaseHTTPSConnection
jpayne@68 60
jpayne@68 61 log = logging.getLogger(__name__)
jpayne@68 62
jpayne@68 63 _TYPE_TIMEOUT = typing.Union[Timeout, float, _TYPE_DEFAULT, None]
jpayne@68 64
jpayne@68 65
jpayne@68 66 # Pool objects
jpayne@68 67 class ConnectionPool:
jpayne@68 68 """
jpayne@68 69 Base class for all connection pools, such as
jpayne@68 70 :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
jpayne@68 71
jpayne@68 72 .. note::
jpayne@68 73 ConnectionPool.urlopen() does not normalize or percent-encode target URIs
jpayne@68 74 which is useful if your target server doesn't support percent-encoded
jpayne@68 75 target URIs.
jpayne@68 76 """
jpayne@68 77
jpayne@68 78 scheme: str | None = None
jpayne@68 79 QueueCls = queue.LifoQueue
jpayne@68 80
jpayne@68 81 def __init__(self, host: str, port: int | None = None) -> None:
jpayne@68 82 if not host:
jpayne@68 83 raise LocationValueError("No host specified.")
jpayne@68 84
jpayne@68 85 self.host = _normalize_host(host, scheme=self.scheme)
jpayne@68 86 self.port = port
jpayne@68 87
jpayne@68 88 # This property uses 'normalize_host()' (not '_normalize_host()')
jpayne@68 89 # to avoid removing square braces around IPv6 addresses.
jpayne@68 90 # This value is sent to `HTTPConnection.set_tunnel()` if called
jpayne@68 91 # because square braces are required for HTTP CONNECT tunneling.
jpayne@68 92 self._tunnel_host = normalize_host(host, scheme=self.scheme).lower()
jpayne@68 93
jpayne@68 94 def __str__(self) -> str:
jpayne@68 95 return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
jpayne@68 96
jpayne@68 97 def __enter__(self) -> Self:
jpayne@68 98 return self
jpayne@68 99
jpayne@68 100 def __exit__(
jpayne@68 101 self,
jpayne@68 102 exc_type: type[BaseException] | None,
jpayne@68 103 exc_val: BaseException | None,
jpayne@68 104 exc_tb: TracebackType | None,
jpayne@68 105 ) -> typing.Literal[False]:
jpayne@68 106 self.close()
jpayne@68 107 # Return False to re-raise any potential exceptions
jpayne@68 108 return False
jpayne@68 109
jpayne@68 110 def close(self) -> None:
jpayne@68 111 """
jpayne@68 112 Close all pooled connections and disable the pool.
jpayne@68 113 """
jpayne@68 114
jpayne@68 115
jpayne@68 116 # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
jpayne@68 117 _blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK}
jpayne@68 118
jpayne@68 119
jpayne@68 120 class HTTPConnectionPool(ConnectionPool, RequestMethods):
jpayne@68 121 """
jpayne@68 122 Thread-safe connection pool for one host.
jpayne@68 123
jpayne@68 124 :param host:
jpayne@68 125 Host used for this HTTP Connection (e.g. "localhost"), passed into
jpayne@68 126 :class:`http.client.HTTPConnection`.
jpayne@68 127
jpayne@68 128 :param port:
jpayne@68 129 Port used for this HTTP Connection (None is equivalent to 80), passed
jpayne@68 130 into :class:`http.client.HTTPConnection`.
jpayne@68 131
jpayne@68 132 :param timeout:
jpayne@68 133 Socket timeout in seconds for each individual connection. This can
jpayne@68 134 be a float or integer, which sets the timeout for the HTTP request,
jpayne@68 135 or an instance of :class:`urllib3.util.Timeout` which gives you more
jpayne@68 136 fine-grained control over request timeouts. After the constructor has
jpayne@68 137 been parsed, this is always a `urllib3.util.Timeout` object.
jpayne@68 138
jpayne@68 139 :param maxsize:
jpayne@68 140 Number of connections to save that can be reused. More than 1 is useful
jpayne@68 141 in multithreaded situations. If ``block`` is set to False, more
jpayne@68 142 connections will be created but they will not be saved once they've
jpayne@68 143 been used.
jpayne@68 144
jpayne@68 145 :param block:
jpayne@68 146 If set to True, no more than ``maxsize`` connections will be used at
jpayne@68 147 a time. When no free connections are available, the call will block
jpayne@68 148 until a connection has been released. This is a useful side effect for
jpayne@68 149 particular multithreaded situations where one does not want to use more
jpayne@68 150 than maxsize connections per host to prevent flooding.
jpayne@68 151
jpayne@68 152 :param headers:
jpayne@68 153 Headers to include with all requests, unless other headers are given
jpayne@68 154 explicitly.
jpayne@68 155
jpayne@68 156 :param retries:
jpayne@68 157 Retry configuration to use by default with requests in this pool.
jpayne@68 158
jpayne@68 159 :param _proxy:
jpayne@68 160 Parsed proxy URL, should not be used directly, instead, see
jpayne@68 161 :class:`urllib3.ProxyManager`
jpayne@68 162
jpayne@68 163 :param _proxy_headers:
jpayne@68 164 A dictionary with proxy headers, should not be used directly,
jpayne@68 165 instead, see :class:`urllib3.ProxyManager`
jpayne@68 166
jpayne@68 167 :param \\**conn_kw:
jpayne@68 168 Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
jpayne@68 169 :class:`urllib3.connection.HTTPSConnection` instances.
jpayne@68 170 """
jpayne@68 171
jpayne@68 172 scheme = "http"
jpayne@68 173 ConnectionCls: (
jpayne@68 174 type[BaseHTTPConnection] | type[BaseHTTPSConnection]
jpayne@68 175 ) = HTTPConnection
jpayne@68 176
jpayne@68 177 def __init__(
jpayne@68 178 self,
jpayne@68 179 host: str,
jpayne@68 180 port: int | None = None,
jpayne@68 181 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
jpayne@68 182 maxsize: int = 1,
jpayne@68 183 block: bool = False,
jpayne@68 184 headers: typing.Mapping[str, str] | None = None,
jpayne@68 185 retries: Retry | bool | int | None = None,
jpayne@68 186 _proxy: Url | None = None,
jpayne@68 187 _proxy_headers: typing.Mapping[str, str] | None = None,
jpayne@68 188 _proxy_config: ProxyConfig | None = None,
jpayne@68 189 **conn_kw: typing.Any,
jpayne@68 190 ):
jpayne@68 191 ConnectionPool.__init__(self, host, port)
jpayne@68 192 RequestMethods.__init__(self, headers)
jpayne@68 193
jpayne@68 194 if not isinstance(timeout, Timeout):
jpayne@68 195 timeout = Timeout.from_float(timeout)
jpayne@68 196
jpayne@68 197 if retries is None:
jpayne@68 198 retries = Retry.DEFAULT
jpayne@68 199
jpayne@68 200 self.timeout = timeout
jpayne@68 201 self.retries = retries
jpayne@68 202
jpayne@68 203 self.pool: queue.LifoQueue[typing.Any] | None = self.QueueCls(maxsize)
jpayne@68 204 self.block = block
jpayne@68 205
jpayne@68 206 self.proxy = _proxy
jpayne@68 207 self.proxy_headers = _proxy_headers or {}
jpayne@68 208 self.proxy_config = _proxy_config
jpayne@68 209
jpayne@68 210 # Fill the queue up so that doing get() on it will block properly
jpayne@68 211 for _ in range(maxsize):
jpayne@68 212 self.pool.put(None)
jpayne@68 213
jpayne@68 214 # These are mostly for testing and debugging purposes.
jpayne@68 215 self.num_connections = 0
jpayne@68 216 self.num_requests = 0
jpayne@68 217 self.conn_kw = conn_kw
jpayne@68 218
jpayne@68 219 if self.proxy:
jpayne@68 220 # Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
jpayne@68 221 # We cannot know if the user has added default socket options, so we cannot replace the
jpayne@68 222 # list.
jpayne@68 223 self.conn_kw.setdefault("socket_options", [])
jpayne@68 224
jpayne@68 225 self.conn_kw["proxy"] = self.proxy
jpayne@68 226 self.conn_kw["proxy_config"] = self.proxy_config
jpayne@68 227
jpayne@68 228 # Do not pass 'self' as callback to 'finalize'.
jpayne@68 229 # Then the 'finalize' would keep an endless living (leak) to self.
jpayne@68 230 # By just passing a reference to the pool allows the garbage collector
jpayne@68 231 # to free self if nobody else has a reference to it.
jpayne@68 232 pool = self.pool
jpayne@68 233
jpayne@68 234 # Close all the HTTPConnections in the pool before the
jpayne@68 235 # HTTPConnectionPool object is garbage collected.
jpayne@68 236 weakref.finalize(self, _close_pool_connections, pool)
jpayne@68 237
jpayne@68 238 def _new_conn(self) -> BaseHTTPConnection:
jpayne@68 239 """
jpayne@68 240 Return a fresh :class:`HTTPConnection`.
jpayne@68 241 """
jpayne@68 242 self.num_connections += 1
jpayne@68 243 log.debug(
jpayne@68 244 "Starting new HTTP connection (%d): %s:%s",
jpayne@68 245 self.num_connections,
jpayne@68 246 self.host,
jpayne@68 247 self.port or "80",
jpayne@68 248 )
jpayne@68 249
jpayne@68 250 conn = self.ConnectionCls(
jpayne@68 251 host=self.host,
jpayne@68 252 port=self.port,
jpayne@68 253 timeout=self.timeout.connect_timeout,
jpayne@68 254 **self.conn_kw,
jpayne@68 255 )
jpayne@68 256 return conn
jpayne@68 257
jpayne@68 258 def _get_conn(self, timeout: float | None = None) -> BaseHTTPConnection:
jpayne@68 259 """
jpayne@68 260 Get a connection. Will return a pooled connection if one is available.
jpayne@68 261
jpayne@68 262 If no connections are available and :prop:`.block` is ``False``, then a
jpayne@68 263 fresh connection is returned.
jpayne@68 264
jpayne@68 265 :param timeout:
jpayne@68 266 Seconds to wait before giving up and raising
jpayne@68 267 :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and
jpayne@68 268 :prop:`.block` is ``True``.
jpayne@68 269 """
jpayne@68 270 conn = None
jpayne@68 271
jpayne@68 272 if self.pool is None:
jpayne@68 273 raise ClosedPoolError(self, "Pool is closed.")
jpayne@68 274
jpayne@68 275 try:
jpayne@68 276 conn = self.pool.get(block=self.block, timeout=timeout)
jpayne@68 277
jpayne@68 278 except AttributeError: # self.pool is None
jpayne@68 279 raise ClosedPoolError(self, "Pool is closed.") from None # Defensive:
jpayne@68 280
jpayne@68 281 except queue.Empty:
jpayne@68 282 if self.block:
jpayne@68 283 raise EmptyPoolError(
jpayne@68 284 self,
jpayne@68 285 "Pool is empty and a new connection can't be opened due to blocking mode.",
jpayne@68 286 ) from None
jpayne@68 287 pass # Oh well, we'll create a new connection then
jpayne@68 288
jpayne@68 289 # If this is a persistent connection, check if it got disconnected
jpayne@68 290 if conn and is_connection_dropped(conn):
jpayne@68 291 log.debug("Resetting dropped connection: %s", self.host)
jpayne@68 292 conn.close()
jpayne@68 293
jpayne@68 294 return conn or self._new_conn()
jpayne@68 295
jpayne@68 296 def _put_conn(self, conn: BaseHTTPConnection | None) -> None:
jpayne@68 297 """
jpayne@68 298 Put a connection back into the pool.
jpayne@68 299
jpayne@68 300 :param conn:
jpayne@68 301 Connection object for the current host and port as returned by
jpayne@68 302 :meth:`._new_conn` or :meth:`._get_conn`.
jpayne@68 303
jpayne@68 304 If the pool is already full, the connection is closed and discarded
jpayne@68 305 because we exceeded maxsize. If connections are discarded frequently,
jpayne@68 306 then maxsize should be increased.
jpayne@68 307
jpayne@68 308 If the pool is closed, then the connection will be closed and discarded.
jpayne@68 309 """
jpayne@68 310 if self.pool is not None:
jpayne@68 311 try:
jpayne@68 312 self.pool.put(conn, block=False)
jpayne@68 313 return # Everything is dandy, done.
jpayne@68 314 except AttributeError:
jpayne@68 315 # self.pool is None.
jpayne@68 316 pass
jpayne@68 317 except queue.Full:
jpayne@68 318 # Connection never got put back into the pool, close it.
jpayne@68 319 if conn:
jpayne@68 320 conn.close()
jpayne@68 321
jpayne@68 322 if self.block:
jpayne@68 323 # This should never happen if you got the conn from self._get_conn
jpayne@68 324 raise FullPoolError(
jpayne@68 325 self,
jpayne@68 326 "Pool reached maximum size and no more connections are allowed.",
jpayne@68 327 ) from None
jpayne@68 328
jpayne@68 329 log.warning(
jpayne@68 330 "Connection pool is full, discarding connection: %s. Connection pool size: %s",
jpayne@68 331 self.host,
jpayne@68 332 self.pool.qsize(),
jpayne@68 333 )
jpayne@68 334
jpayne@68 335 # Connection never got put back into the pool, close it.
jpayne@68 336 if conn:
jpayne@68 337 conn.close()
jpayne@68 338
jpayne@68 339 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
jpayne@68 340 """
jpayne@68 341 Called right before a request is made, after the socket is created.
jpayne@68 342 """
jpayne@68 343
jpayne@68 344 def _prepare_proxy(self, conn: BaseHTTPConnection) -> None:
jpayne@68 345 # Nothing to do for HTTP connections.
jpayne@68 346 pass
jpayne@68 347
jpayne@68 348 def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
jpayne@68 349 """Helper that always returns a :class:`urllib3.util.Timeout`"""
jpayne@68 350 if timeout is _DEFAULT_TIMEOUT:
jpayne@68 351 return self.timeout.clone()
jpayne@68 352
jpayne@68 353 if isinstance(timeout, Timeout):
jpayne@68 354 return timeout.clone()
jpayne@68 355 else:
jpayne@68 356 # User passed us an int/float. This is for backwards compatibility,
jpayne@68 357 # can be removed later
jpayne@68 358 return Timeout.from_float(timeout)
jpayne@68 359
jpayne@68 360 def _raise_timeout(
jpayne@68 361 self,
jpayne@68 362 err: BaseSSLError | OSError | SocketTimeout,
jpayne@68 363 url: str,
jpayne@68 364 timeout_value: _TYPE_TIMEOUT | None,
jpayne@68 365 ) -> None:
jpayne@68 366 """Is the error actually a timeout? Will raise a ReadTimeout or pass"""
jpayne@68 367
jpayne@68 368 if isinstance(err, SocketTimeout):
jpayne@68 369 raise ReadTimeoutError(
jpayne@68 370 self, url, f"Read timed out. (read timeout={timeout_value})"
jpayne@68 371 ) from err
jpayne@68 372
jpayne@68 373 # See the above comment about EAGAIN in Python 3.
jpayne@68 374 if hasattr(err, "errno") and err.errno in _blocking_errnos:
jpayne@68 375 raise ReadTimeoutError(
jpayne@68 376 self, url, f"Read timed out. (read timeout={timeout_value})"
jpayne@68 377 ) from err
jpayne@68 378
jpayne@68 379 def _make_request(
jpayne@68 380 self,
jpayne@68 381 conn: BaseHTTPConnection,
jpayne@68 382 method: str,
jpayne@68 383 url: str,
jpayne@68 384 body: _TYPE_BODY | None = None,
jpayne@68 385 headers: typing.Mapping[str, str] | None = None,
jpayne@68 386 retries: Retry | None = None,
jpayne@68 387 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
jpayne@68 388 chunked: bool = False,
jpayne@68 389 response_conn: BaseHTTPConnection | None = None,
jpayne@68 390 preload_content: bool = True,
jpayne@68 391 decode_content: bool = True,
jpayne@68 392 enforce_content_length: bool = True,
jpayne@68 393 ) -> BaseHTTPResponse:
jpayne@68 394 """
jpayne@68 395 Perform a request on a given urllib connection object taken from our
jpayne@68 396 pool.
jpayne@68 397
jpayne@68 398 :param conn:
jpayne@68 399 a connection from one of our connection pools
jpayne@68 400
jpayne@68 401 :param method:
jpayne@68 402 HTTP request method (such as GET, POST, PUT, etc.)
jpayne@68 403
jpayne@68 404 :param url:
jpayne@68 405 The URL to perform the request on.
jpayne@68 406
jpayne@68 407 :param body:
jpayne@68 408 Data to send in the request body, either :class:`str`, :class:`bytes`,
jpayne@68 409 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
jpayne@68 410
jpayne@68 411 :param headers:
jpayne@68 412 Dictionary of custom headers to send, such as User-Agent,
jpayne@68 413 If-None-Match, etc. If None, pool headers are used. If provided,
jpayne@68 414 these headers completely replace any pool-specific headers.
jpayne@68 415
jpayne@68 416 :param retries:
jpayne@68 417 Configure the number of retries to allow before raising a
jpayne@68 418 :class:`~urllib3.exceptions.MaxRetryError` exception.
jpayne@68 419
jpayne@68 420 Pass ``None`` to retry until you receive a response. Pass a
jpayne@68 421 :class:`~urllib3.util.retry.Retry` object for fine-grained control
jpayne@68 422 over different types of retries.
jpayne@68 423 Pass an integer number to retry connection errors that many times,
jpayne@68 424 but no other types of errors. Pass zero to never retry.
jpayne@68 425
jpayne@68 426 If ``False``, then retries are disabled and any exception is raised
jpayne@68 427 immediately. Also, instead of raising a MaxRetryError on redirects,
jpayne@68 428 the redirect response will be returned.
jpayne@68 429
jpayne@68 430 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
jpayne@68 431
jpayne@68 432 :param timeout:
jpayne@68 433 If specified, overrides the default timeout for this one
jpayne@68 434 request. It may be a float (in seconds) or an instance of
jpayne@68 435 :class:`urllib3.util.Timeout`.
jpayne@68 436
jpayne@68 437 :param chunked:
jpayne@68 438 If True, urllib3 will send the body using chunked transfer
jpayne@68 439 encoding. Otherwise, urllib3 will send the body using the standard
jpayne@68 440 content-length form. Defaults to False.
jpayne@68 441
jpayne@68 442 :param response_conn:
jpayne@68 443 Set this to ``None`` if you will handle releasing the connection or
jpayne@68 444 set the connection to have the response release it.
jpayne@68 445
jpayne@68 446 :param preload_content:
jpayne@68 447 If True, the response's body will be preloaded during construction.
jpayne@68 448
jpayne@68 449 :param decode_content:
jpayne@68 450 If True, will attempt to decode the body based on the
jpayne@68 451 'content-encoding' header.
jpayne@68 452
jpayne@68 453 :param enforce_content_length:
jpayne@68 454 Enforce content length checking. Body returned by server must match
jpayne@68 455 value of Content-Length header, if present. Otherwise, raise error.
jpayne@68 456 """
jpayne@68 457 self.num_requests += 1
jpayne@68 458
jpayne@68 459 timeout_obj = self._get_timeout(timeout)
jpayne@68 460 timeout_obj.start_connect()
jpayne@68 461 conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
jpayne@68 462
jpayne@68 463 try:
jpayne@68 464 # Trigger any extra validation we need to do.
jpayne@68 465 try:
jpayne@68 466 self._validate_conn(conn)
jpayne@68 467 except (SocketTimeout, BaseSSLError) as e:
jpayne@68 468 self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
jpayne@68 469 raise
jpayne@68 470
jpayne@68 471 # _validate_conn() starts the connection to an HTTPS proxy
jpayne@68 472 # so we need to wrap errors with 'ProxyError' here too.
jpayne@68 473 except (
jpayne@68 474 OSError,
jpayne@68 475 NewConnectionError,
jpayne@68 476 TimeoutError,
jpayne@68 477 BaseSSLError,
jpayne@68 478 CertificateError,
jpayne@68 479 SSLError,
jpayne@68 480 ) as e:
jpayne@68 481 new_e: Exception = e
jpayne@68 482 if isinstance(e, (BaseSSLError, CertificateError)):
jpayne@68 483 new_e = SSLError(e)
jpayne@68 484 # If the connection didn't successfully connect to it's proxy
jpayne@68 485 # then there
jpayne@68 486 if isinstance(
jpayne@68 487 new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
jpayne@68 488 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
jpayne@68 489 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
jpayne@68 490 raise new_e
jpayne@68 491
jpayne@68 492 # conn.request() calls http.client.*.request, not the method in
jpayne@68 493 # urllib3.request. It also calls makefile (recv) on the socket.
jpayne@68 494 try:
jpayne@68 495 conn.request(
jpayne@68 496 method,
jpayne@68 497 url,
jpayne@68 498 body=body,
jpayne@68 499 headers=headers,
jpayne@68 500 chunked=chunked,
jpayne@68 501 preload_content=preload_content,
jpayne@68 502 decode_content=decode_content,
jpayne@68 503 enforce_content_length=enforce_content_length,
jpayne@68 504 )
jpayne@68 505
jpayne@68 506 # We are swallowing BrokenPipeError (errno.EPIPE) since the server is
jpayne@68 507 # legitimately able to close the connection after sending a valid response.
jpayne@68 508 # With this behaviour, the received response is still readable.
jpayne@68 509 except BrokenPipeError:
jpayne@68 510 pass
jpayne@68 511 except OSError as e:
jpayne@68 512 # MacOS/Linux
jpayne@68 513 # EPROTOTYPE and ECONNRESET are needed on macOS
jpayne@68 514 # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
jpayne@68 515 # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
jpayne@68 516 if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
jpayne@68 517 raise
jpayne@68 518
jpayne@68 519 # Reset the timeout for the recv() on the socket
jpayne@68 520 read_timeout = timeout_obj.read_timeout
jpayne@68 521
jpayne@68 522 if not conn.is_closed:
jpayne@68 523 # In Python 3 socket.py will catch EAGAIN and return None when you
jpayne@68 524 # try and read into the file pointer created by http.client, which
jpayne@68 525 # instead raises a BadStatusLine exception. Instead of catching
jpayne@68 526 # the exception and assuming all BadStatusLine exceptions are read
jpayne@68 527 # timeouts, check for a zero timeout before making the request.
jpayne@68 528 if read_timeout == 0:
jpayne@68 529 raise ReadTimeoutError(
jpayne@68 530 self, url, f"Read timed out. (read timeout={read_timeout})"
jpayne@68 531 )
jpayne@68 532 conn.timeout = read_timeout
jpayne@68 533
jpayne@68 534 # Receive the response from the server
jpayne@68 535 try:
jpayne@68 536 response = conn.getresponse()
jpayne@68 537 except (BaseSSLError, OSError) as e:
jpayne@68 538 self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
jpayne@68 539 raise
jpayne@68 540
jpayne@68 541 # Set properties that are used by the pooling layer.
jpayne@68 542 response.retries = retries
jpayne@68 543 response._connection = response_conn # type: ignore[attr-defined]
jpayne@68 544 response._pool = self # type: ignore[attr-defined]
jpayne@68 545
jpayne@68 546 log.debug(
jpayne@68 547 '%s://%s:%s "%s %s HTTP/%s" %s %s',
jpayne@68 548 self.scheme,
jpayne@68 549 self.host,
jpayne@68 550 self.port,
jpayne@68 551 method,
jpayne@68 552 url,
jpayne@68 553 response.version,
jpayne@68 554 response.status,
jpayne@68 555 response.length_remaining,
jpayne@68 556 )
jpayne@68 557
jpayne@68 558 return response
jpayne@68 559
jpayne@68 560 def close(self) -> None:
jpayne@68 561 """
jpayne@68 562 Close all pooled connections and disable the pool.
jpayne@68 563 """
jpayne@68 564 if self.pool is None:
jpayne@68 565 return
jpayne@68 566 # Disable access to the pool
jpayne@68 567 old_pool, self.pool = self.pool, None
jpayne@68 568
jpayne@68 569 # Close all the HTTPConnections in the pool.
jpayne@68 570 _close_pool_connections(old_pool)
jpayne@68 571
jpayne@68 572 def is_same_host(self, url: str) -> bool:
jpayne@68 573 """
jpayne@68 574 Check if the given ``url`` is a member of the same host as this
jpayne@68 575 connection pool.
jpayne@68 576 """
jpayne@68 577 if url.startswith("/"):
jpayne@68 578 return True
jpayne@68 579
jpayne@68 580 # TODO: Add optional support for socket.gethostbyname checking.
jpayne@68 581 scheme, _, host, port, *_ = parse_url(url)
jpayne@68 582 scheme = scheme or "http"
jpayne@68 583 if host is not None:
jpayne@68 584 host = _normalize_host(host, scheme=scheme)
jpayne@68 585
jpayne@68 586 # Use explicit default port for comparison when none is given
jpayne@68 587 if self.port and not port:
jpayne@68 588 port = port_by_scheme.get(scheme)
jpayne@68 589 elif not self.port and port == port_by_scheme.get(scheme):
jpayne@68 590 port = None
jpayne@68 591
jpayne@68 592 return (scheme, host, port) == (self.scheme, self.host, self.port)
jpayne@68 593
jpayne@68 594 def urlopen( # type: ignore[override]
jpayne@68 595 self,
jpayne@68 596 method: str,
jpayne@68 597 url: str,
jpayne@68 598 body: _TYPE_BODY | None = None,
jpayne@68 599 headers: typing.Mapping[str, str] | None = None,
jpayne@68 600 retries: Retry | bool | int | None = None,
jpayne@68 601 redirect: bool = True,
jpayne@68 602 assert_same_host: bool = True,
jpayne@68 603 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
jpayne@68 604 pool_timeout: int | None = None,
jpayne@68 605 release_conn: bool | None = None,
jpayne@68 606 chunked: bool = False,
jpayne@68 607 body_pos: _TYPE_BODY_POSITION | None = None,
jpayne@68 608 preload_content: bool = True,
jpayne@68 609 decode_content: bool = True,
jpayne@68 610 **response_kw: typing.Any,
jpayne@68 611 ) -> BaseHTTPResponse:
jpayne@68 612 """
jpayne@68 613 Get a connection from the pool and perform an HTTP request. This is the
jpayne@68 614 lowest level call for making a request, so you'll need to specify all
jpayne@68 615 the raw details.
jpayne@68 616
jpayne@68 617 .. note::
jpayne@68 618
jpayne@68 619 More commonly, it's appropriate to use a convenience method
jpayne@68 620 such as :meth:`request`.
jpayne@68 621
jpayne@68 622 .. note::
jpayne@68 623
jpayne@68 624 `release_conn` will only behave as expected if
jpayne@68 625 `preload_content=False` because we want to make
jpayne@68 626 `preload_content=False` the default behaviour someday soon without
jpayne@68 627 breaking backwards compatibility.
jpayne@68 628
jpayne@68 629 :param method:
jpayne@68 630 HTTP request method (such as GET, POST, PUT, etc.)
jpayne@68 631
jpayne@68 632 :param url:
jpayne@68 633 The URL to perform the request on.
jpayne@68 634
jpayne@68 635 :param body:
jpayne@68 636 Data to send in the request body, either :class:`str`, :class:`bytes`,
jpayne@68 637 an iterable of :class:`str`/:class:`bytes`, or a file-like object.
jpayne@68 638
jpayne@68 639 :param headers:
jpayne@68 640 Dictionary of custom headers to send, such as User-Agent,
jpayne@68 641 If-None-Match, etc. If None, pool headers are used. If provided,
jpayne@68 642 these headers completely replace any pool-specific headers.
jpayne@68 643
jpayne@68 644 :param retries:
jpayne@68 645 Configure the number of retries to allow before raising a
jpayne@68 646 :class:`~urllib3.exceptions.MaxRetryError` exception.
jpayne@68 647
jpayne@68 648 If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
jpayne@68 649 :class:`~urllib3.util.retry.Retry` object for fine-grained control
jpayne@68 650 over different types of retries.
jpayne@68 651 Pass an integer number to retry connection errors that many times,
jpayne@68 652 but no other types of errors. Pass zero to never retry.
jpayne@68 653
jpayne@68 654 If ``False``, then retries are disabled and any exception is raised
jpayne@68 655 immediately. Also, instead of raising a MaxRetryError on redirects,
jpayne@68 656 the redirect response will be returned.
jpayne@68 657
jpayne@68 658 :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
jpayne@68 659
jpayne@68 660 :param redirect:
jpayne@68 661 If True, automatically handle redirects (status codes 301, 302,
jpayne@68 662 303, 307, 308). Each redirect counts as a retry. Disabling retries
jpayne@68 663 will disable redirect, too.
jpayne@68 664
jpayne@68 665 :param assert_same_host:
jpayne@68 666 If ``True``, will make sure that the host of the pool requests is
jpayne@68 667 consistent else will raise HostChangedError. When ``False``, you can
jpayne@68 668 use the pool on an HTTP proxy and request foreign hosts.
jpayne@68 669
jpayne@68 670 :param timeout:
jpayne@68 671 If specified, overrides the default timeout for this one
jpayne@68 672 request. It may be a float (in seconds) or an instance of
jpayne@68 673 :class:`urllib3.util.Timeout`.
jpayne@68 674
jpayne@68 675 :param pool_timeout:
jpayne@68 676 If set and the pool is set to block=True, then this method will
jpayne@68 677 block for ``pool_timeout`` seconds and raise EmptyPoolError if no
jpayne@68 678 connection is available within the time period.
jpayne@68 679
jpayne@68 680 :param bool preload_content:
jpayne@68 681 If True, the response's body will be preloaded into memory.
jpayne@68 682
jpayne@68 683 :param bool decode_content:
jpayne@68 684 If True, will attempt to decode the body based on the
jpayne@68 685 'content-encoding' header.
jpayne@68 686
jpayne@68 687 :param release_conn:
jpayne@68 688 If False, then the urlopen call will not release the connection
jpayne@68 689 back into the pool once a response is received (but will release if
jpayne@68 690 you read the entire contents of the response such as when
jpayne@68 691 `preload_content=True`). This is useful if you're not preloading
jpayne@68 692 the response's content immediately. You will need to call
jpayne@68 693 ``r.release_conn()`` on the response ``r`` to return the connection
jpayne@68 694 back into the pool. If None, it takes the value of ``preload_content``
jpayne@68 695 which defaults to ``True``.
jpayne@68 696
jpayne@68 697 :param bool chunked:
jpayne@68 698 If True, urllib3 will send the body using chunked transfer
jpayne@68 699 encoding. Otherwise, urllib3 will send the body using the standard
jpayne@68 700 content-length form. Defaults to False.
jpayne@68 701
jpayne@68 702 :param int body_pos:
jpayne@68 703 Position to seek to in file-like body in the event of a retry or
jpayne@68 704 redirect. Typically this won't need to be set because urllib3 will
jpayne@68 705 auto-populate the value when needed.
jpayne@68 706 """
jpayne@68 707 parsed_url = parse_url(url)
jpayne@68 708 destination_scheme = parsed_url.scheme
jpayne@68 709
jpayne@68 710 if headers is None:
jpayne@68 711 headers = self.headers
jpayne@68 712
jpayne@68 713 if not isinstance(retries, Retry):
jpayne@68 714 retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
jpayne@68 715
jpayne@68 716 if release_conn is None:
jpayne@68 717 release_conn = preload_content
jpayne@68 718
jpayne@68 719 # Check host
jpayne@68 720 if assert_same_host and not self.is_same_host(url):
jpayne@68 721 raise HostChangedError(self, url, retries)
jpayne@68 722
jpayne@68 723 # Ensure that the URL we're connecting to is properly encoded
jpayne@68 724 if url.startswith("/"):
jpayne@68 725 url = to_str(_encode_target(url))
jpayne@68 726 else:
jpayne@68 727 url = to_str(parsed_url.url)
jpayne@68 728
jpayne@68 729 conn = None
jpayne@68 730
jpayne@68 731 # Track whether `conn` needs to be released before
jpayne@68 732 # returning/raising/recursing. Update this variable if necessary, and
jpayne@68 733 # leave `release_conn` constant throughout the function. That way, if
jpayne@68 734 # the function recurses, the original value of `release_conn` will be
jpayne@68 735 # passed down into the recursive call, and its value will be respected.
jpayne@68 736 #
jpayne@68 737 # See issue #651 [1] for details.
jpayne@68 738 #
jpayne@68 739 # [1] <https://github.com/urllib3/urllib3/issues/651>
jpayne@68 740 release_this_conn = release_conn
jpayne@68 741
jpayne@68 742 http_tunnel_required = connection_requires_http_tunnel(
jpayne@68 743 self.proxy, self.proxy_config, destination_scheme
jpayne@68 744 )
jpayne@68 745
jpayne@68 746 # Merge the proxy headers. Only done when not using HTTP CONNECT. We
jpayne@68 747 # have to copy the headers dict so we can safely change it without those
jpayne@68 748 # changes being reflected in anyone else's copy.
jpayne@68 749 if not http_tunnel_required:
jpayne@68 750 headers = headers.copy() # type: ignore[attr-defined]
jpayne@68 751 headers.update(self.proxy_headers) # type: ignore[union-attr]
jpayne@68 752
jpayne@68 753 # Must keep the exception bound to a separate variable or else Python 3
jpayne@68 754 # complains about UnboundLocalError.
jpayne@68 755 err = None
jpayne@68 756
jpayne@68 757 # Keep track of whether we cleanly exited the except block. This
jpayne@68 758 # ensures we do proper cleanup in finally.
jpayne@68 759 clean_exit = False
jpayne@68 760
jpayne@68 761 # Rewind body position, if needed. Record current position
jpayne@68 762 # for future rewinds in the event of a redirect/retry.
jpayne@68 763 body_pos = set_file_position(body, body_pos)
jpayne@68 764
jpayne@68 765 try:
jpayne@68 766 # Request a connection from the queue.
jpayne@68 767 timeout_obj = self._get_timeout(timeout)
jpayne@68 768 conn = self._get_conn(timeout=pool_timeout)
jpayne@68 769
jpayne@68 770 conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
jpayne@68 771
jpayne@68 772 # Is this a closed/new connection that requires CONNECT tunnelling?
jpayne@68 773 if self.proxy is not None and http_tunnel_required and conn.is_closed:
jpayne@68 774 try:
jpayne@68 775 self._prepare_proxy(conn)
jpayne@68 776 except (BaseSSLError, OSError, SocketTimeout) as e:
jpayne@68 777 self._raise_timeout(
jpayne@68 778 err=e, url=self.proxy.url, timeout_value=conn.timeout
jpayne@68 779 )
jpayne@68 780 raise
jpayne@68 781
jpayne@68 782 # If we're going to release the connection in ``finally:``, then
jpayne@68 783 # the response doesn't need to know about the connection. Otherwise
jpayne@68 784 # it will also try to release it and we'll have a double-release
jpayne@68 785 # mess.
jpayne@68 786 response_conn = conn if not release_conn else None
jpayne@68 787
jpayne@68 788 # Make the request on the HTTPConnection object
jpayne@68 789 response = self._make_request(
jpayne@68 790 conn,
jpayne@68 791 method,
jpayne@68 792 url,
jpayne@68 793 timeout=timeout_obj,
jpayne@68 794 body=body,
jpayne@68 795 headers=headers,
jpayne@68 796 chunked=chunked,
jpayne@68 797 retries=retries,
jpayne@68 798 response_conn=response_conn,
jpayne@68 799 preload_content=preload_content,
jpayne@68 800 decode_content=decode_content,
jpayne@68 801 **response_kw,
jpayne@68 802 )
jpayne@68 803
jpayne@68 804 # Everything went great!
jpayne@68 805 clean_exit = True
jpayne@68 806
jpayne@68 807 except EmptyPoolError:
jpayne@68 808 # Didn't get a connection from the pool, no need to clean up
jpayne@68 809 clean_exit = True
jpayne@68 810 release_this_conn = False
jpayne@68 811 raise
jpayne@68 812
jpayne@68 813 except (
jpayne@68 814 TimeoutError,
jpayne@68 815 HTTPException,
jpayne@68 816 OSError,
jpayne@68 817 ProtocolError,
jpayne@68 818 BaseSSLError,
jpayne@68 819 SSLError,
jpayne@68 820 CertificateError,
jpayne@68 821 ProxyError,
jpayne@68 822 ) as e:
jpayne@68 823 # Discard the connection for these exceptions. It will be
jpayne@68 824 # replaced during the next _get_conn() call.
jpayne@68 825 clean_exit = False
jpayne@68 826 new_e: Exception = e
jpayne@68 827 if isinstance(e, (BaseSSLError, CertificateError)):
jpayne@68 828 new_e = SSLError(e)
jpayne@68 829 if isinstance(
jpayne@68 830 new_e,
jpayne@68 831 (
jpayne@68 832 OSError,
jpayne@68 833 NewConnectionError,
jpayne@68 834 TimeoutError,
jpayne@68 835 SSLError,
jpayne@68 836 HTTPException,
jpayne@68 837 ),
jpayne@68 838 ) and (conn and conn.proxy and not conn.has_connected_to_proxy):
jpayne@68 839 new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
jpayne@68 840 elif isinstance(new_e, (OSError, HTTPException)):
jpayne@68 841 new_e = ProtocolError("Connection aborted.", new_e)
jpayne@68 842
jpayne@68 843 retries = retries.increment(
jpayne@68 844 method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
jpayne@68 845 )
jpayne@68 846 retries.sleep()
jpayne@68 847
jpayne@68 848 # Keep track of the error for the retry warning.
jpayne@68 849 err = e
jpayne@68 850
jpayne@68 851 finally:
jpayne@68 852 if not clean_exit:
jpayne@68 853 # We hit some kind of exception, handled or otherwise. We need
jpayne@68 854 # to throw the connection away unless explicitly told not to.
jpayne@68 855 # Close the connection, set the variable to None, and make sure
jpayne@68 856 # we put the None back in the pool to avoid leaking it.
jpayne@68 857 if conn:
jpayne@68 858 conn.close()
jpayne@68 859 conn = None
jpayne@68 860 release_this_conn = True
jpayne@68 861
jpayne@68 862 if release_this_conn:
jpayne@68 863 # Put the connection back to be reused. If the connection is
jpayne@68 864 # expired then it will be None, which will get replaced with a
jpayne@68 865 # fresh connection during _get_conn.
jpayne@68 866 self._put_conn(conn)
jpayne@68 867
jpayne@68 868 if not conn:
jpayne@68 869 # Try again
jpayne@68 870 log.warning(
jpayne@68 871 "Retrying (%r) after connection broken by '%r': %s", retries, err, url
jpayne@68 872 )
jpayne@68 873 return self.urlopen(
jpayne@68 874 method,
jpayne@68 875 url,
jpayne@68 876 body,
jpayne@68 877 headers,
jpayne@68 878 retries,
jpayne@68 879 redirect,
jpayne@68 880 assert_same_host,
jpayne@68 881 timeout=timeout,
jpayne@68 882 pool_timeout=pool_timeout,
jpayne@68 883 release_conn=release_conn,
jpayne@68 884 chunked=chunked,
jpayne@68 885 body_pos=body_pos,
jpayne@68 886 preload_content=preload_content,
jpayne@68 887 decode_content=decode_content,
jpayne@68 888 **response_kw,
jpayne@68 889 )
jpayne@68 890
jpayne@68 891 # Handle redirect?
jpayne@68 892 redirect_location = redirect and response.get_redirect_location()
jpayne@68 893 if redirect_location:
jpayne@68 894 if response.status == 303:
jpayne@68 895 # Change the method according to RFC 9110, Section 15.4.4.
jpayne@68 896 method = "GET"
jpayne@68 897 # And lose the body not to transfer anything sensitive.
jpayne@68 898 body = None
jpayne@68 899 headers = HTTPHeaderDict(headers)._prepare_for_method_change()
jpayne@68 900
jpayne@68 901 try:
jpayne@68 902 retries = retries.increment(method, url, response=response, _pool=self)
jpayne@68 903 except MaxRetryError:
jpayne@68 904 if retries.raise_on_redirect:
jpayne@68 905 response.drain_conn()
jpayne@68 906 raise
jpayne@68 907 return response
jpayne@68 908
jpayne@68 909 response.drain_conn()
jpayne@68 910 retries.sleep_for_retry(response)
jpayne@68 911 log.debug("Redirecting %s -> %s", url, redirect_location)
jpayne@68 912 return self.urlopen(
jpayne@68 913 method,
jpayne@68 914 redirect_location,
jpayne@68 915 body,
jpayne@68 916 headers,
jpayne@68 917 retries=retries,
jpayne@68 918 redirect=redirect,
jpayne@68 919 assert_same_host=assert_same_host,
jpayne@68 920 timeout=timeout,
jpayne@68 921 pool_timeout=pool_timeout,
jpayne@68 922 release_conn=release_conn,
jpayne@68 923 chunked=chunked,
jpayne@68 924 body_pos=body_pos,
jpayne@68 925 preload_content=preload_content,
jpayne@68 926 decode_content=decode_content,
jpayne@68 927 **response_kw,
jpayne@68 928 )
jpayne@68 929
jpayne@68 930 # Check if we should retry the HTTP response.
jpayne@68 931 has_retry_after = bool(response.headers.get("Retry-After"))
jpayne@68 932 if retries.is_retry(method, response.status, has_retry_after):
jpayne@68 933 try:
jpayne@68 934 retries = retries.increment(method, url, response=response, _pool=self)
jpayne@68 935 except MaxRetryError:
jpayne@68 936 if retries.raise_on_status:
jpayne@68 937 response.drain_conn()
jpayne@68 938 raise
jpayne@68 939 return response
jpayne@68 940
jpayne@68 941 response.drain_conn()
jpayne@68 942 retries.sleep(response)
jpayne@68 943 log.debug("Retry: %s", url)
jpayne@68 944 return self.urlopen(
jpayne@68 945 method,
jpayne@68 946 url,
jpayne@68 947 body,
jpayne@68 948 headers,
jpayne@68 949 retries=retries,
jpayne@68 950 redirect=redirect,
jpayne@68 951 assert_same_host=assert_same_host,
jpayne@68 952 timeout=timeout,
jpayne@68 953 pool_timeout=pool_timeout,
jpayne@68 954 release_conn=release_conn,
jpayne@68 955 chunked=chunked,
jpayne@68 956 body_pos=body_pos,
jpayne@68 957 preload_content=preload_content,
jpayne@68 958 decode_content=decode_content,
jpayne@68 959 **response_kw,
jpayne@68 960 )
jpayne@68 961
jpayne@68 962 return response
jpayne@68 963
jpayne@68 964
jpayne@68 965 class HTTPSConnectionPool(HTTPConnectionPool):
jpayne@68 966 """
jpayne@68 967 Same as :class:`.HTTPConnectionPool`, but HTTPS.
jpayne@68 968
jpayne@68 969 :class:`.HTTPSConnection` uses one of ``assert_fingerprint``,
jpayne@68 970 ``assert_hostname`` and ``host`` in this order to verify connections.
jpayne@68 971 If ``assert_hostname`` is False, no verification is done.
jpayne@68 972
jpayne@68 973 The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``,
jpayne@68 974 ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl`
jpayne@68 975 is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade
jpayne@68 976 the connection socket into an SSL socket.
jpayne@68 977 """
jpayne@68 978
jpayne@68 979 scheme = "https"
jpayne@68 980 ConnectionCls: type[BaseHTTPSConnection] = HTTPSConnection
jpayne@68 981
jpayne@68 982 def __init__(
jpayne@68 983 self,
jpayne@68 984 host: str,
jpayne@68 985 port: int | None = None,
jpayne@68 986 timeout: _TYPE_TIMEOUT | None = _DEFAULT_TIMEOUT,
jpayne@68 987 maxsize: int = 1,
jpayne@68 988 block: bool = False,
jpayne@68 989 headers: typing.Mapping[str, str] | None = None,
jpayne@68 990 retries: Retry | bool | int | None = None,
jpayne@68 991 _proxy: Url | None = None,
jpayne@68 992 _proxy_headers: typing.Mapping[str, str] | None = None,
jpayne@68 993 key_file: str | None = None,
jpayne@68 994 cert_file: str | None = None,
jpayne@68 995 cert_reqs: int | str | None = None,
jpayne@68 996 key_password: str | None = None,
jpayne@68 997 ca_certs: str | None = None,
jpayne@68 998 ssl_version: int | str | None = None,
jpayne@68 999 ssl_minimum_version: ssl.TLSVersion | None = None,
jpayne@68 1000 ssl_maximum_version: ssl.TLSVersion | None = None,
jpayne@68 1001 assert_hostname: str | typing.Literal[False] | None = None,
jpayne@68 1002 assert_fingerprint: str | None = None,
jpayne@68 1003 ca_cert_dir: str | None = None,
jpayne@68 1004 **conn_kw: typing.Any,
jpayne@68 1005 ) -> None:
jpayne@68 1006 super().__init__(
jpayne@68 1007 host,
jpayne@68 1008 port,
jpayne@68 1009 timeout,
jpayne@68 1010 maxsize,
jpayne@68 1011 block,
jpayne@68 1012 headers,
jpayne@68 1013 retries,
jpayne@68 1014 _proxy,
jpayne@68 1015 _proxy_headers,
jpayne@68 1016 **conn_kw,
jpayne@68 1017 )
jpayne@68 1018
jpayne@68 1019 self.key_file = key_file
jpayne@68 1020 self.cert_file = cert_file
jpayne@68 1021 self.cert_reqs = cert_reqs
jpayne@68 1022 self.key_password = key_password
jpayne@68 1023 self.ca_certs = ca_certs
jpayne@68 1024 self.ca_cert_dir = ca_cert_dir
jpayne@68 1025 self.ssl_version = ssl_version
jpayne@68 1026 self.ssl_minimum_version = ssl_minimum_version
jpayne@68 1027 self.ssl_maximum_version = ssl_maximum_version
jpayne@68 1028 self.assert_hostname = assert_hostname
jpayne@68 1029 self.assert_fingerprint = assert_fingerprint
jpayne@68 1030
jpayne@68 1031 def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore[override]
jpayne@68 1032 """Establishes a tunnel connection through HTTP CONNECT."""
jpayne@68 1033 if self.proxy and self.proxy.scheme == "https":
jpayne@68 1034 tunnel_scheme = "https"
jpayne@68 1035 else:
jpayne@68 1036 tunnel_scheme = "http"
jpayne@68 1037
jpayne@68 1038 conn.set_tunnel(
jpayne@68 1039 scheme=tunnel_scheme,
jpayne@68 1040 host=self._tunnel_host,
jpayne@68 1041 port=self.port,
jpayne@68 1042 headers=self.proxy_headers,
jpayne@68 1043 )
jpayne@68 1044 conn.connect()
jpayne@68 1045
jpayne@68 1046 def _new_conn(self) -> BaseHTTPSConnection:
jpayne@68 1047 """
jpayne@68 1048 Return a fresh :class:`urllib3.connection.HTTPConnection`.
jpayne@68 1049 """
jpayne@68 1050 self.num_connections += 1
jpayne@68 1051 log.debug(
jpayne@68 1052 "Starting new HTTPS connection (%d): %s:%s",
jpayne@68 1053 self.num_connections,
jpayne@68 1054 self.host,
jpayne@68 1055 self.port or "443",
jpayne@68 1056 )
jpayne@68 1057
jpayne@68 1058 if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore[comparison-overlap]
jpayne@68 1059 raise ImportError(
jpayne@68 1060 "Can't connect to HTTPS URL because the SSL module is not available."
jpayne@68 1061 )
jpayne@68 1062
jpayne@68 1063 actual_host: str = self.host
jpayne@68 1064 actual_port = self.port
jpayne@68 1065 if self.proxy is not None and self.proxy.host is not None:
jpayne@68 1066 actual_host = self.proxy.host
jpayne@68 1067 actual_port = self.proxy.port
jpayne@68 1068
jpayne@68 1069 return self.ConnectionCls(
jpayne@68 1070 host=actual_host,
jpayne@68 1071 port=actual_port,
jpayne@68 1072 timeout=self.timeout.connect_timeout,
jpayne@68 1073 cert_file=self.cert_file,
jpayne@68 1074 key_file=self.key_file,
jpayne@68 1075 key_password=self.key_password,
jpayne@68 1076 cert_reqs=self.cert_reqs,
jpayne@68 1077 ca_certs=self.ca_certs,
jpayne@68 1078 ca_cert_dir=self.ca_cert_dir,
jpayne@68 1079 assert_hostname=self.assert_hostname,
jpayne@68 1080 assert_fingerprint=self.assert_fingerprint,
jpayne@68 1081 ssl_version=self.ssl_version,
jpayne@68 1082 ssl_minimum_version=self.ssl_minimum_version,
jpayne@68 1083 ssl_maximum_version=self.ssl_maximum_version,
jpayne@68 1084 **self.conn_kw,
jpayne@68 1085 )
jpayne@68 1086
jpayne@68 1087 def _validate_conn(self, conn: BaseHTTPConnection) -> None:
jpayne@68 1088 """
jpayne@68 1089 Called right before a request is made, after the socket is created.
jpayne@68 1090 """
jpayne@68 1091 super()._validate_conn(conn)
jpayne@68 1092
jpayne@68 1093 # Force connect early to allow us to validate the connection.
jpayne@68 1094 if conn.is_closed:
jpayne@68 1095 conn.connect()
jpayne@68 1096
jpayne@68 1097 # TODO revise this, see https://github.com/urllib3/urllib3/issues/2791
jpayne@68 1098 if not conn.is_verified and not conn.proxy_is_verified:
jpayne@68 1099 warnings.warn(
jpayne@68 1100 (
jpayne@68 1101 f"Unverified HTTPS request is being made to host '{conn.host}'. "
jpayne@68 1102 "Adding certificate verification is strongly advised. See: "
jpayne@68 1103 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
jpayne@68 1104 "#tls-warnings"
jpayne@68 1105 ),
jpayne@68 1106 InsecureRequestWarning,
jpayne@68 1107 )
jpayne@68 1108
jpayne@68 1109
jpayne@68 1110 def connection_from_url(url: str, **kw: typing.Any) -> HTTPConnectionPool:
jpayne@68 1111 """
jpayne@68 1112 Given a url, return an :class:`.ConnectionPool` instance of its host.
jpayne@68 1113
jpayne@68 1114 This is a shortcut for not having to parse out the scheme, host, and port
jpayne@68 1115 of the url before creating an :class:`.ConnectionPool` instance.
jpayne@68 1116
jpayne@68 1117 :param url:
jpayne@68 1118 Absolute URL string that must include the scheme. Port is optional.
jpayne@68 1119
jpayne@68 1120 :param \\**kw:
jpayne@68 1121 Passes additional parameters to the constructor of the appropriate
jpayne@68 1122 :class:`.ConnectionPool`. Useful for specifying things like
jpayne@68 1123 timeout, maxsize, headers, etc.
jpayne@68 1124
jpayne@68 1125 Example::
jpayne@68 1126
jpayne@68 1127 >>> conn = connection_from_url('http://google.com/')
jpayne@68 1128 >>> r = conn.request('GET', '/')
jpayne@68 1129 """
jpayne@68 1130 scheme, _, host, port, *_ = parse_url(url)
jpayne@68 1131 scheme = scheme or "http"
jpayne@68 1132 port = port or port_by_scheme.get(scheme, 80)
jpayne@68 1133 if scheme == "https":
jpayne@68 1134 return HTTPSConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
jpayne@68 1135 else:
jpayne@68 1136 return HTTPConnectionPool(host, port=port, **kw) # type: ignore[arg-type]
jpayne@68 1137
jpayne@68 1138
jpayne@68 1139 @typing.overload
jpayne@68 1140 def _normalize_host(host: None, scheme: str | None) -> None:
jpayne@68 1141 ...
jpayne@68 1142
jpayne@68 1143
jpayne@68 1144 @typing.overload
jpayne@68 1145 def _normalize_host(host: str, scheme: str | None) -> str:
jpayne@68 1146 ...
jpayne@68 1147
jpayne@68 1148
jpayne@68 1149 def _normalize_host(host: str | None, scheme: str | None) -> str | None:
jpayne@68 1150 """
jpayne@68 1151 Normalize hosts for comparisons and use with sockets.
jpayne@68 1152 """
jpayne@68 1153
jpayne@68 1154 host = normalize_host(host, scheme)
jpayne@68 1155
jpayne@68 1156 # httplib doesn't like it when we include brackets in IPv6 addresses
jpayne@68 1157 # Specifically, if we include brackets but also pass the port then
jpayne@68 1158 # httplib crazily doubles up the square brackets on the Host header.
jpayne@68 1159 # Instead, we need to make sure we never pass ``None`` as the port.
jpayne@68 1160 # However, for backward compatibility reasons we can't actually
jpayne@68 1161 # *assert* that. See http://bugs.python.org/issue28539
jpayne@68 1162 if host and host.startswith("[") and host.endswith("]"):
jpayne@68 1163 host = host[1:-1]
jpayne@68 1164 return host
jpayne@68 1165
jpayne@68 1166
jpayne@68 1167 def _url_from_pool(
jpayne@68 1168 pool: HTTPConnectionPool | HTTPSConnectionPool, path: str | None = None
jpayne@68 1169 ) -> str:
jpayne@68 1170 """Returns the URL from a given connection pool. This is mainly used for testing and logging."""
jpayne@68 1171 return Url(scheme=pool.scheme, host=pool.host, port=pool.port, path=path).url
jpayne@68 1172
jpayne@68 1173
jpayne@68 1174 def _close_pool_connections(pool: queue.LifoQueue[typing.Any]) -> None:
jpayne@68 1175 """Drains a queue of connections and closes each one."""
jpayne@68 1176 try:
jpayne@68 1177 while True:
jpayne@68 1178 conn = pool.get(block=False)
jpayne@68 1179 if conn:
jpayne@68 1180 conn.close()
jpayne@68 1181 except queue.Empty:
jpayne@68 1182 pass # Done.