Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/site-packages/urllib3/poolmanager.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 from __future__ import annotations
jpayne@69 2
jpayne@69 3 import functools
jpayne@69 4 import logging
jpayne@69 5 import typing
jpayne@69 6 import warnings
jpayne@69 7 from types import TracebackType
jpayne@69 8 from urllib.parse import urljoin
jpayne@69 9
jpayne@69 10 from ._collections import HTTPHeaderDict, RecentlyUsedContainer
jpayne@69 11 from ._request_methods import RequestMethods
jpayne@69 12 from .connection import ProxyConfig
jpayne@69 13 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
jpayne@69 14 from .exceptions import (
jpayne@69 15 LocationValueError,
jpayne@69 16 MaxRetryError,
jpayne@69 17 ProxySchemeUnknown,
jpayne@69 18 URLSchemeUnknown,
jpayne@69 19 )
jpayne@69 20 from .response import BaseHTTPResponse
jpayne@69 21 from .util.connection import _TYPE_SOCKET_OPTIONS
jpayne@69 22 from .util.proxy import connection_requires_http_tunnel
jpayne@69 23 from .util.retry import Retry
jpayne@69 24 from .util.timeout import Timeout
jpayne@69 25 from .util.url import Url, parse_url
jpayne@69 26
jpayne@69 27 if typing.TYPE_CHECKING:
jpayne@69 28 import ssl
jpayne@69 29
jpayne@69 30 from typing_extensions import Self
jpayne@69 31
jpayne@69 32 __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
jpayne@69 33
jpayne@69 34
jpayne@69 35 log = logging.getLogger(__name__)
jpayne@69 36
jpayne@69 37 SSL_KEYWORDS = (
jpayne@69 38 "key_file",
jpayne@69 39 "cert_file",
jpayne@69 40 "cert_reqs",
jpayne@69 41 "ca_certs",
jpayne@69 42 "ca_cert_data",
jpayne@69 43 "ssl_version",
jpayne@69 44 "ssl_minimum_version",
jpayne@69 45 "ssl_maximum_version",
jpayne@69 46 "ca_cert_dir",
jpayne@69 47 "ssl_context",
jpayne@69 48 "key_password",
jpayne@69 49 "server_hostname",
jpayne@69 50 )
jpayne@69 51 # Default value for `blocksize` - a new parameter introduced to
jpayne@69 52 # http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
jpayne@69 53 _DEFAULT_BLOCKSIZE = 16384
jpayne@69 54
jpayne@69 55
jpayne@69 56 class PoolKey(typing.NamedTuple):
jpayne@69 57 """
jpayne@69 58 All known keyword arguments that could be provided to the pool manager, its
jpayne@69 59 pools, or the underlying connections.
jpayne@69 60
jpayne@69 61 All custom key schemes should include the fields in this key at a minimum.
jpayne@69 62 """
jpayne@69 63
jpayne@69 64 key_scheme: str
jpayne@69 65 key_host: str
jpayne@69 66 key_port: int | None
jpayne@69 67 key_timeout: Timeout | float | int | None
jpayne@69 68 key_retries: Retry | bool | int | None
jpayne@69 69 key_block: bool | None
jpayne@69 70 key_source_address: tuple[str, int] | None
jpayne@69 71 key_key_file: str | None
jpayne@69 72 key_key_password: str | None
jpayne@69 73 key_cert_file: str | None
jpayne@69 74 key_cert_reqs: str | None
jpayne@69 75 key_ca_certs: str | None
jpayne@69 76 key_ca_cert_data: str | bytes | None
jpayne@69 77 key_ssl_version: int | str | None
jpayne@69 78 key_ssl_minimum_version: ssl.TLSVersion | None
jpayne@69 79 key_ssl_maximum_version: ssl.TLSVersion | None
jpayne@69 80 key_ca_cert_dir: str | None
jpayne@69 81 key_ssl_context: ssl.SSLContext | None
jpayne@69 82 key_maxsize: int | None
jpayne@69 83 key_headers: frozenset[tuple[str, str]] | None
jpayne@69 84 key__proxy: Url | None
jpayne@69 85 key__proxy_headers: frozenset[tuple[str, str]] | None
jpayne@69 86 key__proxy_config: ProxyConfig | None
jpayne@69 87 key_socket_options: _TYPE_SOCKET_OPTIONS | None
jpayne@69 88 key__socks_options: frozenset[tuple[str, str]] | None
jpayne@69 89 key_assert_hostname: bool | str | None
jpayne@69 90 key_assert_fingerprint: str | None
jpayne@69 91 key_server_hostname: str | None
jpayne@69 92 key_blocksize: int | None
jpayne@69 93
jpayne@69 94
jpayne@69 95 def _default_key_normalizer(
jpayne@69 96 key_class: type[PoolKey], request_context: dict[str, typing.Any]
jpayne@69 97 ) -> PoolKey:
jpayne@69 98 """
jpayne@69 99 Create a pool key out of a request context dictionary.
jpayne@69 100
jpayne@69 101 According to RFC 3986, both the scheme and host are case-insensitive.
jpayne@69 102 Therefore, this function normalizes both before constructing the pool
jpayne@69 103 key for an HTTPS request. If you wish to change this behaviour, provide
jpayne@69 104 alternate callables to ``key_fn_by_scheme``.
jpayne@69 105
jpayne@69 106 :param key_class:
jpayne@69 107 The class to use when constructing the key. This should be a namedtuple
jpayne@69 108 with the ``scheme`` and ``host`` keys at a minimum.
jpayne@69 109 :type key_class: namedtuple
jpayne@69 110 :param request_context:
jpayne@69 111 A dictionary-like object that contain the context for a request.
jpayne@69 112 :type request_context: dict
jpayne@69 113
jpayne@69 114 :return: A namedtuple that can be used as a connection pool key.
jpayne@69 115 :rtype: PoolKey
jpayne@69 116 """
jpayne@69 117 # Since we mutate the dictionary, make a copy first
jpayne@69 118 context = request_context.copy()
jpayne@69 119 context["scheme"] = context["scheme"].lower()
jpayne@69 120 context["host"] = context["host"].lower()
jpayne@69 121
jpayne@69 122 # These are both dictionaries and need to be transformed into frozensets
jpayne@69 123 for key in ("headers", "_proxy_headers", "_socks_options"):
jpayne@69 124 if key in context and context[key] is not None:
jpayne@69 125 context[key] = frozenset(context[key].items())
jpayne@69 126
jpayne@69 127 # The socket_options key may be a list and needs to be transformed into a
jpayne@69 128 # tuple.
jpayne@69 129 socket_opts = context.get("socket_options")
jpayne@69 130 if socket_opts is not None:
jpayne@69 131 context["socket_options"] = tuple(socket_opts)
jpayne@69 132
jpayne@69 133 # Map the kwargs to the names in the namedtuple - this is necessary since
jpayne@69 134 # namedtuples can't have fields starting with '_'.
jpayne@69 135 for key in list(context.keys()):
jpayne@69 136 context["key_" + key] = context.pop(key)
jpayne@69 137
jpayne@69 138 # Default to ``None`` for keys missing from the context
jpayne@69 139 for field in key_class._fields:
jpayne@69 140 if field not in context:
jpayne@69 141 context[field] = None
jpayne@69 142
jpayne@69 143 # Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
jpayne@69 144 if context.get("key_blocksize") is None:
jpayne@69 145 context["key_blocksize"] = _DEFAULT_BLOCKSIZE
jpayne@69 146
jpayne@69 147 return key_class(**context)
jpayne@69 148
jpayne@69 149
jpayne@69 150 #: A dictionary that maps a scheme to a callable that creates a pool key.
jpayne@69 151 #: This can be used to alter the way pool keys are constructed, if desired.
jpayne@69 152 #: Each PoolManager makes a copy of this dictionary so they can be configured
jpayne@69 153 #: globally here, or individually on the instance.
jpayne@69 154 key_fn_by_scheme = {
jpayne@69 155 "http": functools.partial(_default_key_normalizer, PoolKey),
jpayne@69 156 "https": functools.partial(_default_key_normalizer, PoolKey),
jpayne@69 157 }
jpayne@69 158
jpayne@69 159 pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool}
jpayne@69 160
jpayne@69 161
jpayne@69 162 class PoolManager(RequestMethods):
jpayne@69 163 """
jpayne@69 164 Allows for arbitrary requests while transparently keeping track of
jpayne@69 165 necessary connection pools for you.
jpayne@69 166
jpayne@69 167 :param num_pools:
jpayne@69 168 Number of connection pools to cache before discarding the least
jpayne@69 169 recently used pool.
jpayne@69 170
jpayne@69 171 :param headers:
jpayne@69 172 Headers to include with all requests, unless other headers are given
jpayne@69 173 explicitly.
jpayne@69 174
jpayne@69 175 :param \\**connection_pool_kw:
jpayne@69 176 Additional parameters are used to create fresh
jpayne@69 177 :class:`urllib3.connectionpool.ConnectionPool` instances.
jpayne@69 178
jpayne@69 179 Example:
jpayne@69 180
jpayne@69 181 .. code-block:: python
jpayne@69 182
jpayne@69 183 import urllib3
jpayne@69 184
jpayne@69 185 http = urllib3.PoolManager(num_pools=2)
jpayne@69 186
jpayne@69 187 resp1 = http.request("GET", "https://google.com/")
jpayne@69 188 resp2 = http.request("GET", "https://google.com/mail")
jpayne@69 189 resp3 = http.request("GET", "https://yahoo.com/")
jpayne@69 190
jpayne@69 191 print(len(http.pools))
jpayne@69 192 # 2
jpayne@69 193
jpayne@69 194 """
jpayne@69 195
jpayne@69 196 proxy: Url | None = None
jpayne@69 197 proxy_config: ProxyConfig | None = None
jpayne@69 198
jpayne@69 199 def __init__(
jpayne@69 200 self,
jpayne@69 201 num_pools: int = 10,
jpayne@69 202 headers: typing.Mapping[str, str] | None = None,
jpayne@69 203 **connection_pool_kw: typing.Any,
jpayne@69 204 ) -> None:
jpayne@69 205 super().__init__(headers)
jpayne@69 206 self.connection_pool_kw = connection_pool_kw
jpayne@69 207
jpayne@69 208 self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
jpayne@69 209 self.pools = RecentlyUsedContainer(num_pools)
jpayne@69 210
jpayne@69 211 # Locally set the pool classes and keys so other PoolManagers can
jpayne@69 212 # override them.
jpayne@69 213 self.pool_classes_by_scheme = pool_classes_by_scheme
jpayne@69 214 self.key_fn_by_scheme = key_fn_by_scheme.copy()
jpayne@69 215
jpayne@69 216 def __enter__(self) -> Self:
jpayne@69 217 return self
jpayne@69 218
jpayne@69 219 def __exit__(
jpayne@69 220 self,
jpayne@69 221 exc_type: type[BaseException] | None,
jpayne@69 222 exc_val: BaseException | None,
jpayne@69 223 exc_tb: TracebackType | None,
jpayne@69 224 ) -> typing.Literal[False]:
jpayne@69 225 self.clear()
jpayne@69 226 # Return False to re-raise any potential exceptions
jpayne@69 227 return False
jpayne@69 228
jpayne@69 229 def _new_pool(
jpayne@69 230 self,
jpayne@69 231 scheme: str,
jpayne@69 232 host: str,
jpayne@69 233 port: int,
jpayne@69 234 request_context: dict[str, typing.Any] | None = None,
jpayne@69 235 ) -> HTTPConnectionPool:
jpayne@69 236 """
jpayne@69 237 Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
jpayne@69 238 any additional pool keyword arguments.
jpayne@69 239
jpayne@69 240 If ``request_context`` is provided, it is provided as keyword arguments
jpayne@69 241 to the pool class used. This method is used to actually create the
jpayne@69 242 connection pools handed out by :meth:`connection_from_url` and
jpayne@69 243 companion methods. It is intended to be overridden for customization.
jpayne@69 244 """
jpayne@69 245 pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
jpayne@69 246 if request_context is None:
jpayne@69 247 request_context = self.connection_pool_kw.copy()
jpayne@69 248
jpayne@69 249 # Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
jpayne@69 250 # set to 'None' in the request_context.
jpayne@69 251 if request_context.get("blocksize") is None:
jpayne@69 252 request_context["blocksize"] = _DEFAULT_BLOCKSIZE
jpayne@69 253
jpayne@69 254 # Although the context has everything necessary to create the pool,
jpayne@69 255 # this function has historically only used the scheme, host, and port
jpayne@69 256 # in the positional args. When an API change is acceptable these can
jpayne@69 257 # be removed.
jpayne@69 258 for key in ("scheme", "host", "port"):
jpayne@69 259 request_context.pop(key, None)
jpayne@69 260
jpayne@69 261 if scheme == "http":
jpayne@69 262 for kw in SSL_KEYWORDS:
jpayne@69 263 request_context.pop(kw, None)
jpayne@69 264
jpayne@69 265 return pool_cls(host, port, **request_context)
jpayne@69 266
jpayne@69 267 def clear(self) -> None:
jpayne@69 268 """
jpayne@69 269 Empty our store of pools and direct them all to close.
jpayne@69 270
jpayne@69 271 This will not affect in-flight connections, but they will not be
jpayne@69 272 re-used after completion.
jpayne@69 273 """
jpayne@69 274 self.pools.clear()
jpayne@69 275
jpayne@69 276 def connection_from_host(
jpayne@69 277 self,
jpayne@69 278 host: str | None,
jpayne@69 279 port: int | None = None,
jpayne@69 280 scheme: str | None = "http",
jpayne@69 281 pool_kwargs: dict[str, typing.Any] | None = None,
jpayne@69 282 ) -> HTTPConnectionPool:
jpayne@69 283 """
jpayne@69 284 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
jpayne@69 285
jpayne@69 286 If ``port`` isn't given, it will be derived from the ``scheme`` using
jpayne@69 287 ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is
jpayne@69 288 provided, it is merged with the instance's ``connection_pool_kw``
jpayne@69 289 variable and used to create the new connection pool, if one is
jpayne@69 290 needed.
jpayne@69 291 """
jpayne@69 292
jpayne@69 293 if not host:
jpayne@69 294 raise LocationValueError("No host specified.")
jpayne@69 295
jpayne@69 296 request_context = self._merge_pool_kwargs(pool_kwargs)
jpayne@69 297 request_context["scheme"] = scheme or "http"
jpayne@69 298 if not port:
jpayne@69 299 port = port_by_scheme.get(request_context["scheme"].lower(), 80)
jpayne@69 300 request_context["port"] = port
jpayne@69 301 request_context["host"] = host
jpayne@69 302
jpayne@69 303 return self.connection_from_context(request_context)
jpayne@69 304
jpayne@69 305 def connection_from_context(
jpayne@69 306 self, request_context: dict[str, typing.Any]
jpayne@69 307 ) -> HTTPConnectionPool:
jpayne@69 308 """
jpayne@69 309 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
jpayne@69 310
jpayne@69 311 ``request_context`` must at least contain the ``scheme`` key and its
jpayne@69 312 value must be a key in ``key_fn_by_scheme`` instance variable.
jpayne@69 313 """
jpayne@69 314 if "strict" in request_context:
jpayne@69 315 warnings.warn(
jpayne@69 316 "The 'strict' parameter is no longer needed on Python 3+. "
jpayne@69 317 "This will raise an error in urllib3 v2.1.0.",
jpayne@69 318 DeprecationWarning,
jpayne@69 319 )
jpayne@69 320 request_context.pop("strict")
jpayne@69 321
jpayne@69 322 scheme = request_context["scheme"].lower()
jpayne@69 323 pool_key_constructor = self.key_fn_by_scheme.get(scheme)
jpayne@69 324 if not pool_key_constructor:
jpayne@69 325 raise URLSchemeUnknown(scheme)
jpayne@69 326 pool_key = pool_key_constructor(request_context)
jpayne@69 327
jpayne@69 328 return self.connection_from_pool_key(pool_key, request_context=request_context)
jpayne@69 329
jpayne@69 330 def connection_from_pool_key(
jpayne@69 331 self, pool_key: PoolKey, request_context: dict[str, typing.Any]
jpayne@69 332 ) -> HTTPConnectionPool:
jpayne@69 333 """
jpayne@69 334 Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
jpayne@69 335
jpayne@69 336 ``pool_key`` should be a namedtuple that only contains immutable
jpayne@69 337 objects. At a minimum it must have the ``scheme``, ``host``, and
jpayne@69 338 ``port`` fields.
jpayne@69 339 """
jpayne@69 340 with self.pools.lock:
jpayne@69 341 # If the scheme, host, or port doesn't match existing open
jpayne@69 342 # connections, open a new ConnectionPool.
jpayne@69 343 pool = self.pools.get(pool_key)
jpayne@69 344 if pool:
jpayne@69 345 return pool
jpayne@69 346
jpayne@69 347 # Make a fresh ConnectionPool of the desired type
jpayne@69 348 scheme = request_context["scheme"]
jpayne@69 349 host = request_context["host"]
jpayne@69 350 port = request_context["port"]
jpayne@69 351 pool = self._new_pool(scheme, host, port, request_context=request_context)
jpayne@69 352 self.pools[pool_key] = pool
jpayne@69 353
jpayne@69 354 return pool
jpayne@69 355
jpayne@69 356 def connection_from_url(
jpayne@69 357 self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
jpayne@69 358 ) -> HTTPConnectionPool:
jpayne@69 359 """
jpayne@69 360 Similar to :func:`urllib3.connectionpool.connection_from_url`.
jpayne@69 361
jpayne@69 362 If ``pool_kwargs`` is not provided and a new pool needs to be
jpayne@69 363 constructed, ``self.connection_pool_kw`` is used to initialize
jpayne@69 364 the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs``
jpayne@69 365 is provided, it is used instead. Note that if a new pool does not
jpayne@69 366 need to be created for the request, the provided ``pool_kwargs`` are
jpayne@69 367 not used.
jpayne@69 368 """
jpayne@69 369 u = parse_url(url)
jpayne@69 370 return self.connection_from_host(
jpayne@69 371 u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
jpayne@69 372 )
jpayne@69 373
jpayne@69 374 def _merge_pool_kwargs(
jpayne@69 375 self, override: dict[str, typing.Any] | None
jpayne@69 376 ) -> dict[str, typing.Any]:
jpayne@69 377 """
jpayne@69 378 Merge a dictionary of override values for self.connection_pool_kw.
jpayne@69 379
jpayne@69 380 This does not modify self.connection_pool_kw and returns a new dict.
jpayne@69 381 Any keys in the override dictionary with a value of ``None`` are
jpayne@69 382 removed from the merged dictionary.
jpayne@69 383 """
jpayne@69 384 base_pool_kwargs = self.connection_pool_kw.copy()
jpayne@69 385 if override:
jpayne@69 386 for key, value in override.items():
jpayne@69 387 if value is None:
jpayne@69 388 try:
jpayne@69 389 del base_pool_kwargs[key]
jpayne@69 390 except KeyError:
jpayne@69 391 pass
jpayne@69 392 else:
jpayne@69 393 base_pool_kwargs[key] = value
jpayne@69 394 return base_pool_kwargs
jpayne@69 395
jpayne@69 396 def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
jpayne@69 397 """
jpayne@69 398 Indicates if the proxy requires the complete destination URL in the
jpayne@69 399 request. Normally this is only needed when not using an HTTP CONNECT
jpayne@69 400 tunnel.
jpayne@69 401 """
jpayne@69 402 if self.proxy is None:
jpayne@69 403 return False
jpayne@69 404
jpayne@69 405 return not connection_requires_http_tunnel(
jpayne@69 406 self.proxy, self.proxy_config, parsed_url.scheme
jpayne@69 407 )
jpayne@69 408
jpayne@69 409 def urlopen( # type: ignore[override]
jpayne@69 410 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
jpayne@69 411 ) -> BaseHTTPResponse:
jpayne@69 412 """
jpayne@69 413 Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
jpayne@69 414 with custom cross-host redirect logic and only sends the request-uri
jpayne@69 415 portion of the ``url``.
jpayne@69 416
jpayne@69 417 The given ``url`` parameter must be absolute, such that an appropriate
jpayne@69 418 :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
jpayne@69 419 """
jpayne@69 420 u = parse_url(url)
jpayne@69 421
jpayne@69 422 if u.scheme is None:
jpayne@69 423 warnings.warn(
jpayne@69 424 "URLs without a scheme (ie 'https://') are deprecated and will raise an error "
jpayne@69 425 "in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs "
jpayne@69 426 "start with 'https://' or 'http://'. Read more in this issue: "
jpayne@69 427 "https://github.com/urllib3/urllib3/issues/2920",
jpayne@69 428 category=DeprecationWarning,
jpayne@69 429 stacklevel=2,
jpayne@69 430 )
jpayne@69 431
jpayne@69 432 conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
jpayne@69 433
jpayne@69 434 kw["assert_same_host"] = False
jpayne@69 435 kw["redirect"] = False
jpayne@69 436
jpayne@69 437 if "headers" not in kw:
jpayne@69 438 kw["headers"] = self.headers
jpayne@69 439
jpayne@69 440 if self._proxy_requires_url_absolute_form(u):
jpayne@69 441 response = conn.urlopen(method, url, **kw)
jpayne@69 442 else:
jpayne@69 443 response = conn.urlopen(method, u.request_uri, **kw)
jpayne@69 444
jpayne@69 445 redirect_location = redirect and response.get_redirect_location()
jpayne@69 446 if not redirect_location:
jpayne@69 447 return response
jpayne@69 448
jpayne@69 449 # Support relative URLs for redirecting.
jpayne@69 450 redirect_location = urljoin(url, redirect_location)
jpayne@69 451
jpayne@69 452 if response.status == 303:
jpayne@69 453 # Change the method according to RFC 9110, Section 15.4.4.
jpayne@69 454 method = "GET"
jpayne@69 455 # And lose the body not to transfer anything sensitive.
jpayne@69 456 kw["body"] = None
jpayne@69 457 kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
jpayne@69 458
jpayne@69 459 retries = kw.get("retries")
jpayne@69 460 if not isinstance(retries, Retry):
jpayne@69 461 retries = Retry.from_int(retries, redirect=redirect)
jpayne@69 462
jpayne@69 463 # Strip headers marked as unsafe to forward to the redirected location.
jpayne@69 464 # Check remove_headers_on_redirect to avoid a potential network call within
jpayne@69 465 # conn.is_same_host() which may use socket.gethostbyname() in the future.
jpayne@69 466 if retries.remove_headers_on_redirect and not conn.is_same_host(
jpayne@69 467 redirect_location
jpayne@69 468 ):
jpayne@69 469 new_headers = kw["headers"].copy()
jpayne@69 470 for header in kw["headers"]:
jpayne@69 471 if header.lower() in retries.remove_headers_on_redirect:
jpayne@69 472 new_headers.pop(header, None)
jpayne@69 473 kw["headers"] = new_headers
jpayne@69 474
jpayne@69 475 try:
jpayne@69 476 retries = retries.increment(method, url, response=response, _pool=conn)
jpayne@69 477 except MaxRetryError:
jpayne@69 478 if retries.raise_on_redirect:
jpayne@69 479 response.drain_conn()
jpayne@69 480 raise
jpayne@69 481 return response
jpayne@69 482
jpayne@69 483 kw["retries"] = retries
jpayne@69 484 kw["redirect"] = redirect
jpayne@69 485
jpayne@69 486 log.info("Redirecting %s -> %s", url, redirect_location)
jpayne@69 487
jpayne@69 488 response.drain_conn()
jpayne@69 489 return self.urlopen(method, redirect_location, **kw)
jpayne@69 490
jpayne@69 491
jpayne@69 492 class ProxyManager(PoolManager):
jpayne@69 493 """
jpayne@69 494 Behaves just like :class:`PoolManager`, but sends all requests through
jpayne@69 495 the defined proxy, using the CONNECT method for HTTPS URLs.
jpayne@69 496
jpayne@69 497 :param proxy_url:
jpayne@69 498 The URL of the proxy to be used.
jpayne@69 499
jpayne@69 500 :param proxy_headers:
jpayne@69 501 A dictionary containing headers that will be sent to the proxy. In case
jpayne@69 502 of HTTP they are being sent with each request, while in the
jpayne@69 503 HTTPS/CONNECT case they are sent only once. Could be used for proxy
jpayne@69 504 authentication.
jpayne@69 505
jpayne@69 506 :param proxy_ssl_context:
jpayne@69 507 The proxy SSL context is used to establish the TLS connection to the
jpayne@69 508 proxy when using HTTPS proxies.
jpayne@69 509
jpayne@69 510 :param use_forwarding_for_https:
jpayne@69 511 (Defaults to False) If set to True will forward requests to the HTTPS
jpayne@69 512 proxy to be made on behalf of the client instead of creating a TLS
jpayne@69 513 tunnel via the CONNECT method. **Enabling this flag means that request
jpayne@69 514 and response headers and content will be visible from the HTTPS proxy**
jpayne@69 515 whereas tunneling keeps request and response headers and content
jpayne@69 516 private. IP address, target hostname, SNI, and port are always visible
jpayne@69 517 to an HTTPS proxy even when this flag is disabled.
jpayne@69 518
jpayne@69 519 :param proxy_assert_hostname:
jpayne@69 520 The hostname of the certificate to verify against.
jpayne@69 521
jpayne@69 522 :param proxy_assert_fingerprint:
jpayne@69 523 The fingerprint of the certificate to verify against.
jpayne@69 524
jpayne@69 525 Example:
jpayne@69 526
jpayne@69 527 .. code-block:: python
jpayne@69 528
jpayne@69 529 import urllib3
jpayne@69 530
jpayne@69 531 proxy = urllib3.ProxyManager("https://localhost:3128/")
jpayne@69 532
jpayne@69 533 resp1 = proxy.request("GET", "https://google.com/")
jpayne@69 534 resp2 = proxy.request("GET", "https://httpbin.org/")
jpayne@69 535
jpayne@69 536 print(len(proxy.pools))
jpayne@69 537 # 1
jpayne@69 538
jpayne@69 539 resp3 = proxy.request("GET", "https://httpbin.org/")
jpayne@69 540 resp4 = proxy.request("GET", "https://twitter.com/")
jpayne@69 541
jpayne@69 542 print(len(proxy.pools))
jpayne@69 543 # 3
jpayne@69 544
jpayne@69 545 """
jpayne@69 546
jpayne@69 547 def __init__(
jpayne@69 548 self,
jpayne@69 549 proxy_url: str,
jpayne@69 550 num_pools: int = 10,
jpayne@69 551 headers: typing.Mapping[str, str] | None = None,
jpayne@69 552 proxy_headers: typing.Mapping[str, str] | None = None,
jpayne@69 553 proxy_ssl_context: ssl.SSLContext | None = None,
jpayne@69 554 use_forwarding_for_https: bool = False,
jpayne@69 555 proxy_assert_hostname: None | str | typing.Literal[False] = None,
jpayne@69 556 proxy_assert_fingerprint: str | None = None,
jpayne@69 557 **connection_pool_kw: typing.Any,
jpayne@69 558 ) -> None:
jpayne@69 559 if isinstance(proxy_url, HTTPConnectionPool):
jpayne@69 560 str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
jpayne@69 561 else:
jpayne@69 562 str_proxy_url = proxy_url
jpayne@69 563 proxy = parse_url(str_proxy_url)
jpayne@69 564
jpayne@69 565 if proxy.scheme not in ("http", "https"):
jpayne@69 566 raise ProxySchemeUnknown(proxy.scheme)
jpayne@69 567
jpayne@69 568 if not proxy.port:
jpayne@69 569 port = port_by_scheme.get(proxy.scheme, 80)
jpayne@69 570 proxy = proxy._replace(port=port)
jpayne@69 571
jpayne@69 572 self.proxy = proxy
jpayne@69 573 self.proxy_headers = proxy_headers or {}
jpayne@69 574 self.proxy_ssl_context = proxy_ssl_context
jpayne@69 575 self.proxy_config = ProxyConfig(
jpayne@69 576 proxy_ssl_context,
jpayne@69 577 use_forwarding_for_https,
jpayne@69 578 proxy_assert_hostname,
jpayne@69 579 proxy_assert_fingerprint,
jpayne@69 580 )
jpayne@69 581
jpayne@69 582 connection_pool_kw["_proxy"] = self.proxy
jpayne@69 583 connection_pool_kw["_proxy_headers"] = self.proxy_headers
jpayne@69 584 connection_pool_kw["_proxy_config"] = self.proxy_config
jpayne@69 585
jpayne@69 586 super().__init__(num_pools, headers, **connection_pool_kw)
jpayne@69 587
jpayne@69 588 def connection_from_host(
jpayne@69 589 self,
jpayne@69 590 host: str | None,
jpayne@69 591 port: int | None = None,
jpayne@69 592 scheme: str | None = "http",
jpayne@69 593 pool_kwargs: dict[str, typing.Any] | None = None,
jpayne@69 594 ) -> HTTPConnectionPool:
jpayne@69 595 if scheme == "https":
jpayne@69 596 return super().connection_from_host(
jpayne@69 597 host, port, scheme, pool_kwargs=pool_kwargs
jpayne@69 598 )
jpayne@69 599
jpayne@69 600 return super().connection_from_host(
jpayne@69 601 self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
jpayne@69 602 )
jpayne@69 603
jpayne@69 604 def _set_proxy_headers(
jpayne@69 605 self, url: str, headers: typing.Mapping[str, str] | None = None
jpayne@69 606 ) -> typing.Mapping[str, str]:
jpayne@69 607 """
jpayne@69 608 Sets headers needed by proxies: specifically, the Accept and Host
jpayne@69 609 headers. Only sets headers not provided by the user.
jpayne@69 610 """
jpayne@69 611 headers_ = {"Accept": "*/*"}
jpayne@69 612
jpayne@69 613 netloc = parse_url(url).netloc
jpayne@69 614 if netloc:
jpayne@69 615 headers_["Host"] = netloc
jpayne@69 616
jpayne@69 617 if headers:
jpayne@69 618 headers_.update(headers)
jpayne@69 619 return headers_
jpayne@69 620
jpayne@69 621 def urlopen( # type: ignore[override]
jpayne@69 622 self, method: str, url: str, redirect: bool = True, **kw: typing.Any
jpayne@69 623 ) -> BaseHTTPResponse:
jpayne@69 624 "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
jpayne@69 625 u = parse_url(url)
jpayne@69 626 if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
jpayne@69 627 # For connections using HTTP CONNECT, httplib sets the necessary
jpayne@69 628 # headers on the CONNECT to the proxy. If we're not using CONNECT,
jpayne@69 629 # we'll definitely need to set 'Host' at the very least.
jpayne@69 630 headers = kw.get("headers", self.headers)
jpayne@69 631 kw["headers"] = self._set_proxy_headers(url, headers)
jpayne@69 632
jpayne@69 633 return super().urlopen(method, url, redirect=redirect, **kw)
jpayne@69 634
jpayne@69 635
jpayne@69 636 def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
jpayne@69 637 return ProxyManager(proxy_url=url, **kw)