Mercurial > repos > rliterman > csp2

comparison CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/site-packages/urllib3/connection.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
comparison
equal deleted inserted replaced
67:0e9998148a16 69:33d812a61356
1 from __future__ import annotations
2
3 import datetime
4 import http.client
5 import logging
6 import os
7 import re
8 import socket
9 import sys
10 import threading
11 import typing
12 import warnings
13 from http.client import HTTPConnection as _HTTPConnection
14 from http.client import HTTPException as HTTPException # noqa: F401
15 from http.client import ResponseNotReady
16 from socket import timeout as SocketTimeout
17
18 if typing.TYPE_CHECKING:
19 from .response import HTTPResponse
20 from .util.ssl_ import _TYPE_PEER_CERT_RET_DICT
21 from .util.ssltransport import SSLTransport
22
23 from ._collections import HTTPHeaderDict
24 from .http2 import probe as http2_probe
25 from .util.response import assert_header_parsing
26 from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
27 from .util.util import to_str
28 from .util.wait import wait_for_read
29
30 try: # Compiled with SSL?
31 import ssl
32
33 BaseSSLError = ssl.SSLError
34 except (ImportError, AttributeError):
35 ssl = None # type: ignore[assignment]
36
37 class BaseSSLError(BaseException): # type: ignore[no-redef]
38 pass
39
40
41 from ._base_connection import _TYPE_BODY
42 from ._base_connection import ProxyConfig as ProxyConfig
43 from ._base_connection import _ResponseOptions as _ResponseOptions
44 from ._version import __version__
45 from .exceptions import (
46 ConnectTimeoutError,
47 HeaderParsingError,
48 NameResolutionError,
49 NewConnectionError,
50 ProxyError,
51 SystemTimeWarning,
52 )
53 from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
54 from .util.request import body_to_chunks
55 from .util.ssl_ import assert_fingerprint as _assert_fingerprint
56 from .util.ssl_ import (
57 create_urllib3_context,
58 is_ipaddress,
59 resolve_cert_reqs,
60 resolve_ssl_version,
61 ssl_wrap_socket,
62 )
63 from .util.ssl_match_hostname import CertificateError, match_hostname
64 from .util.url import Url
65
66 # Not a no-op, we're adding this to the namespace so it can be imported.
67 ConnectionError = ConnectionError
68 BrokenPipeError = BrokenPipeError
69
70
71 log = logging.getLogger(__name__)
72
73 port_by_scheme = {"http": 80, "https": 443}
74
75 # When it comes time to update this value as a part of regular maintenance
76 # (ie test_recent_date is failing) update it to ~6 months before the current date.
77 RECENT_DATE = datetime.date(2023, 6, 1)
78
79 _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]")
80
81 _HAS_SYS_AUDIT = hasattr(sys, "audit")
82
83
84 class HTTPConnection(_HTTPConnection):
85 """
86 Based on :class:`http.client.HTTPConnection` but provides an extra constructor
87 backwards-compatibility layer between older and newer Pythons.
88
89 Additional keyword parameters are used to configure attributes of the connection.
90 Accepted parameters include:
91
92 - ``source_address``: Set the source address for the current connection.
93 - ``socket_options``: Set specific options on the underlying socket. If not specified, then
94 defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
95 Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
96
97 For example, if you wish to enable TCP Keep Alive in addition to the defaults,
98 you might pass:
99
100 .. code-block:: python
101
102 HTTPConnection.default_socket_options + [
103 (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
104 ]
105
106 Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
107 """
108
109 default_port: typing.ClassVar[int] = port_by_scheme["http"] # type: ignore[misc]
110
111 #: Disable Nagle's algorithm by default.
112 #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
113 default_socket_options: typing.ClassVar[connection._TYPE_SOCKET_OPTIONS] = [
114 (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
115 ]
116
117 #: Whether this connection verifies the host's certificate.
118 is_verified: bool = False
119
120 #: Whether this proxy connection verified the proxy host's certificate.
121 # If no proxy is currently connected to the value will be ``None``.
122 proxy_is_verified: bool | None = None
123
124 blocksize: int
125 source_address: tuple[str, int] | None
126 socket_options: connection._TYPE_SOCKET_OPTIONS | None
127
128 _has_connected_to_proxy: bool
129 _response_options: _ResponseOptions | None
130 _tunnel_host: str | None
131 _tunnel_port: int | None
132 _tunnel_scheme: str | None
133
134 def __init__(
135 self,
136 host: str,
137 port: int | None = None,
138 *,
139 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
140 source_address: tuple[str, int] | None = None,
141 blocksize: int = 16384,
142 socket_options: None
143 | (connection._TYPE_SOCKET_OPTIONS) = default_socket_options,
144 proxy: Url | None = None,
145 proxy_config: ProxyConfig | None = None,
146 ) -> None:
147 super().__init__(
148 host=host,
149 port=port,
150 timeout=Timeout.resolve_default_timeout(timeout),
151 source_address=source_address,
152 blocksize=blocksize,
153 )
154 self.socket_options = socket_options
155 self.proxy = proxy
156 self.proxy_config = proxy_config
157
158 self._has_connected_to_proxy = False
159 self._response_options = None
160 self._tunnel_host: str | None = None
161 self._tunnel_port: int | None = None
162 self._tunnel_scheme: str | None = None
163
164 @property
165 def host(self) -> str:
166 """
167 Getter method to remove any trailing dots that indicate the hostname is an FQDN.
168
169 In general, SSL certificates don't include the trailing dot indicating a
170 fully-qualified domain name, and thus, they don't validate properly when
171 checked against a domain name that includes the dot. In addition, some
172 servers may not expect to receive the trailing dot when provided.
173
174 However, the hostname with trailing dot is critical to DNS resolution; doing a
175 lookup with the trailing dot will properly only resolve the appropriate FQDN,
176 whereas a lookup without a trailing dot will search the system's search domain
177 list. Thus, it's important to keep the original host around for use only in
178 those cases where it's appropriate (i.e., when doing DNS lookup to establish the
179 actual TCP connection across which we're going to send HTTP requests).
180 """
181 return self._dns_host.rstrip(".")
182
183 @host.setter
184 def host(self, value: str) -> None:
185 """
186 Setter for the `host` property.
187
188 We assume that only urllib3 uses the _dns_host attribute; httplib itself
189 only uses `host`, and it seems reasonable that other libraries follow suit.
190 """
191 self._dns_host = value
192
193 def _new_conn(self) -> socket.socket:
194 """Establish a socket connection and set nodelay settings on it.
195
196 :return: New socket connection.
197 """
198 try:
199 sock = connection.create_connection(
200 (self._dns_host, self.port),
201 self.timeout,
202 source_address=self.source_address,
203 socket_options=self.socket_options,
204 )
205 except socket.gaierror as e:
206 raise NameResolutionError(self.host, self, e) from e
207 except SocketTimeout as e:
208 raise ConnectTimeoutError(
209 self,
210 f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
211 ) from e
212
213 except OSError as e:
214 raise NewConnectionError(
215 self, f"Failed to establish a new connection: {e}"
216 ) from e
217
218 # Audit hooks are only available in Python 3.8+
219 if _HAS_SYS_AUDIT:
220 sys.audit("http.client.connect", self, self.host, self.port)
221
222 return sock
223
224 def set_tunnel(
225 self,
226 host: str,
227 port: int | None = None,
228 headers: typing.Mapping[str, str] | None = None,
229 scheme: str = "http",
230 ) -> None:
231 if scheme not in ("http", "https"):
232 raise ValueError(
233 f"Invalid proxy scheme for tunneling: {scheme!r}, must be either 'http' or 'https'"
234 )
235 super().set_tunnel(host, port=port, headers=headers)
236 self._tunnel_scheme = scheme
237
238 if sys.version_info < (3, 11, 4):
239
240 def _tunnel(self) -> None:
241 _MAXLINE = http.client._MAXLINE # type: ignore[attr-defined]
242 connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( # type: ignore[str-format]
243 self._tunnel_host.encode("ascii"), # type: ignore[union-attr]
244 self._tunnel_port,
245 )
246 headers = [connect]
247 for header, value in self._tunnel_headers.items(): # type: ignore[attr-defined]
248 headers.append(f"{header}: {value}\r\n".encode("latin-1"))
249 headers.append(b"\r\n")
250 # Making a single send() call instead of one per line encourages
251 # the host OS to use a more optimal packet size instead of
252 # potentially emitting a series of small packets.
253 self.send(b"".join(headers))
254 del headers
255
256 response = self.response_class(self.sock, method=self._method) # type: ignore[attr-defined]
257 try:
258 (version, code, message) = response._read_status() # type: ignore[attr-defined]
259
260 if code != http.HTTPStatus.OK:
261 self.close()
262 raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
263 while True:
264 line = response.fp.readline(_MAXLINE + 1)
265 if len(line) > _MAXLINE:
266 raise http.client.LineTooLong("header line")
267 if not line:
268 # for sites which EOF without sending a trailer
269 break
270 if line in (b"\r\n", b"\n", b""):
271 break
272
273 if self.debuglevel > 0:
274 print("header:", line.decode())
275 finally:
276 response.close()
277
278 def connect(self) -> None:
279 self.sock = self._new_conn()
280 if self._tunnel_host:
281 # If we're tunneling it means we're connected to our proxy.
282 self._has_connected_to_proxy = True
283
284 # TODO: Fix tunnel so it doesn't depend on self.sock state.
285 self._tunnel()
286
287 # If there's a proxy to be connected to we are fully connected.
288 # This is set twice (once above and here) due to forwarding proxies
289 # not using tunnelling.
290 self._has_connected_to_proxy = bool(self.proxy)
291
292 if self._has_connected_to_proxy:
293 self.proxy_is_verified = False
294
295 @property
296 def is_closed(self) -> bool:
297 return self.sock is None
298
299 @property
300 def is_connected(self) -> bool:
301 if self.sock is None:
302 return False
303 return not wait_for_read(self.sock, timeout=0.0)
304
305 @property
306 def has_connected_to_proxy(self) -> bool:
307 return self._has_connected_to_proxy
308
309 @property
310 def proxy_is_forwarding(self) -> bool:
311 """
312 Return True if a forwarding proxy is configured, else return False
313 """
314 return bool(self.proxy) and self._tunnel_host is None
315
316 def close(self) -> None:
317 try:
318 super().close()
319 finally:
320 # Reset all stateful properties so connection
321 # can be re-used without leaking prior configs.
322 self.sock = None
323 self.is_verified = False
324 self.proxy_is_verified = None
325 self._has_connected_to_proxy = False
326 self._response_options = None
327 self._tunnel_host = None
328 self._tunnel_port = None
329 self._tunnel_scheme = None
330
331 def putrequest(
332 self,
333 method: str,
334 url: str,
335 skip_host: bool = False,
336 skip_accept_encoding: bool = False,
337 ) -> None:
338 """"""
339 # Empty docstring because the indentation of CPython's implementation
340 # is broken but we don't want this method in our documentation.
341 match = _CONTAINS_CONTROL_CHAR_RE.search(method)
342 if match:
343 raise ValueError(
344 f"Method cannot contain non-token characters {method!r} (found at least {match.group()!r})"
345 )
346
347 return super().putrequest(
348 method, url, skip_host=skip_host, skip_accept_encoding=skip_accept_encoding
349 )
350
351 def putheader(self, header: str, *values: str) -> None: # type: ignore[override]
352 """"""
353 if not any(isinstance(v, str) and v == SKIP_HEADER for v in values):
354 super().putheader(header, *values)
355 elif to_str(header.lower()) not in SKIPPABLE_HEADERS:
356 skippable_headers = "', '".join(
357 [str.title(header) for header in sorted(SKIPPABLE_HEADERS)]
358 )
359 raise ValueError(
360 f"urllib3.util.SKIP_HEADER only supports '{skippable_headers}'"
361 )
362
363 # `request` method's signature intentionally violates LSP.
364 # urllib3's API is different from `http.client.HTTPConnection` and the subclassing is only incidental.
365 def request( # type: ignore[override]
366 self,
367 method: str,
368 url: str,
369 body: _TYPE_BODY | None = None,
370 headers: typing.Mapping[str, str] | None = None,
371 *,
372 chunked: bool = False,
373 preload_content: bool = True,
374 decode_content: bool = True,
375 enforce_content_length: bool = True,
376 ) -> None:
377 # Update the inner socket's timeout value to send the request.
378 # This only triggers if the connection is re-used.
379 if self.sock is not None:
380 self.sock.settimeout(self.timeout)
381
382 # Store these values to be fed into the HTTPResponse
383 # object later. TODO: Remove this in favor of a real
384 # HTTP lifecycle mechanism.
385
386 # We have to store these before we call .request()
387 # because sometimes we can still salvage a response
388 # off the wire even if we aren't able to completely
389 # send the request body.
390 self._response_options = _ResponseOptions(
391 request_method=method,
392 request_url=url,
393 preload_content=preload_content,
394 decode_content=decode_content,
395 enforce_content_length=enforce_content_length,
396 )
397
398 if headers is None:
399 headers = {}
400 header_keys = frozenset(to_str(k.lower()) for k in headers)
401 skip_accept_encoding = "accept-encoding" in header_keys
402 skip_host = "host" in header_keys
403 self.putrequest(
404 method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
405 )
406
407 # Transform the body into an iterable of sendall()-able chunks
408 # and detect if an explicit Content-Length is doable.
409 chunks_and_cl = body_to_chunks(body, method=method, blocksize=self.blocksize)
410 chunks = chunks_and_cl.chunks
411 content_length = chunks_and_cl.content_length
412
413 # When chunked is explicit set to 'True' we respect that.
414 if chunked:
415 if "transfer-encoding" not in header_keys:
416 self.putheader("Transfer-Encoding", "chunked")
417 else:
418 # Detect whether a framing mechanism is already in use. If so
419 # we respect that value, otherwise we pick chunked vs content-length
420 # depending on the type of 'body'.
421 if "content-length" in header_keys:
422 chunked = False
423 elif "transfer-encoding" in header_keys:
424 chunked = True
425
426 # Otherwise we go off the recommendation of 'body_to_chunks()'.
427 else:
428 chunked = False
429 if content_length is None:
430 if chunks is not None:
431 chunked = True
432 self.putheader("Transfer-Encoding", "chunked")
433 else:
434 self.putheader("Content-Length", str(content_length))
435
436 # Now that framing headers are out of the way we send all the other headers.
437 if "user-agent" not in header_keys:
438 self.putheader("User-Agent", _get_default_user_agent())
439 for header, value in headers.items():
440 self.putheader(header, value)
441 self.endheaders()
442
443 # If we're given a body we start sending that in chunks.
444 if chunks is not None:
445 for chunk in chunks:
446 # Sending empty chunks isn't allowed for TE: chunked
447 # as it indicates the end of the body.
448 if not chunk:
449 continue
450 if isinstance(chunk, str):
451 chunk = chunk.encode("utf-8")
452 if chunked:
453 self.send(b"%x\r\n%b\r\n" % (len(chunk), chunk))
454 else:
455 self.send(chunk)
456
457 # Regardless of whether we have a body or not, if we're in
458 # chunked mode we want to send an explicit empty chunk.
459 if chunked:
460 self.send(b"0\r\n\r\n")
461
462 def request_chunked(
463 self,
464 method: str,
465 url: str,
466 body: _TYPE_BODY | None = None,
467 headers: typing.Mapping[str, str] | None = None,
468 ) -> None:
469 """
470 Alternative to the common request method, which sends the
471 body with chunked encoding and not as one block
472 """
473 warnings.warn(
474 "HTTPConnection.request_chunked() is deprecated and will be removed "
475 "in urllib3 v2.1.0. Instead use HTTPConnection.request(..., chunked=True).",
476 category=DeprecationWarning,
477 stacklevel=2,
478 )
479 self.request(method, url, body=body, headers=headers, chunked=True)
480
481 def getresponse( # type: ignore[override]
482 self,
483 ) -> HTTPResponse:
484 """
485 Get the response from the server.
486
487 If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.
488
489 If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.
490 """
491 # Raise the same error as http.client.HTTPConnection
492 if self._response_options is None:
493 raise ResponseNotReady()
494
495 # Reset this attribute for being used again.
496 resp_options = self._response_options
497 self._response_options = None
498
499 # Since the connection's timeout value may have been updated
500 # we need to set the timeout on the socket.
501 self.sock.settimeout(self.timeout)
502
503 # This is needed here to avoid circular import errors
504 from .response import HTTPResponse
505
506 # Get the response from http.client.HTTPConnection
507 httplib_response = super().getresponse()
508
509 try:
510 assert_header_parsing(httplib_response.msg)
511 except (HeaderParsingError, TypeError) as hpe:
512 log.warning(
513 "Failed to parse headers (url=%s): %s",
514 _url_from_connection(self, resp_options.request_url),
515 hpe,
516 exc_info=True,
517 )
518
519 headers = HTTPHeaderDict(httplib_response.msg.items())
520
521 response = HTTPResponse(
522 body=httplib_response,
523 headers=headers,
524 status=httplib_response.status,
525 version=httplib_response.version,
526 version_string=getattr(self, "_http_vsn_str", "HTTP/?"),
527 reason=httplib_response.reason,
528 preload_content=resp_options.preload_content,
529 decode_content=resp_options.decode_content,
530 original_response=httplib_response,
531 enforce_content_length=resp_options.enforce_content_length,
532 request_method=resp_options.request_method,
533 request_url=resp_options.request_url,
534 )
535 return response
536
537
538 class HTTPSConnection(HTTPConnection):
539 """
540 Many of the parameters to this constructor are passed to the underlying SSL
541 socket by means of :py:func:`urllib3.util.ssl_wrap_socket`.
542 """
543
544 default_port = port_by_scheme["https"] # type: ignore[misc]
545
546 cert_reqs: int | str | None = None
547 ca_certs: str | None = None
548 ca_cert_dir: str | None = None
549 ca_cert_data: None | str | bytes = None
550 ssl_version: int | str | None = None
551 ssl_minimum_version: int | None = None
552 ssl_maximum_version: int | None = None
553 assert_fingerprint: str | None = None
554 _connect_callback: typing.Callable[..., None] | None = None
555
556 def __init__(
557 self,
558 host: str,
559 port: int | None = None,
560 *,
561 timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
562 source_address: tuple[str, int] | None = None,
563 blocksize: int = 16384,
564 socket_options: None
565 | (connection._TYPE_SOCKET_OPTIONS) = HTTPConnection.default_socket_options,
566 proxy: Url | None = None,
567 proxy_config: ProxyConfig | None = None,
568 cert_reqs: int | str | None = None,
569 assert_hostname: None | str | typing.Literal[False] = None,
570 assert_fingerprint: str | None = None,
571 server_hostname: str | None = None,
572 ssl_context: ssl.SSLContext | None = None,
573 ca_certs: str | None = None,
574 ca_cert_dir: str | None = None,
575 ca_cert_data: None | str | bytes = None,
576 ssl_minimum_version: int | None = None,
577 ssl_maximum_version: int | None = None,
578 ssl_version: int | str | None = None, # Deprecated
579 cert_file: str | None = None,
580 key_file: str | None = None,
581 key_password: str | None = None,
582 ) -> None:
583 super().__init__(
584 host,
585 port=port,
586 timeout=timeout,
587 source_address=source_address,
588 blocksize=blocksize,
589 socket_options=socket_options,
590 proxy=proxy,
591 proxy_config=proxy_config,
592 )
593
594 self.key_file = key_file
595 self.cert_file = cert_file
596 self.key_password = key_password
597 self.ssl_context = ssl_context
598 self.server_hostname = server_hostname
599 self.assert_hostname = assert_hostname
600 self.assert_fingerprint = assert_fingerprint
601 self.ssl_version = ssl_version
602 self.ssl_minimum_version = ssl_minimum_version
603 self.ssl_maximum_version = ssl_maximum_version
604 self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
605 self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
606 self.ca_cert_data = ca_cert_data
607
608 # cert_reqs depends on ssl_context so calculate last.
609 if cert_reqs is None:
610 if self.ssl_context is not None:
611 cert_reqs = self.ssl_context.verify_mode
612 else:
613 cert_reqs = resolve_cert_reqs(None)
614 self.cert_reqs = cert_reqs
615 self._connect_callback = None
616
617 def set_cert(
618 self,
619 key_file: str | None = None,
620 cert_file: str | None = None,
621 cert_reqs: int | str | None = None,
622 key_password: str | None = None,
623 ca_certs: str | None = None,
624 assert_hostname: None | str | typing.Literal[False] = None,
625 assert_fingerprint: str | None = None,
626 ca_cert_dir: str | None = None,
627 ca_cert_data: None | str | bytes = None,
628 ) -> None:
629 """
630 This method should only be called once, before the connection is used.
631 """
632 warnings.warn(
633 "HTTPSConnection.set_cert() is deprecated and will be removed "
634 "in urllib3 v2.1.0. Instead provide the parameters to the "
635 "HTTPSConnection constructor.",
636 category=DeprecationWarning,
637 stacklevel=2,
638 )
639
640 # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
641 # have an SSLContext object in which case we'll use its verify_mode.
642 if cert_reqs is None:
643 if self.ssl_context is not None:
644 cert_reqs = self.ssl_context.verify_mode
645 else:
646 cert_reqs = resolve_cert_reqs(None)
647
648 self.key_file = key_file
649 self.cert_file = cert_file
650 self.cert_reqs = cert_reqs
651 self.key_password = key_password
652 self.assert_hostname = assert_hostname
653 self.assert_fingerprint = assert_fingerprint
654 self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
655 self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
656 self.ca_cert_data = ca_cert_data
657
658 def connect(self) -> None:
659 # Today we don't need to be doing this step before the /actual/ socket
660 # connection, however in the future we'll need to decide whether to
661 # create a new socket or re-use an existing "shared" socket as a part
662 # of the HTTP/2 handshake dance.
663 if self._tunnel_host is not None and self._tunnel_port is not None:
664 probe_http2_host = self._tunnel_host
665 probe_http2_port = self._tunnel_port
666 else:
667 probe_http2_host = self.host
668 probe_http2_port = self.port
669
670 # Check if the target origin supports HTTP/2.
671 # If the value comes back as 'None' it means that the current thread
672 # is probing for HTTP/2 support. Otherwise, we're waiting for another
673 # probe to complete, or we get a value right away.
674 target_supports_http2: bool | None
675 if "h2" in ssl_.ALPN_PROTOCOLS:
676 target_supports_http2 = http2_probe.acquire_and_get(
677 host=probe_http2_host, port=probe_http2_port
678 )
679 else:
680 # If HTTP/2 isn't going to be offered it doesn't matter if
681 # the target supports HTTP/2. Don't want to make a probe.
682 target_supports_http2 = False
683
684 if self._connect_callback is not None:
685 self._connect_callback(
686 "before connect",
687 thread_id=threading.get_ident(),
688 target_supports_http2=target_supports_http2,
689 )
690
691 try:
692 sock: socket.socket | ssl.SSLSocket
693 self.sock = sock = self._new_conn()
694 server_hostname: str = self.host
695 tls_in_tls = False
696
697 # Do we need to establish a tunnel?
698 if self._tunnel_host is not None:
699 # We're tunneling to an HTTPS origin so need to do TLS-in-TLS.
700 if self._tunnel_scheme == "https":
701 # _connect_tls_proxy will verify and assign proxy_is_verified
702 self.sock = sock = self._connect_tls_proxy(self.host, sock)
703 tls_in_tls = True
704 elif self._tunnel_scheme == "http":
705 self.proxy_is_verified = False
706
707 # If we're tunneling it means we're connected to our proxy.
708 self._has_connected_to_proxy = True
709
710 self._tunnel()
711 # Override the host with the one we're requesting data from.
712 server_hostname = self._tunnel_host
713
714 if self.server_hostname is not None:
715 server_hostname = self.server_hostname
716
717 is_time_off = datetime.date.today() < RECENT_DATE
718 if is_time_off:
719 warnings.warn(
720 (
721 f"System time is way off (before {RECENT_DATE}). This will probably "
722 "lead to SSL verification errors"
723 ),
724 SystemTimeWarning,
725 )
726
727 # Remove trailing '.' from fqdn hostnames to allow certificate validation
728 server_hostname_rm_dot = server_hostname.rstrip(".")
729
730 sock_and_verified = _ssl_wrap_socket_and_match_hostname(
731 sock=sock,
732 cert_reqs=self.cert_reqs,
733 ssl_version=self.ssl_version,
734 ssl_minimum_version=self.ssl_minimum_version,
735 ssl_maximum_version=self.ssl_maximum_version,
736 ca_certs=self.ca_certs,
737 ca_cert_dir=self.ca_cert_dir,
738 ca_cert_data=self.ca_cert_data,
739 cert_file=self.cert_file,
740 key_file=self.key_file,
741 key_password=self.key_password,
742 server_hostname=server_hostname_rm_dot,
743 ssl_context=self.ssl_context,
744 tls_in_tls=tls_in_tls,
745 assert_hostname=self.assert_hostname,
746 assert_fingerprint=self.assert_fingerprint,
747 )
748 self.sock = sock_and_verified.socket
749
750 # If an error occurs during connection/handshake we may need to release
751 # our lock so another connection can probe the origin.
752 except BaseException:
753 if self._connect_callback is not None:
754 self._connect_callback(
755 "after connect failure",
756 thread_id=threading.get_ident(),
757 target_supports_http2=target_supports_http2,
758 )
759
760 if target_supports_http2 is None:
761 http2_probe.set_and_release(
762 host=probe_http2_host, port=probe_http2_port, supports_http2=None
763 )
764 raise
765
766 # If this connection doesn't know if the origin supports HTTP/2
767 # we report back to the HTTP/2 probe our result.
768 if target_supports_http2 is None:
769 supports_http2 = sock_and_verified.socket.selected_alpn_protocol() == "h2"
770 http2_probe.set_and_release(
771 host=probe_http2_host,
772 port=probe_http2_port,
773 supports_http2=supports_http2,
774 )
775
776 # Forwarding proxies can never have a verified target since
777 # the proxy is the one doing the verification. Should instead
778 # use a CONNECT tunnel in order to verify the target.
779 # See: https://github.com/urllib3/urllib3/issues/3267.
780 if self.proxy_is_forwarding:
781 self.is_verified = False
782 else:
783 self.is_verified = sock_and_verified.is_verified
784
785 # If there's a proxy to be connected to we are fully connected.
786 # This is set twice (once above and here) due to forwarding proxies
787 # not using tunnelling.
788 self._has_connected_to_proxy = bool(self.proxy)
789
790 # Set `self.proxy_is_verified` unless it's already set while
791 # establishing a tunnel.
792 if self._has_connected_to_proxy and self.proxy_is_verified is None:
793 self.proxy_is_verified = sock_and_verified.is_verified
794
795 def _connect_tls_proxy(self, hostname: str, sock: socket.socket) -> ssl.SSLSocket:
796 """
797 Establish a TLS connection to the proxy using the provided SSL context.
798 """
799 # `_connect_tls_proxy` is called when self._tunnel_host is truthy.
800 proxy_config = typing.cast(ProxyConfig, self.proxy_config)
801 ssl_context = proxy_config.ssl_context
802 sock_and_verified = _ssl_wrap_socket_and_match_hostname(
803 sock,
804 cert_reqs=self.cert_reqs,
805 ssl_version=self.ssl_version,
806 ssl_minimum_version=self.ssl_minimum_version,
807 ssl_maximum_version=self.ssl_maximum_version,
808 ca_certs=self.ca_certs,
809 ca_cert_dir=self.ca_cert_dir,
810 ca_cert_data=self.ca_cert_data,
811 server_hostname=hostname,
812 ssl_context=ssl_context,
813 assert_hostname=proxy_config.assert_hostname,
814 assert_fingerprint=proxy_config.assert_fingerprint,
815 # Features that aren't implemented for proxies yet:
816 cert_file=None,
817 key_file=None,
818 key_password=None,
819 tls_in_tls=False,
820 )
821 self.proxy_is_verified = sock_and_verified.is_verified
822 return sock_and_verified.socket # type: ignore[return-value]
823
824
825 class _WrappedAndVerifiedSocket(typing.NamedTuple):
826 """
827 Wrapped socket and whether the connection is
828 verified after the TLS handshake
829 """
830
831 socket: ssl.SSLSocket | SSLTransport
832 is_verified: bool
833
834
835 def _ssl_wrap_socket_and_match_hostname(
836 sock: socket.socket,
837 *,
838 cert_reqs: None | str | int,
839 ssl_version: None | str | int,
840 ssl_minimum_version: int | None,
841 ssl_maximum_version: int | None,
842 cert_file: str | None,
843 key_file: str | None,
844 key_password: str | None,
845 ca_certs: str | None,
846 ca_cert_dir: str | None,
847 ca_cert_data: None | str | bytes,
848 assert_hostname: None | str | typing.Literal[False],
849 assert_fingerprint: str | None,
850 server_hostname: str | None,
851 ssl_context: ssl.SSLContext | None,
852 tls_in_tls: bool = False,
853 ) -> _WrappedAndVerifiedSocket:
854 """Logic for constructing an SSLContext from all TLS parameters, passing
855 that down into ssl_wrap_socket, and then doing certificate verification
856 either via hostname or fingerprint. This function exists to guarantee
857 that both proxies and targets have the same behavior when connecting via TLS.
858 """
859 default_ssl_context = False
860 if ssl_context is None:
861 default_ssl_context = True
862 context = create_urllib3_context(
863 ssl_version=resolve_ssl_version(ssl_version),
864 ssl_minimum_version=ssl_minimum_version,
865 ssl_maximum_version=ssl_maximum_version,
866 cert_reqs=resolve_cert_reqs(cert_reqs),
867 )
868 else:
869 context = ssl_context
870
871 context.verify_mode = resolve_cert_reqs(cert_reqs)
872
873 # In some cases, we want to verify hostnames ourselves
874 if (
875 # `ssl` can't verify fingerprints or alternate hostnames
876 assert_fingerprint
877 or assert_hostname
878 # assert_hostname can be set to False to disable hostname checking
879 or assert_hostname is False
880 # We still support OpenSSL 1.0.2, which prevents us from verifying
881 # hostnames easily: https://github.com/pyca/pyopenssl/pull/933
882 or ssl_.IS_PYOPENSSL
883 or not ssl_.HAS_NEVER_CHECK_COMMON_NAME
884 ):
885 context.check_hostname = False
886
887 # Try to load OS default certs if none are given. We need to do the hasattr() check
888 # for custom pyOpenSSL SSLContext objects because they don't support
889 # load_default_certs().
890 if (
891 not ca_certs
892 and not ca_cert_dir
893 and not ca_cert_data
894 and default_ssl_context
895 and hasattr(context, "load_default_certs")
896 ):
897 context.load_default_certs()
898
899 # Ensure that IPv6 addresses are in the proper format and don't have a
900 # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses
901 # and interprets them as DNS hostnames.
902 if server_hostname is not None:
903 normalized = server_hostname.strip("[]")
904 if "%" in normalized:
905 normalized = normalized[: normalized.rfind("%")]
906 if is_ipaddress(normalized):
907 server_hostname = normalized
908
909 ssl_sock = ssl_wrap_socket(
910 sock=sock,
911 keyfile=key_file,
912 certfile=cert_file,
913 key_password=key_password,
914 ca_certs=ca_certs,
915 ca_cert_dir=ca_cert_dir,
916 ca_cert_data=ca_cert_data,
917 server_hostname=server_hostname,
918 ssl_context=context,
919 tls_in_tls=tls_in_tls,
920 )
921
922 try:
923 if assert_fingerprint:
924 _assert_fingerprint(
925 ssl_sock.getpeercert(binary_form=True), assert_fingerprint
926 )
927 elif (
928 context.verify_mode != ssl.CERT_NONE
929 and not context.check_hostname
930 and assert_hostname is not False
931 ):
932 cert: _TYPE_PEER_CERT_RET_DICT = ssl_sock.getpeercert() # type: ignore[assignment]
933
934 # Need to signal to our match_hostname whether to use 'commonName' or not.
935 # If we're using our own constructed SSLContext we explicitly set 'False'
936 # because PyPy hard-codes 'True' from SSLContext.hostname_checks_common_name.
937 if default_ssl_context:
938 hostname_checks_common_name = False
939 else:
940 hostname_checks_common_name = (
941 getattr(context, "hostname_checks_common_name", False) or False
942 )
943
944 _match_hostname(
945 cert,
946 assert_hostname or server_hostname, # type: ignore[arg-type]
947 hostname_checks_common_name,
948 )
949
950 return _WrappedAndVerifiedSocket(
951 socket=ssl_sock,
952 is_verified=context.verify_mode == ssl.CERT_REQUIRED
953 or bool(assert_fingerprint),
954 )
955 except BaseException:
956 ssl_sock.close()
957 raise
958
959
960 def _match_hostname(
961 cert: _TYPE_PEER_CERT_RET_DICT | None,
962 asserted_hostname: str,
963 hostname_checks_common_name: bool = False,
964 ) -> None:
965 # Our upstream implementation of ssl.match_hostname()
966 # only applies this normalization to IP addresses so it doesn't
967 # match DNS SANs so we do the same thing!
968 stripped_hostname = asserted_hostname.strip("[]")
969 if is_ipaddress(stripped_hostname):
970 asserted_hostname = stripped_hostname
971
972 try:
973 match_hostname(cert, asserted_hostname, hostname_checks_common_name)
974 except CertificateError as e:
975 log.warning(
976 "Certificate did not match expected hostname: %s. Certificate: %s",
977 asserted_hostname,
978 cert,
979 )
980 # Add cert to exception and reraise so client code can inspect
981 # the cert when catching the exception, if they want to
982 e._peer_cert = cert # type: ignore[attr-defined]
983 raise
984
985
986 def _wrap_proxy_error(err: Exception, proxy_scheme: str | None) -> ProxyError:
987 # Look for the phrase 'wrong version number', if found
988 # then we should warn the user that we're very sure that
989 # this proxy is HTTP-only and they have a configuration issue.
990 error_normalized = " ".join(re.split("[^a-z]", str(err).lower()))
991 is_likely_http_proxy = (
992 "wrong version number" in error_normalized
993 or "unknown protocol" in error_normalized
994 or "record layer failure" in error_normalized
995 )
996 http_proxy_warning = (
997 ". Your proxy appears to only use HTTP and not HTTPS, "
998 "try changing your proxy URL to be HTTP. See: "
999 "https://urllib3.readthedocs.io/en/latest/advanced-usage.html"
1000 "#https-proxy-error-http-proxy"
1001 )
1002 new_err = ProxyError(
1003 f"Unable to connect to proxy"
1004 f"{http_proxy_warning if is_likely_http_proxy and proxy_scheme == 'https' else ''}",
1005 err,
1006 )
1007 new_err.__cause__ = err
1008 return new_err
1009
1010
1011 def _get_default_user_agent() -> str:
1012 return f"python-urllib3/{__version__}"
1013
1014
1015 class DummyConnection:
1016 """Used to detect a failed ConnectionCls import."""
1017
1018
1019 if not ssl:
1020 HTTPSConnection = DummyConnection # type: ignore[misc, assignment] # noqa: F811
1021
1022
1023 VerifiedHTTPSConnection = HTTPSConnection
1024
1025
1026 def _url_from_connection(
1027 conn: HTTPConnection | HTTPSConnection, path: str | None = None
1028 ) -> str:
1029 """Returns the URL from a given connection. This is mainly used for testing and logging."""
1030
1031 scheme = "https" if isinstance(conn, HTTPSConnection) else "http"
1032
1033 return Url(scheme=scheme, host=conn.host, port=conn.port, path=path).url