Mercurial > repos > jpayne > bioproject_to_srr_2

comparison requests/adapters.py @ 7:5eb2d5e3bf22

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload for repository https://toolrepo.galaxytrakr.org/view/jpayne/bioproject_to_srr_2/556cac4fb538
author jpayne
date Sun, 05 May 2024 23:32:17 -0400
parents
children
comparison
equal deleted inserted replaced
6:b2745907b1eb 7:5eb2d5e3bf22
1 """
2 requests.adapters
3 ~~~~~~~~~~~~~~~~~
4
5 This module contains the transport adapters that Requests uses to define
6 and maintain connections.
7 """
8
9 import os.path
10 import socket # noqa: F401
11
12 from urllib3.exceptions import ClosedPoolError, ConnectTimeoutError
13 from urllib3.exceptions import HTTPError as _HTTPError
14 from urllib3.exceptions import InvalidHeader as _InvalidHeader
15 from urllib3.exceptions import (
16 LocationValueError,
17 MaxRetryError,
18 NewConnectionError,
19 ProtocolError,
20 )
21 from urllib3.exceptions import ProxyError as _ProxyError
22 from urllib3.exceptions import ReadTimeoutError, ResponseError
23 from urllib3.exceptions import SSLError as _SSLError
24 from urllib3.poolmanager import PoolManager, proxy_from_url
25 from urllib3.util import Timeout as TimeoutSauce
26 from urllib3.util import parse_url
27 from urllib3.util.retry import Retry
28
29 from .auth import _basic_auth_str
30 from .compat import basestring, urlparse
31 from .cookies import extract_cookies_to_jar
32 from .exceptions import (
33 ConnectionError,
34 ConnectTimeout,
35 InvalidHeader,
36 InvalidProxyURL,
37 InvalidSchema,
38 InvalidURL,
39 ProxyError,
40 ReadTimeout,
41 RetryError,
42 SSLError,
43 )
44 from .models import Response
45 from .structures import CaseInsensitiveDict
46 from .utils import (
47 DEFAULT_CA_BUNDLE_PATH,
48 extract_zipped_paths,
49 get_auth_from_url,
50 get_encoding_from_headers,
51 prepend_scheme_if_needed,
52 select_proxy,
53 urldefragauth,
54 )
55
56 try:
57 from urllib3.contrib.socks import SOCKSProxyManager
58 except ImportError:
59
60 def SOCKSProxyManager(*args, **kwargs):
61 raise InvalidSchema("Missing dependencies for SOCKS support.")
62
63
64 DEFAULT_POOLBLOCK = False
65 DEFAULT_POOLSIZE = 10
66 DEFAULT_RETRIES = 0
67 DEFAULT_POOL_TIMEOUT = None
68
69
70 class BaseAdapter:
71 """The Base Transport Adapter"""
72
73 def __init__(self):
74 super().__init__()
75
76 def send(
77 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
78 ):
79 """Sends PreparedRequest object. Returns Response object.
80
81 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
82 :param stream: (optional) Whether to stream the request content.
83 :param timeout: (optional) How long to wait for the server to send
84 data before giving up, as a float, or a :ref:`(connect timeout,
85 read timeout) <timeouts>` tuple.
86 :type timeout: float or tuple
87 :param verify: (optional) Either a boolean, in which case it controls whether we verify
88 the server's TLS certificate, or a string, in which case it must be a path
89 to a CA bundle to use
90 :param cert: (optional) Any user-provided SSL certificate to be trusted.
91 :param proxies: (optional) The proxies dictionary to apply to the request.
92 """
93 raise NotImplementedError
94
95 def close(self):
96 """Cleans up adapter specific items."""
97 raise NotImplementedError
98
99
100 class HTTPAdapter(BaseAdapter):
101 """The built-in HTTP Adapter for urllib3.
102
103 Provides a general-case interface for Requests sessions to contact HTTP and
104 HTTPS urls by implementing the Transport Adapter interface. This class will
105 usually be created by the :class:`Session <Session>` class under the
106 covers.
107
108 :param pool_connections: The number of urllib3 connection pools to cache.
109 :param pool_maxsize: The maximum number of connections to save in the pool.
110 :param max_retries: The maximum number of retries each connection
111 should attempt. Note, this applies only to failed DNS lookups, socket
112 connections and connection timeouts, never to requests where data has
113 made it to the server. By default, Requests does not retry failed
114 connections. If you need granular control over the conditions under
115 which we retry a request, import urllib3's ``Retry`` class and pass
116 that instead.
117 :param pool_block: Whether the connection pool should block for connections.
118
119 Usage::
120
121 >>> import requests
122 >>> s = requests.Session()
123 >>> a = requests.adapters.HTTPAdapter(max_retries=3)
124 >>> s.mount('http://', a)
125 """
126
127 __attrs__ = [
128 "max_retries",
129 "config",
130 "_pool_connections",
131 "_pool_maxsize",
132 "_pool_block",
133 ]
134
135 def __init__(
136 self,
137 pool_connections=DEFAULT_POOLSIZE,
138 pool_maxsize=DEFAULT_POOLSIZE,
139 max_retries=DEFAULT_RETRIES,
140 pool_block=DEFAULT_POOLBLOCK,
141 ):
142 if max_retries == DEFAULT_RETRIES:
143 self.max_retries = Retry(0, read=False)
144 else:
145 self.max_retries = Retry.from_int(max_retries)
146 self.config = {}
147 self.proxy_manager = {}
148
149 super().__init__()
150
151 self._pool_connections = pool_connections
152 self._pool_maxsize = pool_maxsize
153 self._pool_block = pool_block
154
155 self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block)
156
157 def __getstate__(self):
158 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
159
160 def __setstate__(self, state):
161 # Can't handle by adding 'proxy_manager' to self.__attrs__ because
162 # self.poolmanager uses a lambda function, which isn't pickleable.
163 self.proxy_manager = {}
164 self.config = {}
165
166 for attr, value in state.items():
167 setattr(self, attr, value)
168
169 self.init_poolmanager(
170 self._pool_connections, self._pool_maxsize, block=self._pool_block
171 )
172
173 def init_poolmanager(
174 self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs
175 ):
176 """Initializes a urllib3 PoolManager.
177
178 This method should not be called from user code, and is only
179 exposed for use when subclassing the
180 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
181
182 :param connections: The number of urllib3 connection pools to cache.
183 :param maxsize: The maximum number of connections to save in the pool.
184 :param block: Block when no free connections are available.
185 :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
186 """
187 # save these values for pickling
188 self._pool_connections = connections
189 self._pool_maxsize = maxsize
190 self._pool_block = block
191
192 self.poolmanager = PoolManager(
193 num_pools=connections,
194 maxsize=maxsize,
195 block=block,
196 **pool_kwargs,
197 )
198
199 def proxy_manager_for(self, proxy, **proxy_kwargs):
200 """Return urllib3 ProxyManager for the given proxy.
201
202 This method should not be called from user code, and is only
203 exposed for use when subclassing the
204 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
205
206 :param proxy: The proxy to return a urllib3 ProxyManager for.
207 :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
208 :returns: ProxyManager
209 :rtype: urllib3.ProxyManager
210 """
211 if proxy in self.proxy_manager:
212 manager = self.proxy_manager[proxy]
213 elif proxy.lower().startswith("socks"):
214 username, password = get_auth_from_url(proxy)
215 manager = self.proxy_manager[proxy] = SOCKSProxyManager(
216 proxy,
217 username=username,
218 password=password,
219 num_pools=self._pool_connections,
220 maxsize=self._pool_maxsize,
221 block=self._pool_block,
222 **proxy_kwargs,
223 )
224 else:
225 proxy_headers = self.proxy_headers(proxy)
226 manager = self.proxy_manager[proxy] = proxy_from_url(
227 proxy,
228 proxy_headers=proxy_headers,
229 num_pools=self._pool_connections,
230 maxsize=self._pool_maxsize,
231 block=self._pool_block,
232 **proxy_kwargs,
233 )
234
235 return manager
236
237 def cert_verify(self, conn, url, verify, cert):
238 """Verify a SSL certificate. This method should not be called from user
239 code, and is only exposed for use when subclassing the
240 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
241
242 :param conn: The urllib3 connection object associated with the cert.
243 :param url: The requested URL.
244 :param verify: Either a boolean, in which case it controls whether we verify
245 the server's TLS certificate, or a string, in which case it must be a path
246 to a CA bundle to use
247 :param cert: The SSL certificate to verify.
248 """
249 if url.lower().startswith("https") and verify:
250
251 cert_loc = None
252
253 # Allow self-specified cert location.
254 if verify is not True:
255 cert_loc = verify
256
257 if not cert_loc:
258 cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH)
259
260 if not cert_loc or not os.path.exists(cert_loc):
261 raise OSError(
262 f"Could not find a suitable TLS CA certificate bundle, "
263 f"invalid path: {cert_loc}"
264 )
265
266 conn.cert_reqs = "CERT_REQUIRED"
267
268 if not os.path.isdir(cert_loc):
269 conn.ca_certs = cert_loc
270 else:
271 conn.ca_cert_dir = cert_loc
272 else:
273 conn.cert_reqs = "CERT_NONE"
274 conn.ca_certs = None
275 conn.ca_cert_dir = None
276
277 if cert:
278 if not isinstance(cert, basestring):
279 conn.cert_file = cert[0]
280 conn.key_file = cert[1]
281 else:
282 conn.cert_file = cert
283 conn.key_file = None
284 if conn.cert_file and not os.path.exists(conn.cert_file):
285 raise OSError(
286 f"Could not find the TLS certificate file, "
287 f"invalid path: {conn.cert_file}"
288 )
289 if conn.key_file and not os.path.exists(conn.key_file):
290 raise OSError(
291 f"Could not find the TLS key file, invalid path: {conn.key_file}"
292 )
293
294 def build_response(self, req, resp):
295 """Builds a :class:`Response <requests.Response>` object from a urllib3
296 response. This should not be called from user code, and is only exposed
297 for use when subclassing the
298 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`
299
300 :param req: The :class:`PreparedRequest <PreparedRequest>` used to generate the response.
301 :param resp: The urllib3 response object.
302 :rtype: requests.Response
303 """
304 response = Response()
305
306 # Fallback to None if there's no status_code, for whatever reason.
307 response.status_code = getattr(resp, "status", None)
308
309 # Make headers case-insensitive.
310 response.headers = CaseInsensitiveDict(getattr(resp, "headers", {}))
311
312 # Set encoding.
313 response.encoding = get_encoding_from_headers(response.headers)
314 response.raw = resp
315 response.reason = response.raw.reason
316
317 if isinstance(req.url, bytes):
318 response.url = req.url.decode("utf-8")
319 else:
320 response.url = req.url
321
322 # Add new cookies from the server.
323 extract_cookies_to_jar(response.cookies, req, resp)
324
325 # Give the Response some context.
326 response.request = req
327 response.connection = self
328
329 return response
330
331 def get_connection(self, url, proxies=None):
332 """Returns a urllib3 connection for the given URL. This should not be
333 called from user code, and is only exposed for use when subclassing the
334 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
335
336 :param url: The URL to connect to.
337 :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
338 :rtype: urllib3.ConnectionPool
339 """
340 proxy = select_proxy(url, proxies)
341
342 if proxy:
343 proxy = prepend_scheme_if_needed(proxy, "http")
344 proxy_url = parse_url(proxy)
345 if not proxy_url.host:
346 raise InvalidProxyURL(
347 "Please check proxy URL. It is malformed "
348 "and could be missing the host."
349 )
350 proxy_manager = self.proxy_manager_for(proxy)
351 conn = proxy_manager.connection_from_url(url)
352 else:
353 # Only scheme should be lower case
354 parsed = urlparse(url)
355 url = parsed.geturl()
356 conn = self.poolmanager.connection_from_url(url)
357
358 return conn
359
360 def close(self):
361 """Disposes of any internal state.
362
363 Currently, this closes the PoolManager and any active ProxyManager,
364 which closes any pooled connections.
365 """
366 self.poolmanager.clear()
367 for proxy in self.proxy_manager.values():
368 proxy.clear()
369
370 def request_url(self, request, proxies):
371 """Obtain the url to use when making the final request.
372
373 If the message is being sent through a HTTP proxy, the full URL has to
374 be used. Otherwise, we should only use the path portion of the URL.
375
376 This should not be called from user code, and is only exposed for use
377 when subclassing the
378 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
379
380 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
381 :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
382 :rtype: str
383 """
384 proxy = select_proxy(request.url, proxies)
385 scheme = urlparse(request.url).scheme
386
387 is_proxied_http_request = proxy and scheme != "https"
388 using_socks_proxy = False
389 if proxy:
390 proxy_scheme = urlparse(proxy).scheme.lower()
391 using_socks_proxy = proxy_scheme.startswith("socks")
392
393 url = request.path_url
394 if is_proxied_http_request and not using_socks_proxy:
395 url = urldefragauth(request.url)
396
397 return url
398
399 def add_headers(self, request, **kwargs):
400 """Add any headers needed by the connection. As of v2.0 this does
401 nothing by default, but is left for overriding by users that subclass
402 the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
403
404 This should not be called from user code, and is only exposed for use
405 when subclassing the
406 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
407
408 :param request: The :class:`PreparedRequest <PreparedRequest>` to add headers to.
409 :param kwargs: The keyword arguments from the call to send().
410 """
411 pass
412
413 def proxy_headers(self, proxy):
414 """Returns a dictionary of the headers to add to any request sent
415 through a proxy. This works with urllib3 magic to ensure that they are
416 correctly sent to the proxy, rather than in a tunnelled request if
417 CONNECT is being used.
418
419 This should not be called from user code, and is only exposed for use
420 when subclassing the
421 :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
422
423 :param proxy: The url of the proxy being used for this request.
424 :rtype: dict
425 """
426 headers = {}
427 username, password = get_auth_from_url(proxy)
428
429 if username:
430 headers["Proxy-Authorization"] = _basic_auth_str(username, password)
431
432 return headers
433
434 def send(
435 self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None
436 ):
437 """Sends PreparedRequest object. Returns Response object.
438
439 :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
440 :param stream: (optional) Whether to stream the request content.
441 :param timeout: (optional) How long to wait for the server to send
442 data before giving up, as a float, or a :ref:`(connect timeout,
443 read timeout) <timeouts>` tuple.
444 :type timeout: float or tuple or urllib3 Timeout object
445 :param verify: (optional) Either a boolean, in which case it controls whether
446 we verify the server's TLS certificate, or a string, in which case it
447 must be a path to a CA bundle to use
448 :param cert: (optional) Any user-provided SSL certificate to be trusted.
449 :param proxies: (optional) The proxies dictionary to apply to the request.
450 :rtype: requests.Response
451 """
452
453 try:
454 conn = self.get_connection(request.url, proxies)
455 except LocationValueError as e:
456 raise InvalidURL(e, request=request)
457
458 self.cert_verify(conn, request.url, verify, cert)
459 url = self.request_url(request, proxies)
460 self.add_headers(
461 request,
462 stream=stream,
463 timeout=timeout,
464 verify=verify,
465 cert=cert,
466 proxies=proxies,
467 )
468
469 chunked = not (request.body is None or "Content-Length" in request.headers)
470
471 if isinstance(timeout, tuple):
472 try:
473 connect, read = timeout
474 timeout = TimeoutSauce(connect=connect, read=read)
475 except ValueError:
476 raise ValueError(
477 f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
478 f"or a single float to set both timeouts to the same value."
479 )
480 elif isinstance(timeout, TimeoutSauce):
481 pass
482 else:
483 timeout = TimeoutSauce(connect=timeout, read=timeout)
484
485 try:
486 resp = conn.urlopen(
487 method=request.method,
488 url=url,
489 body=request.body,
490 headers=request.headers,
491 redirect=False,
492 assert_same_host=False,
493 preload_content=False,
494 decode_content=False,
495 retries=self.max_retries,
496 timeout=timeout,
497 chunked=chunked,
498 )
499
500 except (ProtocolError, OSError) as err:
501 raise ConnectionError(err, request=request)
502
503 except MaxRetryError as e:
504 if isinstance(e.reason, ConnectTimeoutError):
505 # TODO: Remove this in 3.0.0: see #2811
506 if not isinstance(e.reason, NewConnectionError):
507 raise ConnectTimeout(e, request=request)
508
509 if isinstance(e.reason, ResponseError):
510 raise RetryError(e, request=request)
511
512 if isinstance(e.reason, _ProxyError):
513 raise ProxyError(e, request=request)
514
515 if isinstance(e.reason, _SSLError):
516 # This branch is for urllib3 v1.22 and later.
517 raise SSLError(e, request=request)
518
519 raise ConnectionError(e, request=request)
520
521 except ClosedPoolError as e:
522 raise ConnectionError(e, request=request)
523
524 except _ProxyError as e:
525 raise ProxyError(e)
526
527 except (_SSLError, _HTTPError) as e:
528 if isinstance(e, _SSLError):
529 # This branch is for urllib3 versions earlier than v1.22
530 raise SSLError(e, request=request)
531 elif isinstance(e, ReadTimeoutError):
532 raise ReadTimeout(e, request=request)
533 elif isinstance(e, _InvalidHeader):
534 raise InvalidHeader(e, request=request)
535 else:
536 raise
537
538 return self.build_response(request, resp)