|
jpayne@7
|
1 from __future__ import annotations
|
|
jpayne@7
|
2
|
|
jpayne@7
|
3 import email
|
|
jpayne@7
|
4 import logging
|
|
jpayne@7
|
5 import random
|
|
jpayne@7
|
6 import re
|
|
jpayne@7
|
7 import time
|
|
jpayne@7
|
8 import typing
|
|
jpayne@7
|
9 from itertools import takewhile
|
|
jpayne@7
|
10 from types import TracebackType
|
|
jpayne@7
|
11
|
|
jpayne@7
|
12 from ..exceptions import (
|
|
jpayne@7
|
13 ConnectTimeoutError,
|
|
jpayne@7
|
14 InvalidHeader,
|
|
jpayne@7
|
15 MaxRetryError,
|
|
jpayne@7
|
16 ProtocolError,
|
|
jpayne@7
|
17 ProxyError,
|
|
jpayne@7
|
18 ReadTimeoutError,
|
|
jpayne@7
|
19 ResponseError,
|
|
jpayne@7
|
20 )
|
|
jpayne@7
|
21 from .util import reraise
|
|
jpayne@7
|
22
|
|
jpayne@7
|
23 if typing.TYPE_CHECKING:
|
|
jpayne@7
|
24 from ..connectionpool import ConnectionPool
|
|
jpayne@7
|
25 from ..response import BaseHTTPResponse
|
|
jpayne@7
|
26
|
|
jpayne@7
|
27 log = logging.getLogger(__name__)
|
|
jpayne@7
|
28
|
|
jpayne@7
|
29
|
|
jpayne@7
|
30 # Data structure for representing the metadata of requests that result in a retry.
|
|
jpayne@7
|
31 class RequestHistory(typing.NamedTuple):
|
|
jpayne@7
|
32 method: str | None
|
|
jpayne@7
|
33 url: str | None
|
|
jpayne@7
|
34 error: Exception | None
|
|
jpayne@7
|
35 status: int | None
|
|
jpayne@7
|
36 redirect_location: str | None
|
|
jpayne@7
|
37
|
|
jpayne@7
|
38
|
|
jpayne@7
|
39 class Retry:
|
|
jpayne@7
|
40 """Retry configuration.
|
|
jpayne@7
|
41
|
|
jpayne@7
|
42 Each retry attempt will create a new Retry object with updated values, so
|
|
jpayne@7
|
43 they can be safely reused.
|
|
jpayne@7
|
44
|
|
jpayne@7
|
45 Retries can be defined as a default for a pool:
|
|
jpayne@7
|
46
|
|
jpayne@7
|
47 .. code-block:: python
|
|
jpayne@7
|
48
|
|
jpayne@7
|
49 retries = Retry(connect=5, read=2, redirect=5)
|
|
jpayne@7
|
50 http = PoolManager(retries=retries)
|
|
jpayne@7
|
51 response = http.request("GET", "https://example.com/")
|
|
jpayne@7
|
52
|
|
jpayne@7
|
53 Or per-request (which overrides the default for the pool):
|
|
jpayne@7
|
54
|
|
jpayne@7
|
55 .. code-block:: python
|
|
jpayne@7
|
56
|
|
jpayne@7
|
57 response = http.request("GET", "https://example.com/", retries=Retry(10))
|
|
jpayne@7
|
58
|
|
jpayne@7
|
59 Retries can be disabled by passing ``False``:
|
|
jpayne@7
|
60
|
|
jpayne@7
|
61 .. code-block:: python
|
|
jpayne@7
|
62
|
|
jpayne@7
|
63 response = http.request("GET", "https://example.com/", retries=False)
|
|
jpayne@7
|
64
|
|
jpayne@7
|
65 Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
|
|
jpayne@7
|
66 retries are disabled, in which case the causing exception will be raised.
|
|
jpayne@7
|
67
|
|
jpayne@7
|
68 :param int total:
|
|
jpayne@7
|
69 Total number of retries to allow. Takes precedence over other counts.
|
|
jpayne@7
|
70
|
|
jpayne@7
|
71 Set to ``None`` to remove this constraint and fall back on other
|
|
jpayne@7
|
72 counts.
|
|
jpayne@7
|
73
|
|
jpayne@7
|
74 Set to ``0`` to fail on the first retry.
|
|
jpayne@7
|
75
|
|
jpayne@7
|
76 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
|
jpayne@7
|
77
|
|
jpayne@7
|
78 :param int connect:
|
|
jpayne@7
|
79 How many connection-related errors to retry on.
|
|
jpayne@7
|
80
|
|
jpayne@7
|
81 These are errors raised before the request is sent to the remote server,
|
|
jpayne@7
|
82 which we assume has not triggered the server to process the request.
|
|
jpayne@7
|
83
|
|
jpayne@7
|
84 Set to ``0`` to fail on the first retry of this type.
|
|
jpayne@7
|
85
|
|
jpayne@7
|
86 :param int read:
|
|
jpayne@7
|
87 How many times to retry on read errors.
|
|
jpayne@7
|
88
|
|
jpayne@7
|
89 These errors are raised after the request was sent to the server, so the
|
|
jpayne@7
|
90 request may have side-effects.
|
|
jpayne@7
|
91
|
|
jpayne@7
|
92 Set to ``0`` to fail on the first retry of this type.
|
|
jpayne@7
|
93
|
|
jpayne@7
|
94 :param int redirect:
|
|
jpayne@7
|
95 How many redirects to perform. Limit this to avoid infinite redirect
|
|
jpayne@7
|
96 loops.
|
|
jpayne@7
|
97
|
|
jpayne@7
|
98 A redirect is a HTTP response with a status code 301, 302, 303, 307 or
|
|
jpayne@7
|
99 308.
|
|
jpayne@7
|
100
|
|
jpayne@7
|
101 Set to ``0`` to fail on the first retry of this type.
|
|
jpayne@7
|
102
|
|
jpayne@7
|
103 Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
|
jpayne@7
|
104
|
|
jpayne@7
|
105 :param int status:
|
|
jpayne@7
|
106 How many times to retry on bad status codes.
|
|
jpayne@7
|
107
|
|
jpayne@7
|
108 These are retries made on responses, where status code matches
|
|
jpayne@7
|
109 ``status_forcelist``.
|
|
jpayne@7
|
110
|
|
jpayne@7
|
111 Set to ``0`` to fail on the first retry of this type.
|
|
jpayne@7
|
112
|
|
jpayne@7
|
113 :param int other:
|
|
jpayne@7
|
114 How many times to retry on other errors.
|
|
jpayne@7
|
115
|
|
jpayne@7
|
116 Other errors are errors that are not connect, read, redirect or status errors.
|
|
jpayne@7
|
117 These errors might be raised after the request was sent to the server, so the
|
|
jpayne@7
|
118 request might have side-effects.
|
|
jpayne@7
|
119
|
|
jpayne@7
|
120 Set to ``0`` to fail on the first retry of this type.
|
|
jpayne@7
|
121
|
|
jpayne@7
|
122 If ``total`` is not set, it's a good idea to set this to 0 to account
|
|
jpayne@7
|
123 for unexpected edge cases and avoid infinite retry loops.
|
|
jpayne@7
|
124
|
|
jpayne@7
|
125 :param Collection allowed_methods:
|
|
jpayne@7
|
126 Set of uppercased HTTP method verbs that we should retry on.
|
|
jpayne@7
|
127
|
|
jpayne@7
|
128 By default, we only retry on methods which are considered to be
|
|
jpayne@7
|
129 idempotent (multiple requests with the same parameters end with the
|
|
jpayne@7
|
130 same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
|
|
jpayne@7
|
131
|
|
jpayne@7
|
132 Set to a ``None`` value to retry on any verb.
|
|
jpayne@7
|
133
|
|
jpayne@7
|
134 :param Collection status_forcelist:
|
|
jpayne@7
|
135 A set of integer HTTP status codes that we should force a retry on.
|
|
jpayne@7
|
136 A retry is initiated if the request method is in ``allowed_methods``
|
|
jpayne@7
|
137 and the response status code is in ``status_forcelist``.
|
|
jpayne@7
|
138
|
|
jpayne@7
|
139 By default, this is disabled with ``None``.
|
|
jpayne@7
|
140
|
|
jpayne@7
|
141 :param float backoff_factor:
|
|
jpayne@7
|
142 A backoff factor to apply between attempts after the second try
|
|
jpayne@7
|
143 (most errors are resolved immediately by a second try without a
|
|
jpayne@7
|
144 delay). urllib3 will sleep for::
|
|
jpayne@7
|
145
|
|
jpayne@7
|
146 {backoff factor} * (2 ** ({number of previous retries}))
|
|
jpayne@7
|
147
|
|
jpayne@7
|
148 seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
|
|
jpayne@7
|
149
|
|
jpayne@7
|
150 random.uniform(0, {backoff jitter})
|
|
jpayne@7
|
151
|
|
jpayne@7
|
152 seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
|
|
jpayne@7
|
153 sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
|
|
jpayne@7
|
154 be longer than `backoff_max`.
|
|
jpayne@7
|
155
|
|
jpayne@7
|
156 By default, backoff is disabled (factor set to 0).
|
|
jpayne@7
|
157
|
|
jpayne@7
|
158 :param bool raise_on_redirect: Whether, if the number of redirects is
|
|
jpayne@7
|
159 exhausted, to raise a MaxRetryError, or to return a response with a
|
|
jpayne@7
|
160 response code in the 3xx range.
|
|
jpayne@7
|
161
|
|
jpayne@7
|
162 :param bool raise_on_status: Similar meaning to ``raise_on_redirect``:
|
|
jpayne@7
|
163 whether we should raise an exception, or return a response,
|
|
jpayne@7
|
164 if status falls in ``status_forcelist`` range and retries have
|
|
jpayne@7
|
165 been exhausted.
|
|
jpayne@7
|
166
|
|
jpayne@7
|
167 :param tuple history: The history of the request encountered during
|
|
jpayne@7
|
168 each call to :meth:`~Retry.increment`. The list is in the order
|
|
jpayne@7
|
169 the requests occurred. Each list item is of class :class:`RequestHistory`.
|
|
jpayne@7
|
170
|
|
jpayne@7
|
171 :param bool respect_retry_after_header:
|
|
jpayne@7
|
172 Whether to respect Retry-After header on status codes defined as
|
|
jpayne@7
|
173 :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
|
|
jpayne@7
|
174
|
|
jpayne@7
|
175 :param Collection remove_headers_on_redirect:
|
|
jpayne@7
|
176 Sequence of headers to remove from the request when a response
|
|
jpayne@7
|
177 indicating a redirect is returned before firing off the redirected
|
|
jpayne@7
|
178 request.
|
|
jpayne@7
|
179 """
|
|
jpayne@7
|
180
|
|
jpayne@7
|
181 #: Default methods to be used for ``allowed_methods``
|
|
jpayne@7
|
182 DEFAULT_ALLOWED_METHODS = frozenset(
|
|
jpayne@7
|
183 ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
|
|
jpayne@7
|
184 )
|
|
jpayne@7
|
185
|
|
jpayne@7
|
186 #: Default status codes to be used for ``status_forcelist``
|
|
jpayne@7
|
187 RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
|
|
jpayne@7
|
188
|
|
jpayne@7
|
189 #: Default headers to be used for ``remove_headers_on_redirect``
|
|
jpayne@7
|
190 DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])
|
|
jpayne@7
|
191
|
|
jpayne@7
|
192 #: Default maximum backoff time.
|
|
jpayne@7
|
193 DEFAULT_BACKOFF_MAX = 120
|
|
jpayne@7
|
194
|
|
jpayne@7
|
195 # Backward compatibility; assigned outside of the class.
|
|
jpayne@7
|
196 DEFAULT: typing.ClassVar[Retry]
|
|
jpayne@7
|
197
|
|
jpayne@7
|
198 def __init__(
|
|
jpayne@7
|
199 self,
|
|
jpayne@7
|
200 total: bool | int | None = 10,
|
|
jpayne@7
|
201 connect: int | None = None,
|
|
jpayne@7
|
202 read: int | None = None,
|
|
jpayne@7
|
203 redirect: bool | int | None = None,
|
|
jpayne@7
|
204 status: int | None = None,
|
|
jpayne@7
|
205 other: int | None = None,
|
|
jpayne@7
|
206 allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
|
|
jpayne@7
|
207 status_forcelist: typing.Collection[int] | None = None,
|
|
jpayne@7
|
208 backoff_factor: float = 0,
|
|
jpayne@7
|
209 backoff_max: float = DEFAULT_BACKOFF_MAX,
|
|
jpayne@7
|
210 raise_on_redirect: bool = True,
|
|
jpayne@7
|
211 raise_on_status: bool = True,
|
|
jpayne@7
|
212 history: tuple[RequestHistory, ...] | None = None,
|
|
jpayne@7
|
213 respect_retry_after_header: bool = True,
|
|
jpayne@7
|
214 remove_headers_on_redirect: typing.Collection[
|
|
jpayne@7
|
215 str
|
|
jpayne@7
|
216 ] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
|
|
jpayne@7
|
217 backoff_jitter: float = 0.0,
|
|
jpayne@7
|
218 ) -> None:
|
|
jpayne@7
|
219 self.total = total
|
|
jpayne@7
|
220 self.connect = connect
|
|
jpayne@7
|
221 self.read = read
|
|
jpayne@7
|
222 self.status = status
|
|
jpayne@7
|
223 self.other = other
|
|
jpayne@7
|
224
|
|
jpayne@7
|
225 if redirect is False or total is False:
|
|
jpayne@7
|
226 redirect = 0
|
|
jpayne@7
|
227 raise_on_redirect = False
|
|
jpayne@7
|
228
|
|
jpayne@7
|
229 self.redirect = redirect
|
|
jpayne@7
|
230 self.status_forcelist = status_forcelist or set()
|
|
jpayne@7
|
231 self.allowed_methods = allowed_methods
|
|
jpayne@7
|
232 self.backoff_factor = backoff_factor
|
|
jpayne@7
|
233 self.backoff_max = backoff_max
|
|
jpayne@7
|
234 self.raise_on_redirect = raise_on_redirect
|
|
jpayne@7
|
235 self.raise_on_status = raise_on_status
|
|
jpayne@7
|
236 self.history = history or ()
|
|
jpayne@7
|
237 self.respect_retry_after_header = respect_retry_after_header
|
|
jpayne@7
|
238 self.remove_headers_on_redirect = frozenset(
|
|
jpayne@7
|
239 h.lower() for h in remove_headers_on_redirect
|
|
jpayne@7
|
240 )
|
|
jpayne@7
|
241 self.backoff_jitter = backoff_jitter
|
|
jpayne@7
|
242
|
|
jpayne@7
|
243 def new(self, **kw: typing.Any) -> Retry:
|
|
jpayne@7
|
244 params = dict(
|
|
jpayne@7
|
245 total=self.total,
|
|
jpayne@7
|
246 connect=self.connect,
|
|
jpayne@7
|
247 read=self.read,
|
|
jpayne@7
|
248 redirect=self.redirect,
|
|
jpayne@7
|
249 status=self.status,
|
|
jpayne@7
|
250 other=self.other,
|
|
jpayne@7
|
251 allowed_methods=self.allowed_methods,
|
|
jpayne@7
|
252 status_forcelist=self.status_forcelist,
|
|
jpayne@7
|
253 backoff_factor=self.backoff_factor,
|
|
jpayne@7
|
254 backoff_max=self.backoff_max,
|
|
jpayne@7
|
255 raise_on_redirect=self.raise_on_redirect,
|
|
jpayne@7
|
256 raise_on_status=self.raise_on_status,
|
|
jpayne@7
|
257 history=self.history,
|
|
jpayne@7
|
258 remove_headers_on_redirect=self.remove_headers_on_redirect,
|
|
jpayne@7
|
259 respect_retry_after_header=self.respect_retry_after_header,
|
|
jpayne@7
|
260 backoff_jitter=self.backoff_jitter,
|
|
jpayne@7
|
261 )
|
|
jpayne@7
|
262
|
|
jpayne@7
|
263 params.update(kw)
|
|
jpayne@7
|
264 return type(self)(**params) # type: ignore[arg-type]
|
|
jpayne@7
|
265
|
|
jpayne@7
|
266 @classmethod
|
|
jpayne@7
|
267 def from_int(
|
|
jpayne@7
|
268 cls,
|
|
jpayne@7
|
269 retries: Retry | bool | int | None,
|
|
jpayne@7
|
270 redirect: bool | int | None = True,
|
|
jpayne@7
|
271 default: Retry | bool | int | None = None,
|
|
jpayne@7
|
272 ) -> Retry:
|
|
jpayne@7
|
273 """Backwards-compatibility for the old retries format."""
|
|
jpayne@7
|
274 if retries is None:
|
|
jpayne@7
|
275 retries = default if default is not None else cls.DEFAULT
|
|
jpayne@7
|
276
|
|
jpayne@7
|
277 if isinstance(retries, Retry):
|
|
jpayne@7
|
278 return retries
|
|
jpayne@7
|
279
|
|
jpayne@7
|
280 redirect = bool(redirect) and None
|
|
jpayne@7
|
281 new_retries = cls(retries, redirect=redirect)
|
|
jpayne@7
|
282 log.debug("Converted retries value: %r -> %r", retries, new_retries)
|
|
jpayne@7
|
283 return new_retries
|
|
jpayne@7
|
284
|
|
jpayne@7
|
285 def get_backoff_time(self) -> float:
|
|
jpayne@7
|
286 """Formula for computing the current backoff
|
|
jpayne@7
|
287
|
|
jpayne@7
|
288 :rtype: float
|
|
jpayne@7
|
289 """
|
|
jpayne@7
|
290 # We want to consider only the last consecutive errors sequence (Ignore redirects).
|
|
jpayne@7
|
291 consecutive_errors_len = len(
|
|
jpayne@7
|
292 list(
|
|
jpayne@7
|
293 takewhile(lambda x: x.redirect_location is None, reversed(self.history))
|
|
jpayne@7
|
294 )
|
|
jpayne@7
|
295 )
|
|
jpayne@7
|
296 if consecutive_errors_len <= 1:
|
|
jpayne@7
|
297 return 0
|
|
jpayne@7
|
298
|
|
jpayne@7
|
299 backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
|
|
jpayne@7
|
300 if self.backoff_jitter != 0.0:
|
|
jpayne@7
|
301 backoff_value += random.random() * self.backoff_jitter
|
|
jpayne@7
|
302 return float(max(0, min(self.backoff_max, backoff_value)))
|
|
jpayne@7
|
303
|
|
jpayne@7
|
304 def parse_retry_after(self, retry_after: str) -> float:
|
|
jpayne@7
|
305 seconds: float
|
|
jpayne@7
|
306 # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
|
|
jpayne@7
|
307 if re.match(r"^\s*[0-9]+\s*$", retry_after):
|
|
jpayne@7
|
308 seconds = int(retry_after)
|
|
jpayne@7
|
309 else:
|
|
jpayne@7
|
310 retry_date_tuple = email.utils.parsedate_tz(retry_after)
|
|
jpayne@7
|
311 if retry_date_tuple is None:
|
|
jpayne@7
|
312 raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
|
|
jpayne@7
|
313
|
|
jpayne@7
|
314 retry_date = email.utils.mktime_tz(retry_date_tuple)
|
|
jpayne@7
|
315 seconds = retry_date - time.time()
|
|
jpayne@7
|
316
|
|
jpayne@7
|
317 seconds = max(seconds, 0)
|
|
jpayne@7
|
318
|
|
jpayne@7
|
319 return seconds
|
|
jpayne@7
|
320
|
|
jpayne@7
|
321 def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
|
|
jpayne@7
|
322 """Get the value of Retry-After in seconds."""
|
|
jpayne@7
|
323
|
|
jpayne@7
|
324 retry_after = response.headers.get("Retry-After")
|
|
jpayne@7
|
325
|
|
jpayne@7
|
326 if retry_after is None:
|
|
jpayne@7
|
327 return None
|
|
jpayne@7
|
328
|
|
jpayne@7
|
329 return self.parse_retry_after(retry_after)
|
|
jpayne@7
|
330
|
|
jpayne@7
|
331 def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
|
|
jpayne@7
|
332 retry_after = self.get_retry_after(response)
|
|
jpayne@7
|
333 if retry_after:
|
|
jpayne@7
|
334 time.sleep(retry_after)
|
|
jpayne@7
|
335 return True
|
|
jpayne@7
|
336
|
|
jpayne@7
|
337 return False
|
|
jpayne@7
|
338
|
|
jpayne@7
|
339 def _sleep_backoff(self) -> None:
|
|
jpayne@7
|
340 backoff = self.get_backoff_time()
|
|
jpayne@7
|
341 if backoff <= 0:
|
|
jpayne@7
|
342 return
|
|
jpayne@7
|
343 time.sleep(backoff)
|
|
jpayne@7
|
344
|
|
jpayne@7
|
345 def sleep(self, response: BaseHTTPResponse | None = None) -> None:
|
|
jpayne@7
|
346 """Sleep between retry attempts.
|
|
jpayne@7
|
347
|
|
jpayne@7
|
348 This method will respect a server's ``Retry-After`` response header
|
|
jpayne@7
|
349 and sleep the duration of the time requested. If that is not present, it
|
|
jpayne@7
|
350 will use an exponential backoff. By default, the backoff factor is 0 and
|
|
jpayne@7
|
351 this method will return immediately.
|
|
jpayne@7
|
352 """
|
|
jpayne@7
|
353
|
|
jpayne@7
|
354 if self.respect_retry_after_header and response:
|
|
jpayne@7
|
355 slept = self.sleep_for_retry(response)
|
|
jpayne@7
|
356 if slept:
|
|
jpayne@7
|
357 return
|
|
jpayne@7
|
358
|
|
jpayne@7
|
359 self._sleep_backoff()
|
|
jpayne@7
|
360
|
|
jpayne@7
|
361 def _is_connection_error(self, err: Exception) -> bool:
|
|
jpayne@7
|
362 """Errors when we're fairly sure that the server did not receive the
|
|
jpayne@7
|
363 request, so it should be safe to retry.
|
|
jpayne@7
|
364 """
|
|
jpayne@7
|
365 if isinstance(err, ProxyError):
|
|
jpayne@7
|
366 err = err.original_error
|
|
jpayne@7
|
367 return isinstance(err, ConnectTimeoutError)
|
|
jpayne@7
|
368
|
|
jpayne@7
|
369 def _is_read_error(self, err: Exception) -> bool:
|
|
jpayne@7
|
370 """Errors that occur after the request has been started, so we should
|
|
jpayne@7
|
371 assume that the server began processing it.
|
|
jpayne@7
|
372 """
|
|
jpayne@7
|
373 return isinstance(err, (ReadTimeoutError, ProtocolError))
|
|
jpayne@7
|
374
|
|
jpayne@7
|
375 def _is_method_retryable(self, method: str) -> bool:
|
|
jpayne@7
|
376 """Checks if a given HTTP method should be retried upon, depending if
|
|
jpayne@7
|
377 it is included in the allowed_methods
|
|
jpayne@7
|
378 """
|
|
jpayne@7
|
379 if self.allowed_methods and method.upper() not in self.allowed_methods:
|
|
jpayne@7
|
380 return False
|
|
jpayne@7
|
381 return True
|
|
jpayne@7
|
382
|
|
jpayne@7
|
383 def is_retry(
|
|
jpayne@7
|
384 self, method: str, status_code: int, has_retry_after: bool = False
|
|
jpayne@7
|
385 ) -> bool:
|
|
jpayne@7
|
386 """Is this method/status code retryable? (Based on allowlists and control
|
|
jpayne@7
|
387 variables such as the number of total retries to allow, whether to
|
|
jpayne@7
|
388 respect the Retry-After header, whether this header is present, and
|
|
jpayne@7
|
389 whether the returned status code is on the list of status codes to
|
|
jpayne@7
|
390 be retried upon on the presence of the aforementioned header)
|
|
jpayne@7
|
391 """
|
|
jpayne@7
|
392 if not self._is_method_retryable(method):
|
|
jpayne@7
|
393 return False
|
|
jpayne@7
|
394
|
|
jpayne@7
|
395 if self.status_forcelist and status_code in self.status_forcelist:
|
|
jpayne@7
|
396 return True
|
|
jpayne@7
|
397
|
|
jpayne@7
|
398 return bool(
|
|
jpayne@7
|
399 self.total
|
|
jpayne@7
|
400 and self.respect_retry_after_header
|
|
jpayne@7
|
401 and has_retry_after
|
|
jpayne@7
|
402 and (status_code in self.RETRY_AFTER_STATUS_CODES)
|
|
jpayne@7
|
403 )
|
|
jpayne@7
|
404
|
|
jpayne@7
|
405 def is_exhausted(self) -> bool:
|
|
jpayne@7
|
406 """Are we out of retries?"""
|
|
jpayne@7
|
407 retry_counts = [
|
|
jpayne@7
|
408 x
|
|
jpayne@7
|
409 for x in (
|
|
jpayne@7
|
410 self.total,
|
|
jpayne@7
|
411 self.connect,
|
|
jpayne@7
|
412 self.read,
|
|
jpayne@7
|
413 self.redirect,
|
|
jpayne@7
|
414 self.status,
|
|
jpayne@7
|
415 self.other,
|
|
jpayne@7
|
416 )
|
|
jpayne@7
|
417 if x
|
|
jpayne@7
|
418 ]
|
|
jpayne@7
|
419 if not retry_counts:
|
|
jpayne@7
|
420 return False
|
|
jpayne@7
|
421
|
|
jpayne@7
|
422 return min(retry_counts) < 0
|
|
jpayne@7
|
423
|
|
jpayne@7
|
424 def increment(
|
|
jpayne@7
|
425 self,
|
|
jpayne@7
|
426 method: str | None = None,
|
|
jpayne@7
|
427 url: str | None = None,
|
|
jpayne@7
|
428 response: BaseHTTPResponse | None = None,
|
|
jpayne@7
|
429 error: Exception | None = None,
|
|
jpayne@7
|
430 _pool: ConnectionPool | None = None,
|
|
jpayne@7
|
431 _stacktrace: TracebackType | None = None,
|
|
jpayne@7
|
432 ) -> Retry:
|
|
jpayne@7
|
433 """Return a new Retry object with incremented retry counters.
|
|
jpayne@7
|
434
|
|
jpayne@7
|
435 :param response: A response object, or None, if the server did not
|
|
jpayne@7
|
436 return a response.
|
|
jpayne@7
|
437 :type response: :class:`~urllib3.response.BaseHTTPResponse`
|
|
jpayne@7
|
438 :param Exception error: An error encountered during the request, or
|
|
jpayne@7
|
439 None if the response was received successfully.
|
|
jpayne@7
|
440
|
|
jpayne@7
|
441 :return: A new ``Retry`` object.
|
|
jpayne@7
|
442 """
|
|
jpayne@7
|
443 if self.total is False and error:
|
|
jpayne@7
|
444 # Disabled, indicate to re-raise the error.
|
|
jpayne@7
|
445 raise reraise(type(error), error, _stacktrace)
|
|
jpayne@7
|
446
|
|
jpayne@7
|
447 total = self.total
|
|
jpayne@7
|
448 if total is not None:
|
|
jpayne@7
|
449 total -= 1
|
|
jpayne@7
|
450
|
|
jpayne@7
|
451 connect = self.connect
|
|
jpayne@7
|
452 read = self.read
|
|
jpayne@7
|
453 redirect = self.redirect
|
|
jpayne@7
|
454 status_count = self.status
|
|
jpayne@7
|
455 other = self.other
|
|
jpayne@7
|
456 cause = "unknown"
|
|
jpayne@7
|
457 status = None
|
|
jpayne@7
|
458 redirect_location = None
|
|
jpayne@7
|
459
|
|
jpayne@7
|
460 if error and self._is_connection_error(error):
|
|
jpayne@7
|
461 # Connect retry?
|
|
jpayne@7
|
462 if connect is False:
|
|
jpayne@7
|
463 raise reraise(type(error), error, _stacktrace)
|
|
jpayne@7
|
464 elif connect is not None:
|
|
jpayne@7
|
465 connect -= 1
|
|
jpayne@7
|
466
|
|
jpayne@7
|
467 elif error and self._is_read_error(error):
|
|
jpayne@7
|
468 # Read retry?
|
|
jpayne@7
|
469 if read is False or method is None or not self._is_method_retryable(method):
|
|
jpayne@7
|
470 raise reraise(type(error), error, _stacktrace)
|
|
jpayne@7
|
471 elif read is not None:
|
|
jpayne@7
|
472 read -= 1
|
|
jpayne@7
|
473
|
|
jpayne@7
|
474 elif error:
|
|
jpayne@7
|
475 # Other retry?
|
|
jpayne@7
|
476 if other is not None:
|
|
jpayne@7
|
477 other -= 1
|
|
jpayne@7
|
478
|
|
jpayne@7
|
479 elif response and response.get_redirect_location():
|
|
jpayne@7
|
480 # Redirect retry?
|
|
jpayne@7
|
481 if redirect is not None:
|
|
jpayne@7
|
482 redirect -= 1
|
|
jpayne@7
|
483 cause = "too many redirects"
|
|
jpayne@7
|
484 response_redirect_location = response.get_redirect_location()
|
|
jpayne@7
|
485 if response_redirect_location:
|
|
jpayne@7
|
486 redirect_location = response_redirect_location
|
|
jpayne@7
|
487 status = response.status
|
|
jpayne@7
|
488
|
|
jpayne@7
|
489 else:
|
|
jpayne@7
|
490 # Incrementing because of a server error like a 500 in
|
|
jpayne@7
|
491 # status_forcelist and the given method is in the allowed_methods
|
|
jpayne@7
|
492 cause = ResponseError.GENERIC_ERROR
|
|
jpayne@7
|
493 if response and response.status:
|
|
jpayne@7
|
494 if status_count is not None:
|
|
jpayne@7
|
495 status_count -= 1
|
|
jpayne@7
|
496 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
|
|
jpayne@7
|
497 status = response.status
|
|
jpayne@7
|
498
|
|
jpayne@7
|
499 history = self.history + (
|
|
jpayne@7
|
500 RequestHistory(method, url, error, status, redirect_location),
|
|
jpayne@7
|
501 )
|
|
jpayne@7
|
502
|
|
jpayne@7
|
503 new_retry = self.new(
|
|
jpayne@7
|
504 total=total,
|
|
jpayne@7
|
505 connect=connect,
|
|
jpayne@7
|
506 read=read,
|
|
jpayne@7
|
507 redirect=redirect,
|
|
jpayne@7
|
508 status=status_count,
|
|
jpayne@7
|
509 other=other,
|
|
jpayne@7
|
510 history=history,
|
|
jpayne@7
|
511 )
|
|
jpayne@7
|
512
|
|
jpayne@7
|
513 if new_retry.is_exhausted():
|
|
jpayne@7
|
514 reason = error or ResponseError(cause)
|
|
jpayne@7
|
515 raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
|
|
jpayne@7
|
516
|
|
jpayne@7
|
517 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
|
|
jpayne@7
|
518
|
|
jpayne@7
|
519 return new_retry
|
|
jpayne@7
|
520
|
|
jpayne@7
|
521 def __repr__(self) -> str:
|
|
jpayne@7
|
522 return (
|
|
jpayne@7
|
523 f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
|
|
jpayne@7
|
524 f"read={self.read}, redirect={self.redirect}, status={self.status})"
|
|
jpayne@7
|
525 )
|
|
jpayne@7
|
526
|
|
jpayne@7
|
527
|
|
jpayne@7
|
528 # For backwards compatibility (equivalent to pre-v1.9):
|
|
jpayne@7
|
529 Retry.DEFAULT = Retry(3)
|
