|
jpayne@7
|
1 """
|
|
jpayne@7
|
2 requests.models
|
|
jpayne@7
|
3 ~~~~~~~~~~~~~~~
|
|
jpayne@7
|
4
|
|
jpayne@7
|
5 This module contains the primary objects that power Requests.
|
|
jpayne@7
|
6 """
|
|
jpayne@7
|
7
|
|
jpayne@7
|
8 import datetime
|
|
jpayne@7
|
9
|
|
jpayne@7
|
10 # Import encoding now, to avoid implicit import later.
|
|
jpayne@7
|
11 # Implicit import within threads may cause LookupError when standard library is in a ZIP,
|
|
jpayne@7
|
12 # such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
|
|
jpayne@7
|
13 import encodings.idna # noqa: F401
|
|
jpayne@7
|
14 from io import UnsupportedOperation
|
|
jpayne@7
|
15
|
|
jpayne@7
|
16 from urllib3.exceptions import (
|
|
jpayne@7
|
17 DecodeError,
|
|
jpayne@7
|
18 LocationParseError,
|
|
jpayne@7
|
19 ProtocolError,
|
|
jpayne@7
|
20 ReadTimeoutError,
|
|
jpayne@7
|
21 SSLError,
|
|
jpayne@7
|
22 )
|
|
jpayne@7
|
23 from urllib3.fields import RequestField
|
|
jpayne@7
|
24 from urllib3.filepost import encode_multipart_formdata
|
|
jpayne@7
|
25 from urllib3.util import parse_url
|
|
jpayne@7
|
26
|
|
jpayne@7
|
27 from ._internal_utils import to_native_string, unicode_is_ascii
|
|
jpayne@7
|
28 from .auth import HTTPBasicAuth
|
|
jpayne@7
|
29 from .compat import (
|
|
jpayne@7
|
30 Callable,
|
|
jpayne@7
|
31 JSONDecodeError,
|
|
jpayne@7
|
32 Mapping,
|
|
jpayne@7
|
33 basestring,
|
|
jpayne@7
|
34 builtin_str,
|
|
jpayne@7
|
35 chardet,
|
|
jpayne@7
|
36 cookielib,
|
|
jpayne@7
|
37 )
|
|
jpayne@7
|
38 from .compat import json as complexjson
|
|
jpayne@7
|
39 from .compat import urlencode, urlsplit, urlunparse
|
|
jpayne@7
|
40 from .cookies import _copy_cookie_jar, cookiejar_from_dict, get_cookie_header
|
|
jpayne@7
|
41 from .exceptions import (
|
|
jpayne@7
|
42 ChunkedEncodingError,
|
|
jpayne@7
|
43 ConnectionError,
|
|
jpayne@7
|
44 ContentDecodingError,
|
|
jpayne@7
|
45 HTTPError,
|
|
jpayne@7
|
46 InvalidJSONError,
|
|
jpayne@7
|
47 InvalidURL,
|
|
jpayne@7
|
48 )
|
|
jpayne@7
|
49 from .exceptions import JSONDecodeError as RequestsJSONDecodeError
|
|
jpayne@7
|
50 from .exceptions import MissingSchema
|
|
jpayne@7
|
51 from .exceptions import SSLError as RequestsSSLError
|
|
jpayne@7
|
52 from .exceptions import StreamConsumedError
|
|
jpayne@7
|
53 from .hooks import default_hooks
|
|
jpayne@7
|
54 from .status_codes import codes
|
|
jpayne@7
|
55 from .structures import CaseInsensitiveDict
|
|
jpayne@7
|
56 from .utils import (
|
|
jpayne@7
|
57 check_header_validity,
|
|
jpayne@7
|
58 get_auth_from_url,
|
|
jpayne@7
|
59 guess_filename,
|
|
jpayne@7
|
60 guess_json_utf,
|
|
jpayne@7
|
61 iter_slices,
|
|
jpayne@7
|
62 parse_header_links,
|
|
jpayne@7
|
63 requote_uri,
|
|
jpayne@7
|
64 stream_decode_response_unicode,
|
|
jpayne@7
|
65 super_len,
|
|
jpayne@7
|
66 to_key_val_list,
|
|
jpayne@7
|
67 )
|
|
jpayne@7
|
68
|
|
jpayne@7
|
69 #: The set of HTTP status codes that indicate an automatically
|
|
jpayne@7
|
70 #: processable redirect.
|
|
jpayne@7
|
71 REDIRECT_STATI = (
|
|
jpayne@7
|
72 codes.moved, # 301
|
|
jpayne@7
|
73 codes.found, # 302
|
|
jpayne@7
|
74 codes.other, # 303
|
|
jpayne@7
|
75 codes.temporary_redirect, # 307
|
|
jpayne@7
|
76 codes.permanent_redirect, # 308
|
|
jpayne@7
|
77 )
|
|
jpayne@7
|
78
|
|
jpayne@7
|
79 DEFAULT_REDIRECT_LIMIT = 30
|
|
jpayne@7
|
80 CONTENT_CHUNK_SIZE = 10 * 1024
|
|
jpayne@7
|
81 ITER_CHUNK_SIZE = 512
|
|
jpayne@7
|
82
|
|
jpayne@7
|
83
|
|
jpayne@7
|
84 class RequestEncodingMixin:
|
|
jpayne@7
|
85 @property
|
|
jpayne@7
|
86 def path_url(self):
|
|
jpayne@7
|
87 """Build the path URL to use."""
|
|
jpayne@7
|
88
|
|
jpayne@7
|
89 url = []
|
|
jpayne@7
|
90
|
|
jpayne@7
|
91 p = urlsplit(self.url)
|
|
jpayne@7
|
92
|
|
jpayne@7
|
93 path = p.path
|
|
jpayne@7
|
94 if not path:
|
|
jpayne@7
|
95 path = "/"
|
|
jpayne@7
|
96
|
|
jpayne@7
|
97 url.append(path)
|
|
jpayne@7
|
98
|
|
jpayne@7
|
99 query = p.query
|
|
jpayne@7
|
100 if query:
|
|
jpayne@7
|
101 url.append("?")
|
|
jpayne@7
|
102 url.append(query)
|
|
jpayne@7
|
103
|
|
jpayne@7
|
104 return "".join(url)
|
|
jpayne@7
|
105
|
|
jpayne@7
|
106 @staticmethod
|
|
jpayne@7
|
107 def _encode_params(data):
|
|
jpayne@7
|
108 """Encode parameters in a piece of data.
|
|
jpayne@7
|
109
|
|
jpayne@7
|
110 Will successfully encode parameters when passed as a dict or a list of
|
|
jpayne@7
|
111 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
|
|
jpayne@7
|
112 if parameters are supplied as a dict.
|
|
jpayne@7
|
113 """
|
|
jpayne@7
|
114
|
|
jpayne@7
|
115 if isinstance(data, (str, bytes)):
|
|
jpayne@7
|
116 return data
|
|
jpayne@7
|
117 elif hasattr(data, "read"):
|
|
jpayne@7
|
118 return data
|
|
jpayne@7
|
119 elif hasattr(data, "__iter__"):
|
|
jpayne@7
|
120 result = []
|
|
jpayne@7
|
121 for k, vs in to_key_val_list(data):
|
|
jpayne@7
|
122 if isinstance(vs, basestring) or not hasattr(vs, "__iter__"):
|
|
jpayne@7
|
123 vs = [vs]
|
|
jpayne@7
|
124 for v in vs:
|
|
jpayne@7
|
125 if v is not None:
|
|
jpayne@7
|
126 result.append(
|
|
jpayne@7
|
127 (
|
|
jpayne@7
|
128 k.encode("utf-8") if isinstance(k, str) else k,
|
|
jpayne@7
|
129 v.encode("utf-8") if isinstance(v, str) else v,
|
|
jpayne@7
|
130 )
|
|
jpayne@7
|
131 )
|
|
jpayne@7
|
132 return urlencode(result, doseq=True)
|
|
jpayne@7
|
133 else:
|
|
jpayne@7
|
134 return data
|
|
jpayne@7
|
135
|
|
jpayne@7
|
136 @staticmethod
|
|
jpayne@7
|
137 def _encode_files(files, data):
|
|
jpayne@7
|
138 """Build the body for a multipart/form-data request.
|
|
jpayne@7
|
139
|
|
jpayne@7
|
140 Will successfully encode files when passed as a dict or a list of
|
|
jpayne@7
|
141 tuples. Order is retained if data is a list of tuples but arbitrary
|
|
jpayne@7
|
142 if parameters are supplied as a dict.
|
|
jpayne@7
|
143 The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
|
|
jpayne@7
|
144 or 4-tuples (filename, fileobj, contentype, custom_headers).
|
|
jpayne@7
|
145 """
|
|
jpayne@7
|
146 if not files:
|
|
jpayne@7
|
147 raise ValueError("Files must be provided.")
|
|
jpayne@7
|
148 elif isinstance(data, basestring):
|
|
jpayne@7
|
149 raise ValueError("Data must not be a string.")
|
|
jpayne@7
|
150
|
|
jpayne@7
|
151 new_fields = []
|
|
jpayne@7
|
152 fields = to_key_val_list(data or {})
|
|
jpayne@7
|
153 files = to_key_val_list(files or {})
|
|
jpayne@7
|
154
|
|
jpayne@7
|
155 for field, val in fields:
|
|
jpayne@7
|
156 if isinstance(val, basestring) or not hasattr(val, "__iter__"):
|
|
jpayne@7
|
157 val = [val]
|
|
jpayne@7
|
158 for v in val:
|
|
jpayne@7
|
159 if v is not None:
|
|
jpayne@7
|
160 # Don't call str() on bytestrings: in Py3 it all goes wrong.
|
|
jpayne@7
|
161 if not isinstance(v, bytes):
|
|
jpayne@7
|
162 v = str(v)
|
|
jpayne@7
|
163
|
|
jpayne@7
|
164 new_fields.append(
|
|
jpayne@7
|
165 (
|
|
jpayne@7
|
166 field.decode("utf-8")
|
|
jpayne@7
|
167 if isinstance(field, bytes)
|
|
jpayne@7
|
168 else field,
|
|
jpayne@7
|
169 v.encode("utf-8") if isinstance(v, str) else v,
|
|
jpayne@7
|
170 )
|
|
jpayne@7
|
171 )
|
|
jpayne@7
|
172
|
|
jpayne@7
|
173 for (k, v) in files:
|
|
jpayne@7
|
174 # support for explicit filename
|
|
jpayne@7
|
175 ft = None
|
|
jpayne@7
|
176 fh = None
|
|
jpayne@7
|
177 if isinstance(v, (tuple, list)):
|
|
jpayne@7
|
178 if len(v) == 2:
|
|
jpayne@7
|
179 fn, fp = v
|
|
jpayne@7
|
180 elif len(v) == 3:
|
|
jpayne@7
|
181 fn, fp, ft = v
|
|
jpayne@7
|
182 else:
|
|
jpayne@7
|
183 fn, fp, ft, fh = v
|
|
jpayne@7
|
184 else:
|
|
jpayne@7
|
185 fn = guess_filename(v) or k
|
|
jpayne@7
|
186 fp = v
|
|
jpayne@7
|
187
|
|
jpayne@7
|
188 if isinstance(fp, (str, bytes, bytearray)):
|
|
jpayne@7
|
189 fdata = fp
|
|
jpayne@7
|
190 elif hasattr(fp, "read"):
|
|
jpayne@7
|
191 fdata = fp.read()
|
|
jpayne@7
|
192 elif fp is None:
|
|
jpayne@7
|
193 continue
|
|
jpayne@7
|
194 else:
|
|
jpayne@7
|
195 fdata = fp
|
|
jpayne@7
|
196
|
|
jpayne@7
|
197 rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
|
|
jpayne@7
|
198 rf.make_multipart(content_type=ft)
|
|
jpayne@7
|
199 new_fields.append(rf)
|
|
jpayne@7
|
200
|
|
jpayne@7
|
201 body, content_type = encode_multipart_formdata(new_fields)
|
|
jpayne@7
|
202
|
|
jpayne@7
|
203 return body, content_type
|
|
jpayne@7
|
204
|
|
jpayne@7
|
205
|
|
jpayne@7
|
206 class RequestHooksMixin:
|
|
jpayne@7
|
207 def register_hook(self, event, hook):
|
|
jpayne@7
|
208 """Properly register a hook."""
|
|
jpayne@7
|
209
|
|
jpayne@7
|
210 if event not in self.hooks:
|
|
jpayne@7
|
211 raise ValueError(f'Unsupported event specified, with event name "{event}"')
|
|
jpayne@7
|
212
|
|
jpayne@7
|
213 if isinstance(hook, Callable):
|
|
jpayne@7
|
214 self.hooks[event].append(hook)
|
|
jpayne@7
|
215 elif hasattr(hook, "__iter__"):
|
|
jpayne@7
|
216 self.hooks[event].extend(h for h in hook if isinstance(h, Callable))
|
|
jpayne@7
|
217
|
|
jpayne@7
|
218 def deregister_hook(self, event, hook):
|
|
jpayne@7
|
219 """Deregister a previously registered hook.
|
|
jpayne@7
|
220 Returns True if the hook existed, False if not.
|
|
jpayne@7
|
221 """
|
|
jpayne@7
|
222
|
|
jpayne@7
|
223 try:
|
|
jpayne@7
|
224 self.hooks[event].remove(hook)
|
|
jpayne@7
|
225 return True
|
|
jpayne@7
|
226 except ValueError:
|
|
jpayne@7
|
227 return False
|
|
jpayne@7
|
228
|
|
jpayne@7
|
229
|
|
jpayne@7
|
230 class Request(RequestHooksMixin):
|
|
jpayne@7
|
231 """A user-created :class:`Request <Request>` object.
|
|
jpayne@7
|
232
|
|
jpayne@7
|
233 Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
|
|
jpayne@7
|
234
|
|
jpayne@7
|
235 :param method: HTTP method to use.
|
|
jpayne@7
|
236 :param url: URL to send.
|
|
jpayne@7
|
237 :param headers: dictionary of headers to send.
|
|
jpayne@7
|
238 :param files: dictionary of {filename: fileobject} files to multipart upload.
|
|
jpayne@7
|
239 :param data: the body to attach to the request. If a dictionary or
|
|
jpayne@7
|
240 list of tuples ``[(key, value)]`` is provided, form-encoding will
|
|
jpayne@7
|
241 take place.
|
|
jpayne@7
|
242 :param json: json for the body to attach to the request (if files or data is not specified).
|
|
jpayne@7
|
243 :param params: URL parameters to append to the URL. If a dictionary or
|
|
jpayne@7
|
244 list of tuples ``[(key, value)]`` is provided, form-encoding will
|
|
jpayne@7
|
245 take place.
|
|
jpayne@7
|
246 :param auth: Auth handler or (user, pass) tuple.
|
|
jpayne@7
|
247 :param cookies: dictionary or CookieJar of cookies to attach to this request.
|
|
jpayne@7
|
248 :param hooks: dictionary of callback hooks, for internal usage.
|
|
jpayne@7
|
249
|
|
jpayne@7
|
250 Usage::
|
|
jpayne@7
|
251
|
|
jpayne@7
|
252 >>> import requests
|
|
jpayne@7
|
253 >>> req = requests.Request('GET', 'https://httpbin.org/get')
|
|
jpayne@7
|
254 >>> req.prepare()
|
|
jpayne@7
|
255 <PreparedRequest [GET]>
|
|
jpayne@7
|
256 """
|
|
jpayne@7
|
257
|
|
jpayne@7
|
258 def __init__(
|
|
jpayne@7
|
259 self,
|
|
jpayne@7
|
260 method=None,
|
|
jpayne@7
|
261 url=None,
|
|
jpayne@7
|
262 headers=None,
|
|
jpayne@7
|
263 files=None,
|
|
jpayne@7
|
264 data=None,
|
|
jpayne@7
|
265 params=None,
|
|
jpayne@7
|
266 auth=None,
|
|
jpayne@7
|
267 cookies=None,
|
|
jpayne@7
|
268 hooks=None,
|
|
jpayne@7
|
269 json=None,
|
|
jpayne@7
|
270 ):
|
|
jpayne@7
|
271
|
|
jpayne@7
|
272 # Default empty dicts for dict params.
|
|
jpayne@7
|
273 data = [] if data is None else data
|
|
jpayne@7
|
274 files = [] if files is None else files
|
|
jpayne@7
|
275 headers = {} if headers is None else headers
|
|
jpayne@7
|
276 params = {} if params is None else params
|
|
jpayne@7
|
277 hooks = {} if hooks is None else hooks
|
|
jpayne@7
|
278
|
|
jpayne@7
|
279 self.hooks = default_hooks()
|
|
jpayne@7
|
280 for (k, v) in list(hooks.items()):
|
|
jpayne@7
|
281 self.register_hook(event=k, hook=v)
|
|
jpayne@7
|
282
|
|
jpayne@7
|
283 self.method = method
|
|
jpayne@7
|
284 self.url = url
|
|
jpayne@7
|
285 self.headers = headers
|
|
jpayne@7
|
286 self.files = files
|
|
jpayne@7
|
287 self.data = data
|
|
jpayne@7
|
288 self.json = json
|
|
jpayne@7
|
289 self.params = params
|
|
jpayne@7
|
290 self.auth = auth
|
|
jpayne@7
|
291 self.cookies = cookies
|
|
jpayne@7
|
292
|
|
jpayne@7
|
293 def __repr__(self):
|
|
jpayne@7
|
294 return f"<Request [{self.method}]>"
|
|
jpayne@7
|
295
|
|
jpayne@7
|
296 def prepare(self):
|
|
jpayne@7
|
297 """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
|
|
jpayne@7
|
298 p = PreparedRequest()
|
|
jpayne@7
|
299 p.prepare(
|
|
jpayne@7
|
300 method=self.method,
|
|
jpayne@7
|
301 url=self.url,
|
|
jpayne@7
|
302 headers=self.headers,
|
|
jpayne@7
|
303 files=self.files,
|
|
jpayne@7
|
304 data=self.data,
|
|
jpayne@7
|
305 json=self.json,
|
|
jpayne@7
|
306 params=self.params,
|
|
jpayne@7
|
307 auth=self.auth,
|
|
jpayne@7
|
308 cookies=self.cookies,
|
|
jpayne@7
|
309 hooks=self.hooks,
|
|
jpayne@7
|
310 )
|
|
jpayne@7
|
311 return p
|
|
jpayne@7
|
312
|
|
jpayne@7
|
313
|
|
jpayne@7
|
314 class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|
jpayne@7
|
315 """The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
|
|
jpayne@7
|
316 containing the exact bytes that will be sent to the server.
|
|
jpayne@7
|
317
|
|
jpayne@7
|
318 Instances are generated from a :class:`Request <Request>` object, and
|
|
jpayne@7
|
319 should not be instantiated manually; doing so may produce undesirable
|
|
jpayne@7
|
320 effects.
|
|
jpayne@7
|
321
|
|
jpayne@7
|
322 Usage::
|
|
jpayne@7
|
323
|
|
jpayne@7
|
324 >>> import requests
|
|
jpayne@7
|
325 >>> req = requests.Request('GET', 'https://httpbin.org/get')
|
|
jpayne@7
|
326 >>> r = req.prepare()
|
|
jpayne@7
|
327 >>> r
|
|
jpayne@7
|
328 <PreparedRequest [GET]>
|
|
jpayne@7
|
329
|
|
jpayne@7
|
330 >>> s = requests.Session()
|
|
jpayne@7
|
331 >>> s.send(r)
|
|
jpayne@7
|
332 <Response [200]>
|
|
jpayne@7
|
333 """
|
|
jpayne@7
|
334
|
|
jpayne@7
|
335 def __init__(self):
|
|
jpayne@7
|
336 #: HTTP verb to send to the server.
|
|
jpayne@7
|
337 self.method = None
|
|
jpayne@7
|
338 #: HTTP URL to send the request to.
|
|
jpayne@7
|
339 self.url = None
|
|
jpayne@7
|
340 #: dictionary of HTTP headers.
|
|
jpayne@7
|
341 self.headers = None
|
|
jpayne@7
|
342 # The `CookieJar` used to create the Cookie header will be stored here
|
|
jpayne@7
|
343 # after prepare_cookies is called
|
|
jpayne@7
|
344 self._cookies = None
|
|
jpayne@7
|
345 #: request body to send to the server.
|
|
jpayne@7
|
346 self.body = None
|
|
jpayne@7
|
347 #: dictionary of callback hooks, for internal usage.
|
|
jpayne@7
|
348 self.hooks = default_hooks()
|
|
jpayne@7
|
349 #: integer denoting starting position of a readable file-like body.
|
|
jpayne@7
|
350 self._body_position = None
|
|
jpayne@7
|
351
|
|
jpayne@7
|
352 def prepare(
|
|
jpayne@7
|
353 self,
|
|
jpayne@7
|
354 method=None,
|
|
jpayne@7
|
355 url=None,
|
|
jpayne@7
|
356 headers=None,
|
|
jpayne@7
|
357 files=None,
|
|
jpayne@7
|
358 data=None,
|
|
jpayne@7
|
359 params=None,
|
|
jpayne@7
|
360 auth=None,
|
|
jpayne@7
|
361 cookies=None,
|
|
jpayne@7
|
362 hooks=None,
|
|
jpayne@7
|
363 json=None,
|
|
jpayne@7
|
364 ):
|
|
jpayne@7
|
365 """Prepares the entire request with the given parameters."""
|
|
jpayne@7
|
366
|
|
jpayne@7
|
367 self.prepare_method(method)
|
|
jpayne@7
|
368 self.prepare_url(url, params)
|
|
jpayne@7
|
369 self.prepare_headers(headers)
|
|
jpayne@7
|
370 self.prepare_cookies(cookies)
|
|
jpayne@7
|
371 self.prepare_body(data, files, json)
|
|
jpayne@7
|
372 self.prepare_auth(auth, url)
|
|
jpayne@7
|
373
|
|
jpayne@7
|
374 # Note that prepare_auth must be last to enable authentication schemes
|
|
jpayne@7
|
375 # such as OAuth to work on a fully prepared request.
|
|
jpayne@7
|
376
|
|
jpayne@7
|
377 # This MUST go after prepare_auth. Authenticators could add a hook
|
|
jpayne@7
|
378 self.prepare_hooks(hooks)
|
|
jpayne@7
|
379
|
|
jpayne@7
|
380 def __repr__(self):
|
|
jpayne@7
|
381 return f"<PreparedRequest [{self.method}]>"
|
|
jpayne@7
|
382
|
|
jpayne@7
|
383 def copy(self):
|
|
jpayne@7
|
384 p = PreparedRequest()
|
|
jpayne@7
|
385 p.method = self.method
|
|
jpayne@7
|
386 p.url = self.url
|
|
jpayne@7
|
387 p.headers = self.headers.copy() if self.headers is not None else None
|
|
jpayne@7
|
388 p._cookies = _copy_cookie_jar(self._cookies)
|
|
jpayne@7
|
389 p.body = self.body
|
|
jpayne@7
|
390 p.hooks = self.hooks
|
|
jpayne@7
|
391 p._body_position = self._body_position
|
|
jpayne@7
|
392 return p
|
|
jpayne@7
|
393
|
|
jpayne@7
|
394 def prepare_method(self, method):
|
|
jpayne@7
|
395 """Prepares the given HTTP method."""
|
|
jpayne@7
|
396 self.method = method
|
|
jpayne@7
|
397 if self.method is not None:
|
|
jpayne@7
|
398 self.method = to_native_string(self.method.upper())
|
|
jpayne@7
|
399
|
|
jpayne@7
|
400 @staticmethod
|
|
jpayne@7
|
401 def _get_idna_encoded_host(host):
|
|
jpayne@7
|
402 import idna
|
|
jpayne@7
|
403
|
|
jpayne@7
|
404 try:
|
|
jpayne@7
|
405 host = idna.encode(host, uts46=True).decode("utf-8")
|
|
jpayne@7
|
406 except idna.IDNAError:
|
|
jpayne@7
|
407 raise UnicodeError
|
|
jpayne@7
|
408 return host
|
|
jpayne@7
|
409
|
|
jpayne@7
|
410 def prepare_url(self, url, params):
|
|
jpayne@7
|
411 """Prepares the given HTTP URL."""
|
|
jpayne@7
|
412 #: Accept objects that have string representations.
|
|
jpayne@7
|
413 #: We're unable to blindly call unicode/str functions
|
|
jpayne@7
|
414 #: as this will include the bytestring indicator (b'')
|
|
jpayne@7
|
415 #: on python 3.x.
|
|
jpayne@7
|
416 #: https://github.com/psf/requests/pull/2238
|
|
jpayne@7
|
417 if isinstance(url, bytes):
|
|
jpayne@7
|
418 url = url.decode("utf8")
|
|
jpayne@7
|
419 else:
|
|
jpayne@7
|
420 url = str(url)
|
|
jpayne@7
|
421
|
|
jpayne@7
|
422 # Remove leading whitespaces from url
|
|
jpayne@7
|
423 url = url.lstrip()
|
|
jpayne@7
|
424
|
|
jpayne@7
|
425 # Don't do any URL preparation for non-HTTP schemes like `mailto`,
|
|
jpayne@7
|
426 # `data` etc to work around exceptions from `url_parse`, which
|
|
jpayne@7
|
427 # handles RFC 3986 only.
|
|
jpayne@7
|
428 if ":" in url and not url.lower().startswith("http"):
|
|
jpayne@7
|
429 self.url = url
|
|
jpayne@7
|
430 return
|
|
jpayne@7
|
431
|
|
jpayne@7
|
432 # Support for unicode domain names and paths.
|
|
jpayne@7
|
433 try:
|
|
jpayne@7
|
434 scheme, auth, host, port, path, query, fragment = parse_url(url)
|
|
jpayne@7
|
435 except LocationParseError as e:
|
|
jpayne@7
|
436 raise InvalidURL(*e.args)
|
|
jpayne@7
|
437
|
|
jpayne@7
|
438 if not scheme:
|
|
jpayne@7
|
439 raise MissingSchema(
|
|
jpayne@7
|
440 f"Invalid URL {url!r}: No scheme supplied. "
|
|
jpayne@7
|
441 f"Perhaps you meant https://{url}?"
|
|
jpayne@7
|
442 )
|
|
jpayne@7
|
443
|
|
jpayne@7
|
444 if not host:
|
|
jpayne@7
|
445 raise InvalidURL(f"Invalid URL {url!r}: No host supplied")
|
|
jpayne@7
|
446
|
|
jpayne@7
|
447 # In general, we want to try IDNA encoding the hostname if the string contains
|
|
jpayne@7
|
448 # non-ASCII characters. This allows users to automatically get the correct IDNA
|
|
jpayne@7
|
449 # behaviour. For strings containing only ASCII characters, we need to also verify
|
|
jpayne@7
|
450 # it doesn't start with a wildcard (*), before allowing the unencoded hostname.
|
|
jpayne@7
|
451 if not unicode_is_ascii(host):
|
|
jpayne@7
|
452 try:
|
|
jpayne@7
|
453 host = self._get_idna_encoded_host(host)
|
|
jpayne@7
|
454 except UnicodeError:
|
|
jpayne@7
|
455 raise InvalidURL("URL has an invalid label.")
|
|
jpayne@7
|
456 elif host.startswith(("*", ".")):
|
|
jpayne@7
|
457 raise InvalidURL("URL has an invalid label.")
|
|
jpayne@7
|
458
|
|
jpayne@7
|
459 # Carefully reconstruct the network location
|
|
jpayne@7
|
460 netloc = auth or ""
|
|
jpayne@7
|
461 if netloc:
|
|
jpayne@7
|
462 netloc += "@"
|
|
jpayne@7
|
463 netloc += host
|
|
jpayne@7
|
464 if port:
|
|
jpayne@7
|
465 netloc += f":{port}"
|
|
jpayne@7
|
466
|
|
jpayne@7
|
467 # Bare domains aren't valid URLs.
|
|
jpayne@7
|
468 if not path:
|
|
jpayne@7
|
469 path = "/"
|
|
jpayne@7
|
470
|
|
jpayne@7
|
471 if isinstance(params, (str, bytes)):
|
|
jpayne@7
|
472 params = to_native_string(params)
|
|
jpayne@7
|
473
|
|
jpayne@7
|
474 enc_params = self._encode_params(params)
|
|
jpayne@7
|
475 if enc_params:
|
|
jpayne@7
|
476 if query:
|
|
jpayne@7
|
477 query = f"{query}&{enc_params}"
|
|
jpayne@7
|
478 else:
|
|
jpayne@7
|
479 query = enc_params
|
|
jpayne@7
|
480
|
|
jpayne@7
|
481 url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
|
|
jpayne@7
|
482 self.url = url
|
|
jpayne@7
|
483
|
|
jpayne@7
|
484 def prepare_headers(self, headers):
|
|
jpayne@7
|
485 """Prepares the given HTTP headers."""
|
|
jpayne@7
|
486
|
|
jpayne@7
|
487 self.headers = CaseInsensitiveDict()
|
|
jpayne@7
|
488 if headers:
|
|
jpayne@7
|
489 for header in headers.items():
|
|
jpayne@7
|
490 # Raise exception on invalid header value.
|
|
jpayne@7
|
491 check_header_validity(header)
|
|
jpayne@7
|
492 name, value = header
|
|
jpayne@7
|
493 self.headers[to_native_string(name)] = value
|
|
jpayne@7
|
494
|
|
jpayne@7
|
495 def prepare_body(self, data, files, json=None):
|
|
jpayne@7
|
496 """Prepares the given HTTP body data."""
|
|
jpayne@7
|
497
|
|
jpayne@7
|
498 # Check if file, fo, generator, iterator.
|
|
jpayne@7
|
499 # If not, run through normal process.
|
|
jpayne@7
|
500
|
|
jpayne@7
|
501 # Nottin' on you.
|
|
jpayne@7
|
502 body = None
|
|
jpayne@7
|
503 content_type = None
|
|
jpayne@7
|
504
|
|
jpayne@7
|
505 if not data and json is not None:
|
|
jpayne@7
|
506 # urllib3 requires a bytes-like body. Python 2's json.dumps
|
|
jpayne@7
|
507 # provides this natively, but Python 3 gives a Unicode string.
|
|
jpayne@7
|
508 content_type = "application/json"
|
|
jpayne@7
|
509
|
|
jpayne@7
|
510 try:
|
|
jpayne@7
|
511 body = complexjson.dumps(json, allow_nan=False)
|
|
jpayne@7
|
512 except ValueError as ve:
|
|
jpayne@7
|
513 raise InvalidJSONError(ve, request=self)
|
|
jpayne@7
|
514
|
|
jpayne@7
|
515 if not isinstance(body, bytes):
|
|
jpayne@7
|
516 body = body.encode("utf-8")
|
|
jpayne@7
|
517
|
|
jpayne@7
|
518 is_stream = all(
|
|
jpayne@7
|
519 [
|
|
jpayne@7
|
520 hasattr(data, "__iter__"),
|
|
jpayne@7
|
521 not isinstance(data, (basestring, list, tuple, Mapping)),
|
|
jpayne@7
|
522 ]
|
|
jpayne@7
|
523 )
|
|
jpayne@7
|
524
|
|
jpayne@7
|
525 if is_stream:
|
|
jpayne@7
|
526 try:
|
|
jpayne@7
|
527 length = super_len(data)
|
|
jpayne@7
|
528 except (TypeError, AttributeError, UnsupportedOperation):
|
|
jpayne@7
|
529 length = None
|
|
jpayne@7
|
530
|
|
jpayne@7
|
531 body = data
|
|
jpayne@7
|
532
|
|
jpayne@7
|
533 if getattr(body, "tell", None) is not None:
|
|
jpayne@7
|
534 # Record the current file position before reading.
|
|
jpayne@7
|
535 # This will allow us to rewind a file in the event
|
|
jpayne@7
|
536 # of a redirect.
|
|
jpayne@7
|
537 try:
|
|
jpayne@7
|
538 self._body_position = body.tell()
|
|
jpayne@7
|
539 except OSError:
|
|
jpayne@7
|
540 # This differentiates from None, allowing us to catch
|
|
jpayne@7
|
541 # a failed `tell()` later when trying to rewind the body
|
|
jpayne@7
|
542 self._body_position = object()
|
|
jpayne@7
|
543
|
|
jpayne@7
|
544 if files:
|
|
jpayne@7
|
545 raise NotImplementedError(
|
|
jpayne@7
|
546 "Streamed bodies and files are mutually exclusive."
|
|
jpayne@7
|
547 )
|
|
jpayne@7
|
548
|
|
jpayne@7
|
549 if length:
|
|
jpayne@7
|
550 self.headers["Content-Length"] = builtin_str(length)
|
|
jpayne@7
|
551 else:
|
|
jpayne@7
|
552 self.headers["Transfer-Encoding"] = "chunked"
|
|
jpayne@7
|
553 else:
|
|
jpayne@7
|
554 # Multi-part file uploads.
|
|
jpayne@7
|
555 if files:
|
|
jpayne@7
|
556 (body, content_type) = self._encode_files(files, data)
|
|
jpayne@7
|
557 else:
|
|
jpayne@7
|
558 if data:
|
|
jpayne@7
|
559 body = self._encode_params(data)
|
|
jpayne@7
|
560 if isinstance(data, basestring) or hasattr(data, "read"):
|
|
jpayne@7
|
561 content_type = None
|
|
jpayne@7
|
562 else:
|
|
jpayne@7
|
563 content_type = "application/x-www-form-urlencoded"
|
|
jpayne@7
|
564
|
|
jpayne@7
|
565 self.prepare_content_length(body)
|
|
jpayne@7
|
566
|
|
jpayne@7
|
567 # Add content-type if it wasn't explicitly provided.
|
|
jpayne@7
|
568 if content_type and ("content-type" not in self.headers):
|
|
jpayne@7
|
569 self.headers["Content-Type"] = content_type
|
|
jpayne@7
|
570
|
|
jpayne@7
|
571 self.body = body
|
|
jpayne@7
|
572
|
|
jpayne@7
|
573 def prepare_content_length(self, body):
|
|
jpayne@7
|
574 """Prepare Content-Length header based on request method and body"""
|
|
jpayne@7
|
575 if body is not None:
|
|
jpayne@7
|
576 length = super_len(body)
|
|
jpayne@7
|
577 if length:
|
|
jpayne@7
|
578 # If length exists, set it. Otherwise, we fallback
|
|
jpayne@7
|
579 # to Transfer-Encoding: chunked.
|
|
jpayne@7
|
580 self.headers["Content-Length"] = builtin_str(length)
|
|
jpayne@7
|
581 elif (
|
|
jpayne@7
|
582 self.method not in ("GET", "HEAD")
|
|
jpayne@7
|
583 and self.headers.get("Content-Length") is None
|
|
jpayne@7
|
584 ):
|
|
jpayne@7
|
585 # Set Content-Length to 0 for methods that can have a body
|
|
jpayne@7
|
586 # but don't provide one. (i.e. not GET or HEAD)
|
|
jpayne@7
|
587 self.headers["Content-Length"] = "0"
|
|
jpayne@7
|
588
|
|
jpayne@7
|
589 def prepare_auth(self, auth, url=""):
|
|
jpayne@7
|
590 """Prepares the given HTTP auth data."""
|
|
jpayne@7
|
591
|
|
jpayne@7
|
592 # If no Auth is explicitly provided, extract it from the URL first.
|
|
jpayne@7
|
593 if auth is None:
|
|
jpayne@7
|
594 url_auth = get_auth_from_url(self.url)
|
|
jpayne@7
|
595 auth = url_auth if any(url_auth) else None
|
|
jpayne@7
|
596
|
|
jpayne@7
|
597 if auth:
|
|
jpayne@7
|
598 if isinstance(auth, tuple) and len(auth) == 2:
|
|
jpayne@7
|
599 # special-case basic HTTP auth
|
|
jpayne@7
|
600 auth = HTTPBasicAuth(*auth)
|
|
jpayne@7
|
601
|
|
jpayne@7
|
602 # Allow auth to make its changes.
|
|
jpayne@7
|
603 r = auth(self)
|
|
jpayne@7
|
604
|
|
jpayne@7
|
605 # Update self to reflect the auth changes.
|
|
jpayne@7
|
606 self.__dict__.update(r.__dict__)
|
|
jpayne@7
|
607
|
|
jpayne@7
|
608 # Recompute Content-Length
|
|
jpayne@7
|
609 self.prepare_content_length(self.body)
|
|
jpayne@7
|
610
|
|
jpayne@7
|
611 def prepare_cookies(self, cookies):
|
|
jpayne@7
|
612 """Prepares the given HTTP cookie data.
|
|
jpayne@7
|
613
|
|
jpayne@7
|
614 This function eventually generates a ``Cookie`` header from the
|
|
jpayne@7
|
615 given cookies using cookielib. Due to cookielib's design, the header
|
|
jpayne@7
|
616 will not be regenerated if it already exists, meaning this function
|
|
jpayne@7
|
617 can only be called once for the life of the
|
|
jpayne@7
|
618 :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
|
|
jpayne@7
|
619 to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
|
|
jpayne@7
|
620 header is removed beforehand.
|
|
jpayne@7
|
621 """
|
|
jpayne@7
|
622 if isinstance(cookies, cookielib.CookieJar):
|
|
jpayne@7
|
623 self._cookies = cookies
|
|
jpayne@7
|
624 else:
|
|
jpayne@7
|
625 self._cookies = cookiejar_from_dict(cookies)
|
|
jpayne@7
|
626
|
|
jpayne@7
|
627 cookie_header = get_cookie_header(self._cookies, self)
|
|
jpayne@7
|
628 if cookie_header is not None:
|
|
jpayne@7
|
629 self.headers["Cookie"] = cookie_header
|
|
jpayne@7
|
630
|
|
jpayne@7
|
631 def prepare_hooks(self, hooks):
|
|
jpayne@7
|
632 """Prepares the given hooks."""
|
|
jpayne@7
|
633 # hooks can be passed as None to the prepare method and to this
|
|
jpayne@7
|
634 # method. To prevent iterating over None, simply use an empty list
|
|
jpayne@7
|
635 # if hooks is False-y
|
|
jpayne@7
|
636 hooks = hooks or []
|
|
jpayne@7
|
637 for event in hooks:
|
|
jpayne@7
|
638 self.register_hook(event, hooks[event])
|
|
jpayne@7
|
639
|
|
jpayne@7
|
640
|
|
jpayne@7
|
641 class Response:
|
|
jpayne@7
|
642 """The :class:`Response <Response>` object, which contains a
|
|
jpayne@7
|
643 server's response to an HTTP request.
|
|
jpayne@7
|
644 """
|
|
jpayne@7
|
645
|
|
jpayne@7
|
646 __attrs__ = [
|
|
jpayne@7
|
647 "_content",
|
|
jpayne@7
|
648 "status_code",
|
|
jpayne@7
|
649 "headers",
|
|
jpayne@7
|
650 "url",
|
|
jpayne@7
|
651 "history",
|
|
jpayne@7
|
652 "encoding",
|
|
jpayne@7
|
653 "reason",
|
|
jpayne@7
|
654 "cookies",
|
|
jpayne@7
|
655 "elapsed",
|
|
jpayne@7
|
656 "request",
|
|
jpayne@7
|
657 ]
|
|
jpayne@7
|
658
|
|
jpayne@7
|
659 def __init__(self):
|
|
jpayne@7
|
660 self._content = False
|
|
jpayne@7
|
661 self._content_consumed = False
|
|
jpayne@7
|
662 self._next = None
|
|
jpayne@7
|
663
|
|
jpayne@7
|
664 #: Integer Code of responded HTTP Status, e.g. 404 or 200.
|
|
jpayne@7
|
665 self.status_code = None
|
|
jpayne@7
|
666
|
|
jpayne@7
|
667 #: Case-insensitive Dictionary of Response Headers.
|
|
jpayne@7
|
668 #: For example, ``headers['content-encoding']`` will return the
|
|
jpayne@7
|
669 #: value of a ``'Content-Encoding'`` response header.
|
|
jpayne@7
|
670 self.headers = CaseInsensitiveDict()
|
|
jpayne@7
|
671
|
|
jpayne@7
|
672 #: File-like object representation of response (for advanced usage).
|
|
jpayne@7
|
673 #: Use of ``raw`` requires that ``stream=True`` be set on the request.
|
|
jpayne@7
|
674 #: This requirement does not apply for use internally to Requests.
|
|
jpayne@7
|
675 self.raw = None
|
|
jpayne@7
|
676
|
|
jpayne@7
|
677 #: Final URL location of Response.
|
|
jpayne@7
|
678 self.url = None
|
|
jpayne@7
|
679
|
|
jpayne@7
|
680 #: Encoding to decode with when accessing r.text.
|
|
jpayne@7
|
681 self.encoding = None
|
|
jpayne@7
|
682
|
|
jpayne@7
|
683 #: A list of :class:`Response <Response>` objects from
|
|
jpayne@7
|
684 #: the history of the Request. Any redirect responses will end
|
|
jpayne@7
|
685 #: up here. The list is sorted from the oldest to the most recent request.
|
|
jpayne@7
|
686 self.history = []
|
|
jpayne@7
|
687
|
|
jpayne@7
|
688 #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
|
|
jpayne@7
|
689 self.reason = None
|
|
jpayne@7
|
690
|
|
jpayne@7
|
691 #: A CookieJar of Cookies the server sent back.
|
|
jpayne@7
|
692 self.cookies = cookiejar_from_dict({})
|
|
jpayne@7
|
693
|
|
jpayne@7
|
694 #: The amount of time elapsed between sending the request
|
|
jpayne@7
|
695 #: and the arrival of the response (as a timedelta).
|
|
jpayne@7
|
696 #: This property specifically measures the time taken between sending
|
|
jpayne@7
|
697 #: the first byte of the request and finishing parsing the headers. It
|
|
jpayne@7
|
698 #: is therefore unaffected by consuming the response content or the
|
|
jpayne@7
|
699 #: value of the ``stream`` keyword argument.
|
|
jpayne@7
|
700 self.elapsed = datetime.timedelta(0)
|
|
jpayne@7
|
701
|
|
jpayne@7
|
702 #: The :class:`PreparedRequest <PreparedRequest>` object to which this
|
|
jpayne@7
|
703 #: is a response.
|
|
jpayne@7
|
704 self.request = None
|
|
jpayne@7
|
705
|
|
jpayne@7
|
706 def __enter__(self):
|
|
jpayne@7
|
707 return self
|
|
jpayne@7
|
708
|
|
jpayne@7
|
709 def __exit__(self, *args):
|
|
jpayne@7
|
710 self.close()
|
|
jpayne@7
|
711
|
|
jpayne@7
|
712 def __getstate__(self):
|
|
jpayne@7
|
713 # Consume everything; accessing the content attribute makes
|
|
jpayne@7
|
714 # sure the content has been fully read.
|
|
jpayne@7
|
715 if not self._content_consumed:
|
|
jpayne@7
|
716 self.content
|
|
jpayne@7
|
717
|
|
jpayne@7
|
718 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
|
|
jpayne@7
|
719
|
|
jpayne@7
|
720 def __setstate__(self, state):
|
|
jpayne@7
|
721 for name, value in state.items():
|
|
jpayne@7
|
722 setattr(self, name, value)
|
|
jpayne@7
|
723
|
|
jpayne@7
|
724 # pickled objects do not have .raw
|
|
jpayne@7
|
725 setattr(self, "_content_consumed", True)
|
|
jpayne@7
|
726 setattr(self, "raw", None)
|
|
jpayne@7
|
727
|
|
jpayne@7
|
728 def __repr__(self):
|
|
jpayne@7
|
729 return f"<Response [{self.status_code}]>"
|
|
jpayne@7
|
730
|
|
jpayne@7
|
731 def __bool__(self):
|
|
jpayne@7
|
732 """Returns True if :attr:`status_code` is less than 400.
|
|
jpayne@7
|
733
|
|
jpayne@7
|
734 This attribute checks if the status code of the response is between
|
|
jpayne@7
|
735 400 and 600 to see if there was a client error or a server error. If
|
|
jpayne@7
|
736 the status code, is between 200 and 400, this will return True. This
|
|
jpayne@7
|
737 is **not** a check to see if the response code is ``200 OK``.
|
|
jpayne@7
|
738 """
|
|
jpayne@7
|
739 return self.ok
|
|
jpayne@7
|
740
|
|
jpayne@7
|
741 def __nonzero__(self):
|
|
jpayne@7
|
742 """Returns True if :attr:`status_code` is less than 400.
|
|
jpayne@7
|
743
|
|
jpayne@7
|
744 This attribute checks if the status code of the response is between
|
|
jpayne@7
|
745 400 and 600 to see if there was a client error or a server error. If
|
|
jpayne@7
|
746 the status code, is between 200 and 400, this will return True. This
|
|
jpayne@7
|
747 is **not** a check to see if the response code is ``200 OK``.
|
|
jpayne@7
|
748 """
|
|
jpayne@7
|
749 return self.ok
|
|
jpayne@7
|
750
|
|
jpayne@7
|
751 def __iter__(self):
|
|
jpayne@7
|
752 """Allows you to use a response as an iterator."""
|
|
jpayne@7
|
753 return self.iter_content(128)
|
|
jpayne@7
|
754
|
|
jpayne@7
|
755 @property
|
|
jpayne@7
|
756 def ok(self):
|
|
jpayne@7
|
757 """Returns True if :attr:`status_code` is less than 400, False if not.
|
|
jpayne@7
|
758
|
|
jpayne@7
|
759 This attribute checks if the status code of the response is between
|
|
jpayne@7
|
760 400 and 600 to see if there was a client error or a server error. If
|
|
jpayne@7
|
761 the status code is between 200 and 400, this will return True. This
|
|
jpayne@7
|
762 is **not** a check to see if the response code is ``200 OK``.
|
|
jpayne@7
|
763 """
|
|
jpayne@7
|
764 try:
|
|
jpayne@7
|
765 self.raise_for_status()
|
|
jpayne@7
|
766 except HTTPError:
|
|
jpayne@7
|
767 return False
|
|
jpayne@7
|
768 return True
|
|
jpayne@7
|
769
|
|
jpayne@7
|
770 @property
|
|
jpayne@7
|
771 def is_redirect(self):
|
|
jpayne@7
|
772 """True if this Response is a well-formed HTTP redirect that could have
|
|
jpayne@7
|
773 been processed automatically (by :meth:`Session.resolve_redirects`).
|
|
jpayne@7
|
774 """
|
|
jpayne@7
|
775 return "location" in self.headers and self.status_code in REDIRECT_STATI
|
|
jpayne@7
|
776
|
|
jpayne@7
|
777 @property
|
|
jpayne@7
|
778 def is_permanent_redirect(self):
|
|
jpayne@7
|
779 """True if this Response one of the permanent versions of redirect."""
|
|
jpayne@7
|
780 return "location" in self.headers and self.status_code in (
|
|
jpayne@7
|
781 codes.moved_permanently,
|
|
jpayne@7
|
782 codes.permanent_redirect,
|
|
jpayne@7
|
783 )
|
|
jpayne@7
|
784
|
|
jpayne@7
|
785 @property
|
|
jpayne@7
|
786 def next(self):
|
|
jpayne@7
|
787 """Returns a PreparedRequest for the next request in a redirect chain, if there is one."""
|
|
jpayne@7
|
788 return self._next
|
|
jpayne@7
|
789
|
|
jpayne@7
|
790 @property
|
|
jpayne@7
|
791 def apparent_encoding(self):
|
|
jpayne@7
|
792 """The apparent encoding, provided by the charset_normalizer or chardet libraries."""
|
|
jpayne@7
|
793 return chardet.detect(self.content)["encoding"]
|
|
jpayne@7
|
794
|
|
jpayne@7
|
795 def iter_content(self, chunk_size=1, decode_unicode=False):
|
|
jpayne@7
|
796 """Iterates over the response data. When stream=True is set on the
|
|
jpayne@7
|
797 request, this avoids reading the content at once into memory for
|
|
jpayne@7
|
798 large responses. The chunk size is the number of bytes it should
|
|
jpayne@7
|
799 read into memory. This is not necessarily the length of each item
|
|
jpayne@7
|
800 returned as decoding can take place.
|
|
jpayne@7
|
801
|
|
jpayne@7
|
802 chunk_size must be of type int or None. A value of None will
|
|
jpayne@7
|
803 function differently depending on the value of `stream`.
|
|
jpayne@7
|
804 stream=True will read data as it arrives in whatever size the
|
|
jpayne@7
|
805 chunks are received. If stream=False, data is returned as
|
|
jpayne@7
|
806 a single chunk.
|
|
jpayne@7
|
807
|
|
jpayne@7
|
808 If decode_unicode is True, content will be decoded using the best
|
|
jpayne@7
|
809 available encoding based on the response.
|
|
jpayne@7
|
810 """
|
|
jpayne@7
|
811
|
|
jpayne@7
|
812 def generate():
|
|
jpayne@7
|
813 # Special case for urllib3.
|
|
jpayne@7
|
814 if hasattr(self.raw, "stream"):
|
|
jpayne@7
|
815 try:
|
|
jpayne@7
|
816 yield from self.raw.stream(chunk_size, decode_content=True)
|
|
jpayne@7
|
817 except ProtocolError as e:
|
|
jpayne@7
|
818 raise ChunkedEncodingError(e)
|
|
jpayne@7
|
819 except DecodeError as e:
|
|
jpayne@7
|
820 raise ContentDecodingError(e)
|
|
jpayne@7
|
821 except ReadTimeoutError as e:
|
|
jpayne@7
|
822 raise ConnectionError(e)
|
|
jpayne@7
|
823 except SSLError as e:
|
|
jpayne@7
|
824 raise RequestsSSLError(e)
|
|
jpayne@7
|
825 else:
|
|
jpayne@7
|
826 # Standard file-like object.
|
|
jpayne@7
|
827 while True:
|
|
jpayne@7
|
828 chunk = self.raw.read(chunk_size)
|
|
jpayne@7
|
829 if not chunk:
|
|
jpayne@7
|
830 break
|
|
jpayne@7
|
831 yield chunk
|
|
jpayne@7
|
832
|
|
jpayne@7
|
833 self._content_consumed = True
|
|
jpayne@7
|
834
|
|
jpayne@7
|
835 if self._content_consumed and isinstance(self._content, bool):
|
|
jpayne@7
|
836 raise StreamConsumedError()
|
|
jpayne@7
|
837 elif chunk_size is not None and not isinstance(chunk_size, int):
|
|
jpayne@7
|
838 raise TypeError(
|
|
jpayne@7
|
839 f"chunk_size must be an int, it is instead a {type(chunk_size)}."
|
|
jpayne@7
|
840 )
|
|
jpayne@7
|
841 # simulate reading small chunks of the content
|
|
jpayne@7
|
842 reused_chunks = iter_slices(self._content, chunk_size)
|
|
jpayne@7
|
843
|
|
jpayne@7
|
844 stream_chunks = generate()
|
|
jpayne@7
|
845
|
|
jpayne@7
|
846 chunks = reused_chunks if self._content_consumed else stream_chunks
|
|
jpayne@7
|
847
|
|
jpayne@7
|
848 if decode_unicode:
|
|
jpayne@7
|
849 chunks = stream_decode_response_unicode(chunks, self)
|
|
jpayne@7
|
850
|
|
jpayne@7
|
851 return chunks
|
|
jpayne@7
|
852
|
|
jpayne@7
|
853 def iter_lines(
|
|
jpayne@7
|
854 self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None
|
|
jpayne@7
|
855 ):
|
|
jpayne@7
|
856 """Iterates over the response data, one line at a time. When
|
|
jpayne@7
|
857 stream=True is set on the request, this avoids reading the
|
|
jpayne@7
|
858 content at once into memory for large responses.
|
|
jpayne@7
|
859
|
|
jpayne@7
|
860 .. note:: This method is not reentrant safe.
|
|
jpayne@7
|
861 """
|
|
jpayne@7
|
862
|
|
jpayne@7
|
863 pending = None
|
|
jpayne@7
|
864
|
|
jpayne@7
|
865 for chunk in self.iter_content(
|
|
jpayne@7
|
866 chunk_size=chunk_size, decode_unicode=decode_unicode
|
|
jpayne@7
|
867 ):
|
|
jpayne@7
|
868
|
|
jpayne@7
|
869 if pending is not None:
|
|
jpayne@7
|
870 chunk = pending + chunk
|
|
jpayne@7
|
871
|
|
jpayne@7
|
872 if delimiter:
|
|
jpayne@7
|
873 lines = chunk.split(delimiter)
|
|
jpayne@7
|
874 else:
|
|
jpayne@7
|
875 lines = chunk.splitlines()
|
|
jpayne@7
|
876
|
|
jpayne@7
|
877 if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
|
|
jpayne@7
|
878 pending = lines.pop()
|
|
jpayne@7
|
879 else:
|
|
jpayne@7
|
880 pending = None
|
|
jpayne@7
|
881
|
|
jpayne@7
|
882 yield from lines
|
|
jpayne@7
|
883
|
|
jpayne@7
|
884 if pending is not None:
|
|
jpayne@7
|
885 yield pending
|
|
jpayne@7
|
886
|
|
jpayne@7
|
887 @property
|
|
jpayne@7
|
888 def content(self):
|
|
jpayne@7
|
889 """Content of the response, in bytes."""
|
|
jpayne@7
|
890
|
|
jpayne@7
|
891 if self._content is False:
|
|
jpayne@7
|
892 # Read the contents.
|
|
jpayne@7
|
893 if self._content_consumed:
|
|
jpayne@7
|
894 raise RuntimeError("The content for this response was already consumed")
|
|
jpayne@7
|
895
|
|
jpayne@7
|
896 if self.status_code == 0 or self.raw is None:
|
|
jpayne@7
|
897 self._content = None
|
|
jpayne@7
|
898 else:
|
|
jpayne@7
|
899 self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
|
|
jpayne@7
|
900
|
|
jpayne@7
|
901 self._content_consumed = True
|
|
jpayne@7
|
902 # don't need to release the connection; that's been handled by urllib3
|
|
jpayne@7
|
903 # since we exhausted the data.
|
|
jpayne@7
|
904 return self._content
|
|
jpayne@7
|
905
|
|
jpayne@7
|
906 @property
|
|
jpayne@7
|
907 def text(self):
|
|
jpayne@7
|
908 """Content of the response, in unicode.
|
|
jpayne@7
|
909
|
|
jpayne@7
|
910 If Response.encoding is None, encoding will be guessed using
|
|
jpayne@7
|
911 ``charset_normalizer`` or ``chardet``.
|
|
jpayne@7
|
912
|
|
jpayne@7
|
913 The encoding of the response content is determined based solely on HTTP
|
|
jpayne@7
|
914 headers, following RFC 2616 to the letter. If you can take advantage of
|
|
jpayne@7
|
915 non-HTTP knowledge to make a better guess at the encoding, you should
|
|
jpayne@7
|
916 set ``r.encoding`` appropriately before accessing this property.
|
|
jpayne@7
|
917 """
|
|
jpayne@7
|
918
|
|
jpayne@7
|
919 # Try charset from content-type
|
|
jpayne@7
|
920 content = None
|
|
jpayne@7
|
921 encoding = self.encoding
|
|
jpayne@7
|
922
|
|
jpayne@7
|
923 if not self.content:
|
|
jpayne@7
|
924 return ""
|
|
jpayne@7
|
925
|
|
jpayne@7
|
926 # Fallback to auto-detected encoding.
|
|
jpayne@7
|
927 if self.encoding is None:
|
|
jpayne@7
|
928 encoding = self.apparent_encoding
|
|
jpayne@7
|
929
|
|
jpayne@7
|
930 # Decode unicode from given encoding.
|
|
jpayne@7
|
931 try:
|
|
jpayne@7
|
932 content = str(self.content, encoding, errors="replace")
|
|
jpayne@7
|
933 except (LookupError, TypeError):
|
|
jpayne@7
|
934 # A LookupError is raised if the encoding was not found which could
|
|
jpayne@7
|
935 # indicate a misspelling or similar mistake.
|
|
jpayne@7
|
936 #
|
|
jpayne@7
|
937 # A TypeError can be raised if encoding is None
|
|
jpayne@7
|
938 #
|
|
jpayne@7
|
939 # So we try blindly encoding.
|
|
jpayne@7
|
940 content = str(self.content, errors="replace")
|
|
jpayne@7
|
941
|
|
jpayne@7
|
942 return content
|
|
jpayne@7
|
943
|
|
jpayne@7
|
944 def json(self, **kwargs):
|
|
jpayne@7
|
945 r"""Returns the json-encoded content of a response, if any.
|
|
jpayne@7
|
946
|
|
jpayne@7
|
947 :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
|
|
jpayne@7
|
948 :raises requests.exceptions.JSONDecodeError: If the response body does not
|
|
jpayne@7
|
949 contain valid json.
|
|
jpayne@7
|
950 """
|
|
jpayne@7
|
951
|
|
jpayne@7
|
952 if not self.encoding and self.content and len(self.content) > 3:
|
|
jpayne@7
|
953 # No encoding set. JSON RFC 4627 section 3 states we should expect
|
|
jpayne@7
|
954 # UTF-8, -16 or -32. Detect which one to use; If the detection or
|
|
jpayne@7
|
955 # decoding fails, fall back to `self.text` (using charset_normalizer to make
|
|
jpayne@7
|
956 # a best guess).
|
|
jpayne@7
|
957 encoding = guess_json_utf(self.content)
|
|
jpayne@7
|
958 if encoding is not None:
|
|
jpayne@7
|
959 try:
|
|
jpayne@7
|
960 return complexjson.loads(self.content.decode(encoding), **kwargs)
|
|
jpayne@7
|
961 except UnicodeDecodeError:
|
|
jpayne@7
|
962 # Wrong UTF codec detected; usually because it's not UTF-8
|
|
jpayne@7
|
963 # but some other 8-bit codec. This is an RFC violation,
|
|
jpayne@7
|
964 # and the server didn't bother to tell us what codec *was*
|
|
jpayne@7
|
965 # used.
|
|
jpayne@7
|
966 pass
|
|
jpayne@7
|
967 except JSONDecodeError as e:
|
|
jpayne@7
|
968 raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
|
|
jpayne@7
|
969
|
|
jpayne@7
|
970 try:
|
|
jpayne@7
|
971 return complexjson.loads(self.text, **kwargs)
|
|
jpayne@7
|
972 except JSONDecodeError as e:
|
|
jpayne@7
|
973 # Catch JSON-related errors and raise as requests.JSONDecodeError
|
|
jpayne@7
|
974 # This aliases json.JSONDecodeError and simplejson.JSONDecodeError
|
|
jpayne@7
|
975 raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
|
|
jpayne@7
|
976
|
|
jpayne@7
|
977 @property
|
|
jpayne@7
|
978 def links(self):
|
|
jpayne@7
|
979 """Returns the parsed header links of the response, if any."""
|
|
jpayne@7
|
980
|
|
jpayne@7
|
981 header = self.headers.get("link")
|
|
jpayne@7
|
982
|
|
jpayne@7
|
983 resolved_links = {}
|
|
jpayne@7
|
984
|
|
jpayne@7
|
985 if header:
|
|
jpayne@7
|
986 links = parse_header_links(header)
|
|
jpayne@7
|
987
|
|
jpayne@7
|
988 for link in links:
|
|
jpayne@7
|
989 key = link.get("rel") or link.get("url")
|
|
jpayne@7
|
990 resolved_links[key] = link
|
|
jpayne@7
|
991
|
|
jpayne@7
|
992 return resolved_links
|
|
jpayne@7
|
993
|
|
jpayne@7
|
994 def raise_for_status(self):
|
|
jpayne@7
|
995 """Raises :class:`HTTPError`, if one occurred."""
|
|
jpayne@7
|
996
|
|
jpayne@7
|
997 http_error_msg = ""
|
|
jpayne@7
|
998 if isinstance(self.reason, bytes):
|
|
jpayne@7
|
999 # We attempt to decode utf-8 first because some servers
|
|
jpayne@7
|
1000 # choose to localize their reason strings. If the string
|
|
jpayne@7
|
1001 # isn't utf-8, we fall back to iso-8859-1 for all other
|
|
jpayne@7
|
1002 # encodings. (See PR #3538)
|
|
jpayne@7
|
1003 try:
|
|
jpayne@7
|
1004 reason = self.reason.decode("utf-8")
|
|
jpayne@7
|
1005 except UnicodeDecodeError:
|
|
jpayne@7
|
1006 reason = self.reason.decode("iso-8859-1")
|
|
jpayne@7
|
1007 else:
|
|
jpayne@7
|
1008 reason = self.reason
|
|
jpayne@7
|
1009
|
|
jpayne@7
|
1010 if 400 <= self.status_code < 500:
|
|
jpayne@7
|
1011 http_error_msg = (
|
|
jpayne@7
|
1012 f"{self.status_code} Client Error: {reason} for url: {self.url}"
|
|
jpayne@7
|
1013 )
|
|
jpayne@7
|
1014
|
|
jpayne@7
|
1015 elif 500 <= self.status_code < 600:
|
|
jpayne@7
|
1016 http_error_msg = (
|
|
jpayne@7
|
1017 f"{self.status_code} Server Error: {reason} for url: {self.url}"
|
|
jpayne@7
|
1018 )
|
|
jpayne@7
|
1019
|
|
jpayne@7
|
1020 if http_error_msg:
|
|
jpayne@7
|
1021 raise HTTPError(http_error_msg, response=self)
|
|
jpayne@7
|
1022
|
|
jpayne@7
|
1023 def close(self):
|
|
jpayne@7
|
1024 """Releases the connection back to the pool. Once this method has been
|
|
jpayne@7
|
1025 called the underlying ``raw`` object must not be accessed again.
|
|
jpayne@7
|
1026
|
|
jpayne@7
|
1027 *Note: Should not normally need to be called explicitly.*
|
|
jpayne@7
|
1028 """
|
|
jpayne@7
|
1029 if not self._content_consumed:
|
|
jpayne@7
|
1030 self.raw.close()
|
|
jpayne@7
|
1031
|
|
jpayne@7
|
1032 release_conn = getattr(self.raw, "release_conn", None)
|
|
jpayne@7
|
1033 if release_conn is not None:
|
|
jpayne@7
|
1034 release_conn()
|
