Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/site-packages/requests/models.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 """
jpayne@69 2 requests.models
jpayne@69 3 ~~~~~~~~~~~~~~~
jpayne@69 4
jpayne@69 5 This module contains the primary objects that power Requests.
jpayne@69 6 """
jpayne@69 7
jpayne@69 8 import datetime
jpayne@69 9
jpayne@69 10 # Import encoding now, to avoid implicit import later.
jpayne@69 11 # Implicit import within threads may cause LookupError when standard library is in a ZIP,
jpayne@69 12 # such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
jpayne@69 13 import encodings.idna # noqa: F401
jpayne@69 14 from io import UnsupportedOperation
jpayne@69 15
jpayne@69 16 from urllib3.exceptions import (
jpayne@69 17 DecodeError,
jpayne@69 18 LocationParseError,
jpayne@69 19 ProtocolError,
jpayne@69 20 ReadTimeoutError,
jpayne@69 21 SSLError,
jpayne@69 22 )
jpayne@69 23 from urllib3.fields import RequestField
jpayne@69 24 from urllib3.filepost import encode_multipart_formdata
jpayne@69 25 from urllib3.util import parse_url
jpayne@69 26
jpayne@69 27 from ._internal_utils import to_native_string, unicode_is_ascii
jpayne@69 28 from .auth import HTTPBasicAuth
jpayne@69 29 from .compat import (
jpayne@69 30 Callable,
jpayne@69 31 JSONDecodeError,
jpayne@69 32 Mapping,
jpayne@69 33 basestring,
jpayne@69 34 builtin_str,
jpayne@69 35 chardet,
jpayne@69 36 cookielib,
jpayne@69 37 )
jpayne@69 38 from .compat import json as complexjson
jpayne@69 39 from .compat import urlencode, urlsplit, urlunparse
jpayne@69 40 from .cookies import _copy_cookie_jar, cookiejar_from_dict, get_cookie_header
jpayne@69 41 from .exceptions import (
jpayne@69 42 ChunkedEncodingError,
jpayne@69 43 ConnectionError,
jpayne@69 44 ContentDecodingError,
jpayne@69 45 HTTPError,
jpayne@69 46 InvalidJSONError,
jpayne@69 47 InvalidURL,
jpayne@69 48 )
jpayne@69 49 from .exceptions import JSONDecodeError as RequestsJSONDecodeError
jpayne@69 50 from .exceptions import MissingSchema
jpayne@69 51 from .exceptions import SSLError as RequestsSSLError
jpayne@69 52 from .exceptions import StreamConsumedError
jpayne@69 53 from .hooks import default_hooks
jpayne@69 54 from .status_codes import codes
jpayne@69 55 from .structures import CaseInsensitiveDict
jpayne@69 56 from .utils import (
jpayne@69 57 check_header_validity,
jpayne@69 58 get_auth_from_url,
jpayne@69 59 guess_filename,
jpayne@69 60 guess_json_utf,
jpayne@69 61 iter_slices,
jpayne@69 62 parse_header_links,
jpayne@69 63 requote_uri,
jpayne@69 64 stream_decode_response_unicode,
jpayne@69 65 super_len,
jpayne@69 66 to_key_val_list,
jpayne@69 67 )
jpayne@69 68
jpayne@69 69 #: The set of HTTP status codes that indicate an automatically
jpayne@69 70 #: processable redirect.
jpayne@69 71 REDIRECT_STATI = (
jpayne@69 72 codes.moved, # 301
jpayne@69 73 codes.found, # 302
jpayne@69 74 codes.other, # 303
jpayne@69 75 codes.temporary_redirect, # 307
jpayne@69 76 codes.permanent_redirect, # 308
jpayne@69 77 )
jpayne@69 78
jpayne@69 79 DEFAULT_REDIRECT_LIMIT = 30
jpayne@69 80 CONTENT_CHUNK_SIZE = 10 * 1024
jpayne@69 81 ITER_CHUNK_SIZE = 512
jpayne@69 82
jpayne@69 83
jpayne@69 84 class RequestEncodingMixin:
jpayne@69 85 @property
jpayne@69 86 def path_url(self):
jpayne@69 87 """Build the path URL to use."""
jpayne@69 88
jpayne@69 89 url = []
jpayne@69 90
jpayne@69 91 p = urlsplit(self.url)
jpayne@69 92
jpayne@69 93 path = p.path
jpayne@69 94 if not path:
jpayne@69 95 path = "/"
jpayne@69 96
jpayne@69 97 url.append(path)
jpayne@69 98
jpayne@69 99 query = p.query
jpayne@69 100 if query:
jpayne@69 101 url.append("?")
jpayne@69 102 url.append(query)
jpayne@69 103
jpayne@69 104 return "".join(url)
jpayne@69 105
jpayne@69 106 @staticmethod
jpayne@69 107 def _encode_params(data):
jpayne@69 108 """Encode parameters in a piece of data.
jpayne@69 109
jpayne@69 110 Will successfully encode parameters when passed as a dict or a list of
jpayne@69 111 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
jpayne@69 112 if parameters are supplied as a dict.
jpayne@69 113 """
jpayne@69 114
jpayne@69 115 if isinstance(data, (str, bytes)):
jpayne@69 116 return data
jpayne@69 117 elif hasattr(data, "read"):
jpayne@69 118 return data
jpayne@69 119 elif hasattr(data, "__iter__"):
jpayne@69 120 result = []
jpayne@69 121 for k, vs in to_key_val_list(data):
jpayne@69 122 if isinstance(vs, basestring) or not hasattr(vs, "__iter__"):
jpayne@69 123 vs = [vs]
jpayne@69 124 for v in vs:
jpayne@69 125 if v is not None:
jpayne@69 126 result.append(
jpayne@69 127 (
jpayne@69 128 k.encode("utf-8") if isinstance(k, str) else k,
jpayne@69 129 v.encode("utf-8") if isinstance(v, str) else v,
jpayne@69 130 )
jpayne@69 131 )
jpayne@69 132 return urlencode(result, doseq=True)
jpayne@69 133 else:
jpayne@69 134 return data
jpayne@69 135
jpayne@69 136 @staticmethod
jpayne@69 137 def _encode_files(files, data):
jpayne@69 138 """Build the body for a multipart/form-data request.
jpayne@69 139
jpayne@69 140 Will successfully encode files when passed as a dict or a list of
jpayne@69 141 tuples. Order is retained if data is a list of tuples but arbitrary
jpayne@69 142 if parameters are supplied as a dict.
jpayne@69 143 The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
jpayne@69 144 or 4-tuples (filename, fileobj, contentype, custom_headers).
jpayne@69 145 """
jpayne@69 146 if not files:
jpayne@69 147 raise ValueError("Files must be provided.")
jpayne@69 148 elif isinstance(data, basestring):
jpayne@69 149 raise ValueError("Data must not be a string.")
jpayne@69 150
jpayne@69 151 new_fields = []
jpayne@69 152 fields = to_key_val_list(data or {})
jpayne@69 153 files = to_key_val_list(files or {})
jpayne@69 154
jpayne@69 155 for field, val in fields:
jpayne@69 156 if isinstance(val, basestring) or not hasattr(val, "__iter__"):
jpayne@69 157 val = [val]
jpayne@69 158 for v in val:
jpayne@69 159 if v is not None:
jpayne@69 160 # Don't call str() on bytestrings: in Py3 it all goes wrong.
jpayne@69 161 if not isinstance(v, bytes):
jpayne@69 162 v = str(v)
jpayne@69 163
jpayne@69 164 new_fields.append(
jpayne@69 165 (
jpayne@69 166 field.decode("utf-8")
jpayne@69 167 if isinstance(field, bytes)
jpayne@69 168 else field,
jpayne@69 169 v.encode("utf-8") if isinstance(v, str) else v,
jpayne@69 170 )
jpayne@69 171 )
jpayne@69 172
jpayne@69 173 for k, v in files:
jpayne@69 174 # support for explicit filename
jpayne@69 175 ft = None
jpayne@69 176 fh = None
jpayne@69 177 if isinstance(v, (tuple, list)):
jpayne@69 178 if len(v) == 2:
jpayne@69 179 fn, fp = v
jpayne@69 180 elif len(v) == 3:
jpayne@69 181 fn, fp, ft = v
jpayne@69 182 else:
jpayne@69 183 fn, fp, ft, fh = v
jpayne@69 184 else:
jpayne@69 185 fn = guess_filename(v) or k
jpayne@69 186 fp = v
jpayne@69 187
jpayne@69 188 if isinstance(fp, (str, bytes, bytearray)):
jpayne@69 189 fdata = fp
jpayne@69 190 elif hasattr(fp, "read"):
jpayne@69 191 fdata = fp.read()
jpayne@69 192 elif fp is None:
jpayne@69 193 continue
jpayne@69 194 else:
jpayne@69 195 fdata = fp
jpayne@69 196
jpayne@69 197 rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
jpayne@69 198 rf.make_multipart(content_type=ft)
jpayne@69 199 new_fields.append(rf)
jpayne@69 200
jpayne@69 201 body, content_type = encode_multipart_formdata(new_fields)
jpayne@69 202
jpayne@69 203 return body, content_type
jpayne@69 204
jpayne@69 205
jpayne@69 206 class RequestHooksMixin:
jpayne@69 207 def register_hook(self, event, hook):
jpayne@69 208 """Properly register a hook."""
jpayne@69 209
jpayne@69 210 if event not in self.hooks:
jpayne@69 211 raise ValueError(f'Unsupported event specified, with event name "{event}"')
jpayne@69 212
jpayne@69 213 if isinstance(hook, Callable):
jpayne@69 214 self.hooks[event].append(hook)
jpayne@69 215 elif hasattr(hook, "__iter__"):
jpayne@69 216 self.hooks[event].extend(h for h in hook if isinstance(h, Callable))
jpayne@69 217
jpayne@69 218 def deregister_hook(self, event, hook):
jpayne@69 219 """Deregister a previously registered hook.
jpayne@69 220 Returns True if the hook existed, False if not.
jpayne@69 221 """
jpayne@69 222
jpayne@69 223 try:
jpayne@69 224 self.hooks[event].remove(hook)
jpayne@69 225 return True
jpayne@69 226 except ValueError:
jpayne@69 227 return False
jpayne@69 228
jpayne@69 229
jpayne@69 230 class Request(RequestHooksMixin):
jpayne@69 231 """A user-created :class:`Request <Request>` object.
jpayne@69 232
jpayne@69 233 Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
jpayne@69 234
jpayne@69 235 :param method: HTTP method to use.
jpayne@69 236 :param url: URL to send.
jpayne@69 237 :param headers: dictionary of headers to send.
jpayne@69 238 :param files: dictionary of {filename: fileobject} files to multipart upload.
jpayne@69 239 :param data: the body to attach to the request. If a dictionary or
jpayne@69 240 list of tuples ``[(key, value)]`` is provided, form-encoding will
jpayne@69 241 take place.
jpayne@69 242 :param json: json for the body to attach to the request (if files or data is not specified).
jpayne@69 243 :param params: URL parameters to append to the URL. If a dictionary or
jpayne@69 244 list of tuples ``[(key, value)]`` is provided, form-encoding will
jpayne@69 245 take place.
jpayne@69 246 :param auth: Auth handler or (user, pass) tuple.
jpayne@69 247 :param cookies: dictionary or CookieJar of cookies to attach to this request.
jpayne@69 248 :param hooks: dictionary of callback hooks, for internal usage.
jpayne@69 249
jpayne@69 250 Usage::
jpayne@69 251
jpayne@69 252 >>> import requests
jpayne@69 253 >>> req = requests.Request('GET', 'https://httpbin.org/get')
jpayne@69 254 >>> req.prepare()
jpayne@69 255 <PreparedRequest [GET]>
jpayne@69 256 """
jpayne@69 257
jpayne@69 258 def __init__(
jpayne@69 259 self,
jpayne@69 260 method=None,
jpayne@69 261 url=None,
jpayne@69 262 headers=None,
jpayne@69 263 files=None,
jpayne@69 264 data=None,
jpayne@69 265 params=None,
jpayne@69 266 auth=None,
jpayne@69 267 cookies=None,
jpayne@69 268 hooks=None,
jpayne@69 269 json=None,
jpayne@69 270 ):
jpayne@69 271 # Default empty dicts for dict params.
jpayne@69 272 data = [] if data is None else data
jpayne@69 273 files = [] if files is None else files
jpayne@69 274 headers = {} if headers is None else headers
jpayne@69 275 params = {} if params is None else params
jpayne@69 276 hooks = {} if hooks is None else hooks
jpayne@69 277
jpayne@69 278 self.hooks = default_hooks()
jpayne@69 279 for k, v in list(hooks.items()):
jpayne@69 280 self.register_hook(event=k, hook=v)
jpayne@69 281
jpayne@69 282 self.method = method
jpayne@69 283 self.url = url
jpayne@69 284 self.headers = headers
jpayne@69 285 self.files = files
jpayne@69 286 self.data = data
jpayne@69 287 self.json = json
jpayne@69 288 self.params = params
jpayne@69 289 self.auth = auth
jpayne@69 290 self.cookies = cookies
jpayne@69 291
jpayne@69 292 def __repr__(self):
jpayne@69 293 return f"<Request [{self.method}]>"
jpayne@69 294
jpayne@69 295 def prepare(self):
jpayne@69 296 """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
jpayne@69 297 p = PreparedRequest()
jpayne@69 298 p.prepare(
jpayne@69 299 method=self.method,
jpayne@69 300 url=self.url,
jpayne@69 301 headers=self.headers,
jpayne@69 302 files=self.files,
jpayne@69 303 data=self.data,
jpayne@69 304 json=self.json,
jpayne@69 305 params=self.params,
jpayne@69 306 auth=self.auth,
jpayne@69 307 cookies=self.cookies,
jpayne@69 308 hooks=self.hooks,
jpayne@69 309 )
jpayne@69 310 return p
jpayne@69 311
jpayne@69 312
jpayne@69 313 class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
jpayne@69 314 """The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
jpayne@69 315 containing the exact bytes that will be sent to the server.
jpayne@69 316
jpayne@69 317 Instances are generated from a :class:`Request <Request>` object, and
jpayne@69 318 should not be instantiated manually; doing so may produce undesirable
jpayne@69 319 effects.
jpayne@69 320
jpayne@69 321 Usage::
jpayne@69 322
jpayne@69 323 >>> import requests
jpayne@69 324 >>> req = requests.Request('GET', 'https://httpbin.org/get')
jpayne@69 325 >>> r = req.prepare()
jpayne@69 326 >>> r
jpayne@69 327 <PreparedRequest [GET]>
jpayne@69 328
jpayne@69 329 >>> s = requests.Session()
jpayne@69 330 >>> s.send(r)
jpayne@69 331 <Response [200]>
jpayne@69 332 """
jpayne@69 333
jpayne@69 334 def __init__(self):
jpayne@69 335 #: HTTP verb to send to the server.
jpayne@69 336 self.method = None
jpayne@69 337 #: HTTP URL to send the request to.
jpayne@69 338 self.url = None
jpayne@69 339 #: dictionary of HTTP headers.
jpayne@69 340 self.headers = None
jpayne@69 341 # The `CookieJar` used to create the Cookie header will be stored here
jpayne@69 342 # after prepare_cookies is called
jpayne@69 343 self._cookies = None
jpayne@69 344 #: request body to send to the server.
jpayne@69 345 self.body = None
jpayne@69 346 #: dictionary of callback hooks, for internal usage.
jpayne@69 347 self.hooks = default_hooks()
jpayne@69 348 #: integer denoting starting position of a readable file-like body.
jpayne@69 349 self._body_position = None
jpayne@69 350
jpayne@69 351 def prepare(
jpayne@69 352 self,
jpayne@69 353 method=None,
jpayne@69 354 url=None,
jpayne@69 355 headers=None,
jpayne@69 356 files=None,
jpayne@69 357 data=None,
jpayne@69 358 params=None,
jpayne@69 359 auth=None,
jpayne@69 360 cookies=None,
jpayne@69 361 hooks=None,
jpayne@69 362 json=None,
jpayne@69 363 ):
jpayne@69 364 """Prepares the entire request with the given parameters."""
jpayne@69 365
jpayne@69 366 self.prepare_method(method)
jpayne@69 367 self.prepare_url(url, params)
jpayne@69 368 self.prepare_headers(headers)
jpayne@69 369 self.prepare_cookies(cookies)
jpayne@69 370 self.prepare_body(data, files, json)
jpayne@69 371 self.prepare_auth(auth, url)
jpayne@69 372
jpayne@69 373 # Note that prepare_auth must be last to enable authentication schemes
jpayne@69 374 # such as OAuth to work on a fully prepared request.
jpayne@69 375
jpayne@69 376 # This MUST go after prepare_auth. Authenticators could add a hook
jpayne@69 377 self.prepare_hooks(hooks)
jpayne@69 378
jpayne@69 379 def __repr__(self):
jpayne@69 380 return f"<PreparedRequest [{self.method}]>"
jpayne@69 381
jpayne@69 382 def copy(self):
jpayne@69 383 p = PreparedRequest()
jpayne@69 384 p.method = self.method
jpayne@69 385 p.url = self.url
jpayne@69 386 p.headers = self.headers.copy() if self.headers is not None else None
jpayne@69 387 p._cookies = _copy_cookie_jar(self._cookies)
jpayne@69 388 p.body = self.body
jpayne@69 389 p.hooks = self.hooks
jpayne@69 390 p._body_position = self._body_position
jpayne@69 391 return p
jpayne@69 392
jpayne@69 393 def prepare_method(self, method):
jpayne@69 394 """Prepares the given HTTP method."""
jpayne@69 395 self.method = method
jpayne@69 396 if self.method is not None:
jpayne@69 397 self.method = to_native_string(self.method.upper())
jpayne@69 398
jpayne@69 399 @staticmethod
jpayne@69 400 def _get_idna_encoded_host(host):
jpayne@69 401 import idna
jpayne@69 402
jpayne@69 403 try:
jpayne@69 404 host = idna.encode(host, uts46=True).decode("utf-8")
jpayne@69 405 except idna.IDNAError:
jpayne@69 406 raise UnicodeError
jpayne@69 407 return host
jpayne@69 408
jpayne@69 409 def prepare_url(self, url, params):
jpayne@69 410 """Prepares the given HTTP URL."""
jpayne@69 411 #: Accept objects that have string representations.
jpayne@69 412 #: We're unable to blindly call unicode/str functions
jpayne@69 413 #: as this will include the bytestring indicator (b'')
jpayne@69 414 #: on python 3.x.
jpayne@69 415 #: https://github.com/psf/requests/pull/2238
jpayne@69 416 if isinstance(url, bytes):
jpayne@69 417 url = url.decode("utf8")
jpayne@69 418 else:
jpayne@69 419 url = str(url)
jpayne@69 420
jpayne@69 421 # Remove leading whitespaces from url
jpayne@69 422 url = url.lstrip()
jpayne@69 423
jpayne@69 424 # Don't do any URL preparation for non-HTTP schemes like `mailto`,
jpayne@69 425 # `data` etc to work around exceptions from `url_parse`, which
jpayne@69 426 # handles RFC 3986 only.
jpayne@69 427 if ":" in url and not url.lower().startswith("http"):
jpayne@69 428 self.url = url
jpayne@69 429 return
jpayne@69 430
jpayne@69 431 # Support for unicode domain names and paths.
jpayne@69 432 try:
jpayne@69 433 scheme, auth, host, port, path, query, fragment = parse_url(url)
jpayne@69 434 except LocationParseError as e:
jpayne@69 435 raise InvalidURL(*e.args)
jpayne@69 436
jpayne@69 437 if not scheme:
jpayne@69 438 raise MissingSchema(
jpayne@69 439 f"Invalid URL {url!r}: No scheme supplied. "
jpayne@69 440 f"Perhaps you meant https://{url}?"
jpayne@69 441 )
jpayne@69 442
jpayne@69 443 if not host:
jpayne@69 444 raise InvalidURL(f"Invalid URL {url!r}: No host supplied")
jpayne@69 445
jpayne@69 446 # In general, we want to try IDNA encoding the hostname if the string contains
jpayne@69 447 # non-ASCII characters. This allows users to automatically get the correct IDNA
jpayne@69 448 # behaviour. For strings containing only ASCII characters, we need to also verify
jpayne@69 449 # it doesn't start with a wildcard (*), before allowing the unencoded hostname.
jpayne@69 450 if not unicode_is_ascii(host):
jpayne@69 451 try:
jpayne@69 452 host = self._get_idna_encoded_host(host)
jpayne@69 453 except UnicodeError:
jpayne@69 454 raise InvalidURL("URL has an invalid label.")
jpayne@69 455 elif host.startswith(("*", ".")):
jpayne@69 456 raise InvalidURL("URL has an invalid label.")
jpayne@69 457
jpayne@69 458 # Carefully reconstruct the network location
jpayne@69 459 netloc = auth or ""
jpayne@69 460 if netloc:
jpayne@69 461 netloc += "@"
jpayne@69 462 netloc += host
jpayne@69 463 if port:
jpayne@69 464 netloc += f":{port}"
jpayne@69 465
jpayne@69 466 # Bare domains aren't valid URLs.
jpayne@69 467 if not path:
jpayne@69 468 path = "/"
jpayne@69 469
jpayne@69 470 if isinstance(params, (str, bytes)):
jpayne@69 471 params = to_native_string(params)
jpayne@69 472
jpayne@69 473 enc_params = self._encode_params(params)
jpayne@69 474 if enc_params:
jpayne@69 475 if query:
jpayne@69 476 query = f"{query}&{enc_params}"
jpayne@69 477 else:
jpayne@69 478 query = enc_params
jpayne@69 479
jpayne@69 480 url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
jpayne@69 481 self.url = url
jpayne@69 482
jpayne@69 483 def prepare_headers(self, headers):
jpayne@69 484 """Prepares the given HTTP headers."""
jpayne@69 485
jpayne@69 486 self.headers = CaseInsensitiveDict()
jpayne@69 487 if headers:
jpayne@69 488 for header in headers.items():
jpayne@69 489 # Raise exception on invalid header value.
jpayne@69 490 check_header_validity(header)
jpayne@69 491 name, value = header
jpayne@69 492 self.headers[to_native_string(name)] = value
jpayne@69 493
jpayne@69 494 def prepare_body(self, data, files, json=None):
jpayne@69 495 """Prepares the given HTTP body data."""
jpayne@69 496
jpayne@69 497 # Check if file, fo, generator, iterator.
jpayne@69 498 # If not, run through normal process.
jpayne@69 499
jpayne@69 500 # Nottin' on you.
jpayne@69 501 body = None
jpayne@69 502 content_type = None
jpayne@69 503
jpayne@69 504 if not data and json is not None:
jpayne@69 505 # urllib3 requires a bytes-like body. Python 2's json.dumps
jpayne@69 506 # provides this natively, but Python 3 gives a Unicode string.
jpayne@69 507 content_type = "application/json"
jpayne@69 508
jpayne@69 509 try:
jpayne@69 510 body = complexjson.dumps(json, allow_nan=False)
jpayne@69 511 except ValueError as ve:
jpayne@69 512 raise InvalidJSONError(ve, request=self)
jpayne@69 513
jpayne@69 514 if not isinstance(body, bytes):
jpayne@69 515 body = body.encode("utf-8")
jpayne@69 516
jpayne@69 517 is_stream = all(
jpayne@69 518 [
jpayne@69 519 hasattr(data, "__iter__"),
jpayne@69 520 not isinstance(data, (basestring, list, tuple, Mapping)),
jpayne@69 521 ]
jpayne@69 522 )
jpayne@69 523
jpayne@69 524 if is_stream:
jpayne@69 525 try:
jpayne@69 526 length = super_len(data)
jpayne@69 527 except (TypeError, AttributeError, UnsupportedOperation):
jpayne@69 528 length = None
jpayne@69 529
jpayne@69 530 body = data
jpayne@69 531
jpayne@69 532 if getattr(body, "tell", None) is not None:
jpayne@69 533 # Record the current file position before reading.
jpayne@69 534 # This will allow us to rewind a file in the event
jpayne@69 535 # of a redirect.
jpayne@69 536 try:
jpayne@69 537 self._body_position = body.tell()
jpayne@69 538 except OSError:
jpayne@69 539 # This differentiates from None, allowing us to catch
jpayne@69 540 # a failed `tell()` later when trying to rewind the body
jpayne@69 541 self._body_position = object()
jpayne@69 542
jpayne@69 543 if files:
jpayne@69 544 raise NotImplementedError(
jpayne@69 545 "Streamed bodies and files are mutually exclusive."
jpayne@69 546 )
jpayne@69 547
jpayne@69 548 if length:
jpayne@69 549 self.headers["Content-Length"] = builtin_str(length)
jpayne@69 550 else:
jpayne@69 551 self.headers["Transfer-Encoding"] = "chunked"
jpayne@69 552 else:
jpayne@69 553 # Multi-part file uploads.
jpayne@69 554 if files:
jpayne@69 555 (body, content_type) = self._encode_files(files, data)
jpayne@69 556 else:
jpayne@69 557 if data:
jpayne@69 558 body = self._encode_params(data)
jpayne@69 559 if isinstance(data, basestring) or hasattr(data, "read"):
jpayne@69 560 content_type = None
jpayne@69 561 else:
jpayne@69 562 content_type = "application/x-www-form-urlencoded"
jpayne@69 563
jpayne@69 564 self.prepare_content_length(body)
jpayne@69 565
jpayne@69 566 # Add content-type if it wasn't explicitly provided.
jpayne@69 567 if content_type and ("content-type" not in self.headers):
jpayne@69 568 self.headers["Content-Type"] = content_type
jpayne@69 569
jpayne@69 570 self.body = body
jpayne@69 571
jpayne@69 572 def prepare_content_length(self, body):
jpayne@69 573 """Prepare Content-Length header based on request method and body"""
jpayne@69 574 if body is not None:
jpayne@69 575 length = super_len(body)
jpayne@69 576 if length:
jpayne@69 577 # If length exists, set it. Otherwise, we fallback
jpayne@69 578 # to Transfer-Encoding: chunked.
jpayne@69 579 self.headers["Content-Length"] = builtin_str(length)
jpayne@69 580 elif (
jpayne@69 581 self.method not in ("GET", "HEAD")
jpayne@69 582 and self.headers.get("Content-Length") is None
jpayne@69 583 ):
jpayne@69 584 # Set Content-Length to 0 for methods that can have a body
jpayne@69 585 # but don't provide one. (i.e. not GET or HEAD)
jpayne@69 586 self.headers["Content-Length"] = "0"
jpayne@69 587
jpayne@69 588 def prepare_auth(self, auth, url=""):
jpayne@69 589 """Prepares the given HTTP auth data."""
jpayne@69 590
jpayne@69 591 # If no Auth is explicitly provided, extract it from the URL first.
jpayne@69 592 if auth is None:
jpayne@69 593 url_auth = get_auth_from_url(self.url)
jpayne@69 594 auth = url_auth if any(url_auth) else None
jpayne@69 595
jpayne@69 596 if auth:
jpayne@69 597 if isinstance(auth, tuple) and len(auth) == 2:
jpayne@69 598 # special-case basic HTTP auth
jpayne@69 599 auth = HTTPBasicAuth(*auth)
jpayne@69 600
jpayne@69 601 # Allow auth to make its changes.
jpayne@69 602 r = auth(self)
jpayne@69 603
jpayne@69 604 # Update self to reflect the auth changes.
jpayne@69 605 self.__dict__.update(r.__dict__)
jpayne@69 606
jpayne@69 607 # Recompute Content-Length
jpayne@69 608 self.prepare_content_length(self.body)
jpayne@69 609
jpayne@69 610 def prepare_cookies(self, cookies):
jpayne@69 611 """Prepares the given HTTP cookie data.
jpayne@69 612
jpayne@69 613 This function eventually generates a ``Cookie`` header from the
jpayne@69 614 given cookies using cookielib. Due to cookielib's design, the header
jpayne@69 615 will not be regenerated if it already exists, meaning this function
jpayne@69 616 can only be called once for the life of the
jpayne@69 617 :class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
jpayne@69 618 to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
jpayne@69 619 header is removed beforehand.
jpayne@69 620 """
jpayne@69 621 if isinstance(cookies, cookielib.CookieJar):
jpayne@69 622 self._cookies = cookies
jpayne@69 623 else:
jpayne@69 624 self._cookies = cookiejar_from_dict(cookies)
jpayne@69 625
jpayne@69 626 cookie_header = get_cookie_header(self._cookies, self)
jpayne@69 627 if cookie_header is not None:
jpayne@69 628 self.headers["Cookie"] = cookie_header
jpayne@69 629
jpayne@69 630 def prepare_hooks(self, hooks):
jpayne@69 631 """Prepares the given hooks."""
jpayne@69 632 # hooks can be passed as None to the prepare method and to this
jpayne@69 633 # method. To prevent iterating over None, simply use an empty list
jpayne@69 634 # if hooks is False-y
jpayne@69 635 hooks = hooks or []
jpayne@69 636 for event in hooks:
jpayne@69 637 self.register_hook(event, hooks[event])
jpayne@69 638
jpayne@69 639
jpayne@69 640 class Response:
jpayne@69 641 """The :class:`Response <Response>` object, which contains a
jpayne@69 642 server's response to an HTTP request.
jpayne@69 643 """
jpayne@69 644
jpayne@69 645 __attrs__ = [
jpayne@69 646 "_content",
jpayne@69 647 "status_code",
jpayne@69 648 "headers",
jpayne@69 649 "url",
jpayne@69 650 "history",
jpayne@69 651 "encoding",
jpayne@69 652 "reason",
jpayne@69 653 "cookies",
jpayne@69 654 "elapsed",
jpayne@69 655 "request",
jpayne@69 656 ]
jpayne@69 657
jpayne@69 658 def __init__(self):
jpayne@69 659 self._content = False
jpayne@69 660 self._content_consumed = False
jpayne@69 661 self._next = None
jpayne@69 662
jpayne@69 663 #: Integer Code of responded HTTP Status, e.g. 404 or 200.
jpayne@69 664 self.status_code = None
jpayne@69 665
jpayne@69 666 #: Case-insensitive Dictionary of Response Headers.
jpayne@69 667 #: For example, ``headers['content-encoding']`` will return the
jpayne@69 668 #: value of a ``'Content-Encoding'`` response header.
jpayne@69 669 self.headers = CaseInsensitiveDict()
jpayne@69 670
jpayne@69 671 #: File-like object representation of response (for advanced usage).
jpayne@69 672 #: Use of ``raw`` requires that ``stream=True`` be set on the request.
jpayne@69 673 #: This requirement does not apply for use internally to Requests.
jpayne@69 674 self.raw = None
jpayne@69 675
jpayne@69 676 #: Final URL location of Response.
jpayne@69 677 self.url = None
jpayne@69 678
jpayne@69 679 #: Encoding to decode with when accessing r.text.
jpayne@69 680 self.encoding = None
jpayne@69 681
jpayne@69 682 #: A list of :class:`Response <Response>` objects from
jpayne@69 683 #: the history of the Request. Any redirect responses will end
jpayne@69 684 #: up here. The list is sorted from the oldest to the most recent request.
jpayne@69 685 self.history = []
jpayne@69 686
jpayne@69 687 #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
jpayne@69 688 self.reason = None
jpayne@69 689
jpayne@69 690 #: A CookieJar of Cookies the server sent back.
jpayne@69 691 self.cookies = cookiejar_from_dict({})
jpayne@69 692
jpayne@69 693 #: The amount of time elapsed between sending the request
jpayne@69 694 #: and the arrival of the response (as a timedelta).
jpayne@69 695 #: This property specifically measures the time taken between sending
jpayne@69 696 #: the first byte of the request and finishing parsing the headers. It
jpayne@69 697 #: is therefore unaffected by consuming the response content or the
jpayne@69 698 #: value of the ``stream`` keyword argument.
jpayne@69 699 self.elapsed = datetime.timedelta(0)
jpayne@69 700
jpayne@69 701 #: The :class:`PreparedRequest <PreparedRequest>` object to which this
jpayne@69 702 #: is a response.
jpayne@69 703 self.request = None
jpayne@69 704
jpayne@69 705 def __enter__(self):
jpayne@69 706 return self
jpayne@69 707
jpayne@69 708 def __exit__(self, *args):
jpayne@69 709 self.close()
jpayne@69 710
jpayne@69 711 def __getstate__(self):
jpayne@69 712 # Consume everything; accessing the content attribute makes
jpayne@69 713 # sure the content has been fully read.
jpayne@69 714 if not self._content_consumed:
jpayne@69 715 self.content
jpayne@69 716
jpayne@69 717 return {attr: getattr(self, attr, None) for attr in self.__attrs__}
jpayne@69 718
jpayne@69 719 def __setstate__(self, state):
jpayne@69 720 for name, value in state.items():
jpayne@69 721 setattr(self, name, value)
jpayne@69 722
jpayne@69 723 # pickled objects do not have .raw
jpayne@69 724 setattr(self, "_content_consumed", True)
jpayne@69 725 setattr(self, "raw", None)
jpayne@69 726
jpayne@69 727 def __repr__(self):
jpayne@69 728 return f"<Response [{self.status_code}]>"
jpayne@69 729
jpayne@69 730 def __bool__(self):
jpayne@69 731 """Returns True if :attr:`status_code` is less than 400.
jpayne@69 732
jpayne@69 733 This attribute checks if the status code of the response is between
jpayne@69 734 400 and 600 to see if there was a client error or a server error. If
jpayne@69 735 the status code, is between 200 and 400, this will return True. This
jpayne@69 736 is **not** a check to see if the response code is ``200 OK``.
jpayne@69 737 """
jpayne@69 738 return self.ok
jpayne@69 739
jpayne@69 740 def __nonzero__(self):
jpayne@69 741 """Returns True if :attr:`status_code` is less than 400.
jpayne@69 742
jpayne@69 743 This attribute checks if the status code of the response is between
jpayne@69 744 400 and 600 to see if there was a client error or a server error. If
jpayne@69 745 the status code, is between 200 and 400, this will return True. This
jpayne@69 746 is **not** a check to see if the response code is ``200 OK``.
jpayne@69 747 """
jpayne@69 748 return self.ok
jpayne@69 749
jpayne@69 750 def __iter__(self):
jpayne@69 751 """Allows you to use a response as an iterator."""
jpayne@69 752 return self.iter_content(128)
jpayne@69 753
jpayne@69 754 @property
jpayne@69 755 def ok(self):
jpayne@69 756 """Returns True if :attr:`status_code` is less than 400, False if not.
jpayne@69 757
jpayne@69 758 This attribute checks if the status code of the response is between
jpayne@69 759 400 and 600 to see if there was a client error or a server error. If
jpayne@69 760 the status code is between 200 and 400, this will return True. This
jpayne@69 761 is **not** a check to see if the response code is ``200 OK``.
jpayne@69 762 """
jpayne@69 763 try:
jpayne@69 764 self.raise_for_status()
jpayne@69 765 except HTTPError:
jpayne@69 766 return False
jpayne@69 767 return True
jpayne@69 768
jpayne@69 769 @property
jpayne@69 770 def is_redirect(self):
jpayne@69 771 """True if this Response is a well-formed HTTP redirect that could have
jpayne@69 772 been processed automatically (by :meth:`Session.resolve_redirects`).
jpayne@69 773 """
jpayne@69 774 return "location" in self.headers and self.status_code in REDIRECT_STATI
jpayne@69 775
jpayne@69 776 @property
jpayne@69 777 def is_permanent_redirect(self):
jpayne@69 778 """True if this Response one of the permanent versions of redirect."""
jpayne@69 779 return "location" in self.headers and self.status_code in (
jpayne@69 780 codes.moved_permanently,
jpayne@69 781 codes.permanent_redirect,
jpayne@69 782 )
jpayne@69 783
jpayne@69 784 @property
jpayne@69 785 def next(self):
jpayne@69 786 """Returns a PreparedRequest for the next request in a redirect chain, if there is one."""
jpayne@69 787 return self._next
jpayne@69 788
jpayne@69 789 @property
jpayne@69 790 def apparent_encoding(self):
jpayne@69 791 """The apparent encoding, provided by the charset_normalizer or chardet libraries."""
jpayne@69 792 if chardet is not None:
jpayne@69 793 return chardet.detect(self.content)["encoding"]
jpayne@69 794 else:
jpayne@69 795 # If no character detection library is available, we'll fall back
jpayne@69 796 # to a standard Python utf-8 str.
jpayne@69 797 return "utf-8"
jpayne@69 798
jpayne@69 799 def iter_content(self, chunk_size=1, decode_unicode=False):
jpayne@69 800 """Iterates over the response data. When stream=True is set on the
jpayne@69 801 request, this avoids reading the content at once into memory for
jpayne@69 802 large responses. The chunk size is the number of bytes it should
jpayne@69 803 read into memory. This is not necessarily the length of each item
jpayne@69 804 returned as decoding can take place.
jpayne@69 805
jpayne@69 806 chunk_size must be of type int or None. A value of None will
jpayne@69 807 function differently depending on the value of `stream`.
jpayne@69 808 stream=True will read data as it arrives in whatever size the
jpayne@69 809 chunks are received. If stream=False, data is returned as
jpayne@69 810 a single chunk.
jpayne@69 811
jpayne@69 812 If decode_unicode is True, content will be decoded using the best
jpayne@69 813 available encoding based on the response.
jpayne@69 814 """
jpayne@69 815
jpayne@69 816 def generate():
jpayne@69 817 # Special case for urllib3.
jpayne@69 818 if hasattr(self.raw, "stream"):
jpayne@69 819 try:
jpayne@69 820 yield from self.raw.stream(chunk_size, decode_content=True)
jpayne@69 821 except ProtocolError as e:
jpayne@69 822 raise ChunkedEncodingError(e)
jpayne@69 823 except DecodeError as e:
jpayne@69 824 raise ContentDecodingError(e)
jpayne@69 825 except ReadTimeoutError as e:
jpayne@69 826 raise ConnectionError(e)
jpayne@69 827 except SSLError as e:
jpayne@69 828 raise RequestsSSLError(e)
jpayne@69 829 else:
jpayne@69 830 # Standard file-like object.
jpayne@69 831 while True:
jpayne@69 832 chunk = self.raw.read(chunk_size)
jpayne@69 833 if not chunk:
jpayne@69 834 break
jpayne@69 835 yield chunk
jpayne@69 836
jpayne@69 837 self._content_consumed = True
jpayne@69 838
jpayne@69 839 if self._content_consumed and isinstance(self._content, bool):
jpayne@69 840 raise StreamConsumedError()
jpayne@69 841 elif chunk_size is not None and not isinstance(chunk_size, int):
jpayne@69 842 raise TypeError(
jpayne@69 843 f"chunk_size must be an int, it is instead a {type(chunk_size)}."
jpayne@69 844 )
jpayne@69 845 # simulate reading small chunks of the content
jpayne@69 846 reused_chunks = iter_slices(self._content, chunk_size)
jpayne@69 847
jpayne@69 848 stream_chunks = generate()
jpayne@69 849
jpayne@69 850 chunks = reused_chunks if self._content_consumed else stream_chunks
jpayne@69 851
jpayne@69 852 if decode_unicode:
jpayne@69 853 chunks = stream_decode_response_unicode(chunks, self)
jpayne@69 854
jpayne@69 855 return chunks
jpayne@69 856
jpayne@69 857 def iter_lines(
jpayne@69 858 self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None
jpayne@69 859 ):
jpayne@69 860 """Iterates over the response data, one line at a time. When
jpayne@69 861 stream=True is set on the request, this avoids reading the
jpayne@69 862 content at once into memory for large responses.
jpayne@69 863
jpayne@69 864 .. note:: This method is not reentrant safe.
jpayne@69 865 """
jpayne@69 866
jpayne@69 867 pending = None
jpayne@69 868
jpayne@69 869 for chunk in self.iter_content(
jpayne@69 870 chunk_size=chunk_size, decode_unicode=decode_unicode
jpayne@69 871 ):
jpayne@69 872 if pending is not None:
jpayne@69 873 chunk = pending + chunk
jpayne@69 874
jpayne@69 875 if delimiter:
jpayne@69 876 lines = chunk.split(delimiter)
jpayne@69 877 else:
jpayne@69 878 lines = chunk.splitlines()
jpayne@69 879
jpayne@69 880 if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
jpayne@69 881 pending = lines.pop()
jpayne@69 882 else:
jpayne@69 883 pending = None
jpayne@69 884
jpayne@69 885 yield from lines
jpayne@69 886
jpayne@69 887 if pending is not None:
jpayne@69 888 yield pending
jpayne@69 889
jpayne@69 890 @property
jpayne@69 891 def content(self):
jpayne@69 892 """Content of the response, in bytes."""
jpayne@69 893
jpayne@69 894 if self._content is False:
jpayne@69 895 # Read the contents.
jpayne@69 896 if self._content_consumed:
jpayne@69 897 raise RuntimeError("The content for this response was already consumed")
jpayne@69 898
jpayne@69 899 if self.status_code == 0 or self.raw is None:
jpayne@69 900 self._content = None
jpayne@69 901 else:
jpayne@69 902 self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
jpayne@69 903
jpayne@69 904 self._content_consumed = True
jpayne@69 905 # don't need to release the connection; that's been handled by urllib3
jpayne@69 906 # since we exhausted the data.
jpayne@69 907 return self._content
jpayne@69 908
jpayne@69 909 @property
jpayne@69 910 def text(self):
jpayne@69 911 """Content of the response, in unicode.
jpayne@69 912
jpayne@69 913 If Response.encoding is None, encoding will be guessed using
jpayne@69 914 ``charset_normalizer`` or ``chardet``.
jpayne@69 915
jpayne@69 916 The encoding of the response content is determined based solely on HTTP
jpayne@69 917 headers, following RFC 2616 to the letter. If you can take advantage of
jpayne@69 918 non-HTTP knowledge to make a better guess at the encoding, you should
jpayne@69 919 set ``r.encoding`` appropriately before accessing this property.
jpayne@69 920 """
jpayne@69 921
jpayne@69 922 # Try charset from content-type
jpayne@69 923 content = None
jpayne@69 924 encoding = self.encoding
jpayne@69 925
jpayne@69 926 if not self.content:
jpayne@69 927 return ""
jpayne@69 928
jpayne@69 929 # Fallback to auto-detected encoding.
jpayne@69 930 if self.encoding is None:
jpayne@69 931 encoding = self.apparent_encoding
jpayne@69 932
jpayne@69 933 # Decode unicode from given encoding.
jpayne@69 934 try:
jpayne@69 935 content = str(self.content, encoding, errors="replace")
jpayne@69 936 except (LookupError, TypeError):
jpayne@69 937 # A LookupError is raised if the encoding was not found which could
jpayne@69 938 # indicate a misspelling or similar mistake.
jpayne@69 939 #
jpayne@69 940 # A TypeError can be raised if encoding is None
jpayne@69 941 #
jpayne@69 942 # So we try blindly encoding.
jpayne@69 943 content = str(self.content, errors="replace")
jpayne@69 944
jpayne@69 945 return content
jpayne@69 946
jpayne@69 947 def json(self, **kwargs):
jpayne@69 948 r"""Returns the json-encoded content of a response, if any.
jpayne@69 949
jpayne@69 950 :param \*\*kwargs: Optional arguments that ``json.loads`` takes.
jpayne@69 951 :raises requests.exceptions.JSONDecodeError: If the response body does not
jpayne@69 952 contain valid json.
jpayne@69 953 """
jpayne@69 954
jpayne@69 955 if not self.encoding and self.content and len(self.content) > 3:
jpayne@69 956 # No encoding set. JSON RFC 4627 section 3 states we should expect
jpayne@69 957 # UTF-8, -16 or -32. Detect which one to use; If the detection or
jpayne@69 958 # decoding fails, fall back to `self.text` (using charset_normalizer to make
jpayne@69 959 # a best guess).
jpayne@69 960 encoding = guess_json_utf(self.content)
jpayne@69 961 if encoding is not None:
jpayne@69 962 try:
jpayne@69 963 return complexjson.loads(self.content.decode(encoding), **kwargs)
jpayne@69 964 except UnicodeDecodeError:
jpayne@69 965 # Wrong UTF codec detected; usually because it's not UTF-8
jpayne@69 966 # but some other 8-bit codec. This is an RFC violation,
jpayne@69 967 # and the server didn't bother to tell us what codec *was*
jpayne@69 968 # used.
jpayne@69 969 pass
jpayne@69 970 except JSONDecodeError as e:
jpayne@69 971 raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
jpayne@69 972
jpayne@69 973 try:
jpayne@69 974 return complexjson.loads(self.text, **kwargs)
jpayne@69 975 except JSONDecodeError as e:
jpayne@69 976 # Catch JSON-related errors and raise as requests.JSONDecodeError
jpayne@69 977 # This aliases json.JSONDecodeError and simplejson.JSONDecodeError
jpayne@69 978 raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
jpayne@69 979
jpayne@69 980 @property
jpayne@69 981 def links(self):
jpayne@69 982 """Returns the parsed header links of the response, if any."""
jpayne@69 983
jpayne@69 984 header = self.headers.get("link")
jpayne@69 985
jpayne@69 986 resolved_links = {}
jpayne@69 987
jpayne@69 988 if header:
jpayne@69 989 links = parse_header_links(header)
jpayne@69 990
jpayne@69 991 for link in links:
jpayne@69 992 key = link.get("rel") or link.get("url")
jpayne@69 993 resolved_links[key] = link
jpayne@69 994
jpayne@69 995 return resolved_links
jpayne@69 996
jpayne@69 997 def raise_for_status(self):
jpayne@69 998 """Raises :class:`HTTPError`, if one occurred."""
jpayne@69 999
jpayne@69 1000 http_error_msg = ""
jpayne@69 1001 if isinstance(self.reason, bytes):
jpayne@69 1002 # We attempt to decode utf-8 first because some servers
jpayne@69 1003 # choose to localize their reason strings. If the string
jpayne@69 1004 # isn't utf-8, we fall back to iso-8859-1 for all other
jpayne@69 1005 # encodings. (See PR #3538)
jpayne@69 1006 try:
jpayne@69 1007 reason = self.reason.decode("utf-8")
jpayne@69 1008 except UnicodeDecodeError:
jpayne@69 1009 reason = self.reason.decode("iso-8859-1")
jpayne@69 1010 else:
jpayne@69 1011 reason = self.reason
jpayne@69 1012
jpayne@69 1013 if 400 <= self.status_code < 500:
jpayne@69 1014 http_error_msg = (
jpayne@69 1015 f"{self.status_code} Client Error: {reason} for url: {self.url}"
jpayne@69 1016 )
jpayne@69 1017
jpayne@69 1018 elif 500 <= self.status_code < 600:
jpayne@69 1019 http_error_msg = (
jpayne@69 1020 f"{self.status_code} Server Error: {reason} for url: {self.url}"
jpayne@69 1021 )
jpayne@69 1022
jpayne@69 1023 if http_error_msg:
jpayne@69 1024 raise HTTPError(http_error_msg, response=self)
jpayne@69 1025
jpayne@69 1026 def close(self):
jpayne@69 1027 """Releases the connection back to the pool. Once this method has been
jpayne@69 1028 called the underlying ``raw`` object must not be accessed again.
jpayne@69 1029
jpayne@69 1030 *Note: Should not normally need to be called explicitly.*
jpayne@69 1031 """
jpayne@69 1032 if not self._content_consumed:
jpayne@69 1033 self.raw.close()
jpayne@69 1034
jpayne@69 1035 release_conn = getattr(self.raw, "release_conn", None)
jpayne@69 1036 if release_conn is not None:
jpayne@69 1037 release_conn()