Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/http/client.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 r"""HTTP/1.1 client library
jpayne@69 2
jpayne@69 3 <intro stuff goes here>
jpayne@69 4 <other stuff, too>
jpayne@69 5
jpayne@69 6 HTTPConnection goes through a number of "states", which define when a client
jpayne@69 7 may legally make another request or fetch the response for a particular
jpayne@69 8 request. This diagram details these state transitions:
jpayne@69 9
jpayne@69 10 (null)
jpayne@69 11 |
jpayne@69 12 | HTTPConnection()
jpayne@69 13 v
jpayne@69 14 Idle
jpayne@69 15 |
jpayne@69 16 | putrequest()
jpayne@69 17 v
jpayne@69 18 Request-started
jpayne@69 19 |
jpayne@69 20 | ( putheader() )* endheaders()
jpayne@69 21 v
jpayne@69 22 Request-sent
jpayne@69 23 |\_____________________________
jpayne@69 24 | | getresponse() raises
jpayne@69 25 | response = getresponse() | ConnectionError
jpayne@69 26 v v
jpayne@69 27 Unread-response Idle
jpayne@69 28 [Response-headers-read]
jpayne@69 29 |\____________________
jpayne@69 30 | |
jpayne@69 31 | response.read() | putrequest()
jpayne@69 32 v v
jpayne@69 33 Idle Req-started-unread-response
jpayne@69 34 ______/|
jpayne@69 35 / |
jpayne@69 36 response.read() | | ( putheader() )* endheaders()
jpayne@69 37 v v
jpayne@69 38 Request-started Req-sent-unread-response
jpayne@69 39 |
jpayne@69 40 | response.read()
jpayne@69 41 v
jpayne@69 42 Request-sent
jpayne@69 43
jpayne@69 44 This diagram presents the following rules:
jpayne@69 45 -- a second request may not be started until {response-headers-read}
jpayne@69 46 -- a response [object] cannot be retrieved until {request-sent}
jpayne@69 47 -- there is no differentiation between an unread response body and a
jpayne@69 48 partially read response body
jpayne@69 49
jpayne@69 50 Note: this enforcement is applied by the HTTPConnection class. The
jpayne@69 51 HTTPResponse class does not enforce this state machine, which
jpayne@69 52 implies sophisticated clients may accelerate the request/response
jpayne@69 53 pipeline. Caution should be taken, though: accelerating the states
jpayne@69 54 beyond the above pattern may imply knowledge of the server's
jpayne@69 55 connection-close behavior for certain requests. For example, it
jpayne@69 56 is impossible to tell whether the server will close the connection
jpayne@69 57 UNTIL the response headers have been read; this means that further
jpayne@69 58 requests cannot be placed into the pipeline until it is known that
jpayne@69 59 the server will NOT be closing the connection.
jpayne@69 60
jpayne@69 61 Logical State __state __response
jpayne@69 62 ------------- ------- ----------
jpayne@69 63 Idle _CS_IDLE None
jpayne@69 64 Request-started _CS_REQ_STARTED None
jpayne@69 65 Request-sent _CS_REQ_SENT None
jpayne@69 66 Unread-response _CS_IDLE <response_class>
jpayne@69 67 Req-started-unread-response _CS_REQ_STARTED <response_class>
jpayne@69 68 Req-sent-unread-response _CS_REQ_SENT <response_class>
jpayne@69 69 """
jpayne@69 70
jpayne@69 71 import email.parser
jpayne@69 72 import email.message
jpayne@69 73 import http
jpayne@69 74 import io
jpayne@69 75 import re
jpayne@69 76 import socket
jpayne@69 77 import collections.abc
jpayne@69 78 from urllib.parse import urlsplit
jpayne@69 79
jpayne@69 80 # HTTPMessage, parse_headers(), and the HTTP status code constants are
jpayne@69 81 # intentionally omitted for simplicity
jpayne@69 82 __all__ = ["HTTPResponse", "HTTPConnection",
jpayne@69 83 "HTTPException", "NotConnected", "UnknownProtocol",
jpayne@69 84 "UnknownTransferEncoding", "UnimplementedFileMode",
jpayne@69 85 "IncompleteRead", "InvalidURL", "ImproperConnectionState",
jpayne@69 86 "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
jpayne@69 87 "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",
jpayne@69 88 "responses"]
jpayne@69 89
jpayne@69 90 HTTP_PORT = 80
jpayne@69 91 HTTPS_PORT = 443
jpayne@69 92
jpayne@69 93 _UNKNOWN = 'UNKNOWN'
jpayne@69 94
jpayne@69 95 # connection states
jpayne@69 96 _CS_IDLE = 'Idle'
jpayne@69 97 _CS_REQ_STARTED = 'Request-started'
jpayne@69 98 _CS_REQ_SENT = 'Request-sent'
jpayne@69 99
jpayne@69 100
jpayne@69 101 # hack to maintain backwards compatibility
jpayne@69 102 globals().update(http.HTTPStatus.__members__)
jpayne@69 103
jpayne@69 104 # another hack to maintain backwards compatibility
jpayne@69 105 # Mapping status codes to official W3C names
jpayne@69 106 responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}
jpayne@69 107
jpayne@69 108 # maximal line length when calling readline().
jpayne@69 109 _MAXLINE = 65536
jpayne@69 110 _MAXHEADERS = 100
jpayne@69 111
jpayne@69 112 # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
jpayne@69 113 #
jpayne@69 114 # VCHAR = %x21-7E
jpayne@69 115 # obs-text = %x80-FF
jpayne@69 116 # header-field = field-name ":" OWS field-value OWS
jpayne@69 117 # field-name = token
jpayne@69 118 # field-value = *( field-content / obs-fold )
jpayne@69 119 # field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
jpayne@69 120 # field-vchar = VCHAR / obs-text
jpayne@69 121 #
jpayne@69 122 # obs-fold = CRLF 1*( SP / HTAB )
jpayne@69 123 # ; obsolete line folding
jpayne@69 124 # ; see Section 3.2.4
jpayne@69 125
jpayne@69 126 # token = 1*tchar
jpayne@69 127 #
jpayne@69 128 # tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*"
jpayne@69 129 # / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
jpayne@69 130 # / DIGIT / ALPHA
jpayne@69 131 # ; any VCHAR, except delimiters
jpayne@69 132 #
jpayne@69 133 # VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
jpayne@69 134
jpayne@69 135 # the patterns for both name and value are more lenient than RFC
jpayne@69 136 # definitions to allow for backwards compatibility
jpayne@69 137 _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
jpayne@69 138 _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
jpayne@69 139
jpayne@69 140 # These characters are not allowed within HTTP URL paths.
jpayne@69 141 # See https://tools.ietf.org/html/rfc3986#section-3.3 and the
jpayne@69 142 # https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
jpayne@69 143 # Prevents CVE-2019-9740. Includes control characters such as \r\n.
jpayne@69 144 # We don't restrict chars above \x7f as putrequest() limits us to ASCII.
jpayne@69 145 _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
jpayne@69 146 # Arguably only these _should_ allowed:
jpayne@69 147 # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
jpayne@69 148 # We are more lenient for assumed real world compatibility purposes.
jpayne@69 149
jpayne@69 150 # We always set the Content-Length header for these methods because some
jpayne@69 151 # servers will otherwise respond with a 411
jpayne@69 152 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
jpayne@69 153
jpayne@69 154
jpayne@69 155 def _encode(data, name='data'):
jpayne@69 156 """Call data.encode("latin-1") but show a better error message."""
jpayne@69 157 try:
jpayne@69 158 return data.encode("latin-1")
jpayne@69 159 except UnicodeEncodeError as err:
jpayne@69 160 raise UnicodeEncodeError(
jpayne@69 161 err.encoding,
jpayne@69 162 err.object,
jpayne@69 163 err.start,
jpayne@69 164 err.end,
jpayne@69 165 "%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "
jpayne@69 166 "if you want to send it encoded in UTF-8." %
jpayne@69 167 (name.title(), data[err.start:err.end], name)) from None
jpayne@69 168
jpayne@69 169
jpayne@69 170 class HTTPMessage(email.message.Message):
jpayne@69 171 # XXX The only usage of this method is in
jpayne@69 172 # http.server.CGIHTTPRequestHandler. Maybe move the code there so
jpayne@69 173 # that it doesn't need to be part of the public API. The API has
jpayne@69 174 # never been defined so this could cause backwards compatibility
jpayne@69 175 # issues.
jpayne@69 176
jpayne@69 177 def getallmatchingheaders(self, name):
jpayne@69 178 """Find all header lines matching a given header name.
jpayne@69 179
jpayne@69 180 Look through the list of headers and find all lines matching a given
jpayne@69 181 header name (and their continuation lines). A list of the lines is
jpayne@69 182 returned, without interpretation. If the header does not occur, an
jpayne@69 183 empty list is returned. If the header occurs multiple times, all
jpayne@69 184 occurrences are returned. Case is not important in the header name.
jpayne@69 185
jpayne@69 186 """
jpayne@69 187 name = name.lower() + ':'
jpayne@69 188 n = len(name)
jpayne@69 189 lst = []
jpayne@69 190 hit = 0
jpayne@69 191 for line in self.keys():
jpayne@69 192 if line[:n].lower() == name:
jpayne@69 193 hit = 1
jpayne@69 194 elif not line[:1].isspace():
jpayne@69 195 hit = 0
jpayne@69 196 if hit:
jpayne@69 197 lst.append(line)
jpayne@69 198 return lst
jpayne@69 199
jpayne@69 200 def parse_headers(fp, _class=HTTPMessage):
jpayne@69 201 """Parses only RFC2822 headers from a file pointer.
jpayne@69 202
jpayne@69 203 email Parser wants to see strings rather than bytes.
jpayne@69 204 But a TextIOWrapper around self.rfile would buffer too many bytes
jpayne@69 205 from the stream, bytes which we later need to read as bytes.
jpayne@69 206 So we read the correct bytes here, as bytes, for email Parser
jpayne@69 207 to parse.
jpayne@69 208
jpayne@69 209 """
jpayne@69 210 headers = []
jpayne@69 211 while True:
jpayne@69 212 line = fp.readline(_MAXLINE + 1)
jpayne@69 213 if len(line) > _MAXLINE:
jpayne@69 214 raise LineTooLong("header line")
jpayne@69 215 headers.append(line)
jpayne@69 216 if len(headers) > _MAXHEADERS:
jpayne@69 217 raise HTTPException("got more than %d headers" % _MAXHEADERS)
jpayne@69 218 if line in (b'\r\n', b'\n', b''):
jpayne@69 219 break
jpayne@69 220 hstring = b''.join(headers).decode('iso-8859-1')
jpayne@69 221 return email.parser.Parser(_class=_class).parsestr(hstring)
jpayne@69 222
jpayne@69 223
jpayne@69 224 class HTTPResponse(io.BufferedIOBase):
jpayne@69 225
jpayne@69 226 # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
jpayne@69 227
jpayne@69 228 # The bytes from the socket object are iso-8859-1 strings.
jpayne@69 229 # See RFC 2616 sec 2.2 which notes an exception for MIME-encoded
jpayne@69 230 # text following RFC 2047. The basic status line parsing only
jpayne@69 231 # accepts iso-8859-1.
jpayne@69 232
jpayne@69 233 def __init__(self, sock, debuglevel=0, method=None, url=None):
jpayne@69 234 # If the response includes a content-length header, we need to
jpayne@69 235 # make sure that the client doesn't read more than the
jpayne@69 236 # specified number of bytes. If it does, it will block until
jpayne@69 237 # the server times out and closes the connection. This will
jpayne@69 238 # happen if a self.fp.read() is done (without a size) whether
jpayne@69 239 # self.fp is buffered or not. So, no self.fp.read() by
jpayne@69 240 # clients unless they know what they are doing.
jpayne@69 241 self.fp = sock.makefile("rb")
jpayne@69 242 self.debuglevel = debuglevel
jpayne@69 243 self._method = method
jpayne@69 244
jpayne@69 245 # The HTTPResponse object is returned via urllib. The clients
jpayne@69 246 # of http and urllib expect different attributes for the
jpayne@69 247 # headers. headers is used here and supports urllib. msg is
jpayne@69 248 # provided as a backwards compatibility layer for http
jpayne@69 249 # clients.
jpayne@69 250
jpayne@69 251 self.headers = self.msg = None
jpayne@69 252
jpayne@69 253 # from the Status-Line of the response
jpayne@69 254 self.version = _UNKNOWN # HTTP-Version
jpayne@69 255 self.status = _UNKNOWN # Status-Code
jpayne@69 256 self.reason = _UNKNOWN # Reason-Phrase
jpayne@69 257
jpayne@69 258 self.chunked = _UNKNOWN # is "chunked" being used?
jpayne@69 259 self.chunk_left = _UNKNOWN # bytes left to read in current chunk
jpayne@69 260 self.length = _UNKNOWN # number of bytes left in response
jpayne@69 261 self.will_close = _UNKNOWN # conn will close at end of response
jpayne@69 262
jpayne@69 263 def _read_status(self):
jpayne@69 264 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
jpayne@69 265 if len(line) > _MAXLINE:
jpayne@69 266 raise LineTooLong("status line")
jpayne@69 267 if self.debuglevel > 0:
jpayne@69 268 print("reply:", repr(line))
jpayne@69 269 if not line:
jpayne@69 270 # Presumably, the server closed the connection before
jpayne@69 271 # sending a valid response.
jpayne@69 272 raise RemoteDisconnected("Remote end closed connection without"
jpayne@69 273 " response")
jpayne@69 274 try:
jpayne@69 275 version, status, reason = line.split(None, 2)
jpayne@69 276 except ValueError:
jpayne@69 277 try:
jpayne@69 278 version, status = line.split(None, 1)
jpayne@69 279 reason = ""
jpayne@69 280 except ValueError:
jpayne@69 281 # empty version will cause next test to fail.
jpayne@69 282 version = ""
jpayne@69 283 if not version.startswith("HTTP/"):
jpayne@69 284 self._close_conn()
jpayne@69 285 raise BadStatusLine(line)
jpayne@69 286
jpayne@69 287 # The status code is a three-digit number
jpayne@69 288 try:
jpayne@69 289 status = int(status)
jpayne@69 290 if status < 100 or status > 999:
jpayne@69 291 raise BadStatusLine(line)
jpayne@69 292 except ValueError:
jpayne@69 293 raise BadStatusLine(line)
jpayne@69 294 return version, status, reason
jpayne@69 295
jpayne@69 296 def begin(self):
jpayne@69 297 if self.headers is not None:
jpayne@69 298 # we've already started reading the response
jpayne@69 299 return
jpayne@69 300
jpayne@69 301 # read until we get a non-100 response
jpayne@69 302 while True:
jpayne@69 303 version, status, reason = self._read_status()
jpayne@69 304 if status != CONTINUE:
jpayne@69 305 break
jpayne@69 306 # skip the header from the 100 response
jpayne@69 307 while True:
jpayne@69 308 skip = self.fp.readline(_MAXLINE + 1)
jpayne@69 309 if len(skip) > _MAXLINE:
jpayne@69 310 raise LineTooLong("header line")
jpayne@69 311 skip = skip.strip()
jpayne@69 312 if not skip:
jpayne@69 313 break
jpayne@69 314 if self.debuglevel > 0:
jpayne@69 315 print("header:", skip)
jpayne@69 316
jpayne@69 317 self.code = self.status = status
jpayne@69 318 self.reason = reason.strip()
jpayne@69 319 if version in ("HTTP/1.0", "HTTP/0.9"):
jpayne@69 320 # Some servers might still return "0.9", treat it as 1.0 anyway
jpayne@69 321 self.version = 10
jpayne@69 322 elif version.startswith("HTTP/1."):
jpayne@69 323 self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
jpayne@69 324 else:
jpayne@69 325 raise UnknownProtocol(version)
jpayne@69 326
jpayne@69 327 self.headers = self.msg = parse_headers(self.fp)
jpayne@69 328
jpayne@69 329 if self.debuglevel > 0:
jpayne@69 330 for hdr, val in self.headers.items():
jpayne@69 331 print("header:", hdr + ":", val)
jpayne@69 332
jpayne@69 333 # are we using the chunked-style of transfer encoding?
jpayne@69 334 tr_enc = self.headers.get("transfer-encoding")
jpayne@69 335 if tr_enc and tr_enc.lower() == "chunked":
jpayne@69 336 self.chunked = True
jpayne@69 337 self.chunk_left = None
jpayne@69 338 else:
jpayne@69 339 self.chunked = False
jpayne@69 340
jpayne@69 341 # will the connection close at the end of the response?
jpayne@69 342 self.will_close = self._check_close()
jpayne@69 343
jpayne@69 344 # do we have a Content-Length?
jpayne@69 345 # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
jpayne@69 346 self.length = None
jpayne@69 347 length = self.headers.get("content-length")
jpayne@69 348
jpayne@69 349 # are we using the chunked-style of transfer encoding?
jpayne@69 350 tr_enc = self.headers.get("transfer-encoding")
jpayne@69 351 if length and not self.chunked:
jpayne@69 352 try:
jpayne@69 353 self.length = int(length)
jpayne@69 354 except ValueError:
jpayne@69 355 self.length = None
jpayne@69 356 else:
jpayne@69 357 if self.length < 0: # ignore nonsensical negative lengths
jpayne@69 358 self.length = None
jpayne@69 359 else:
jpayne@69 360 self.length = None
jpayne@69 361
jpayne@69 362 # does the body have a fixed length? (of zero)
jpayne@69 363 if (status == NO_CONTENT or status == NOT_MODIFIED or
jpayne@69 364 100 <= status < 200 or # 1xx codes
jpayne@69 365 self._method == "HEAD"):
jpayne@69 366 self.length = 0
jpayne@69 367
jpayne@69 368 # if the connection remains open, and we aren't using chunked, and
jpayne@69 369 # a content-length was not provided, then assume that the connection
jpayne@69 370 # WILL close.
jpayne@69 371 if (not self.will_close and
jpayne@69 372 not self.chunked and
jpayne@69 373 self.length is None):
jpayne@69 374 self.will_close = True
jpayne@69 375
jpayne@69 376 def _check_close(self):
jpayne@69 377 conn = self.headers.get("connection")
jpayne@69 378 if self.version == 11:
jpayne@69 379 # An HTTP/1.1 proxy is assumed to stay open unless
jpayne@69 380 # explicitly closed.
jpayne@69 381 if conn and "close" in conn.lower():
jpayne@69 382 return True
jpayne@69 383 return False
jpayne@69 384
jpayne@69 385 # Some HTTP/1.0 implementations have support for persistent
jpayne@69 386 # connections, using rules different than HTTP/1.1.
jpayne@69 387
jpayne@69 388 # For older HTTP, Keep-Alive indicates persistent connection.
jpayne@69 389 if self.headers.get("keep-alive"):
jpayne@69 390 return False
jpayne@69 391
jpayne@69 392 # At least Akamai returns a "Connection: Keep-Alive" header,
jpayne@69 393 # which was supposed to be sent by the client.
jpayne@69 394 if conn and "keep-alive" in conn.lower():
jpayne@69 395 return False
jpayne@69 396
jpayne@69 397 # Proxy-Connection is a netscape hack.
jpayne@69 398 pconn = self.headers.get("proxy-connection")
jpayne@69 399 if pconn and "keep-alive" in pconn.lower():
jpayne@69 400 return False
jpayne@69 401
jpayne@69 402 # otherwise, assume it will close
jpayne@69 403 return True
jpayne@69 404
jpayne@69 405 def _close_conn(self):
jpayne@69 406 fp = self.fp
jpayne@69 407 self.fp = None
jpayne@69 408 fp.close()
jpayne@69 409
jpayne@69 410 def close(self):
jpayne@69 411 try:
jpayne@69 412 super().close() # set "closed" flag
jpayne@69 413 finally:
jpayne@69 414 if self.fp:
jpayne@69 415 self._close_conn()
jpayne@69 416
jpayne@69 417 # These implementations are for the benefit of io.BufferedReader.
jpayne@69 418
jpayne@69 419 # XXX This class should probably be revised to act more like
jpayne@69 420 # the "raw stream" that BufferedReader expects.
jpayne@69 421
jpayne@69 422 def flush(self):
jpayne@69 423 super().flush()
jpayne@69 424 if self.fp:
jpayne@69 425 self.fp.flush()
jpayne@69 426
jpayne@69 427 def readable(self):
jpayne@69 428 """Always returns True"""
jpayne@69 429 return True
jpayne@69 430
jpayne@69 431 # End of "raw stream" methods
jpayne@69 432
jpayne@69 433 def isclosed(self):
jpayne@69 434 """True if the connection is closed."""
jpayne@69 435 # NOTE: it is possible that we will not ever call self.close(). This
jpayne@69 436 # case occurs when will_close is TRUE, length is None, and we
jpayne@69 437 # read up to the last byte, but NOT past it.
jpayne@69 438 #
jpayne@69 439 # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
jpayne@69 440 # called, meaning self.isclosed() is meaningful.
jpayne@69 441 return self.fp is None
jpayne@69 442
jpayne@69 443 def read(self, amt=None):
jpayne@69 444 if self.fp is None:
jpayne@69 445 return b""
jpayne@69 446
jpayne@69 447 if self._method == "HEAD":
jpayne@69 448 self._close_conn()
jpayne@69 449 return b""
jpayne@69 450
jpayne@69 451 if amt is not None:
jpayne@69 452 # Amount is given, implement using readinto
jpayne@69 453 b = bytearray(amt)
jpayne@69 454 n = self.readinto(b)
jpayne@69 455 return memoryview(b)[:n].tobytes()
jpayne@69 456 else:
jpayne@69 457 # Amount is not given (unbounded read) so we must check self.length
jpayne@69 458 # and self.chunked
jpayne@69 459
jpayne@69 460 if self.chunked:
jpayne@69 461 return self._readall_chunked()
jpayne@69 462
jpayne@69 463 if self.length is None:
jpayne@69 464 s = self.fp.read()
jpayne@69 465 else:
jpayne@69 466 try:
jpayne@69 467 s = self._safe_read(self.length)
jpayne@69 468 except IncompleteRead:
jpayne@69 469 self._close_conn()
jpayne@69 470 raise
jpayne@69 471 self.length = 0
jpayne@69 472 self._close_conn() # we read everything
jpayne@69 473 return s
jpayne@69 474
jpayne@69 475 def readinto(self, b):
jpayne@69 476 """Read up to len(b) bytes into bytearray b and return the number
jpayne@69 477 of bytes read.
jpayne@69 478 """
jpayne@69 479
jpayne@69 480 if self.fp is None:
jpayne@69 481 return 0
jpayne@69 482
jpayne@69 483 if self._method == "HEAD":
jpayne@69 484 self._close_conn()
jpayne@69 485 return 0
jpayne@69 486
jpayne@69 487 if self.chunked:
jpayne@69 488 return self._readinto_chunked(b)
jpayne@69 489
jpayne@69 490 if self.length is not None:
jpayne@69 491 if len(b) > self.length:
jpayne@69 492 # clip the read to the "end of response"
jpayne@69 493 b = memoryview(b)[0:self.length]
jpayne@69 494
jpayne@69 495 # we do not use _safe_read() here because this may be a .will_close
jpayne@69 496 # connection, and the user is reading more bytes than will be provided
jpayne@69 497 # (for example, reading in 1k chunks)
jpayne@69 498 n = self.fp.readinto(b)
jpayne@69 499 if not n and b:
jpayne@69 500 # Ideally, we would raise IncompleteRead if the content-length
jpayne@69 501 # wasn't satisfied, but it might break compatibility.
jpayne@69 502 self._close_conn()
jpayne@69 503 elif self.length is not None:
jpayne@69 504 self.length -= n
jpayne@69 505 if not self.length:
jpayne@69 506 self._close_conn()
jpayne@69 507 return n
jpayne@69 508
jpayne@69 509 def _read_next_chunk_size(self):
jpayne@69 510 # Read the next chunk size from the file
jpayne@69 511 line = self.fp.readline(_MAXLINE + 1)
jpayne@69 512 if len(line) > _MAXLINE:
jpayne@69 513 raise LineTooLong("chunk size")
jpayne@69 514 i = line.find(b";")
jpayne@69 515 if i >= 0:
jpayne@69 516 line = line[:i] # strip chunk-extensions
jpayne@69 517 try:
jpayne@69 518 return int(line, 16)
jpayne@69 519 except ValueError:
jpayne@69 520 # close the connection as protocol synchronisation is
jpayne@69 521 # probably lost
jpayne@69 522 self._close_conn()
jpayne@69 523 raise
jpayne@69 524
jpayne@69 525 def _read_and_discard_trailer(self):
jpayne@69 526 # read and discard trailer up to the CRLF terminator
jpayne@69 527 ### note: we shouldn't have any trailers!
jpayne@69 528 while True:
jpayne@69 529 line = self.fp.readline(_MAXLINE + 1)
jpayne@69 530 if len(line) > _MAXLINE:
jpayne@69 531 raise LineTooLong("trailer line")
jpayne@69 532 if not line:
jpayne@69 533 # a vanishingly small number of sites EOF without
jpayne@69 534 # sending the trailer
jpayne@69 535 break
jpayne@69 536 if line in (b'\r\n', b'\n', b''):
jpayne@69 537 break
jpayne@69 538
jpayne@69 539 def _get_chunk_left(self):
jpayne@69 540 # return self.chunk_left, reading a new chunk if necessary.
jpayne@69 541 # chunk_left == 0: at the end of the current chunk, need to close it
jpayne@69 542 # chunk_left == None: No current chunk, should read next.
jpayne@69 543 # This function returns non-zero or None if the last chunk has
jpayne@69 544 # been read.
jpayne@69 545 chunk_left = self.chunk_left
jpayne@69 546 if not chunk_left: # Can be 0 or None
jpayne@69 547 if chunk_left is not None:
jpayne@69 548 # We are at the end of chunk, discard chunk end
jpayne@69 549 self._safe_read(2) # toss the CRLF at the end of the chunk
jpayne@69 550 try:
jpayne@69 551 chunk_left = self._read_next_chunk_size()
jpayne@69 552 except ValueError:
jpayne@69 553 raise IncompleteRead(b'')
jpayne@69 554 if chunk_left == 0:
jpayne@69 555 # last chunk: 1*("0") [ chunk-extension ] CRLF
jpayne@69 556 self._read_and_discard_trailer()
jpayne@69 557 # we read everything; close the "file"
jpayne@69 558 self._close_conn()
jpayne@69 559 chunk_left = None
jpayne@69 560 self.chunk_left = chunk_left
jpayne@69 561 return chunk_left
jpayne@69 562
jpayne@69 563 def _readall_chunked(self):
jpayne@69 564 assert self.chunked != _UNKNOWN
jpayne@69 565 value = []
jpayne@69 566 try:
jpayne@69 567 while True:
jpayne@69 568 chunk_left = self._get_chunk_left()
jpayne@69 569 if chunk_left is None:
jpayne@69 570 break
jpayne@69 571 value.append(self._safe_read(chunk_left))
jpayne@69 572 self.chunk_left = 0
jpayne@69 573 return b''.join(value)
jpayne@69 574 except IncompleteRead:
jpayne@69 575 raise IncompleteRead(b''.join(value))
jpayne@69 576
jpayne@69 577 def _readinto_chunked(self, b):
jpayne@69 578 assert self.chunked != _UNKNOWN
jpayne@69 579 total_bytes = 0
jpayne@69 580 mvb = memoryview(b)
jpayne@69 581 try:
jpayne@69 582 while True:
jpayne@69 583 chunk_left = self._get_chunk_left()
jpayne@69 584 if chunk_left is None:
jpayne@69 585 return total_bytes
jpayne@69 586
jpayne@69 587 if len(mvb) <= chunk_left:
jpayne@69 588 n = self._safe_readinto(mvb)
jpayne@69 589 self.chunk_left = chunk_left - n
jpayne@69 590 return total_bytes + n
jpayne@69 591
jpayne@69 592 temp_mvb = mvb[:chunk_left]
jpayne@69 593 n = self._safe_readinto(temp_mvb)
jpayne@69 594 mvb = mvb[n:]
jpayne@69 595 total_bytes += n
jpayne@69 596 self.chunk_left = 0
jpayne@69 597
jpayne@69 598 except IncompleteRead:
jpayne@69 599 raise IncompleteRead(bytes(b[0:total_bytes]))
jpayne@69 600
jpayne@69 601 def _safe_read(self, amt):
jpayne@69 602 """Read the number of bytes requested.
jpayne@69 603
jpayne@69 604 This function should be used when <amt> bytes "should" be present for
jpayne@69 605 reading. If the bytes are truly not available (due to EOF), then the
jpayne@69 606 IncompleteRead exception can be used to detect the problem.
jpayne@69 607 """
jpayne@69 608 data = self.fp.read(amt)
jpayne@69 609 if len(data) < amt:
jpayne@69 610 raise IncompleteRead(data, amt-len(data))
jpayne@69 611 return data
jpayne@69 612
jpayne@69 613 def _safe_readinto(self, b):
jpayne@69 614 """Same as _safe_read, but for reading into a buffer."""
jpayne@69 615 amt = len(b)
jpayne@69 616 n = self.fp.readinto(b)
jpayne@69 617 if n < amt:
jpayne@69 618 raise IncompleteRead(bytes(b[:n]), amt-n)
jpayne@69 619 return n
jpayne@69 620
jpayne@69 621 def read1(self, n=-1):
jpayne@69 622 """Read with at most one underlying system call. If at least one
jpayne@69 623 byte is buffered, return that instead.
jpayne@69 624 """
jpayne@69 625 if self.fp is None or self._method == "HEAD":
jpayne@69 626 return b""
jpayne@69 627 if self.chunked:
jpayne@69 628 return self._read1_chunked(n)
jpayne@69 629 if self.length is not None and (n < 0 or n > self.length):
jpayne@69 630 n = self.length
jpayne@69 631 result = self.fp.read1(n)
jpayne@69 632 if not result and n:
jpayne@69 633 self._close_conn()
jpayne@69 634 elif self.length is not None:
jpayne@69 635 self.length -= len(result)
jpayne@69 636 return result
jpayne@69 637
jpayne@69 638 def peek(self, n=-1):
jpayne@69 639 # Having this enables IOBase.readline() to read more than one
jpayne@69 640 # byte at a time
jpayne@69 641 if self.fp is None or self._method == "HEAD":
jpayne@69 642 return b""
jpayne@69 643 if self.chunked:
jpayne@69 644 return self._peek_chunked(n)
jpayne@69 645 return self.fp.peek(n)
jpayne@69 646
jpayne@69 647 def readline(self, limit=-1):
jpayne@69 648 if self.fp is None or self._method == "HEAD":
jpayne@69 649 return b""
jpayne@69 650 if self.chunked:
jpayne@69 651 # Fallback to IOBase readline which uses peek() and read()
jpayne@69 652 return super().readline(limit)
jpayne@69 653 if self.length is not None and (limit < 0 or limit > self.length):
jpayne@69 654 limit = self.length
jpayne@69 655 result = self.fp.readline(limit)
jpayne@69 656 if not result and limit:
jpayne@69 657 self._close_conn()
jpayne@69 658 elif self.length is not None:
jpayne@69 659 self.length -= len(result)
jpayne@69 660 return result
jpayne@69 661
jpayne@69 662 def _read1_chunked(self, n):
jpayne@69 663 # Strictly speaking, _get_chunk_left() may cause more than one read,
jpayne@69 664 # but that is ok, since that is to satisfy the chunked protocol.
jpayne@69 665 chunk_left = self._get_chunk_left()
jpayne@69 666 if chunk_left is None or n == 0:
jpayne@69 667 return b''
jpayne@69 668 if not (0 <= n <= chunk_left):
jpayne@69 669 n = chunk_left # if n is negative or larger than chunk_left
jpayne@69 670 read = self.fp.read1(n)
jpayne@69 671 self.chunk_left -= len(read)
jpayne@69 672 if not read:
jpayne@69 673 raise IncompleteRead(b"")
jpayne@69 674 return read
jpayne@69 675
jpayne@69 676 def _peek_chunked(self, n):
jpayne@69 677 # Strictly speaking, _get_chunk_left() may cause more than one read,
jpayne@69 678 # but that is ok, since that is to satisfy the chunked protocol.
jpayne@69 679 try:
jpayne@69 680 chunk_left = self._get_chunk_left()
jpayne@69 681 except IncompleteRead:
jpayne@69 682 return b'' # peek doesn't worry about protocol
jpayne@69 683 if chunk_left is None:
jpayne@69 684 return b'' # eof
jpayne@69 685 # peek is allowed to return more than requested. Just request the
jpayne@69 686 # entire chunk, and truncate what we get.
jpayne@69 687 return self.fp.peek(chunk_left)[:chunk_left]
jpayne@69 688
jpayne@69 689 def fileno(self):
jpayne@69 690 return self.fp.fileno()
jpayne@69 691
jpayne@69 692 def getheader(self, name, default=None):
jpayne@69 693 '''Returns the value of the header matching *name*.
jpayne@69 694
jpayne@69 695 If there are multiple matching headers, the values are
jpayne@69 696 combined into a single string separated by commas and spaces.
jpayne@69 697
jpayne@69 698 If no matching header is found, returns *default* or None if
jpayne@69 699 the *default* is not specified.
jpayne@69 700
jpayne@69 701 If the headers are unknown, raises http.client.ResponseNotReady.
jpayne@69 702
jpayne@69 703 '''
jpayne@69 704 if self.headers is None:
jpayne@69 705 raise ResponseNotReady()
jpayne@69 706 headers = self.headers.get_all(name) or default
jpayne@69 707 if isinstance(headers, str) or not hasattr(headers, '__iter__'):
jpayne@69 708 return headers
jpayne@69 709 else:
jpayne@69 710 return ', '.join(headers)
jpayne@69 711
jpayne@69 712 def getheaders(self):
jpayne@69 713 """Return list of (header, value) tuples."""
jpayne@69 714 if self.headers is None:
jpayne@69 715 raise ResponseNotReady()
jpayne@69 716 return list(self.headers.items())
jpayne@69 717
jpayne@69 718 # We override IOBase.__iter__ so that it doesn't check for closed-ness
jpayne@69 719
jpayne@69 720 def __iter__(self):
jpayne@69 721 return self
jpayne@69 722
jpayne@69 723 # For compatibility with old-style urllib responses.
jpayne@69 724
jpayne@69 725 def info(self):
jpayne@69 726 '''Returns an instance of the class mimetools.Message containing
jpayne@69 727 meta-information associated with the URL.
jpayne@69 728
jpayne@69 729 When the method is HTTP, these headers are those returned by
jpayne@69 730 the server at the head of the retrieved HTML page (including
jpayne@69 731 Content-Length and Content-Type).
jpayne@69 732
jpayne@69 733 When the method is FTP, a Content-Length header will be
jpayne@69 734 present if (as is now usual) the server passed back a file
jpayne@69 735 length in response to the FTP retrieval request. A
jpayne@69 736 Content-Type header will be present if the MIME type can be
jpayne@69 737 guessed.
jpayne@69 738
jpayne@69 739 When the method is local-file, returned headers will include
jpayne@69 740 a Date representing the file's last-modified time, a
jpayne@69 741 Content-Length giving file size, and a Content-Type
jpayne@69 742 containing a guess at the file's type. See also the
jpayne@69 743 description of the mimetools module.
jpayne@69 744
jpayne@69 745 '''
jpayne@69 746 return self.headers
jpayne@69 747
jpayne@69 748 def geturl(self):
jpayne@69 749 '''Return the real URL of the page.
jpayne@69 750
jpayne@69 751 In some cases, the HTTP server redirects a client to another
jpayne@69 752 URL. The urlopen() function handles this transparently, but in
jpayne@69 753 some cases the caller needs to know which URL the client was
jpayne@69 754 redirected to. The geturl() method can be used to get at this
jpayne@69 755 redirected URL.
jpayne@69 756
jpayne@69 757 '''
jpayne@69 758 return self.url
jpayne@69 759
jpayne@69 760 def getcode(self):
jpayne@69 761 '''Return the HTTP status code that was sent with the response,
jpayne@69 762 or None if the URL is not an HTTP URL.
jpayne@69 763
jpayne@69 764 '''
jpayne@69 765 return self.status
jpayne@69 766
jpayne@69 767 class HTTPConnection:
jpayne@69 768
jpayne@69 769 _http_vsn = 11
jpayne@69 770 _http_vsn_str = 'HTTP/1.1'
jpayne@69 771
jpayne@69 772 response_class = HTTPResponse
jpayne@69 773 default_port = HTTP_PORT
jpayne@69 774 auto_open = 1
jpayne@69 775 debuglevel = 0
jpayne@69 776
jpayne@69 777 @staticmethod
jpayne@69 778 def _is_textIO(stream):
jpayne@69 779 """Test whether a file-like object is a text or a binary stream.
jpayne@69 780 """
jpayne@69 781 return isinstance(stream, io.TextIOBase)
jpayne@69 782
jpayne@69 783 @staticmethod
jpayne@69 784 def _get_content_length(body, method):
jpayne@69 785 """Get the content-length based on the body.
jpayne@69 786
jpayne@69 787 If the body is None, we set Content-Length: 0 for methods that expect
jpayne@69 788 a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
jpayne@69 789 any method if the body is a str or bytes-like object and not a file.
jpayne@69 790 """
jpayne@69 791 if body is None:
jpayne@69 792 # do an explicit check for not None here to distinguish
jpayne@69 793 # between unset and set but empty
jpayne@69 794 if method.upper() in _METHODS_EXPECTING_BODY:
jpayne@69 795 return 0
jpayne@69 796 else:
jpayne@69 797 return None
jpayne@69 798
jpayne@69 799 if hasattr(body, 'read'):
jpayne@69 800 # file-like object.
jpayne@69 801 return None
jpayne@69 802
jpayne@69 803 try:
jpayne@69 804 # does it implement the buffer protocol (bytes, bytearray, array)?
jpayne@69 805 mv = memoryview(body)
jpayne@69 806 return mv.nbytes
jpayne@69 807 except TypeError:
jpayne@69 808 pass
jpayne@69 809
jpayne@69 810 if isinstance(body, str):
jpayne@69 811 return len(body)
jpayne@69 812
jpayne@69 813 return None
jpayne@69 814
jpayne@69 815 def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
jpayne@69 816 source_address=None, blocksize=8192):
jpayne@69 817 self.timeout = timeout
jpayne@69 818 self.source_address = source_address
jpayne@69 819 self.blocksize = blocksize
jpayne@69 820 self.sock = None
jpayne@69 821 self._buffer = []
jpayne@69 822 self.__response = None
jpayne@69 823 self.__state = _CS_IDLE
jpayne@69 824 self._method = None
jpayne@69 825 self._tunnel_host = None
jpayne@69 826 self._tunnel_port = None
jpayne@69 827 self._tunnel_headers = {}
jpayne@69 828
jpayne@69 829 (self.host, self.port) = self._get_hostport(host, port)
jpayne@69 830
jpayne@69 831 # This is stored as an instance variable to allow unit
jpayne@69 832 # tests to replace it with a suitable mockup
jpayne@69 833 self._create_connection = socket.create_connection
jpayne@69 834
jpayne@69 835 def set_tunnel(self, host, port=None, headers=None):
jpayne@69 836 """Set up host and port for HTTP CONNECT tunnelling.
jpayne@69 837
jpayne@69 838 In a connection that uses HTTP CONNECT tunneling, the host passed to the
jpayne@69 839 constructor is used as a proxy server that relays all communication to
jpayne@69 840 the endpoint passed to `set_tunnel`. This done by sending an HTTP
jpayne@69 841 CONNECT request to the proxy server when the connection is established.
jpayne@69 842
jpayne@69 843 This method must be called before the HTML connection has been
jpayne@69 844 established.
jpayne@69 845
jpayne@69 846 The headers argument should be a mapping of extra HTTP headers to send
jpayne@69 847 with the CONNECT request.
jpayne@69 848 """
jpayne@69 849
jpayne@69 850 if self.sock:
jpayne@69 851 raise RuntimeError("Can't set up tunnel for established connection")
jpayne@69 852
jpayne@69 853 self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)
jpayne@69 854 if headers:
jpayne@69 855 self._tunnel_headers = headers
jpayne@69 856 else:
jpayne@69 857 self._tunnel_headers.clear()
jpayne@69 858
jpayne@69 859 def _get_hostport(self, host, port):
jpayne@69 860 if port is None:
jpayne@69 861 i = host.rfind(':')
jpayne@69 862 j = host.rfind(']') # ipv6 addresses have [...]
jpayne@69 863 if i > j:
jpayne@69 864 try:
jpayne@69 865 port = int(host[i+1:])
jpayne@69 866 except ValueError:
jpayne@69 867 if host[i+1:] == "": # http://foo.com:/ == http://foo.com/
jpayne@69 868 port = self.default_port
jpayne@69 869 else:
jpayne@69 870 raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
jpayne@69 871 host = host[:i]
jpayne@69 872 else:
jpayne@69 873 port = self.default_port
jpayne@69 874 if host and host[0] == '[' and host[-1] == ']':
jpayne@69 875 host = host[1:-1]
jpayne@69 876
jpayne@69 877 return (host, port)
jpayne@69 878
jpayne@69 879 def set_debuglevel(self, level):
jpayne@69 880 self.debuglevel = level
jpayne@69 881
jpayne@69 882 def _tunnel(self):
jpayne@69 883 connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
jpayne@69 884 self._tunnel_port)
jpayne@69 885 connect_bytes = connect_str.encode("ascii")
jpayne@69 886 self.send(connect_bytes)
jpayne@69 887 for header, value in self._tunnel_headers.items():
jpayne@69 888 header_str = "%s: %s\r\n" % (header, value)
jpayne@69 889 header_bytes = header_str.encode("latin-1")
jpayne@69 890 self.send(header_bytes)
jpayne@69 891 self.send(b'\r\n')
jpayne@69 892
jpayne@69 893 response = self.response_class(self.sock, method=self._method)
jpayne@69 894 (version, code, message) = response._read_status()
jpayne@69 895
jpayne@69 896 if code != http.HTTPStatus.OK:
jpayne@69 897 self.close()
jpayne@69 898 raise OSError("Tunnel connection failed: %d %s" % (code,
jpayne@69 899 message.strip()))
jpayne@69 900 while True:
jpayne@69 901 line = response.fp.readline(_MAXLINE + 1)
jpayne@69 902 if len(line) > _MAXLINE:
jpayne@69 903 raise LineTooLong("header line")
jpayne@69 904 if not line:
jpayne@69 905 # for sites which EOF without sending a trailer
jpayne@69 906 break
jpayne@69 907 if line in (b'\r\n', b'\n', b''):
jpayne@69 908 break
jpayne@69 909
jpayne@69 910 if self.debuglevel > 0:
jpayne@69 911 print('header:', line.decode())
jpayne@69 912
jpayne@69 913 def connect(self):
jpayne@69 914 """Connect to the host and port specified in __init__."""
jpayne@69 915 self.sock = self._create_connection(
jpayne@69 916 (self.host,self.port), self.timeout, self.source_address)
jpayne@69 917 self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
jpayne@69 918
jpayne@69 919 if self._tunnel_host:
jpayne@69 920 self._tunnel()
jpayne@69 921
jpayne@69 922 def close(self):
jpayne@69 923 """Close the connection to the HTTP server."""
jpayne@69 924 self.__state = _CS_IDLE
jpayne@69 925 try:
jpayne@69 926 sock = self.sock
jpayne@69 927 if sock:
jpayne@69 928 self.sock = None
jpayne@69 929 sock.close() # close it manually... there may be other refs
jpayne@69 930 finally:
jpayne@69 931 response = self.__response
jpayne@69 932 if response:
jpayne@69 933 self.__response = None
jpayne@69 934 response.close()
jpayne@69 935
jpayne@69 936 def send(self, data):
jpayne@69 937 """Send `data' to the server.
jpayne@69 938 ``data`` can be a string object, a bytes object, an array object, a
jpayne@69 939 file-like object that supports a .read() method, or an iterable object.
jpayne@69 940 """
jpayne@69 941
jpayne@69 942 if self.sock is None:
jpayne@69 943 if self.auto_open:
jpayne@69 944 self.connect()
jpayne@69 945 else:
jpayne@69 946 raise NotConnected()
jpayne@69 947
jpayne@69 948 if self.debuglevel > 0:
jpayne@69 949 print("send:", repr(data))
jpayne@69 950 if hasattr(data, "read") :
jpayne@69 951 if self.debuglevel > 0:
jpayne@69 952 print("sendIng a read()able")
jpayne@69 953 encode = self._is_textIO(data)
jpayne@69 954 if encode and self.debuglevel > 0:
jpayne@69 955 print("encoding file using iso-8859-1")
jpayne@69 956 while 1:
jpayne@69 957 datablock = data.read(self.blocksize)
jpayne@69 958 if not datablock:
jpayne@69 959 break
jpayne@69 960 if encode:
jpayne@69 961 datablock = datablock.encode("iso-8859-1")
jpayne@69 962 self.sock.sendall(datablock)
jpayne@69 963 return
jpayne@69 964 try:
jpayne@69 965 self.sock.sendall(data)
jpayne@69 966 except TypeError:
jpayne@69 967 if isinstance(data, collections.abc.Iterable):
jpayne@69 968 for d in data:
jpayne@69 969 self.sock.sendall(d)
jpayne@69 970 else:
jpayne@69 971 raise TypeError("data should be a bytes-like object "
jpayne@69 972 "or an iterable, got %r" % type(data))
jpayne@69 973
jpayne@69 974 def _output(self, s):
jpayne@69 975 """Add a line of output to the current request buffer.
jpayne@69 976
jpayne@69 977 Assumes that the line does *not* end with \\r\\n.
jpayne@69 978 """
jpayne@69 979 self._buffer.append(s)
jpayne@69 980
jpayne@69 981 def _read_readable(self, readable):
jpayne@69 982 if self.debuglevel > 0:
jpayne@69 983 print("sendIng a read()able")
jpayne@69 984 encode = self._is_textIO(readable)
jpayne@69 985 if encode and self.debuglevel > 0:
jpayne@69 986 print("encoding file using iso-8859-1")
jpayne@69 987 while True:
jpayne@69 988 datablock = readable.read(self.blocksize)
jpayne@69 989 if not datablock:
jpayne@69 990 break
jpayne@69 991 if encode:
jpayne@69 992 datablock = datablock.encode("iso-8859-1")
jpayne@69 993 yield datablock
jpayne@69 994
jpayne@69 995 def _send_output(self, message_body=None, encode_chunked=False):
jpayne@69 996 """Send the currently buffered request and clear the buffer.
jpayne@69 997
jpayne@69 998 Appends an extra \\r\\n to the buffer.
jpayne@69 999 A message_body may be specified, to be appended to the request.
jpayne@69 1000 """
jpayne@69 1001 self._buffer.extend((b"", b""))
jpayne@69 1002 msg = b"\r\n".join(self._buffer)
jpayne@69 1003 del self._buffer[:]
jpayne@69 1004 self.send(msg)
jpayne@69 1005
jpayne@69 1006 if message_body is not None:
jpayne@69 1007
jpayne@69 1008 # create a consistent interface to message_body
jpayne@69 1009 if hasattr(message_body, 'read'):
jpayne@69 1010 # Let file-like take precedence over byte-like. This
jpayne@69 1011 # is needed to allow the current position of mmap'ed
jpayne@69 1012 # files to be taken into account.
jpayne@69 1013 chunks = self._read_readable(message_body)
jpayne@69 1014 else:
jpayne@69 1015 try:
jpayne@69 1016 # this is solely to check to see if message_body
jpayne@69 1017 # implements the buffer API. it /would/ be easier
jpayne@69 1018 # to capture if PyObject_CheckBuffer was exposed
jpayne@69 1019 # to Python.
jpayne@69 1020 memoryview(message_body)
jpayne@69 1021 except TypeError:
jpayne@69 1022 try:
jpayne@69 1023 chunks = iter(message_body)
jpayne@69 1024 except TypeError:
jpayne@69 1025 raise TypeError("message_body should be a bytes-like "
jpayne@69 1026 "object or an iterable, got %r"
jpayne@69 1027 % type(message_body))
jpayne@69 1028 else:
jpayne@69 1029 # the object implements the buffer interface and
jpayne@69 1030 # can be passed directly into socket methods
jpayne@69 1031 chunks = (message_body,)
jpayne@69 1032
jpayne@69 1033 for chunk in chunks:
jpayne@69 1034 if not chunk:
jpayne@69 1035 if self.debuglevel > 0:
jpayne@69 1036 print('Zero length chunk ignored')
jpayne@69 1037 continue
jpayne@69 1038
jpayne@69 1039 if encode_chunked and self._http_vsn == 11:
jpayne@69 1040 # chunked encoding
jpayne@69 1041 chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \
jpayne@69 1042 + b'\r\n'
jpayne@69 1043 self.send(chunk)
jpayne@69 1044
jpayne@69 1045 if encode_chunked and self._http_vsn == 11:
jpayne@69 1046 # end chunked transfer
jpayne@69 1047 self.send(b'0\r\n\r\n')
jpayne@69 1048
jpayne@69 1049 def putrequest(self, method, url, skip_host=False,
jpayne@69 1050 skip_accept_encoding=False):
jpayne@69 1051 """Send a request to the server.
jpayne@69 1052
jpayne@69 1053 `method' specifies an HTTP request method, e.g. 'GET'.
jpayne@69 1054 `url' specifies the object being requested, e.g. '/index.html'.
jpayne@69 1055 `skip_host' if True does not add automatically a 'Host:' header
jpayne@69 1056 `skip_accept_encoding' if True does not add automatically an
jpayne@69 1057 'Accept-Encoding:' header
jpayne@69 1058 """
jpayne@69 1059
jpayne@69 1060 # if a prior response has been completed, then forget about it.
jpayne@69 1061 if self.__response and self.__response.isclosed():
jpayne@69 1062 self.__response = None
jpayne@69 1063
jpayne@69 1064
jpayne@69 1065 # in certain cases, we cannot issue another request on this connection.
jpayne@69 1066 # this occurs when:
jpayne@69 1067 # 1) we are in the process of sending a request. (_CS_REQ_STARTED)
jpayne@69 1068 # 2) a response to a previous request has signalled that it is going
jpayne@69 1069 # to close the connection upon completion.
jpayne@69 1070 # 3) the headers for the previous response have not been read, thus
jpayne@69 1071 # we cannot determine whether point (2) is true. (_CS_REQ_SENT)
jpayne@69 1072 #
jpayne@69 1073 # if there is no prior response, then we can request at will.
jpayne@69 1074 #
jpayne@69 1075 # if point (2) is true, then we will have passed the socket to the
jpayne@69 1076 # response (effectively meaning, "there is no prior response"), and
jpayne@69 1077 # will open a new one when a new request is made.
jpayne@69 1078 #
jpayne@69 1079 # Note: if a prior response exists, then we *can* start a new request.
jpayne@69 1080 # We are not allowed to begin fetching the response to this new
jpayne@69 1081 # request, however, until that prior response is complete.
jpayne@69 1082 #
jpayne@69 1083 if self.__state == _CS_IDLE:
jpayne@69 1084 self.__state = _CS_REQ_STARTED
jpayne@69 1085 else:
jpayne@69 1086 raise CannotSendRequest(self.__state)
jpayne@69 1087
jpayne@69 1088 # Save the method for use later in the response phase
jpayne@69 1089 self._method = method
jpayne@69 1090
jpayne@69 1091 url = url or '/'
jpayne@69 1092 self._validate_path(url)
jpayne@69 1093
jpayne@69 1094 request = '%s %s %s' % (method, url, self._http_vsn_str)
jpayne@69 1095
jpayne@69 1096 self._output(self._encode_request(request))
jpayne@69 1097
jpayne@69 1098 if self._http_vsn == 11:
jpayne@69 1099 # Issue some standard headers for better HTTP/1.1 compliance
jpayne@69 1100
jpayne@69 1101 if not skip_host:
jpayne@69 1102 # this header is issued *only* for HTTP/1.1
jpayne@69 1103 # connections. more specifically, this means it is
jpayne@69 1104 # only issued when the client uses the new
jpayne@69 1105 # HTTPConnection() class. backwards-compat clients
jpayne@69 1106 # will be using HTTP/1.0 and those clients may be
jpayne@69 1107 # issuing this header themselves. we should NOT issue
jpayne@69 1108 # it twice; some web servers (such as Apache) barf
jpayne@69 1109 # when they see two Host: headers
jpayne@69 1110
jpayne@69 1111 # If we need a non-standard port,include it in the
jpayne@69 1112 # header. If the request is going through a proxy,
jpayne@69 1113 # but the host of the actual URL, not the host of the
jpayne@69 1114 # proxy.
jpayne@69 1115
jpayne@69 1116 netloc = ''
jpayne@69 1117 if url.startswith('http'):
jpayne@69 1118 nil, netloc, nil, nil, nil = urlsplit(url)
jpayne@69 1119
jpayne@69 1120 if netloc:
jpayne@69 1121 try:
jpayne@69 1122 netloc_enc = netloc.encode("ascii")
jpayne@69 1123 except UnicodeEncodeError:
jpayne@69 1124 netloc_enc = netloc.encode("idna")
jpayne@69 1125 self.putheader('Host', netloc_enc)
jpayne@69 1126 else:
jpayne@69 1127 if self._tunnel_host:
jpayne@69 1128 host = self._tunnel_host
jpayne@69 1129 port = self._tunnel_port
jpayne@69 1130 else:
jpayne@69 1131 host = self.host
jpayne@69 1132 port = self.port
jpayne@69 1133
jpayne@69 1134 try:
jpayne@69 1135 host_enc = host.encode("ascii")
jpayne@69 1136 except UnicodeEncodeError:
jpayne@69 1137 host_enc = host.encode("idna")
jpayne@69 1138
jpayne@69 1139 # As per RFC 273, IPv6 address should be wrapped with []
jpayne@69 1140 # when used as Host header
jpayne@69 1141
jpayne@69 1142 if host.find(':') >= 0:
jpayne@69 1143 host_enc = b'[' + host_enc + b']'
jpayne@69 1144
jpayne@69 1145 if port == self.default_port:
jpayne@69 1146 self.putheader('Host', host_enc)
jpayne@69 1147 else:
jpayne@69 1148 host_enc = host_enc.decode("ascii")
jpayne@69 1149 self.putheader('Host', "%s:%s" % (host_enc, port))
jpayne@69 1150
jpayne@69 1151 # note: we are assuming that clients will not attempt to set these
jpayne@69 1152 # headers since *this* library must deal with the
jpayne@69 1153 # consequences. this also means that when the supporting
jpayne@69 1154 # libraries are updated to recognize other forms, then this
jpayne@69 1155 # code should be changed (removed or updated).
jpayne@69 1156
jpayne@69 1157 # we only want a Content-Encoding of "identity" since we don't
jpayne@69 1158 # support encodings such as x-gzip or x-deflate.
jpayne@69 1159 if not skip_accept_encoding:
jpayne@69 1160 self.putheader('Accept-Encoding', 'identity')
jpayne@69 1161
jpayne@69 1162 # we can accept "chunked" Transfer-Encodings, but no others
jpayne@69 1163 # NOTE: no TE header implies *only* "chunked"
jpayne@69 1164 #self.putheader('TE', 'chunked')
jpayne@69 1165
jpayne@69 1166 # if TE is supplied in the header, then it must appear in a
jpayne@69 1167 # Connection header.
jpayne@69 1168 #self.putheader('Connection', 'TE')
jpayne@69 1169
jpayne@69 1170 else:
jpayne@69 1171 # For HTTP/1.0, the server will assume "not chunked"
jpayne@69 1172 pass
jpayne@69 1173
jpayne@69 1174 def _encode_request(self, request):
jpayne@69 1175 # ASCII also helps prevent CVE-2019-9740.
jpayne@69 1176 return request.encode('ascii')
jpayne@69 1177
jpayne@69 1178 def _validate_path(self, url):
jpayne@69 1179 """Validate a url for putrequest."""
jpayne@69 1180 # Prevent CVE-2019-9740.
jpayne@69 1181 match = _contains_disallowed_url_pchar_re.search(url)
jpayne@69 1182 if match:
jpayne@69 1183 raise InvalidURL(f"URL can't contain control characters. {url!r} "
jpayne@69 1184 f"(found at least {match.group()!r})")
jpayne@69 1185
jpayne@69 1186 def putheader(self, header, *values):
jpayne@69 1187 """Send a request header line to the server.
jpayne@69 1188
jpayne@69 1189 For example: h.putheader('Accept', 'text/html')
jpayne@69 1190 """
jpayne@69 1191 if self.__state != _CS_REQ_STARTED:
jpayne@69 1192 raise CannotSendHeader()
jpayne@69 1193
jpayne@69 1194 if hasattr(header, 'encode'):
jpayne@69 1195 header = header.encode('ascii')
jpayne@69 1196
jpayne@69 1197 if not _is_legal_header_name(header):
jpayne@69 1198 raise ValueError('Invalid header name %r' % (header,))
jpayne@69 1199
jpayne@69 1200 values = list(values)
jpayne@69 1201 for i, one_value in enumerate(values):
jpayne@69 1202 if hasattr(one_value, 'encode'):
jpayne@69 1203 values[i] = one_value.encode('latin-1')
jpayne@69 1204 elif isinstance(one_value, int):
jpayne@69 1205 values[i] = str(one_value).encode('ascii')
jpayne@69 1206
jpayne@69 1207 if _is_illegal_header_value(values[i]):
jpayne@69 1208 raise ValueError('Invalid header value %r' % (values[i],))
jpayne@69 1209
jpayne@69 1210 value = b'\r\n\t'.join(values)
jpayne@69 1211 header = header + b': ' + value
jpayne@69 1212 self._output(header)
jpayne@69 1213
jpayne@69 1214 def endheaders(self, message_body=None, *, encode_chunked=False):
jpayne@69 1215 """Indicate that the last header line has been sent to the server.
jpayne@69 1216
jpayne@69 1217 This method sends the request to the server. The optional message_body
jpayne@69 1218 argument can be used to pass a message body associated with the
jpayne@69 1219 request.
jpayne@69 1220 """
jpayne@69 1221 if self.__state == _CS_REQ_STARTED:
jpayne@69 1222 self.__state = _CS_REQ_SENT
jpayne@69 1223 else:
jpayne@69 1224 raise CannotSendHeader()
jpayne@69 1225 self._send_output(message_body, encode_chunked=encode_chunked)
jpayne@69 1226
jpayne@69 1227 def request(self, method, url, body=None, headers={}, *,
jpayne@69 1228 encode_chunked=False):
jpayne@69 1229 """Send a complete request to the server."""
jpayne@69 1230 self._send_request(method, url, body, headers, encode_chunked)
jpayne@69 1231
jpayne@69 1232 def _send_request(self, method, url, body, headers, encode_chunked):
jpayne@69 1233 # Honor explicitly requested Host: and Accept-Encoding: headers.
jpayne@69 1234 header_names = frozenset(k.lower() for k in headers)
jpayne@69 1235 skips = {}
jpayne@69 1236 if 'host' in header_names:
jpayne@69 1237 skips['skip_host'] = 1
jpayne@69 1238 if 'accept-encoding' in header_names:
jpayne@69 1239 skips['skip_accept_encoding'] = 1
jpayne@69 1240
jpayne@69 1241 self.putrequest(method, url, **skips)
jpayne@69 1242
jpayne@69 1243 # chunked encoding will happen if HTTP/1.1 is used and either
jpayne@69 1244 # the caller passes encode_chunked=True or the following
jpayne@69 1245 # conditions hold:
jpayne@69 1246 # 1. content-length has not been explicitly set
jpayne@69 1247 # 2. the body is a file or iterable, but not a str or bytes-like
jpayne@69 1248 # 3. Transfer-Encoding has NOT been explicitly set by the caller
jpayne@69 1249
jpayne@69 1250 if 'content-length' not in header_names:
jpayne@69 1251 # only chunk body if not explicitly set for backwards
jpayne@69 1252 # compatibility, assuming the client code is already handling the
jpayne@69 1253 # chunking
jpayne@69 1254 if 'transfer-encoding' not in header_names:
jpayne@69 1255 # if content-length cannot be automatically determined, fall
jpayne@69 1256 # back to chunked encoding
jpayne@69 1257 encode_chunked = False
jpayne@69 1258 content_length = self._get_content_length(body, method)
jpayne@69 1259 if content_length is None:
jpayne@69 1260 if body is not None:
jpayne@69 1261 if self.debuglevel > 0:
jpayne@69 1262 print('Unable to determine size of %r' % body)
jpayne@69 1263 encode_chunked = True
jpayne@69 1264 self.putheader('Transfer-Encoding', 'chunked')
jpayne@69 1265 else:
jpayne@69 1266 self.putheader('Content-Length', str(content_length))
jpayne@69 1267 else:
jpayne@69 1268 encode_chunked = False
jpayne@69 1269
jpayne@69 1270 for hdr, value in headers.items():
jpayne@69 1271 self.putheader(hdr, value)
jpayne@69 1272 if isinstance(body, str):
jpayne@69 1273 # RFC 2616 Section 3.7.1 says that text default has a
jpayne@69 1274 # default charset of iso-8859-1.
jpayne@69 1275 body = _encode(body, 'body')
jpayne@69 1276 self.endheaders(body, encode_chunked=encode_chunked)
jpayne@69 1277
jpayne@69 1278 def getresponse(self):
jpayne@69 1279 """Get the response from the server.
jpayne@69 1280
jpayne@69 1281 If the HTTPConnection is in the correct state, returns an
jpayne@69 1282 instance of HTTPResponse or of whatever object is returned by
jpayne@69 1283 the response_class variable.
jpayne@69 1284
jpayne@69 1285 If a request has not been sent or if a previous response has
jpayne@69 1286 not be handled, ResponseNotReady is raised. If the HTTP
jpayne@69 1287 response indicates that the connection should be closed, then
jpayne@69 1288 it will be closed before the response is returned. When the
jpayne@69 1289 connection is closed, the underlying socket is closed.
jpayne@69 1290 """
jpayne@69 1291
jpayne@69 1292 # if a prior response has been completed, then forget about it.
jpayne@69 1293 if self.__response and self.__response.isclosed():
jpayne@69 1294 self.__response = None
jpayne@69 1295
jpayne@69 1296 # if a prior response exists, then it must be completed (otherwise, we
jpayne@69 1297 # cannot read this response's header to determine the connection-close
jpayne@69 1298 # behavior)
jpayne@69 1299 #
jpayne@69 1300 # note: if a prior response existed, but was connection-close, then the
jpayne@69 1301 # socket and response were made independent of this HTTPConnection
jpayne@69 1302 # object since a new request requires that we open a whole new
jpayne@69 1303 # connection
jpayne@69 1304 #
jpayne@69 1305 # this means the prior response had one of two states:
jpayne@69 1306 # 1) will_close: this connection was reset and the prior socket and
jpayne@69 1307 # response operate independently
jpayne@69 1308 # 2) persistent: the response was retained and we await its
jpayne@69 1309 # isclosed() status to become true.
jpayne@69 1310 #
jpayne@69 1311 if self.__state != _CS_REQ_SENT or self.__response:
jpayne@69 1312 raise ResponseNotReady(self.__state)
jpayne@69 1313
jpayne@69 1314 if self.debuglevel > 0:
jpayne@69 1315 response = self.response_class(self.sock, self.debuglevel,
jpayne@69 1316 method=self._method)
jpayne@69 1317 else:
jpayne@69 1318 response = self.response_class(self.sock, method=self._method)
jpayne@69 1319
jpayne@69 1320 try:
jpayne@69 1321 try:
jpayne@69 1322 response.begin()
jpayne@69 1323 except ConnectionError:
jpayne@69 1324 self.close()
jpayne@69 1325 raise
jpayne@69 1326 assert response.will_close != _UNKNOWN
jpayne@69 1327 self.__state = _CS_IDLE
jpayne@69 1328
jpayne@69 1329 if response.will_close:
jpayne@69 1330 # this effectively passes the connection to the response
jpayne@69 1331 self.close()
jpayne@69 1332 else:
jpayne@69 1333 # remember this, so we can tell when it is complete
jpayne@69 1334 self.__response = response
jpayne@69 1335
jpayne@69 1336 return response
jpayne@69 1337 except:
jpayne@69 1338 response.close()
jpayne@69 1339 raise
jpayne@69 1340
jpayne@69 1341 try:
jpayne@69 1342 import ssl
jpayne@69 1343 except ImportError:
jpayne@69 1344 pass
jpayne@69 1345 else:
jpayne@69 1346 class HTTPSConnection(HTTPConnection):
jpayne@69 1347 "This class allows communication via SSL."
jpayne@69 1348
jpayne@69 1349 default_port = HTTPS_PORT
jpayne@69 1350
jpayne@69 1351 # XXX Should key_file and cert_file be deprecated in favour of context?
jpayne@69 1352
jpayne@69 1353 def __init__(self, host, port=None, key_file=None, cert_file=None,
jpayne@69 1354 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
jpayne@69 1355 source_address=None, *, context=None,
jpayne@69 1356 check_hostname=None, blocksize=8192):
jpayne@69 1357 super(HTTPSConnection, self).__init__(host, port, timeout,
jpayne@69 1358 source_address,
jpayne@69 1359 blocksize=blocksize)
jpayne@69 1360 if (key_file is not None or cert_file is not None or
jpayne@69 1361 check_hostname is not None):
jpayne@69 1362 import warnings
jpayne@69 1363 warnings.warn("key_file, cert_file and check_hostname are "
jpayne@69 1364 "deprecated, use a custom context instead.",
jpayne@69 1365 DeprecationWarning, 2)
jpayne@69 1366 self.key_file = key_file
jpayne@69 1367 self.cert_file = cert_file
jpayne@69 1368 if context is None:
jpayne@69 1369 context = ssl._create_default_https_context()
jpayne@69 1370 # enable PHA for TLS 1.3 connections if available
jpayne@69 1371 if context.post_handshake_auth is not None:
jpayne@69 1372 context.post_handshake_auth = True
jpayne@69 1373 will_verify = context.verify_mode != ssl.CERT_NONE
jpayne@69 1374 if check_hostname is None:
jpayne@69 1375 check_hostname = context.check_hostname
jpayne@69 1376 if check_hostname and not will_verify:
jpayne@69 1377 raise ValueError("check_hostname needs a SSL context with "
jpayne@69 1378 "either CERT_OPTIONAL or CERT_REQUIRED")
jpayne@69 1379 if key_file or cert_file:
jpayne@69 1380 context.load_cert_chain(cert_file, key_file)
jpayne@69 1381 # cert and key file means the user wants to authenticate.
jpayne@69 1382 # enable TLS 1.3 PHA implicitly even for custom contexts.
jpayne@69 1383 if context.post_handshake_auth is not None:
jpayne@69 1384 context.post_handshake_auth = True
jpayne@69 1385 self._context = context
jpayne@69 1386 if check_hostname is not None:
jpayne@69 1387 self._context.check_hostname = check_hostname
jpayne@69 1388
jpayne@69 1389 def connect(self):
jpayne@69 1390 "Connect to a host on a given (SSL) port."
jpayne@69 1391
jpayne@69 1392 super().connect()
jpayne@69 1393
jpayne@69 1394 if self._tunnel_host:
jpayne@69 1395 server_hostname = self._tunnel_host
jpayne@69 1396 else:
jpayne@69 1397 server_hostname = self.host
jpayne@69 1398
jpayne@69 1399 self.sock = self._context.wrap_socket(self.sock,
jpayne@69 1400 server_hostname=server_hostname)
jpayne@69 1401
jpayne@69 1402 __all__.append("HTTPSConnection")
jpayne@69 1403
jpayne@69 1404 class HTTPException(Exception):
jpayne@69 1405 # Subclasses that define an __init__ must call Exception.__init__
jpayne@69 1406 # or define self.args. Otherwise, str() will fail.
jpayne@69 1407 pass
jpayne@69 1408
jpayne@69 1409 class NotConnected(HTTPException):
jpayne@69 1410 pass
jpayne@69 1411
jpayne@69 1412 class InvalidURL(HTTPException):
jpayne@69 1413 pass
jpayne@69 1414
jpayne@69 1415 class UnknownProtocol(HTTPException):
jpayne@69 1416 def __init__(self, version):
jpayne@69 1417 self.args = version,
jpayne@69 1418 self.version = version
jpayne@69 1419
jpayne@69 1420 class UnknownTransferEncoding(HTTPException):
jpayne@69 1421 pass
jpayne@69 1422
jpayne@69 1423 class UnimplementedFileMode(HTTPException):
jpayne@69 1424 pass
jpayne@69 1425
jpayne@69 1426 class IncompleteRead(HTTPException):
jpayne@69 1427 def __init__(self, partial, expected=None):
jpayne@69 1428 self.args = partial,
jpayne@69 1429 self.partial = partial
jpayne@69 1430 self.expected = expected
jpayne@69 1431 def __repr__(self):
jpayne@69 1432 if self.expected is not None:
jpayne@69 1433 e = ', %i more expected' % self.expected
jpayne@69 1434 else:
jpayne@69 1435 e = ''
jpayne@69 1436 return '%s(%i bytes read%s)' % (self.__class__.__name__,
jpayne@69 1437 len(self.partial), e)
jpayne@69 1438 __str__ = object.__str__
jpayne@69 1439
jpayne@69 1440 class ImproperConnectionState(HTTPException):
jpayne@69 1441 pass
jpayne@69 1442
jpayne@69 1443 class CannotSendRequest(ImproperConnectionState):
jpayne@69 1444 pass
jpayne@69 1445
jpayne@69 1446 class CannotSendHeader(ImproperConnectionState):
jpayne@69 1447 pass
jpayne@69 1448
jpayne@69 1449 class ResponseNotReady(ImproperConnectionState):
jpayne@69 1450 pass
jpayne@69 1451
jpayne@69 1452 class BadStatusLine(HTTPException):
jpayne@69 1453 def __init__(self, line):
jpayne@69 1454 if not line:
jpayne@69 1455 line = repr(line)
jpayne@69 1456 self.args = line,
jpayne@69 1457 self.line = line
jpayne@69 1458
jpayne@69 1459 class LineTooLong(HTTPException):
jpayne@69 1460 def __init__(self, line_type):
jpayne@69 1461 HTTPException.__init__(self, "got more than %d bytes when reading %s"
jpayne@69 1462 % (_MAXLINE, line_type))
jpayne@69 1463
jpayne@69 1464 class RemoteDisconnected(ConnectionResetError, BadStatusLine):
jpayne@69 1465 def __init__(self, *pos, **kw):
jpayne@69 1466 BadStatusLine.__init__(self, "")
jpayne@69 1467 ConnectionResetError.__init__(self, *pos, **kw)
jpayne@69 1468
jpayne@69 1469 # for backwards compatibility
jpayne@69 1470 error = HTTPException