Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/http/client.py @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 r"""HTTP/1.1 client library
jpayne@68 2
jpayne@68 3 <intro stuff goes here>
jpayne@68 4 <other stuff, too>
jpayne@68 5
jpayne@68 6 HTTPConnection goes through a number of "states", which define when a client
jpayne@68 7 may legally make another request or fetch the response for a particular
jpayne@68 8 request. This diagram details these state transitions:
jpayne@68 9
jpayne@68 10 (null)
jpayne@68 11 |
jpayne@68 12 | HTTPConnection()
jpayne@68 13 v
jpayne@68 14 Idle
jpayne@68 15 |
jpayne@68 16 | putrequest()
jpayne@68 17 v
jpayne@68 18 Request-started
jpayne@68 19 |
jpayne@68 20 | ( putheader() )* endheaders()
jpayne@68 21 v
jpayne@68 22 Request-sent
jpayne@68 23 |\_____________________________
jpayne@68 24 | | getresponse() raises
jpayne@68 25 | response = getresponse() | ConnectionError
jpayne@68 26 v v
jpayne@68 27 Unread-response Idle
jpayne@68 28 [Response-headers-read]
jpayne@68 29 |\____________________
jpayne@68 30 | |
jpayne@68 31 | response.read() | putrequest()
jpayne@68 32 v v
jpayne@68 33 Idle Req-started-unread-response
jpayne@68 34 ______/|
jpayne@68 35 / |
jpayne@68 36 response.read() | | ( putheader() )* endheaders()
jpayne@68 37 v v
jpayne@68 38 Request-started Req-sent-unread-response
jpayne@68 39 |
jpayne@68 40 | response.read()
jpayne@68 41 v
jpayne@68 42 Request-sent
jpayne@68 43
jpayne@68 44 This diagram presents the following rules:
jpayne@68 45 -- a second request may not be started until {response-headers-read}
jpayne@68 46 -- a response [object] cannot be retrieved until {request-sent}
jpayne@68 47 -- there is no differentiation between an unread response body and a
jpayne@68 48 partially read response body
jpayne@68 49
jpayne@68 50 Note: this enforcement is applied by the HTTPConnection class. The
jpayne@68 51 HTTPResponse class does not enforce this state machine, which
jpayne@68 52 implies sophisticated clients may accelerate the request/response
jpayne@68 53 pipeline. Caution should be taken, though: accelerating the states
jpayne@68 54 beyond the above pattern may imply knowledge of the server's
jpayne@68 55 connection-close behavior for certain requests. For example, it
jpayne@68 56 is impossible to tell whether the server will close the connection
jpayne@68 57 UNTIL the response headers have been read; this means that further
jpayne@68 58 requests cannot be placed into the pipeline until it is known that
jpayne@68 59 the server will NOT be closing the connection.
jpayne@68 60
jpayne@68 61 Logical State __state __response
jpayne@68 62 ------------- ------- ----------
jpayne@68 63 Idle _CS_IDLE None
jpayne@68 64 Request-started _CS_REQ_STARTED None
jpayne@68 65 Request-sent _CS_REQ_SENT None
jpayne@68 66 Unread-response _CS_IDLE <response_class>
jpayne@68 67 Req-started-unread-response _CS_REQ_STARTED <response_class>
jpayne@68 68 Req-sent-unread-response _CS_REQ_SENT <response_class>
jpayne@68 69 """
jpayne@68 70
jpayne@68 71 import email.parser
jpayne@68 72 import email.message
jpayne@68 73 import http
jpayne@68 74 import io
jpayne@68 75 import re
jpayne@68 76 import socket
jpayne@68 77 import collections.abc
jpayne@68 78 from urllib.parse import urlsplit
jpayne@68 79
jpayne@68 80 # HTTPMessage, parse_headers(), and the HTTP status code constants are
jpayne@68 81 # intentionally omitted for simplicity
jpayne@68 82 __all__ = ["HTTPResponse", "HTTPConnection",
jpayne@68 83 "HTTPException", "NotConnected", "UnknownProtocol",
jpayne@68 84 "UnknownTransferEncoding", "UnimplementedFileMode",
jpayne@68 85 "IncompleteRead", "InvalidURL", "ImproperConnectionState",
jpayne@68 86 "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
jpayne@68 87 "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",
jpayne@68 88 "responses"]
jpayne@68 89
jpayne@68 90 HTTP_PORT = 80
jpayne@68 91 HTTPS_PORT = 443
jpayne@68 92
jpayne@68 93 _UNKNOWN = 'UNKNOWN'
jpayne@68 94
jpayne@68 95 # connection states
jpayne@68 96 _CS_IDLE = 'Idle'
jpayne@68 97 _CS_REQ_STARTED = 'Request-started'
jpayne@68 98 _CS_REQ_SENT = 'Request-sent'
jpayne@68 99
jpayne@68 100
jpayne@68 101 # hack to maintain backwards compatibility
jpayne@68 102 globals().update(http.HTTPStatus.__members__)
jpayne@68 103
jpayne@68 104 # another hack to maintain backwards compatibility
jpayne@68 105 # Mapping status codes to official W3C names
jpayne@68 106 responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}
jpayne@68 107
jpayne@68 108 # maximal line length when calling readline().
jpayne@68 109 _MAXLINE = 65536
jpayne@68 110 _MAXHEADERS = 100
jpayne@68 111
jpayne@68 112 # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
jpayne@68 113 #
jpayne@68 114 # VCHAR = %x21-7E
jpayne@68 115 # obs-text = %x80-FF
jpayne@68 116 # header-field = field-name ":" OWS field-value OWS
jpayne@68 117 # field-name = token
jpayne@68 118 # field-value = *( field-content / obs-fold )
jpayne@68 119 # field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
jpayne@68 120 # field-vchar = VCHAR / obs-text
jpayne@68 121 #
jpayne@68 122 # obs-fold = CRLF 1*( SP / HTAB )
jpayne@68 123 # ; obsolete line folding
jpayne@68 124 # ; see Section 3.2.4
jpayne@68 125
jpayne@68 126 # token = 1*tchar
jpayne@68 127 #
jpayne@68 128 # tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*"
jpayne@68 129 # / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
jpayne@68 130 # / DIGIT / ALPHA
jpayne@68 131 # ; any VCHAR, except delimiters
jpayne@68 132 #
jpayne@68 133 # VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
jpayne@68 134
jpayne@68 135 # the patterns for both name and value are more lenient than RFC
jpayne@68 136 # definitions to allow for backwards compatibility
jpayne@68 137 _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
jpayne@68 138 _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
jpayne@68 139
jpayne@68 140 # These characters are not allowed within HTTP URL paths.
jpayne@68 141 # See https://tools.ietf.org/html/rfc3986#section-3.3 and the
jpayne@68 142 # https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
jpayne@68 143 # Prevents CVE-2019-9740. Includes control characters such as \r\n.
jpayne@68 144 # We don't restrict chars above \x7f as putrequest() limits us to ASCII.
jpayne@68 145 _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
jpayne@68 146 # Arguably only these _should_ allowed:
jpayne@68 147 # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
jpayne@68 148 # We are more lenient for assumed real world compatibility purposes.
jpayne@68 149
jpayne@68 150 # We always set the Content-Length header for these methods because some
jpayne@68 151 # servers will otherwise respond with a 411
jpayne@68 152 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
jpayne@68 153
jpayne@68 154
jpayne@68 155 def _encode(data, name='data'):
jpayne@68 156 """Call data.encode("latin-1") but show a better error message."""
jpayne@68 157 try:
jpayne@68 158 return data.encode("latin-1")
jpayne@68 159 except UnicodeEncodeError as err:
jpayne@68 160 raise UnicodeEncodeError(
jpayne@68 161 err.encoding,
jpayne@68 162 err.object,
jpayne@68 163 err.start,
jpayne@68 164 err.end,
jpayne@68 165 "%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "
jpayne@68 166 "if you want to send it encoded in UTF-8." %
jpayne@68 167 (name.title(), data[err.start:err.end], name)) from None
jpayne@68 168
jpayne@68 169
jpayne@68 170 class HTTPMessage(email.message.Message):
jpayne@68 171 # XXX The only usage of this method is in
jpayne@68 172 # http.server.CGIHTTPRequestHandler. Maybe move the code there so
jpayne@68 173 # that it doesn't need to be part of the public API. The API has
jpayne@68 174 # never been defined so this could cause backwards compatibility
jpayne@68 175 # issues.
jpayne@68 176
jpayne@68 177 def getallmatchingheaders(self, name):
jpayne@68 178 """Find all header lines matching a given header name.
jpayne@68 179
jpayne@68 180 Look through the list of headers and find all lines matching a given
jpayne@68 181 header name (and their continuation lines). A list of the lines is
jpayne@68 182 returned, without interpretation. If the header does not occur, an
jpayne@68 183 empty list is returned. If the header occurs multiple times, all
jpayne@68 184 occurrences are returned. Case is not important in the header name.
jpayne@68 185
jpayne@68 186 """
jpayne@68 187 name = name.lower() + ':'
jpayne@68 188 n = len(name)
jpayne@68 189 lst = []
jpayne@68 190 hit = 0
jpayne@68 191 for line in self.keys():
jpayne@68 192 if line[:n].lower() == name:
jpayne@68 193 hit = 1
jpayne@68 194 elif not line[:1].isspace():
jpayne@68 195 hit = 0
jpayne@68 196 if hit:
jpayne@68 197 lst.append(line)
jpayne@68 198 return lst
jpayne@68 199
jpayne@68 200 def parse_headers(fp, _class=HTTPMessage):
jpayne@68 201 """Parses only RFC2822 headers from a file pointer.
jpayne@68 202
jpayne@68 203 email Parser wants to see strings rather than bytes.
jpayne@68 204 But a TextIOWrapper around self.rfile would buffer too many bytes
jpayne@68 205 from the stream, bytes which we later need to read as bytes.
jpayne@68 206 So we read the correct bytes here, as bytes, for email Parser
jpayne@68 207 to parse.
jpayne@68 208
jpayne@68 209 """
jpayne@68 210 headers = []
jpayne@68 211 while True:
jpayne@68 212 line = fp.readline(_MAXLINE + 1)
jpayne@68 213 if len(line) > _MAXLINE:
jpayne@68 214 raise LineTooLong("header line")
jpayne@68 215 headers.append(line)
jpayne@68 216 if len(headers) > _MAXHEADERS:
jpayne@68 217 raise HTTPException("got more than %d headers" % _MAXHEADERS)
jpayne@68 218 if line in (b'\r\n', b'\n', b''):
jpayne@68 219 break
jpayne@68 220 hstring = b''.join(headers).decode('iso-8859-1')
jpayne@68 221 return email.parser.Parser(_class=_class).parsestr(hstring)
jpayne@68 222
jpayne@68 223
jpayne@68 224 class HTTPResponse(io.BufferedIOBase):
jpayne@68 225
jpayne@68 226 # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
jpayne@68 227
jpayne@68 228 # The bytes from the socket object are iso-8859-1 strings.
jpayne@68 229 # See RFC 2616 sec 2.2 which notes an exception for MIME-encoded
jpayne@68 230 # text following RFC 2047. The basic status line parsing only
jpayne@68 231 # accepts iso-8859-1.
jpayne@68 232
jpayne@68 233 def __init__(self, sock, debuglevel=0, method=None, url=None):
jpayne@68 234 # If the response includes a content-length header, we need to
jpayne@68 235 # make sure that the client doesn't read more than the
jpayne@68 236 # specified number of bytes. If it does, it will block until
jpayne@68 237 # the server times out and closes the connection. This will
jpayne@68 238 # happen if a self.fp.read() is done (without a size) whether
jpayne@68 239 # self.fp is buffered or not. So, no self.fp.read() by
jpayne@68 240 # clients unless they know what they are doing.
jpayne@68 241 self.fp = sock.makefile("rb")
jpayne@68 242 self.debuglevel = debuglevel
jpayne@68 243 self._method = method
jpayne@68 244
jpayne@68 245 # The HTTPResponse object is returned via urllib. The clients
jpayne@68 246 # of http and urllib expect different attributes for the
jpayne@68 247 # headers. headers is used here and supports urllib. msg is
jpayne@68 248 # provided as a backwards compatibility layer for http
jpayne@68 249 # clients.
jpayne@68 250
jpayne@68 251 self.headers = self.msg = None
jpayne@68 252
jpayne@68 253 # from the Status-Line of the response
jpayne@68 254 self.version = _UNKNOWN # HTTP-Version
jpayne@68 255 self.status = _UNKNOWN # Status-Code
jpayne@68 256 self.reason = _UNKNOWN # Reason-Phrase
jpayne@68 257
jpayne@68 258 self.chunked = _UNKNOWN # is "chunked" being used?
jpayne@68 259 self.chunk_left = _UNKNOWN # bytes left to read in current chunk
jpayne@68 260 self.length = _UNKNOWN # number of bytes left in response
jpayne@68 261 self.will_close = _UNKNOWN # conn will close at end of response
jpayne@68 262
jpayne@68 263 def _read_status(self):
jpayne@68 264 line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
jpayne@68 265 if len(line) > _MAXLINE:
jpayne@68 266 raise LineTooLong("status line")
jpayne@68 267 if self.debuglevel > 0:
jpayne@68 268 print("reply:", repr(line))
jpayne@68 269 if not line:
jpayne@68 270 # Presumably, the server closed the connection before
jpayne@68 271 # sending a valid response.
jpayne@68 272 raise RemoteDisconnected("Remote end closed connection without"
jpayne@68 273 " response")
jpayne@68 274 try:
jpayne@68 275 version, status, reason = line.split(None, 2)
jpayne@68 276 except ValueError:
jpayne@68 277 try:
jpayne@68 278 version, status = line.split(None, 1)
jpayne@68 279 reason = ""
jpayne@68 280 except ValueError:
jpayne@68 281 # empty version will cause next test to fail.
jpayne@68 282 version = ""
jpayne@68 283 if not version.startswith("HTTP/"):
jpayne@68 284 self._close_conn()
jpayne@68 285 raise BadStatusLine(line)
jpayne@68 286
jpayne@68 287 # The status code is a three-digit number
jpayne@68 288 try:
jpayne@68 289 status = int(status)
jpayne@68 290 if status < 100 or status > 999:
jpayne@68 291 raise BadStatusLine(line)
jpayne@68 292 except ValueError:
jpayne@68 293 raise BadStatusLine(line)
jpayne@68 294 return version, status, reason
jpayne@68 295
jpayne@68 296 def begin(self):
jpayne@68 297 if self.headers is not None:
jpayne@68 298 # we've already started reading the response
jpayne@68 299 return
jpayne@68 300
jpayne@68 301 # read until we get a non-100 response
jpayne@68 302 while True:
jpayne@68 303 version, status, reason = self._read_status()
jpayne@68 304 if status != CONTINUE:
jpayne@68 305 break
jpayne@68 306 # skip the header from the 100 response
jpayne@68 307 while True:
jpayne@68 308 skip = self.fp.readline(_MAXLINE + 1)
jpayne@68 309 if len(skip) > _MAXLINE:
jpayne@68 310 raise LineTooLong("header line")
jpayne@68 311 skip = skip.strip()
jpayne@68 312 if not skip:
jpayne@68 313 break
jpayne@68 314 if self.debuglevel > 0:
jpayne@68 315 print("header:", skip)
jpayne@68 316
jpayne@68 317 self.code = self.status = status
jpayne@68 318 self.reason = reason.strip()
jpayne@68 319 if version in ("HTTP/1.0", "HTTP/0.9"):
jpayne@68 320 # Some servers might still return "0.9", treat it as 1.0 anyway
jpayne@68 321 self.version = 10
jpayne@68 322 elif version.startswith("HTTP/1."):
jpayne@68 323 self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
jpayne@68 324 else:
jpayne@68 325 raise UnknownProtocol(version)
jpayne@68 326
jpayne@68 327 self.headers = self.msg = parse_headers(self.fp)
jpayne@68 328
jpayne@68 329 if self.debuglevel > 0:
jpayne@68 330 for hdr, val in self.headers.items():
jpayne@68 331 print("header:", hdr + ":", val)
jpayne@68 332
jpayne@68 333 # are we using the chunked-style of transfer encoding?
jpayne@68 334 tr_enc = self.headers.get("transfer-encoding")
jpayne@68 335 if tr_enc and tr_enc.lower() == "chunked":
jpayne@68 336 self.chunked = True
jpayne@68 337 self.chunk_left = None
jpayne@68 338 else:
jpayne@68 339 self.chunked = False
jpayne@68 340
jpayne@68 341 # will the connection close at the end of the response?
jpayne@68 342 self.will_close = self._check_close()
jpayne@68 343
jpayne@68 344 # do we have a Content-Length?
jpayne@68 345 # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
jpayne@68 346 self.length = None
jpayne@68 347 length = self.headers.get("content-length")
jpayne@68 348
jpayne@68 349 # are we using the chunked-style of transfer encoding?
jpayne@68 350 tr_enc = self.headers.get("transfer-encoding")
jpayne@68 351 if length and not self.chunked:
jpayne@68 352 try:
jpayne@68 353 self.length = int(length)
jpayne@68 354 except ValueError:
jpayne@68 355 self.length = None
jpayne@68 356 else:
jpayne@68 357 if self.length < 0: # ignore nonsensical negative lengths
jpayne@68 358 self.length = None
jpayne@68 359 else:
jpayne@68 360 self.length = None
jpayne@68 361
jpayne@68 362 # does the body have a fixed length? (of zero)
jpayne@68 363 if (status == NO_CONTENT or status == NOT_MODIFIED or
jpayne@68 364 100 <= status < 200 or # 1xx codes
jpayne@68 365 self._method == "HEAD"):
jpayne@68 366 self.length = 0
jpayne@68 367
jpayne@68 368 # if the connection remains open, and we aren't using chunked, and
jpayne@68 369 # a content-length was not provided, then assume that the connection
jpayne@68 370 # WILL close.
jpayne@68 371 if (not self.will_close and
jpayne@68 372 not self.chunked and
jpayne@68 373 self.length is None):
jpayne@68 374 self.will_close = True
jpayne@68 375
jpayne@68 376 def _check_close(self):
jpayne@68 377 conn = self.headers.get("connection")
jpayne@68 378 if self.version == 11:
jpayne@68 379 # An HTTP/1.1 proxy is assumed to stay open unless
jpayne@68 380 # explicitly closed.
jpayne@68 381 if conn and "close" in conn.lower():
jpayne@68 382 return True
jpayne@68 383 return False
jpayne@68 384
jpayne@68 385 # Some HTTP/1.0 implementations have support for persistent
jpayne@68 386 # connections, using rules different than HTTP/1.1.
jpayne@68 387
jpayne@68 388 # For older HTTP, Keep-Alive indicates persistent connection.
jpayne@68 389 if self.headers.get("keep-alive"):
jpayne@68 390 return False
jpayne@68 391
jpayne@68 392 # At least Akamai returns a "Connection: Keep-Alive" header,
jpayne@68 393 # which was supposed to be sent by the client.
jpayne@68 394 if conn and "keep-alive" in conn.lower():
jpayne@68 395 return False
jpayne@68 396
jpayne@68 397 # Proxy-Connection is a netscape hack.
jpayne@68 398 pconn = self.headers.get("proxy-connection")
jpayne@68 399 if pconn and "keep-alive" in pconn.lower():
jpayne@68 400 return False
jpayne@68 401
jpayne@68 402 # otherwise, assume it will close
jpayne@68 403 return True
jpayne@68 404
jpayne@68 405 def _close_conn(self):
jpayne@68 406 fp = self.fp
jpayne@68 407 self.fp = None
jpayne@68 408 fp.close()
jpayne@68 409
jpayne@68 410 def close(self):
jpayne@68 411 try:
jpayne@68 412 super().close() # set "closed" flag
jpayne@68 413 finally:
jpayne@68 414 if self.fp:
jpayne@68 415 self._close_conn()
jpayne@68 416
jpayne@68 417 # These implementations are for the benefit of io.BufferedReader.
jpayne@68 418
jpayne@68 419 # XXX This class should probably be revised to act more like
jpayne@68 420 # the "raw stream" that BufferedReader expects.
jpayne@68 421
jpayne@68 422 def flush(self):
jpayne@68 423 super().flush()
jpayne@68 424 if self.fp:
jpayne@68 425 self.fp.flush()
jpayne@68 426
jpayne@68 427 def readable(self):
jpayne@68 428 """Always returns True"""
jpayne@68 429 return True
jpayne@68 430
jpayne@68 431 # End of "raw stream" methods
jpayne@68 432
jpayne@68 433 def isclosed(self):
jpayne@68 434 """True if the connection is closed."""
jpayne@68 435 # NOTE: it is possible that we will not ever call self.close(). This
jpayne@68 436 # case occurs when will_close is TRUE, length is None, and we
jpayne@68 437 # read up to the last byte, but NOT past it.
jpayne@68 438 #
jpayne@68 439 # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
jpayne@68 440 # called, meaning self.isclosed() is meaningful.
jpayne@68 441 return self.fp is None
jpayne@68 442
jpayne@68 443 def read(self, amt=None):
jpayne@68 444 if self.fp is None:
jpayne@68 445 return b""
jpayne@68 446
jpayne@68 447 if self._method == "HEAD":
jpayne@68 448 self._close_conn()
jpayne@68 449 return b""
jpayne@68 450
jpayne@68 451 if amt is not None:
jpayne@68 452 # Amount is given, implement using readinto
jpayne@68 453 b = bytearray(amt)
jpayne@68 454 n = self.readinto(b)
jpayne@68 455 return memoryview(b)[:n].tobytes()
jpayne@68 456 else:
jpayne@68 457 # Amount is not given (unbounded read) so we must check self.length
jpayne@68 458 # and self.chunked
jpayne@68 459
jpayne@68 460 if self.chunked:
jpayne@68 461 return self._readall_chunked()
jpayne@68 462
jpayne@68 463 if self.length is None:
jpayne@68 464 s = self.fp.read()
jpayne@68 465 else:
jpayne@68 466 try:
jpayne@68 467 s = self._safe_read(self.length)
jpayne@68 468 except IncompleteRead:
jpayne@68 469 self._close_conn()
jpayne@68 470 raise
jpayne@68 471 self.length = 0
jpayne@68 472 self._close_conn() # we read everything
jpayne@68 473 return s
jpayne@68 474
jpayne@68 475 def readinto(self, b):
jpayne@68 476 """Read up to len(b) bytes into bytearray b and return the number
jpayne@68 477 of bytes read.
jpayne@68 478 """
jpayne@68 479
jpayne@68 480 if self.fp is None:
jpayne@68 481 return 0
jpayne@68 482
jpayne@68 483 if self._method == "HEAD":
jpayne@68 484 self._close_conn()
jpayne@68 485 return 0
jpayne@68 486
jpayne@68 487 if self.chunked:
jpayne@68 488 return self._readinto_chunked(b)
jpayne@68 489
jpayne@68 490 if self.length is not None:
jpayne@68 491 if len(b) > self.length:
jpayne@68 492 # clip the read to the "end of response"
jpayne@68 493 b = memoryview(b)[0:self.length]
jpayne@68 494
jpayne@68 495 # we do not use _safe_read() here because this may be a .will_close
jpayne@68 496 # connection, and the user is reading more bytes than will be provided
jpayne@68 497 # (for example, reading in 1k chunks)
jpayne@68 498 n = self.fp.readinto(b)
jpayne@68 499 if not n and b:
jpayne@68 500 # Ideally, we would raise IncompleteRead if the content-length
jpayne@68 501 # wasn't satisfied, but it might break compatibility.
jpayne@68 502 self._close_conn()
jpayne@68 503 elif self.length is not None:
jpayne@68 504 self.length -= n
jpayne@68 505 if not self.length:
jpayne@68 506 self._close_conn()
jpayne@68 507 return n
jpayne@68 508
jpayne@68 509 def _read_next_chunk_size(self):
jpayne@68 510 # Read the next chunk size from the file
jpayne@68 511 line = self.fp.readline(_MAXLINE + 1)
jpayne@68 512 if len(line) > _MAXLINE:
jpayne@68 513 raise LineTooLong("chunk size")
jpayne@68 514 i = line.find(b";")
jpayne@68 515 if i >= 0:
jpayne@68 516 line = line[:i] # strip chunk-extensions
jpayne@68 517 try:
jpayne@68 518 return int(line, 16)
jpayne@68 519 except ValueError:
jpayne@68 520 # close the connection as protocol synchronisation is
jpayne@68 521 # probably lost
jpayne@68 522 self._close_conn()
jpayne@68 523 raise
jpayne@68 524
jpayne@68 525 def _read_and_discard_trailer(self):
jpayne@68 526 # read and discard trailer up to the CRLF terminator
jpayne@68 527 ### note: we shouldn't have any trailers!
jpayne@68 528 while True:
jpayne@68 529 line = self.fp.readline(_MAXLINE + 1)
jpayne@68 530 if len(line) > _MAXLINE:
jpayne@68 531 raise LineTooLong("trailer line")
jpayne@68 532 if not line:
jpayne@68 533 # a vanishingly small number of sites EOF without
jpayne@68 534 # sending the trailer
jpayne@68 535 break
jpayne@68 536 if line in (b'\r\n', b'\n', b''):
jpayne@68 537 break
jpayne@68 538
jpayne@68 539 def _get_chunk_left(self):
jpayne@68 540 # return self.chunk_left, reading a new chunk if necessary.
jpayne@68 541 # chunk_left == 0: at the end of the current chunk, need to close it
jpayne@68 542 # chunk_left == None: No current chunk, should read next.
jpayne@68 543 # This function returns non-zero or None if the last chunk has
jpayne@68 544 # been read.
jpayne@68 545 chunk_left = self.chunk_left
jpayne@68 546 if not chunk_left: # Can be 0 or None
jpayne@68 547 if chunk_left is not None:
jpayne@68 548 # We are at the end of chunk, discard chunk end
jpayne@68 549 self._safe_read(2) # toss the CRLF at the end of the chunk
jpayne@68 550 try:
jpayne@68 551 chunk_left = self._read_next_chunk_size()
jpayne@68 552 except ValueError:
jpayne@68 553 raise IncompleteRead(b'')
jpayne@68 554 if chunk_left == 0:
jpayne@68 555 # last chunk: 1*("0") [ chunk-extension ] CRLF
jpayne@68 556 self._read_and_discard_trailer()
jpayne@68 557 # we read everything; close the "file"
jpayne@68 558 self._close_conn()
jpayne@68 559 chunk_left = None
jpayne@68 560 self.chunk_left = chunk_left
jpayne@68 561 return chunk_left
jpayne@68 562
jpayne@68 563 def _readall_chunked(self):
jpayne@68 564 assert self.chunked != _UNKNOWN
jpayne@68 565 value = []
jpayne@68 566 try:
jpayne@68 567 while True:
jpayne@68 568 chunk_left = self._get_chunk_left()
jpayne@68 569 if chunk_left is None:
jpayne@68 570 break
jpayne@68 571 value.append(self._safe_read(chunk_left))
jpayne@68 572 self.chunk_left = 0
jpayne@68 573 return b''.join(value)
jpayne@68 574 except IncompleteRead:
jpayne@68 575 raise IncompleteRead(b''.join(value))
jpayne@68 576
jpayne@68 577 def _readinto_chunked(self, b):
jpayne@68 578 assert self.chunked != _UNKNOWN
jpayne@68 579 total_bytes = 0
jpayne@68 580 mvb = memoryview(b)
jpayne@68 581 try:
jpayne@68 582 while True:
jpayne@68 583 chunk_left = self._get_chunk_left()
jpayne@68 584 if chunk_left is None:
jpayne@68 585 return total_bytes
jpayne@68 586
jpayne@68 587 if len(mvb) <= chunk_left:
jpayne@68 588 n = self._safe_readinto(mvb)
jpayne@68 589 self.chunk_left = chunk_left - n
jpayne@68 590 return total_bytes + n
jpayne@68 591
jpayne@68 592 temp_mvb = mvb[:chunk_left]
jpayne@68 593 n = self._safe_readinto(temp_mvb)
jpayne@68 594 mvb = mvb[n:]
jpayne@68 595 total_bytes += n
jpayne@68 596 self.chunk_left = 0
jpayne@68 597
jpayne@68 598 except IncompleteRead:
jpayne@68 599 raise IncompleteRead(bytes(b[0:total_bytes]))
jpayne@68 600
jpayne@68 601 def _safe_read(self, amt):
jpayne@68 602 """Read the number of bytes requested.
jpayne@68 603
jpayne@68 604 This function should be used when <amt> bytes "should" be present for
jpayne@68 605 reading. If the bytes are truly not available (due to EOF), then the
jpayne@68 606 IncompleteRead exception can be used to detect the problem.
jpayne@68 607 """
jpayne@68 608 data = self.fp.read(amt)
jpayne@68 609 if len(data) < amt:
jpayne@68 610 raise IncompleteRead(data, amt-len(data))
jpayne@68 611 return data
jpayne@68 612
jpayne@68 613 def _safe_readinto(self, b):
jpayne@68 614 """Same as _safe_read, but for reading into a buffer."""
jpayne@68 615 amt = len(b)
jpayne@68 616 n = self.fp.readinto(b)
jpayne@68 617 if n < amt:
jpayne@68 618 raise IncompleteRead(bytes(b[:n]), amt-n)
jpayne@68 619 return n
jpayne@68 620
jpayne@68 621 def read1(self, n=-1):
jpayne@68 622 """Read with at most one underlying system call. If at least one
jpayne@68 623 byte is buffered, return that instead.
jpayne@68 624 """
jpayne@68 625 if self.fp is None or self._method == "HEAD":
jpayne@68 626 return b""
jpayne@68 627 if self.chunked:
jpayne@68 628 return self._read1_chunked(n)
jpayne@68 629 if self.length is not None and (n < 0 or n > self.length):
jpayne@68 630 n = self.length
jpayne@68 631 result = self.fp.read1(n)
jpayne@68 632 if not result and n:
jpayne@68 633 self._close_conn()
jpayne@68 634 elif self.length is not None:
jpayne@68 635 self.length -= len(result)
jpayne@68 636 return result
jpayne@68 637
jpayne@68 638 def peek(self, n=-1):
jpayne@68 639 # Having this enables IOBase.readline() to read more than one
jpayne@68 640 # byte at a time
jpayne@68 641 if self.fp is None or self._method == "HEAD":
jpayne@68 642 return b""
jpayne@68 643 if self.chunked:
jpayne@68 644 return self._peek_chunked(n)
jpayne@68 645 return self.fp.peek(n)
jpayne@68 646
jpayne@68 647 def readline(self, limit=-1):
jpayne@68 648 if self.fp is None or self._method == "HEAD":
jpayne@68 649 return b""
jpayne@68 650 if self.chunked:
jpayne@68 651 # Fallback to IOBase readline which uses peek() and read()
jpayne@68 652 return super().readline(limit)
jpayne@68 653 if self.length is not None and (limit < 0 or limit > self.length):
jpayne@68 654 limit = self.length
jpayne@68 655 result = self.fp.readline(limit)
jpayne@68 656 if not result and limit:
jpayne@68 657 self._close_conn()
jpayne@68 658 elif self.length is not None:
jpayne@68 659 self.length -= len(result)
jpayne@68 660 return result
jpayne@68 661
jpayne@68 662 def _read1_chunked(self, n):
jpayne@68 663 # Strictly speaking, _get_chunk_left() may cause more than one read,
jpayne@68 664 # but that is ok, since that is to satisfy the chunked protocol.
jpayne@68 665 chunk_left = self._get_chunk_left()
jpayne@68 666 if chunk_left is None or n == 0:
jpayne@68 667 return b''
jpayne@68 668 if not (0 <= n <= chunk_left):
jpayne@68 669 n = chunk_left # if n is negative or larger than chunk_left
jpayne@68 670 read = self.fp.read1(n)
jpayne@68 671 self.chunk_left -= len(read)
jpayne@68 672 if not read:
jpayne@68 673 raise IncompleteRead(b"")
jpayne@68 674 return read
jpayne@68 675
jpayne@68 676 def _peek_chunked(self, n):
jpayne@68 677 # Strictly speaking, _get_chunk_left() may cause more than one read,
jpayne@68 678 # but that is ok, since that is to satisfy the chunked protocol.
jpayne@68 679 try:
jpayne@68 680 chunk_left = self._get_chunk_left()
jpayne@68 681 except IncompleteRead:
jpayne@68 682 return b'' # peek doesn't worry about protocol
jpayne@68 683 if chunk_left is None:
jpayne@68 684 return b'' # eof
jpayne@68 685 # peek is allowed to return more than requested. Just request the
jpayne@68 686 # entire chunk, and truncate what we get.
jpayne@68 687 return self.fp.peek(chunk_left)[:chunk_left]
jpayne@68 688
jpayne@68 689 def fileno(self):
jpayne@68 690 return self.fp.fileno()
jpayne@68 691
jpayne@68 692 def getheader(self, name, default=None):
jpayne@68 693 '''Returns the value of the header matching *name*.
jpayne@68 694
jpayne@68 695 If there are multiple matching headers, the values are
jpayne@68 696 combined into a single string separated by commas and spaces.
jpayne@68 697
jpayne@68 698 If no matching header is found, returns *default* or None if
jpayne@68 699 the *default* is not specified.
jpayne@68 700
jpayne@68 701 If the headers are unknown, raises http.client.ResponseNotReady.
jpayne@68 702
jpayne@68 703 '''
jpayne@68 704 if self.headers is None:
jpayne@68 705 raise ResponseNotReady()
jpayne@68 706 headers = self.headers.get_all(name) or default
jpayne@68 707 if isinstance(headers, str) or not hasattr(headers, '__iter__'):
jpayne@68 708 return headers
jpayne@68 709 else:
jpayne@68 710 return ', '.join(headers)
jpayne@68 711
jpayne@68 712 def getheaders(self):
jpayne@68 713 """Return list of (header, value) tuples."""
jpayne@68 714 if self.headers is None:
jpayne@68 715 raise ResponseNotReady()
jpayne@68 716 return list(self.headers.items())
jpayne@68 717
jpayne@68 718 # We override IOBase.__iter__ so that it doesn't check for closed-ness
jpayne@68 719
jpayne@68 720 def __iter__(self):
jpayne@68 721 return self
jpayne@68 722
jpayne@68 723 # For compatibility with old-style urllib responses.
jpayne@68 724
jpayne@68 725 def info(self):
jpayne@68 726 '''Returns an instance of the class mimetools.Message containing
jpayne@68 727 meta-information associated with the URL.
jpayne@68 728
jpayne@68 729 When the method is HTTP, these headers are those returned by
jpayne@68 730 the server at the head of the retrieved HTML page (including
jpayne@68 731 Content-Length and Content-Type).
jpayne@68 732
jpayne@68 733 When the method is FTP, a Content-Length header will be
jpayne@68 734 present if (as is now usual) the server passed back a file
jpayne@68 735 length in response to the FTP retrieval request. A
jpayne@68 736 Content-Type header will be present if the MIME type can be
jpayne@68 737 guessed.
jpayne@68 738
jpayne@68 739 When the method is local-file, returned headers will include
jpayne@68 740 a Date representing the file's last-modified time, a
jpayne@68 741 Content-Length giving file size, and a Content-Type
jpayne@68 742 containing a guess at the file's type. See also the
jpayne@68 743 description of the mimetools module.
jpayne@68 744
jpayne@68 745 '''
jpayne@68 746 return self.headers
jpayne@68 747
jpayne@68 748 def geturl(self):
jpayne@68 749 '''Return the real URL of the page.
jpayne@68 750
jpayne@68 751 In some cases, the HTTP server redirects a client to another
jpayne@68 752 URL. The urlopen() function handles this transparently, but in
jpayne@68 753 some cases the caller needs to know which URL the client was
jpayne@68 754 redirected to. The geturl() method can be used to get at this
jpayne@68 755 redirected URL.
jpayne@68 756
jpayne@68 757 '''
jpayne@68 758 return self.url
jpayne@68 759
jpayne@68 760 def getcode(self):
jpayne@68 761 '''Return the HTTP status code that was sent with the response,
jpayne@68 762 or None if the URL is not an HTTP URL.
jpayne@68 763
jpayne@68 764 '''
jpayne@68 765 return self.status
jpayne@68 766
jpayne@68 767 class HTTPConnection:
jpayne@68 768
jpayne@68 769 _http_vsn = 11
jpayne@68 770 _http_vsn_str = 'HTTP/1.1'
jpayne@68 771
jpayne@68 772 response_class = HTTPResponse
jpayne@68 773 default_port = HTTP_PORT
jpayne@68 774 auto_open = 1
jpayne@68 775 debuglevel = 0
jpayne@68 776
jpayne@68 777 @staticmethod
jpayne@68 778 def _is_textIO(stream):
jpayne@68 779 """Test whether a file-like object is a text or a binary stream.
jpayne@68 780 """
jpayne@68 781 return isinstance(stream, io.TextIOBase)
jpayne@68 782
jpayne@68 783 @staticmethod
jpayne@68 784 def _get_content_length(body, method):
jpayne@68 785 """Get the content-length based on the body.
jpayne@68 786
jpayne@68 787 If the body is None, we set Content-Length: 0 for methods that expect
jpayne@68 788 a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
jpayne@68 789 any method if the body is a str or bytes-like object and not a file.
jpayne@68 790 """
jpayne@68 791 if body is None:
jpayne@68 792 # do an explicit check for not None here to distinguish
jpayne@68 793 # between unset and set but empty
jpayne@68 794 if method.upper() in _METHODS_EXPECTING_BODY:
jpayne@68 795 return 0
jpayne@68 796 else:
jpayne@68 797 return None
jpayne@68 798
jpayne@68 799 if hasattr(body, 'read'):
jpayne@68 800 # file-like object.
jpayne@68 801 return None
jpayne@68 802
jpayne@68 803 try:
jpayne@68 804 # does it implement the buffer protocol (bytes, bytearray, array)?
jpayne@68 805 mv = memoryview(body)
jpayne@68 806 return mv.nbytes
jpayne@68 807 except TypeError:
jpayne@68 808 pass
jpayne@68 809
jpayne@68 810 if isinstance(body, str):
jpayne@68 811 return len(body)
jpayne@68 812
jpayne@68 813 return None
jpayne@68 814
jpayne@68 815 def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
jpayne@68 816 source_address=None, blocksize=8192):
jpayne@68 817 self.timeout = timeout
jpayne@68 818 self.source_address = source_address
jpayne@68 819 self.blocksize = blocksize
jpayne@68 820 self.sock = None
jpayne@68 821 self._buffer = []
jpayne@68 822 self.__response = None
jpayne@68 823 self.__state = _CS_IDLE
jpayne@68 824 self._method = None
jpayne@68 825 self._tunnel_host = None
jpayne@68 826 self._tunnel_port = None
jpayne@68 827 self._tunnel_headers = {}
jpayne@68 828
jpayne@68 829 (self.host, self.port) = self._get_hostport(host, port)
jpayne@68 830
jpayne@68 831 # This is stored as an instance variable to allow unit
jpayne@68 832 # tests to replace it with a suitable mockup
jpayne@68 833 self._create_connection = socket.create_connection
jpayne@68 834
jpayne@68 835 def set_tunnel(self, host, port=None, headers=None):
jpayne@68 836 """Set up host and port for HTTP CONNECT tunnelling.
jpayne@68 837
jpayne@68 838 In a connection that uses HTTP CONNECT tunneling, the host passed to the
jpayne@68 839 constructor is used as a proxy server that relays all communication to
jpayne@68 840 the endpoint passed to `set_tunnel`. This done by sending an HTTP
jpayne@68 841 CONNECT request to the proxy server when the connection is established.
jpayne@68 842
jpayne@68 843 This method must be called before the HTML connection has been
jpayne@68 844 established.
jpayne@68 845
jpayne@68 846 The headers argument should be a mapping of extra HTTP headers to send
jpayne@68 847 with the CONNECT request.
jpayne@68 848 """
jpayne@68 849
jpayne@68 850 if self.sock:
jpayne@68 851 raise RuntimeError("Can't set up tunnel for established connection")
jpayne@68 852
jpayne@68 853 self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)
jpayne@68 854 if headers:
jpayne@68 855 self._tunnel_headers = headers
jpayne@68 856 else:
jpayne@68 857 self._tunnel_headers.clear()
jpayne@68 858
jpayne@68 859 def _get_hostport(self, host, port):
jpayne@68 860 if port is None:
jpayne@68 861 i = host.rfind(':')
jpayne@68 862 j = host.rfind(']') # ipv6 addresses have [...]
jpayne@68 863 if i > j:
jpayne@68 864 try:
jpayne@68 865 port = int(host[i+1:])
jpayne@68 866 except ValueError:
jpayne@68 867 if host[i+1:] == "": # http://foo.com:/ == http://foo.com/
jpayne@68 868 port = self.default_port
jpayne@68 869 else:
jpayne@68 870 raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
jpayne@68 871 host = host[:i]
jpayne@68 872 else:
jpayne@68 873 port = self.default_port
jpayne@68 874 if host and host[0] == '[' and host[-1] == ']':
jpayne@68 875 host = host[1:-1]
jpayne@68 876
jpayne@68 877 return (host, port)
jpayne@68 878
jpayne@68 879 def set_debuglevel(self, level):
jpayne@68 880 self.debuglevel = level
jpayne@68 881
jpayne@68 882 def _tunnel(self):
jpayne@68 883 connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
jpayne@68 884 self._tunnel_port)
jpayne@68 885 connect_bytes = connect_str.encode("ascii")
jpayne@68 886 self.send(connect_bytes)
jpayne@68 887 for header, value in self._tunnel_headers.items():
jpayne@68 888 header_str = "%s: %s\r\n" % (header, value)
jpayne@68 889 header_bytes = header_str.encode("latin-1")
jpayne@68 890 self.send(header_bytes)
jpayne@68 891 self.send(b'\r\n')
jpayne@68 892
jpayne@68 893 response = self.response_class(self.sock, method=self._method)
jpayne@68 894 (version, code, message) = response._read_status()
jpayne@68 895
jpayne@68 896 if code != http.HTTPStatus.OK:
jpayne@68 897 self.close()
jpayne@68 898 raise OSError("Tunnel connection failed: %d %s" % (code,
jpayne@68 899 message.strip()))
jpayne@68 900 while True:
jpayne@68 901 line = response.fp.readline(_MAXLINE + 1)
jpayne@68 902 if len(line) > _MAXLINE:
jpayne@68 903 raise LineTooLong("header line")
jpayne@68 904 if not line:
jpayne@68 905 # for sites which EOF without sending a trailer
jpayne@68 906 break
jpayne@68 907 if line in (b'\r\n', b'\n', b''):
jpayne@68 908 break
jpayne@68 909
jpayne@68 910 if self.debuglevel > 0:
jpayne@68 911 print('header:', line.decode())
jpayne@68 912
jpayne@68 913 def connect(self):
jpayne@68 914 """Connect to the host and port specified in __init__."""
jpayne@68 915 self.sock = self._create_connection(
jpayne@68 916 (self.host,self.port), self.timeout, self.source_address)
jpayne@68 917 self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
jpayne@68 918
jpayne@68 919 if self._tunnel_host:
jpayne@68 920 self._tunnel()
jpayne@68 921
jpayne@68 922 def close(self):
jpayne@68 923 """Close the connection to the HTTP server."""
jpayne@68 924 self.__state = _CS_IDLE
jpayne@68 925 try:
jpayne@68 926 sock = self.sock
jpayne@68 927 if sock:
jpayne@68 928 self.sock = None
jpayne@68 929 sock.close() # close it manually... there may be other refs
jpayne@68 930 finally:
jpayne@68 931 response = self.__response
jpayne@68 932 if response:
jpayne@68 933 self.__response = None
jpayne@68 934 response.close()
jpayne@68 935
jpayne@68 936 def send(self, data):
jpayne@68 937 """Send `data' to the server.
jpayne@68 938 ``data`` can be a string object, a bytes object, an array object, a
jpayne@68 939 file-like object that supports a .read() method, or an iterable object.
jpayne@68 940 """
jpayne@68 941
jpayne@68 942 if self.sock is None:
jpayne@68 943 if self.auto_open:
jpayne@68 944 self.connect()
jpayne@68 945 else:
jpayne@68 946 raise NotConnected()
jpayne@68 947
jpayne@68 948 if self.debuglevel > 0:
jpayne@68 949 print("send:", repr(data))
jpayne@68 950 if hasattr(data, "read") :
jpayne@68 951 if self.debuglevel > 0:
jpayne@68 952 print("sendIng a read()able")
jpayne@68 953 encode = self._is_textIO(data)
jpayne@68 954 if encode and self.debuglevel > 0:
jpayne@68 955 print("encoding file using iso-8859-1")
jpayne@68 956 while 1:
jpayne@68 957 datablock = data.read(self.blocksize)
jpayne@68 958 if not datablock:
jpayne@68 959 break
jpayne@68 960 if encode:
jpayne@68 961 datablock = datablock.encode("iso-8859-1")
jpayne@68 962 self.sock.sendall(datablock)
jpayne@68 963 return
jpayne@68 964 try:
jpayne@68 965 self.sock.sendall(data)
jpayne@68 966 except TypeError:
jpayne@68 967 if isinstance(data, collections.abc.Iterable):
jpayne@68 968 for d in data:
jpayne@68 969 self.sock.sendall(d)
jpayne@68 970 else:
jpayne@68 971 raise TypeError("data should be a bytes-like object "
jpayne@68 972 "or an iterable, got %r" % type(data))
jpayne@68 973
jpayne@68 974 def _output(self, s):
jpayne@68 975 """Add a line of output to the current request buffer.
jpayne@68 976
jpayne@68 977 Assumes that the line does *not* end with \\r\\n.
jpayne@68 978 """
jpayne@68 979 self._buffer.append(s)
jpayne@68 980
jpayne@68 981 def _read_readable(self, readable):
jpayne@68 982 if self.debuglevel > 0:
jpayne@68 983 print("sendIng a read()able")
jpayne@68 984 encode = self._is_textIO(readable)
jpayne@68 985 if encode and self.debuglevel > 0:
jpayne@68 986 print("encoding file using iso-8859-1")
jpayne@68 987 while True:
jpayne@68 988 datablock = readable.read(self.blocksize)
jpayne@68 989 if not datablock:
jpayne@68 990 break
jpayne@68 991 if encode:
jpayne@68 992 datablock = datablock.encode("iso-8859-1")
jpayne@68 993 yield datablock
jpayne@68 994
jpayne@68 995 def _send_output(self, message_body=None, encode_chunked=False):
jpayne@68 996 """Send the currently buffered request and clear the buffer.
jpayne@68 997
jpayne@68 998 Appends an extra \\r\\n to the buffer.
jpayne@68 999 A message_body may be specified, to be appended to the request.
jpayne@68 1000 """
jpayne@68 1001 self._buffer.extend((b"", b""))
jpayne@68 1002 msg = b"\r\n".join(self._buffer)
jpayne@68 1003 del self._buffer[:]
jpayne@68 1004 self.send(msg)
jpayne@68 1005
jpayne@68 1006 if message_body is not None:
jpayne@68 1007
jpayne@68 1008 # create a consistent interface to message_body
jpayne@68 1009 if hasattr(message_body, 'read'):
jpayne@68 1010 # Let file-like take precedence over byte-like. This
jpayne@68 1011 # is needed to allow the current position of mmap'ed
jpayne@68 1012 # files to be taken into account.
jpayne@68 1013 chunks = self._read_readable(message_body)
jpayne@68 1014 else:
jpayne@68 1015 try:
jpayne@68 1016 # this is solely to check to see if message_body
jpayne@68 1017 # implements the buffer API. it /would/ be easier
jpayne@68 1018 # to capture if PyObject_CheckBuffer was exposed
jpayne@68 1019 # to Python.
jpayne@68 1020 memoryview(message_body)
jpayne@68 1021 except TypeError:
jpayne@68 1022 try:
jpayne@68 1023 chunks = iter(message_body)
jpayne@68 1024 except TypeError:
jpayne@68 1025 raise TypeError("message_body should be a bytes-like "
jpayne@68 1026 "object or an iterable, got %r"
jpayne@68 1027 % type(message_body))
jpayne@68 1028 else:
jpayne@68 1029 # the object implements the buffer interface and
jpayne@68 1030 # can be passed directly into socket methods
jpayne@68 1031 chunks = (message_body,)
jpayne@68 1032
jpayne@68 1033 for chunk in chunks:
jpayne@68 1034 if not chunk:
jpayne@68 1035 if self.debuglevel > 0:
jpayne@68 1036 print('Zero length chunk ignored')
jpayne@68 1037 continue
jpayne@68 1038
jpayne@68 1039 if encode_chunked and self._http_vsn == 11:
jpayne@68 1040 # chunked encoding
jpayne@68 1041 chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \
jpayne@68 1042 + b'\r\n'
jpayne@68 1043 self.send(chunk)
jpayne@68 1044
jpayne@68 1045 if encode_chunked and self._http_vsn == 11:
jpayne@68 1046 # end chunked transfer
jpayne@68 1047 self.send(b'0\r\n\r\n')
jpayne@68 1048
jpayne@68 1049 def putrequest(self, method, url, skip_host=False,
jpayne@68 1050 skip_accept_encoding=False):
jpayne@68 1051 """Send a request to the server.
jpayne@68 1052
jpayne@68 1053 `method' specifies an HTTP request method, e.g. 'GET'.
jpayne@68 1054 `url' specifies the object being requested, e.g. '/index.html'.
jpayne@68 1055 `skip_host' if True does not add automatically a 'Host:' header
jpayne@68 1056 `skip_accept_encoding' if True does not add automatically an
jpayne@68 1057 'Accept-Encoding:' header
jpayne@68 1058 """
jpayne@68 1059
jpayne@68 1060 # if a prior response has been completed, then forget about it.
jpayne@68 1061 if self.__response and self.__response.isclosed():
jpayne@68 1062 self.__response = None
jpayne@68 1063
jpayne@68 1064
jpayne@68 1065 # in certain cases, we cannot issue another request on this connection.
jpayne@68 1066 # this occurs when:
jpayne@68 1067 # 1) we are in the process of sending a request. (_CS_REQ_STARTED)
jpayne@68 1068 # 2) a response to a previous request has signalled that it is going
jpayne@68 1069 # to close the connection upon completion.
jpayne@68 1070 # 3) the headers for the previous response have not been read, thus
jpayne@68 1071 # we cannot determine whether point (2) is true. (_CS_REQ_SENT)
jpayne@68 1072 #
jpayne@68 1073 # if there is no prior response, then we can request at will.
jpayne@68 1074 #
jpayne@68 1075 # if point (2) is true, then we will have passed the socket to the
jpayne@68 1076 # response (effectively meaning, "there is no prior response"), and
jpayne@68 1077 # will open a new one when a new request is made.
jpayne@68 1078 #
jpayne@68 1079 # Note: if a prior response exists, then we *can* start a new request.
jpayne@68 1080 # We are not allowed to begin fetching the response to this new
jpayne@68 1081 # request, however, until that prior response is complete.
jpayne@68 1082 #
jpayne@68 1083 if self.__state == _CS_IDLE:
jpayne@68 1084 self.__state = _CS_REQ_STARTED
jpayne@68 1085 else:
jpayne@68 1086 raise CannotSendRequest(self.__state)
jpayne@68 1087
jpayne@68 1088 # Save the method for use later in the response phase
jpayne@68 1089 self._method = method
jpayne@68 1090
jpayne@68 1091 url = url or '/'
jpayne@68 1092 self._validate_path(url)
jpayne@68 1093
jpayne@68 1094 request = '%s %s %s' % (method, url, self._http_vsn_str)
jpayne@68 1095
jpayne@68 1096 self._output(self._encode_request(request))
jpayne@68 1097
jpayne@68 1098 if self._http_vsn == 11:
jpayne@68 1099 # Issue some standard headers for better HTTP/1.1 compliance
jpayne@68 1100
jpayne@68 1101 if not skip_host:
jpayne@68 1102 # this header is issued *only* for HTTP/1.1
jpayne@68 1103 # connections. more specifically, this means it is
jpayne@68 1104 # only issued when the client uses the new
jpayne@68 1105 # HTTPConnection() class. backwards-compat clients
jpayne@68 1106 # will be using HTTP/1.0 and those clients may be
jpayne@68 1107 # issuing this header themselves. we should NOT issue
jpayne@68 1108 # it twice; some web servers (such as Apache) barf
jpayne@68 1109 # when they see two Host: headers
jpayne@68 1110
jpayne@68 1111 # If we need a non-standard port,include it in the
jpayne@68 1112 # header. If the request is going through a proxy,
jpayne@68 1113 # but the host of the actual URL, not the host of the
jpayne@68 1114 # proxy.
jpayne@68 1115
jpayne@68 1116 netloc = ''
jpayne@68 1117 if url.startswith('http'):
jpayne@68 1118 nil, netloc, nil, nil, nil = urlsplit(url)
jpayne@68 1119
jpayne@68 1120 if netloc:
jpayne@68 1121 try:
jpayne@68 1122 netloc_enc = netloc.encode("ascii")
jpayne@68 1123 except UnicodeEncodeError:
jpayne@68 1124 netloc_enc = netloc.encode("idna")
jpayne@68 1125 self.putheader('Host', netloc_enc)
jpayne@68 1126 else:
jpayne@68 1127 if self._tunnel_host:
jpayne@68 1128 host = self._tunnel_host
jpayne@68 1129 port = self._tunnel_port
jpayne@68 1130 else:
jpayne@68 1131 host = self.host
jpayne@68 1132 port = self.port
jpayne@68 1133
jpayne@68 1134 try:
jpayne@68 1135 host_enc = host.encode("ascii")
jpayne@68 1136 except UnicodeEncodeError:
jpayne@68 1137 host_enc = host.encode("idna")
jpayne@68 1138
jpayne@68 1139 # As per RFC 273, IPv6 address should be wrapped with []
jpayne@68 1140 # when used as Host header
jpayne@68 1141
jpayne@68 1142 if host.find(':') >= 0:
jpayne@68 1143 host_enc = b'[' + host_enc + b']'
jpayne@68 1144
jpayne@68 1145 if port == self.default_port:
jpayne@68 1146 self.putheader('Host', host_enc)
jpayne@68 1147 else:
jpayne@68 1148 host_enc = host_enc.decode("ascii")
jpayne@68 1149 self.putheader('Host', "%s:%s" % (host_enc, port))
jpayne@68 1150
jpayne@68 1151 # note: we are assuming that clients will not attempt to set these
jpayne@68 1152 # headers since *this* library must deal with the
jpayne@68 1153 # consequences. this also means that when the supporting
jpayne@68 1154 # libraries are updated to recognize other forms, then this
jpayne@68 1155 # code should be changed (removed or updated).
jpayne@68 1156
jpayne@68 1157 # we only want a Content-Encoding of "identity" since we don't
jpayne@68 1158 # support encodings such as x-gzip or x-deflate.
jpayne@68 1159 if not skip_accept_encoding:
jpayne@68 1160 self.putheader('Accept-Encoding', 'identity')
jpayne@68 1161
jpayne@68 1162 # we can accept "chunked" Transfer-Encodings, but no others
jpayne@68 1163 # NOTE: no TE header implies *only* "chunked"
jpayne@68 1164 #self.putheader('TE', 'chunked')
jpayne@68 1165
jpayne@68 1166 # if TE is supplied in the header, then it must appear in a
jpayne@68 1167 # Connection header.
jpayne@68 1168 #self.putheader('Connection', 'TE')
jpayne@68 1169
jpayne@68 1170 else:
jpayne@68 1171 # For HTTP/1.0, the server will assume "not chunked"
jpayne@68 1172 pass
jpayne@68 1173
jpayne@68 1174 def _encode_request(self, request):
jpayne@68 1175 # ASCII also helps prevent CVE-2019-9740.
jpayne@68 1176 return request.encode('ascii')
jpayne@68 1177
jpayne@68 1178 def _validate_path(self, url):
jpayne@68 1179 """Validate a url for putrequest."""
jpayne@68 1180 # Prevent CVE-2019-9740.
jpayne@68 1181 match = _contains_disallowed_url_pchar_re.search(url)
jpayne@68 1182 if match:
jpayne@68 1183 raise InvalidURL(f"URL can't contain control characters. {url!r} "
jpayne@68 1184 f"(found at least {match.group()!r})")
jpayne@68 1185
jpayne@68 1186 def putheader(self, header, *values):
jpayne@68 1187 """Send a request header line to the server.
jpayne@68 1188
jpayne@68 1189 For example: h.putheader('Accept', 'text/html')
jpayne@68 1190 """
jpayne@68 1191 if self.__state != _CS_REQ_STARTED:
jpayne@68 1192 raise CannotSendHeader()
jpayne@68 1193
jpayne@68 1194 if hasattr(header, 'encode'):
jpayne@68 1195 header = header.encode('ascii')
jpayne@68 1196
jpayne@68 1197 if not _is_legal_header_name(header):
jpayne@68 1198 raise ValueError('Invalid header name %r' % (header,))
jpayne@68 1199
jpayne@68 1200 values = list(values)
jpayne@68 1201 for i, one_value in enumerate(values):
jpayne@68 1202 if hasattr(one_value, 'encode'):
jpayne@68 1203 values[i] = one_value.encode('latin-1')
jpayne@68 1204 elif isinstance(one_value, int):
jpayne@68 1205 values[i] = str(one_value).encode('ascii')
jpayne@68 1206
jpayne@68 1207 if _is_illegal_header_value(values[i]):
jpayne@68 1208 raise ValueError('Invalid header value %r' % (values[i],))
jpayne@68 1209
jpayne@68 1210 value = b'\r\n\t'.join(values)
jpayne@68 1211 header = header + b': ' + value
jpayne@68 1212 self._output(header)
jpayne@68 1213
jpayne@68 1214 def endheaders(self, message_body=None, *, encode_chunked=False):
jpayne@68 1215 """Indicate that the last header line has been sent to the server.
jpayne@68 1216
jpayne@68 1217 This method sends the request to the server. The optional message_body
jpayne@68 1218 argument can be used to pass a message body associated with the
jpayne@68 1219 request.
jpayne@68 1220 """
jpayne@68 1221 if self.__state == _CS_REQ_STARTED:
jpayne@68 1222 self.__state = _CS_REQ_SENT
jpayne@68 1223 else:
jpayne@68 1224 raise CannotSendHeader()
jpayne@68 1225 self._send_output(message_body, encode_chunked=encode_chunked)
jpayne@68 1226
jpayne@68 1227 def request(self, method, url, body=None, headers={}, *,
jpayne@68 1228 encode_chunked=False):
jpayne@68 1229 """Send a complete request to the server."""
jpayne@68 1230 self._send_request(method, url, body, headers, encode_chunked)
jpayne@68 1231
jpayne@68 1232 def _send_request(self, method, url, body, headers, encode_chunked):
jpayne@68 1233 # Honor explicitly requested Host: and Accept-Encoding: headers.
jpayne@68 1234 header_names = frozenset(k.lower() for k in headers)
jpayne@68 1235 skips = {}
jpayne@68 1236 if 'host' in header_names:
jpayne@68 1237 skips['skip_host'] = 1
jpayne@68 1238 if 'accept-encoding' in header_names:
jpayne@68 1239 skips['skip_accept_encoding'] = 1
jpayne@68 1240
jpayne@68 1241 self.putrequest(method, url, **skips)
jpayne@68 1242
jpayne@68 1243 # chunked encoding will happen if HTTP/1.1 is used and either
jpayne@68 1244 # the caller passes encode_chunked=True or the following
jpayne@68 1245 # conditions hold:
jpayne@68 1246 # 1. content-length has not been explicitly set
jpayne@68 1247 # 2. the body is a file or iterable, but not a str or bytes-like
jpayne@68 1248 # 3. Transfer-Encoding has NOT been explicitly set by the caller
jpayne@68 1249
jpayne@68 1250 if 'content-length' not in header_names:
jpayne@68 1251 # only chunk body if not explicitly set for backwards
jpayne@68 1252 # compatibility, assuming the client code is already handling the
jpayne@68 1253 # chunking
jpayne@68 1254 if 'transfer-encoding' not in header_names:
jpayne@68 1255 # if content-length cannot be automatically determined, fall
jpayne@68 1256 # back to chunked encoding
jpayne@68 1257 encode_chunked = False
jpayne@68 1258 content_length = self._get_content_length(body, method)
jpayne@68 1259 if content_length is None:
jpayne@68 1260 if body is not None:
jpayne@68 1261 if self.debuglevel > 0:
jpayne@68 1262 print('Unable to determine size of %r' % body)
jpayne@68 1263 encode_chunked = True
jpayne@68 1264 self.putheader('Transfer-Encoding', 'chunked')
jpayne@68 1265 else:
jpayne@68 1266 self.putheader('Content-Length', str(content_length))
jpayne@68 1267 else:
jpayne@68 1268 encode_chunked = False
jpayne@68 1269
jpayne@68 1270 for hdr, value in headers.items():
jpayne@68 1271 self.putheader(hdr, value)
jpayne@68 1272 if isinstance(body, str):
jpayne@68 1273 # RFC 2616 Section 3.7.1 says that text default has a
jpayne@68 1274 # default charset of iso-8859-1.
jpayne@68 1275 body = _encode(body, 'body')
jpayne@68 1276 self.endheaders(body, encode_chunked=encode_chunked)
jpayne@68 1277
jpayne@68 1278 def getresponse(self):
jpayne@68 1279 """Get the response from the server.
jpayne@68 1280
jpayne@68 1281 If the HTTPConnection is in the correct state, returns an
jpayne@68 1282 instance of HTTPResponse or of whatever object is returned by
jpayne@68 1283 the response_class variable.
jpayne@68 1284
jpayne@68 1285 If a request has not been sent or if a previous response has
jpayne@68 1286 not be handled, ResponseNotReady is raised. If the HTTP
jpayne@68 1287 response indicates that the connection should be closed, then
jpayne@68 1288 it will be closed before the response is returned. When the
jpayne@68 1289 connection is closed, the underlying socket is closed.
jpayne@68 1290 """
jpayne@68 1291
jpayne@68 1292 # if a prior response has been completed, then forget about it.
jpayne@68 1293 if self.__response and self.__response.isclosed():
jpayne@68 1294 self.__response = None
jpayne@68 1295
jpayne@68 1296 # if a prior response exists, then it must be completed (otherwise, we
jpayne@68 1297 # cannot read this response's header to determine the connection-close
jpayne@68 1298 # behavior)
jpayne@68 1299 #
jpayne@68 1300 # note: if a prior response existed, but was connection-close, then the
jpayne@68 1301 # socket and response were made independent of this HTTPConnection
jpayne@68 1302 # object since a new request requires that we open a whole new
jpayne@68 1303 # connection
jpayne@68 1304 #
jpayne@68 1305 # this means the prior response had one of two states:
jpayne@68 1306 # 1) will_close: this connection was reset and the prior socket and
jpayne@68 1307 # response operate independently
jpayne@68 1308 # 2) persistent: the response was retained and we await its
jpayne@68 1309 # isclosed() status to become true.
jpayne@68 1310 #
jpayne@68 1311 if self.__state != _CS_REQ_SENT or self.__response:
jpayne@68 1312 raise ResponseNotReady(self.__state)
jpayne@68 1313
jpayne@68 1314 if self.debuglevel > 0:
jpayne@68 1315 response = self.response_class(self.sock, self.debuglevel,
jpayne@68 1316 method=self._method)
jpayne@68 1317 else:
jpayne@68 1318 response = self.response_class(self.sock, method=self._method)
jpayne@68 1319
jpayne@68 1320 try:
jpayne@68 1321 try:
jpayne@68 1322 response.begin()
jpayne@68 1323 except ConnectionError:
jpayne@68 1324 self.close()
jpayne@68 1325 raise
jpayne@68 1326 assert response.will_close != _UNKNOWN
jpayne@68 1327 self.__state = _CS_IDLE
jpayne@68 1328
jpayne@68 1329 if response.will_close:
jpayne@68 1330 # this effectively passes the connection to the response
jpayne@68 1331 self.close()
jpayne@68 1332 else:
jpayne@68 1333 # remember this, so we can tell when it is complete
jpayne@68 1334 self.__response = response
jpayne@68 1335
jpayne@68 1336 return response
jpayne@68 1337 except:
jpayne@68 1338 response.close()
jpayne@68 1339 raise
jpayne@68 1340
jpayne@68 1341 try:
jpayne@68 1342 import ssl
jpayne@68 1343 except ImportError:
jpayne@68 1344 pass
jpayne@68 1345 else:
jpayne@68 1346 class HTTPSConnection(HTTPConnection):
jpayne@68 1347 "This class allows communication via SSL."
jpayne@68 1348
jpayne@68 1349 default_port = HTTPS_PORT
jpayne@68 1350
jpayne@68 1351 # XXX Should key_file and cert_file be deprecated in favour of context?
jpayne@68 1352
jpayne@68 1353 def __init__(self, host, port=None, key_file=None, cert_file=None,
jpayne@68 1354 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
jpayne@68 1355 source_address=None, *, context=None,
jpayne@68 1356 check_hostname=None, blocksize=8192):
jpayne@68 1357 super(HTTPSConnection, self).__init__(host, port, timeout,
jpayne@68 1358 source_address,
jpayne@68 1359 blocksize=blocksize)
jpayne@68 1360 if (key_file is not None or cert_file is not None or
jpayne@68 1361 check_hostname is not None):
jpayne@68 1362 import warnings
jpayne@68 1363 warnings.warn("key_file, cert_file and check_hostname are "
jpayne@68 1364 "deprecated, use a custom context instead.",
jpayne@68 1365 DeprecationWarning, 2)
jpayne@68 1366 self.key_file = key_file
jpayne@68 1367 self.cert_file = cert_file
jpayne@68 1368 if context is None:
jpayne@68 1369 context = ssl._create_default_https_context()
jpayne@68 1370 # enable PHA for TLS 1.3 connections if available
jpayne@68 1371 if context.post_handshake_auth is not None:
jpayne@68 1372 context.post_handshake_auth = True
jpayne@68 1373 will_verify = context.verify_mode != ssl.CERT_NONE
jpayne@68 1374 if check_hostname is None:
jpayne@68 1375 check_hostname = context.check_hostname
jpayne@68 1376 if check_hostname and not will_verify:
jpayne@68 1377 raise ValueError("check_hostname needs a SSL context with "
jpayne@68 1378 "either CERT_OPTIONAL or CERT_REQUIRED")
jpayne@68 1379 if key_file or cert_file:
jpayne@68 1380 context.load_cert_chain(cert_file, key_file)
jpayne@68 1381 # cert and key file means the user wants to authenticate.
jpayne@68 1382 # enable TLS 1.3 PHA implicitly even for custom contexts.
jpayne@68 1383 if context.post_handshake_auth is not None:
jpayne@68 1384 context.post_handshake_auth = True
jpayne@68 1385 self._context = context
jpayne@68 1386 if check_hostname is not None:
jpayne@68 1387 self._context.check_hostname = check_hostname
jpayne@68 1388
jpayne@68 1389 def connect(self):
jpayne@68 1390 "Connect to a host on a given (SSL) port."
jpayne@68 1391
jpayne@68 1392 super().connect()
jpayne@68 1393
jpayne@68 1394 if self._tunnel_host:
jpayne@68 1395 server_hostname = self._tunnel_host
jpayne@68 1396 else:
jpayne@68 1397 server_hostname = self.host
jpayne@68 1398
jpayne@68 1399 self.sock = self._context.wrap_socket(self.sock,
jpayne@68 1400 server_hostname=server_hostname)
jpayne@68 1401
jpayne@68 1402 __all__.append("HTTPSConnection")
jpayne@68 1403
jpayne@68 1404 class HTTPException(Exception):
jpayne@68 1405 # Subclasses that define an __init__ must call Exception.__init__
jpayne@68 1406 # or define self.args. Otherwise, str() will fail.
jpayne@68 1407 pass
jpayne@68 1408
jpayne@68 1409 class NotConnected(HTTPException):
jpayne@68 1410 pass
jpayne@68 1411
jpayne@68 1412 class InvalidURL(HTTPException):
jpayne@68 1413 pass
jpayne@68 1414
jpayne@68 1415 class UnknownProtocol(HTTPException):
jpayne@68 1416 def __init__(self, version):
jpayne@68 1417 self.args = version,
jpayne@68 1418 self.version = version
jpayne@68 1419
jpayne@68 1420 class UnknownTransferEncoding(HTTPException):
jpayne@68 1421 pass
jpayne@68 1422
jpayne@68 1423 class UnimplementedFileMode(HTTPException):
jpayne@68 1424 pass
jpayne@68 1425
jpayne@68 1426 class IncompleteRead(HTTPException):
jpayne@68 1427 def __init__(self, partial, expected=None):
jpayne@68 1428 self.args = partial,
jpayne@68 1429 self.partial = partial
jpayne@68 1430 self.expected = expected
jpayne@68 1431 def __repr__(self):
jpayne@68 1432 if self.expected is not None:
jpayne@68 1433 e = ', %i more expected' % self.expected
jpayne@68 1434 else:
jpayne@68 1435 e = ''
jpayne@68 1436 return '%s(%i bytes read%s)' % (self.__class__.__name__,
jpayne@68 1437 len(self.partial), e)
jpayne@68 1438 __str__ = object.__str__
jpayne@68 1439
jpayne@68 1440 class ImproperConnectionState(HTTPException):
jpayne@68 1441 pass
jpayne@68 1442
jpayne@68 1443 class CannotSendRequest(ImproperConnectionState):
jpayne@68 1444 pass
jpayne@68 1445
jpayne@68 1446 class CannotSendHeader(ImproperConnectionState):
jpayne@68 1447 pass
jpayne@68 1448
jpayne@68 1449 class ResponseNotReady(ImproperConnectionState):
jpayne@68 1450 pass
jpayne@68 1451
jpayne@68 1452 class BadStatusLine(HTTPException):
jpayne@68 1453 def __init__(self, line):
jpayne@68 1454 if not line:
jpayne@68 1455 line = repr(line)
jpayne@68 1456 self.args = line,
jpayne@68 1457 self.line = line
jpayne@68 1458
jpayne@68 1459 class LineTooLong(HTTPException):
jpayne@68 1460 def __init__(self, line_type):
jpayne@68 1461 HTTPException.__init__(self, "got more than %d bytes when reading %s"
jpayne@68 1462 % (_MAXLINE, line_type))
jpayne@68 1463
jpayne@68 1464 class RemoteDisconnected(ConnectionResetError, BadStatusLine):
jpayne@68 1465 def __init__(self, *pos, **kw):
jpayne@68 1466 BadStatusLine.__init__(self, "")
jpayne@68 1467 ConnectionResetError.__init__(self, *pos, **kw)
jpayne@68 1468
jpayne@68 1469 # for backwards compatibility
jpayne@68 1470 error = HTTPException