jpayne@68: r"""HTTP/1.1 client library
jpayne@68: 
jpayne@68: <intro stuff goes here>
jpayne@68: <other stuff, too>
jpayne@68: 
jpayne@68: HTTPConnection goes through a number of "states", which define when a client
jpayne@68: may legally make another request or fetch the response for a particular
jpayne@68: request. This diagram details these state transitions:
jpayne@68: 
jpayne@68:     (null)
jpayne@68:       |
jpayne@68:       | HTTPConnection()
jpayne@68:       v
jpayne@68:     Idle
jpayne@68:       |
jpayne@68:       | putrequest()
jpayne@68:       v
jpayne@68:     Request-started
jpayne@68:       |
jpayne@68:       | ( putheader() )*  endheaders()
jpayne@68:       v
jpayne@68:     Request-sent
jpayne@68:       |\_____________________________
jpayne@68:       |                              | getresponse() raises
jpayne@68:       | response = getresponse()     | ConnectionError
jpayne@68:       v                              v
jpayne@68:     Unread-response                Idle
jpayne@68:     [Response-headers-read]
jpayne@68:       |\____________________
jpayne@68:       |                     |
jpayne@68:       | response.read()     | putrequest()
jpayne@68:       v                     v
jpayne@68:     Idle                  Req-started-unread-response
jpayne@68:                      ______/|
jpayne@68:                    /        |
jpayne@68:    response.read() |        | ( putheader() )*  endheaders()
jpayne@68:                    v        v
jpayne@68:        Request-started    Req-sent-unread-response
jpayne@68:                             |
jpayne@68:                             | response.read()
jpayne@68:                             v
jpayne@68:                           Request-sent
jpayne@68: 
jpayne@68: This diagram presents the following rules:
jpayne@68:   -- a second request may not be started until {response-headers-read}
jpayne@68:   -- a response [object] cannot be retrieved until {request-sent}
jpayne@68:   -- there is no differentiation between an unread response body and a
jpayne@68:      partially read response body
jpayne@68: 
jpayne@68: Note: this enforcement is applied by the HTTPConnection class. The
jpayne@68:       HTTPResponse class does not enforce this state machine, which
jpayne@68:       implies sophisticated clients may accelerate the request/response
jpayne@68:       pipeline. Caution should be taken, though: accelerating the states
jpayne@68:       beyond the above pattern may imply knowledge of the server's
jpayne@68:       connection-close behavior for certain requests. For example, it
jpayne@68:       is impossible to tell whether the server will close the connection
jpayne@68:       UNTIL the response headers have been read; this means that further
jpayne@68:       requests cannot be placed into the pipeline until it is known that
jpayne@68:       the server will NOT be closing the connection.
jpayne@68: 
jpayne@68: Logical State                  __state            __response
jpayne@68: -------------                  -------            ----------
jpayne@68: Idle                           _CS_IDLE           None
jpayne@68: Request-started                _CS_REQ_STARTED    None
jpayne@68: Request-sent                   _CS_REQ_SENT       None
jpayne@68: Unread-response                _CS_IDLE           <response_class>
jpayne@68: Req-started-unread-response    _CS_REQ_STARTED    <response_class>
jpayne@68: Req-sent-unread-response       _CS_REQ_SENT       <response_class>
jpayne@68: """
jpayne@68: 
jpayne@68: import email.parser
jpayne@68: import email.message
jpayne@68: import http
jpayne@68: import io
jpayne@68: import re
jpayne@68: import socket
jpayne@68: import collections.abc
jpayne@68: from urllib.parse import urlsplit
jpayne@68: 
jpayne@68: # HTTPMessage, parse_headers(), and the HTTP status code constants are
jpayne@68: # intentionally omitted for simplicity
jpayne@68: __all__ = ["HTTPResponse", "HTTPConnection",
jpayne@68:            "HTTPException", "NotConnected", "UnknownProtocol",
jpayne@68:            "UnknownTransferEncoding", "UnimplementedFileMode",
jpayne@68:            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
jpayne@68:            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
jpayne@68:            "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error",
jpayne@68:            "responses"]
jpayne@68: 
jpayne@68: HTTP_PORT = 80
jpayne@68: HTTPS_PORT = 443
jpayne@68: 
jpayne@68: _UNKNOWN = 'UNKNOWN'
jpayne@68: 
jpayne@68: # connection states
jpayne@68: _CS_IDLE = 'Idle'
jpayne@68: _CS_REQ_STARTED = 'Request-started'
jpayne@68: _CS_REQ_SENT = 'Request-sent'
jpayne@68: 
jpayne@68: 
jpayne@68: # hack to maintain backwards compatibility
jpayne@68: globals().update(http.HTTPStatus.__members__)
jpayne@68: 
jpayne@68: # another hack to maintain backwards compatibility
jpayne@68: # Mapping status codes to official W3C names
jpayne@68: responses = {v: v.phrase for v in http.HTTPStatus.__members__.values()}
jpayne@68: 
jpayne@68: # maximal line length when calling readline().
jpayne@68: _MAXLINE = 65536
jpayne@68: _MAXHEADERS = 100
jpayne@68: 
jpayne@68: # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
jpayne@68: #
jpayne@68: # VCHAR          = %x21-7E
jpayne@68: # obs-text       = %x80-FF
jpayne@68: # header-field   = field-name ":" OWS field-value OWS
jpayne@68: # field-name     = token
jpayne@68: # field-value    = *( field-content / obs-fold )
jpayne@68: # field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
jpayne@68: # field-vchar    = VCHAR / obs-text
jpayne@68: #
jpayne@68: # obs-fold       = CRLF 1*( SP / HTAB )
jpayne@68: #                ; obsolete line folding
jpayne@68: #                ; see Section 3.2.4
jpayne@68: 
jpayne@68: # token          = 1*tchar
jpayne@68: #
jpayne@68: # tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
jpayne@68: #                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
jpayne@68: #                / DIGIT / ALPHA
jpayne@68: #                ; any VCHAR, except delimiters
jpayne@68: #
jpayne@68: # VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
jpayne@68: 
jpayne@68: # the patterns for both name and value are more lenient than RFC
jpayne@68: # definitions to allow for backwards compatibility
jpayne@68: _is_legal_header_name = re.compile(rb'[^:\s][^:\r\n]*').fullmatch
jpayne@68: _is_illegal_header_value = re.compile(rb'\n(?![ \t])|\r(?![ \t\n])').search
jpayne@68: 
jpayne@68: # These characters are not allowed within HTTP URL paths.
jpayne@68: #  See https://tools.ietf.org/html/rfc3986#section-3.3 and the
jpayne@68: #  https://tools.ietf.org/html/rfc3986#appendix-A pchar definition.
jpayne@68: # Prevents CVE-2019-9740.  Includes control characters such as \r\n.
jpayne@68: # We don't restrict chars above \x7f as putrequest() limits us to ASCII.
jpayne@68: _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]')
jpayne@68: # Arguably only these _should_ allowed:
jpayne@68: #  _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$")
jpayne@68: # We are more lenient for assumed real world compatibility purposes.
jpayne@68: 
jpayne@68: # We always set the Content-Length header for these methods because some
jpayne@68: # servers will otherwise respond with a 411
jpayne@68: _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
jpayne@68: 
jpayne@68: 
jpayne@68: def _encode(data, name='data'):
jpayne@68:     """Call data.encode("latin-1") but show a better error message."""
jpayne@68:     try:
jpayne@68:         return data.encode("latin-1")
jpayne@68:     except UnicodeEncodeError as err:
jpayne@68:         raise UnicodeEncodeError(
jpayne@68:             err.encoding,
jpayne@68:             err.object,
jpayne@68:             err.start,
jpayne@68:             err.end,
jpayne@68:             "%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "
jpayne@68:             "if you want to send it encoded in UTF-8." %
jpayne@68:             (name.title(), data[err.start:err.end], name)) from None
jpayne@68: 
jpayne@68: 
jpayne@68: class HTTPMessage(email.message.Message):
jpayne@68:     # XXX The only usage of this method is in
jpayne@68:     # http.server.CGIHTTPRequestHandler.  Maybe move the code there so
jpayne@68:     # that it doesn't need to be part of the public API.  The API has
jpayne@68:     # never been defined so this could cause backwards compatibility
jpayne@68:     # issues.
jpayne@68: 
jpayne@68:     def getallmatchingheaders(self, name):
jpayne@68:         """Find all header lines matching a given header name.
jpayne@68: 
jpayne@68:         Look through the list of headers and find all lines matching a given
jpayne@68:         header name (and their continuation lines).  A list of the lines is
jpayne@68:         returned, without interpretation.  If the header does not occur, an
jpayne@68:         empty list is returned.  If the header occurs multiple times, all
jpayne@68:         occurrences are returned.  Case is not important in the header name.
jpayne@68: 
jpayne@68:         """
jpayne@68:         name = name.lower() + ':'
jpayne@68:         n = len(name)
jpayne@68:         lst = []
jpayne@68:         hit = 0
jpayne@68:         for line in self.keys():
jpayne@68:             if line[:n].lower() == name:
jpayne@68:                 hit = 1
jpayne@68:             elif not line[:1].isspace():
jpayne@68:                 hit = 0
jpayne@68:             if hit:
jpayne@68:                 lst.append(line)
jpayne@68:         return lst
jpayne@68: 
jpayne@68: def parse_headers(fp, _class=HTTPMessage):
jpayne@68:     """Parses only RFC2822 headers from a file pointer.
jpayne@68: 
jpayne@68:     email Parser wants to see strings rather than bytes.
jpayne@68:     But a TextIOWrapper around self.rfile would buffer too many bytes
jpayne@68:     from the stream, bytes which we later need to read as bytes.
jpayne@68:     So we read the correct bytes here, as bytes, for email Parser
jpayne@68:     to parse.
jpayne@68: 
jpayne@68:     """
jpayne@68:     headers = []
jpayne@68:     while True:
jpayne@68:         line = fp.readline(_MAXLINE + 1)
jpayne@68:         if len(line) > _MAXLINE:
jpayne@68:             raise LineTooLong("header line")
jpayne@68:         headers.append(line)
jpayne@68:         if len(headers) > _MAXHEADERS:
jpayne@68:             raise HTTPException("got more than %d headers" % _MAXHEADERS)
jpayne@68:         if line in (b'\r\n', b'\n', b''):
jpayne@68:             break
jpayne@68:     hstring = b''.join(headers).decode('iso-8859-1')
jpayne@68:     return email.parser.Parser(_class=_class).parsestr(hstring)
jpayne@68: 
jpayne@68: 
jpayne@68: class HTTPResponse(io.BufferedIOBase):
jpayne@68: 
jpayne@68:     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
jpayne@68: 
jpayne@68:     # The bytes from the socket object are iso-8859-1 strings.
jpayne@68:     # See RFC 2616 sec 2.2 which notes an exception for MIME-encoded
jpayne@68:     # text following RFC 2047.  The basic status line parsing only
jpayne@68:     # accepts iso-8859-1.
jpayne@68: 
jpayne@68:     def __init__(self, sock, debuglevel=0, method=None, url=None):
jpayne@68:         # If the response includes a content-length header, we need to
jpayne@68:         # make sure that the client doesn't read more than the
jpayne@68:         # specified number of bytes.  If it does, it will block until
jpayne@68:         # the server times out and closes the connection.  This will
jpayne@68:         # happen if a self.fp.read() is done (without a size) whether
jpayne@68:         # self.fp is buffered or not.  So, no self.fp.read() by
jpayne@68:         # clients unless they know what they are doing.
jpayne@68:         self.fp = sock.makefile("rb")
jpayne@68:         self.debuglevel = debuglevel
jpayne@68:         self._method = method
jpayne@68: 
jpayne@68:         # The HTTPResponse object is returned via urllib.  The clients
jpayne@68:         # of http and urllib expect different attributes for the
jpayne@68:         # headers.  headers is used here and supports urllib.  msg is
jpayne@68:         # provided as a backwards compatibility layer for http
jpayne@68:         # clients.
jpayne@68: 
jpayne@68:         self.headers = self.msg = None
jpayne@68: 
jpayne@68:         # from the Status-Line of the response
jpayne@68:         self.version = _UNKNOWN # HTTP-Version
jpayne@68:         self.status = _UNKNOWN  # Status-Code
jpayne@68:         self.reason = _UNKNOWN  # Reason-Phrase
jpayne@68: 
jpayne@68:         self.chunked = _UNKNOWN         # is "chunked" being used?
jpayne@68:         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
jpayne@68:         self.length = _UNKNOWN          # number of bytes left in response
jpayne@68:         self.will_close = _UNKNOWN      # conn will close at end of response
jpayne@68: 
jpayne@68:     def _read_status(self):
jpayne@68:         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
jpayne@68:         if len(line) > _MAXLINE:
jpayne@68:             raise LineTooLong("status line")
jpayne@68:         if self.debuglevel > 0:
jpayne@68:             print("reply:", repr(line))
jpayne@68:         if not line:
jpayne@68:             # Presumably, the server closed the connection before
jpayne@68:             # sending a valid response.
jpayne@68:             raise RemoteDisconnected("Remote end closed connection without"
jpayne@68:                                      " response")
jpayne@68:         try:
jpayne@68:             version, status, reason = line.split(None, 2)
jpayne@68:         except ValueError:
jpayne@68:             try:
jpayne@68:                 version, status = line.split(None, 1)
jpayne@68:                 reason = ""
jpayne@68:             except ValueError:
jpayne@68:                 # empty version will cause next test to fail.
jpayne@68:                 version = ""
jpayne@68:         if not version.startswith("HTTP/"):
jpayne@68:             self._close_conn()
jpayne@68:             raise BadStatusLine(line)
jpayne@68: 
jpayne@68:         # The status code is a three-digit number
jpayne@68:         try:
jpayne@68:             status = int(status)
jpayne@68:             if status < 100 or status > 999:
jpayne@68:                 raise BadStatusLine(line)
jpayne@68:         except ValueError:
jpayne@68:             raise BadStatusLine(line)
jpayne@68:         return version, status, reason
jpayne@68: 
jpayne@68:     def begin(self):
jpayne@68:         if self.headers is not None:
jpayne@68:             # we've already started reading the response
jpayne@68:             return
jpayne@68: 
jpayne@68:         # read until we get a non-100 response
jpayne@68:         while True:
jpayne@68:             version, status, reason = self._read_status()
jpayne@68:             if status != CONTINUE:
jpayne@68:                 break
jpayne@68:             # skip the header from the 100 response
jpayne@68:             while True:
jpayne@68:                 skip = self.fp.readline(_MAXLINE + 1)
jpayne@68:                 if len(skip) > _MAXLINE:
jpayne@68:                     raise LineTooLong("header line")
jpayne@68:                 skip = skip.strip()
jpayne@68:                 if not skip:
jpayne@68:                     break
jpayne@68:                 if self.debuglevel > 0:
jpayne@68:                     print("header:", skip)
jpayne@68: 
jpayne@68:         self.code = self.status = status
jpayne@68:         self.reason = reason.strip()
jpayne@68:         if version in ("HTTP/1.0", "HTTP/0.9"):
jpayne@68:             # Some servers might still return "0.9", treat it as 1.0 anyway
jpayne@68:             self.version = 10
jpayne@68:         elif version.startswith("HTTP/1."):
jpayne@68:             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
jpayne@68:         else:
jpayne@68:             raise UnknownProtocol(version)
jpayne@68: 
jpayne@68:         self.headers = self.msg = parse_headers(self.fp)
jpayne@68: 
jpayne@68:         if self.debuglevel > 0:
jpayne@68:             for hdr, val in self.headers.items():
jpayne@68:                 print("header:", hdr + ":", val)
jpayne@68: 
jpayne@68:         # are we using the chunked-style of transfer encoding?
jpayne@68:         tr_enc = self.headers.get("transfer-encoding")
jpayne@68:         if tr_enc and tr_enc.lower() == "chunked":
jpayne@68:             self.chunked = True
jpayne@68:             self.chunk_left = None
jpayne@68:         else:
jpayne@68:             self.chunked = False
jpayne@68: 
jpayne@68:         # will the connection close at the end of the response?
jpayne@68:         self.will_close = self._check_close()
jpayne@68: 
jpayne@68:         # do we have a Content-Length?
jpayne@68:         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
jpayne@68:         self.length = None
jpayne@68:         length = self.headers.get("content-length")
jpayne@68: 
jpayne@68:          # are we using the chunked-style of transfer encoding?
jpayne@68:         tr_enc = self.headers.get("transfer-encoding")
jpayne@68:         if length and not self.chunked:
jpayne@68:             try:
jpayne@68:                 self.length = int(length)
jpayne@68:             except ValueError:
jpayne@68:                 self.length = None
jpayne@68:             else:
jpayne@68:                 if self.length < 0:  # ignore nonsensical negative lengths
jpayne@68:                     self.length = None
jpayne@68:         else:
jpayne@68:             self.length = None
jpayne@68: 
jpayne@68:         # does the body have a fixed length? (of zero)
jpayne@68:         if (status == NO_CONTENT or status == NOT_MODIFIED or
jpayne@68:             100 <= status < 200 or      # 1xx codes
jpayne@68:             self._method == "HEAD"):
jpayne@68:             self.length = 0
jpayne@68: 
jpayne@68:         # if the connection remains open, and we aren't using chunked, and
jpayne@68:         # a content-length was not provided, then assume that the connection
jpayne@68:         # WILL close.
jpayne@68:         if (not self.will_close and
jpayne@68:             not self.chunked and
jpayne@68:             self.length is None):
jpayne@68:             self.will_close = True
jpayne@68: 
jpayne@68:     def _check_close(self):
jpayne@68:         conn = self.headers.get("connection")
jpayne@68:         if self.version == 11:
jpayne@68:             # An HTTP/1.1 proxy is assumed to stay open unless
jpayne@68:             # explicitly closed.
jpayne@68:             if conn and "close" in conn.lower():
jpayne@68:                 return True
jpayne@68:             return False
jpayne@68: 
jpayne@68:         # Some HTTP/1.0 implementations have support for persistent
jpayne@68:         # connections, using rules different than HTTP/1.1.
jpayne@68: 
jpayne@68:         # For older HTTP, Keep-Alive indicates persistent connection.
jpayne@68:         if self.headers.get("keep-alive"):
jpayne@68:             return False
jpayne@68: 
jpayne@68:         # At least Akamai returns a "Connection: Keep-Alive" header,
jpayne@68:         # which was supposed to be sent by the client.
jpayne@68:         if conn and "keep-alive" in conn.lower():
jpayne@68:             return False
jpayne@68: 
jpayne@68:         # Proxy-Connection is a netscape hack.
jpayne@68:         pconn = self.headers.get("proxy-connection")
jpayne@68:         if pconn and "keep-alive" in pconn.lower():
jpayne@68:             return False
jpayne@68: 
jpayne@68:         # otherwise, assume it will close
jpayne@68:         return True
jpayne@68: 
jpayne@68:     def _close_conn(self):
jpayne@68:         fp = self.fp
jpayne@68:         self.fp = None
jpayne@68:         fp.close()
jpayne@68: 
jpayne@68:     def close(self):
jpayne@68:         try:
jpayne@68:             super().close() # set "closed" flag
jpayne@68:         finally:
jpayne@68:             if self.fp:
jpayne@68:                 self._close_conn()
jpayne@68: 
jpayne@68:     # These implementations are for the benefit of io.BufferedReader.
jpayne@68: 
jpayne@68:     # XXX This class should probably be revised to act more like
jpayne@68:     # the "raw stream" that BufferedReader expects.
jpayne@68: 
jpayne@68:     def flush(self):
jpayne@68:         super().flush()
jpayne@68:         if self.fp:
jpayne@68:             self.fp.flush()
jpayne@68: 
jpayne@68:     def readable(self):
jpayne@68:         """Always returns True"""
jpayne@68:         return True
jpayne@68: 
jpayne@68:     # End of "raw stream" methods
jpayne@68: 
jpayne@68:     def isclosed(self):
jpayne@68:         """True if the connection is closed."""
jpayne@68:         # NOTE: it is possible that we will not ever call self.close(). This
jpayne@68:         #       case occurs when will_close is TRUE, length is None, and we
jpayne@68:         #       read up to the last byte, but NOT past it.
jpayne@68:         #
jpayne@68:         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
jpayne@68:         #          called, meaning self.isclosed() is meaningful.
jpayne@68:         return self.fp is None
jpayne@68: 
jpayne@68:     def read(self, amt=None):
jpayne@68:         if self.fp is None:
jpayne@68:             return b""
jpayne@68: 
jpayne@68:         if self._method == "HEAD":
jpayne@68:             self._close_conn()
jpayne@68:             return b""
jpayne@68: 
jpayne@68:         if amt is not None:
jpayne@68:             # Amount is given, implement using readinto
jpayne@68:             b = bytearray(amt)
jpayne@68:             n = self.readinto(b)
jpayne@68:             return memoryview(b)[:n].tobytes()
jpayne@68:         else:
jpayne@68:             # Amount is not given (unbounded read) so we must check self.length
jpayne@68:             # and self.chunked
jpayne@68: 
jpayne@68:             if self.chunked:
jpayne@68:                 return self._readall_chunked()
jpayne@68: 
jpayne@68:             if self.length is None:
jpayne@68:                 s = self.fp.read()
jpayne@68:             else:
jpayne@68:                 try:
jpayne@68:                     s = self._safe_read(self.length)
jpayne@68:                 except IncompleteRead:
jpayne@68:                     self._close_conn()
jpayne@68:                     raise
jpayne@68:                 self.length = 0
jpayne@68:             self._close_conn()        # we read everything
jpayne@68:             return s
jpayne@68: 
jpayne@68:     def readinto(self, b):
jpayne@68:         """Read up to len(b) bytes into bytearray b and return the number
jpayne@68:         of bytes read.
jpayne@68:         """
jpayne@68: 
jpayne@68:         if self.fp is None:
jpayne@68:             return 0
jpayne@68: 
jpayne@68:         if self._method == "HEAD":
jpayne@68:             self._close_conn()
jpayne@68:             return 0
jpayne@68: 
jpayne@68:         if self.chunked:
jpayne@68:             return self._readinto_chunked(b)
jpayne@68: 
jpayne@68:         if self.length is not None:
jpayne@68:             if len(b) > self.length:
jpayne@68:                 # clip the read to the "end of response"
jpayne@68:                 b = memoryview(b)[0:self.length]
jpayne@68: 
jpayne@68:         # we do not use _safe_read() here because this may be a .will_close
jpayne@68:         # connection, and the user is reading more bytes than will be provided
jpayne@68:         # (for example, reading in 1k chunks)
jpayne@68:         n = self.fp.readinto(b)
jpayne@68:         if not n and b:
jpayne@68:             # Ideally, we would raise IncompleteRead if the content-length
jpayne@68:             # wasn't satisfied, but it might break compatibility.
jpayne@68:             self._close_conn()
jpayne@68:         elif self.length is not None:
jpayne@68:             self.length -= n
jpayne@68:             if not self.length:
jpayne@68:                 self._close_conn()
jpayne@68:         return n
jpayne@68: 
jpayne@68:     def _read_next_chunk_size(self):
jpayne@68:         # Read the next chunk size from the file
jpayne@68:         line = self.fp.readline(_MAXLINE + 1)
jpayne@68:         if len(line) > _MAXLINE:
jpayne@68:             raise LineTooLong("chunk size")
jpayne@68:         i = line.find(b";")
jpayne@68:         if i >= 0:
jpayne@68:             line = line[:i] # strip chunk-extensions
jpayne@68:         try:
jpayne@68:             return int(line, 16)
jpayne@68:         except ValueError:
jpayne@68:             # close the connection as protocol synchronisation is
jpayne@68:             # probably lost
jpayne@68:             self._close_conn()
jpayne@68:             raise
jpayne@68: 
jpayne@68:     def _read_and_discard_trailer(self):
jpayne@68:         # read and discard trailer up to the CRLF terminator
jpayne@68:         ### note: we shouldn't have any trailers!
jpayne@68:         while True:
jpayne@68:             line = self.fp.readline(_MAXLINE + 1)
jpayne@68:             if len(line) > _MAXLINE:
jpayne@68:                 raise LineTooLong("trailer line")
jpayne@68:             if not line:
jpayne@68:                 # a vanishingly small number of sites EOF without
jpayne@68:                 # sending the trailer
jpayne@68:                 break
jpayne@68:             if line in (b'\r\n', b'\n', b''):
jpayne@68:                 break
jpayne@68: 
jpayne@68:     def _get_chunk_left(self):
jpayne@68:         # return self.chunk_left, reading a new chunk if necessary.
jpayne@68:         # chunk_left == 0: at the end of the current chunk, need to close it
jpayne@68:         # chunk_left == None: No current chunk, should read next.
jpayne@68:         # This function returns non-zero or None if the last chunk has
jpayne@68:         # been read.
jpayne@68:         chunk_left = self.chunk_left
jpayne@68:         if not chunk_left: # Can be 0 or None
jpayne@68:             if chunk_left is not None:
jpayne@68:                 # We are at the end of chunk, discard chunk end
jpayne@68:                 self._safe_read(2)  # toss the CRLF at the end of the chunk
jpayne@68:             try:
jpayne@68:                 chunk_left = self._read_next_chunk_size()
jpayne@68:             except ValueError:
jpayne@68:                 raise IncompleteRead(b'')
jpayne@68:             if chunk_left == 0:
jpayne@68:                 # last chunk: 1*("0") [ chunk-extension ] CRLF
jpayne@68:                 self._read_and_discard_trailer()
jpayne@68:                 # we read everything; close the "file"
jpayne@68:                 self._close_conn()
jpayne@68:                 chunk_left = None
jpayne@68:             self.chunk_left = chunk_left
jpayne@68:         return chunk_left
jpayne@68: 
jpayne@68:     def _readall_chunked(self):
jpayne@68:         assert self.chunked != _UNKNOWN
jpayne@68:         value = []
jpayne@68:         try:
jpayne@68:             while True:
jpayne@68:                 chunk_left = self._get_chunk_left()
jpayne@68:                 if chunk_left is None:
jpayne@68:                     break
jpayne@68:                 value.append(self._safe_read(chunk_left))
jpayne@68:                 self.chunk_left = 0
jpayne@68:             return b''.join(value)
jpayne@68:         except IncompleteRead:
jpayne@68:             raise IncompleteRead(b''.join(value))
jpayne@68: 
jpayne@68:     def _readinto_chunked(self, b):
jpayne@68:         assert self.chunked != _UNKNOWN
jpayne@68:         total_bytes = 0
jpayne@68:         mvb = memoryview(b)
jpayne@68:         try:
jpayne@68:             while True:
jpayne@68:                 chunk_left = self._get_chunk_left()
jpayne@68:                 if chunk_left is None:
jpayne@68:                     return total_bytes
jpayne@68: 
jpayne@68:                 if len(mvb) <= chunk_left:
jpayne@68:                     n = self._safe_readinto(mvb)
jpayne@68:                     self.chunk_left = chunk_left - n
jpayne@68:                     return total_bytes + n
jpayne@68: 
jpayne@68:                 temp_mvb = mvb[:chunk_left]
jpayne@68:                 n = self._safe_readinto(temp_mvb)
jpayne@68:                 mvb = mvb[n:]
jpayne@68:                 total_bytes += n
jpayne@68:                 self.chunk_left = 0
jpayne@68: 
jpayne@68:         except IncompleteRead:
jpayne@68:             raise IncompleteRead(bytes(b[0:total_bytes]))
jpayne@68: 
jpayne@68:     def _safe_read(self, amt):
jpayne@68:         """Read the number of bytes requested.
jpayne@68: 
jpayne@68:         This function should be used when <amt> bytes "should" be present for
jpayne@68:         reading. If the bytes are truly not available (due to EOF), then the
jpayne@68:         IncompleteRead exception can be used to detect the problem.
jpayne@68:         """
jpayne@68:         data = self.fp.read(amt)
jpayne@68:         if len(data) < amt:
jpayne@68:             raise IncompleteRead(data, amt-len(data))
jpayne@68:         return data
jpayne@68: 
jpayne@68:     def _safe_readinto(self, b):
jpayne@68:         """Same as _safe_read, but for reading into a buffer."""
jpayne@68:         amt = len(b)
jpayne@68:         n = self.fp.readinto(b)
jpayne@68:         if n < amt:
jpayne@68:             raise IncompleteRead(bytes(b[:n]), amt-n)
jpayne@68:         return n
jpayne@68: 
jpayne@68:     def read1(self, n=-1):
jpayne@68:         """Read with at most one underlying system call.  If at least one
jpayne@68:         byte is buffered, return that instead.
jpayne@68:         """
jpayne@68:         if self.fp is None or self._method == "HEAD":
jpayne@68:             return b""
jpayne@68:         if self.chunked:
jpayne@68:             return self._read1_chunked(n)
jpayne@68:         if self.length is not None and (n < 0 or n > self.length):
jpayne@68:             n = self.length
jpayne@68:         result = self.fp.read1(n)
jpayne@68:         if not result and n:
jpayne@68:             self._close_conn()
jpayne@68:         elif self.length is not None:
jpayne@68:             self.length -= len(result)
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def peek(self, n=-1):
jpayne@68:         # Having this enables IOBase.readline() to read more than one
jpayne@68:         # byte at a time
jpayne@68:         if self.fp is None or self._method == "HEAD":
jpayne@68:             return b""
jpayne@68:         if self.chunked:
jpayne@68:             return self._peek_chunked(n)
jpayne@68:         return self.fp.peek(n)
jpayne@68: 
jpayne@68:     def readline(self, limit=-1):
jpayne@68:         if self.fp is None or self._method == "HEAD":
jpayne@68:             return b""
jpayne@68:         if self.chunked:
jpayne@68:             # Fallback to IOBase readline which uses peek() and read()
jpayne@68:             return super().readline(limit)
jpayne@68:         if self.length is not None and (limit < 0 or limit > self.length):
jpayne@68:             limit = self.length
jpayne@68:         result = self.fp.readline(limit)
jpayne@68:         if not result and limit:
jpayne@68:             self._close_conn()
jpayne@68:         elif self.length is not None:
jpayne@68:             self.length -= len(result)
jpayne@68:         return result
jpayne@68: 
jpayne@68:     def _read1_chunked(self, n):
jpayne@68:         # Strictly speaking, _get_chunk_left() may cause more than one read,
jpayne@68:         # but that is ok, since that is to satisfy the chunked protocol.
jpayne@68:         chunk_left = self._get_chunk_left()
jpayne@68:         if chunk_left is None or n == 0:
jpayne@68:             return b''
jpayne@68:         if not (0 <= n <= chunk_left):
jpayne@68:             n = chunk_left # if n is negative or larger than chunk_left
jpayne@68:         read = self.fp.read1(n)
jpayne@68:         self.chunk_left -= len(read)
jpayne@68:         if not read:
jpayne@68:             raise IncompleteRead(b"")
jpayne@68:         return read
jpayne@68: 
jpayne@68:     def _peek_chunked(self, n):
jpayne@68:         # Strictly speaking, _get_chunk_left() may cause more than one read,
jpayne@68:         # but that is ok, since that is to satisfy the chunked protocol.
jpayne@68:         try:
jpayne@68:             chunk_left = self._get_chunk_left()
jpayne@68:         except IncompleteRead:
jpayne@68:             return b'' # peek doesn't worry about protocol
jpayne@68:         if chunk_left is None:
jpayne@68:             return b'' # eof
jpayne@68:         # peek is allowed to return more than requested.  Just request the
jpayne@68:         # entire chunk, and truncate what we get.
jpayne@68:         return self.fp.peek(chunk_left)[:chunk_left]
jpayne@68: 
jpayne@68:     def fileno(self):
jpayne@68:         return self.fp.fileno()
jpayne@68: 
jpayne@68:     def getheader(self, name, default=None):
jpayne@68:         '''Returns the value of the header matching *name*.
jpayne@68: 
jpayne@68:         If there are multiple matching headers, the values are
jpayne@68:         combined into a single string separated by commas and spaces.
jpayne@68: 
jpayne@68:         If no matching header is found, returns *default* or None if
jpayne@68:         the *default* is not specified.
jpayne@68: 
jpayne@68:         If the headers are unknown, raises http.client.ResponseNotReady.
jpayne@68: 
jpayne@68:         '''
jpayne@68:         if self.headers is None:
jpayne@68:             raise ResponseNotReady()
jpayne@68:         headers = self.headers.get_all(name) or default
jpayne@68:         if isinstance(headers, str) or not hasattr(headers, '__iter__'):
jpayne@68:             return headers
jpayne@68:         else:
jpayne@68:             return ', '.join(headers)
jpayne@68: 
jpayne@68:     def getheaders(self):
jpayne@68:         """Return list of (header, value) tuples."""
jpayne@68:         if self.headers is None:
jpayne@68:             raise ResponseNotReady()
jpayne@68:         return list(self.headers.items())
jpayne@68: 
jpayne@68:     # We override IOBase.__iter__ so that it doesn't check for closed-ness
jpayne@68: 
jpayne@68:     def __iter__(self):
jpayne@68:         return self
jpayne@68: 
jpayne@68:     # For compatibility with old-style urllib responses.
jpayne@68: 
jpayne@68:     def info(self):
jpayne@68:         '''Returns an instance of the class mimetools.Message containing
jpayne@68:         meta-information associated with the URL.
jpayne@68: 
jpayne@68:         When the method is HTTP, these headers are those returned by
jpayne@68:         the server at the head of the retrieved HTML page (including
jpayne@68:         Content-Length and Content-Type).
jpayne@68: 
jpayne@68:         When the method is FTP, a Content-Length header will be
jpayne@68:         present if (as is now usual) the server passed back a file
jpayne@68:         length in response to the FTP retrieval request. A
jpayne@68:         Content-Type header will be present if the MIME type can be
jpayne@68:         guessed.
jpayne@68: 
jpayne@68:         When the method is local-file, returned headers will include
jpayne@68:         a Date representing the file's last-modified time, a
jpayne@68:         Content-Length giving file size, and a Content-Type
jpayne@68:         containing a guess at the file's type. See also the
jpayne@68:         description of the mimetools module.
jpayne@68: 
jpayne@68:         '''
jpayne@68:         return self.headers
jpayne@68: 
jpayne@68:     def geturl(self):
jpayne@68:         '''Return the real URL of the page.
jpayne@68: 
jpayne@68:         In some cases, the HTTP server redirects a client to another
jpayne@68:         URL. The urlopen() function handles this transparently, but in
jpayne@68:         some cases the caller needs to know which URL the client was
jpayne@68:         redirected to. The geturl() method can be used to get at this
jpayne@68:         redirected URL.
jpayne@68: 
jpayne@68:         '''
jpayne@68:         return self.url
jpayne@68: 
jpayne@68:     def getcode(self):
jpayne@68:         '''Return the HTTP status code that was sent with the response,
jpayne@68:         or None if the URL is not an HTTP URL.
jpayne@68: 
jpayne@68:         '''
jpayne@68:         return self.status
jpayne@68: 
jpayne@68: class HTTPConnection:
jpayne@68: 
jpayne@68:     _http_vsn = 11
jpayne@68:     _http_vsn_str = 'HTTP/1.1'
jpayne@68: 
jpayne@68:     response_class = HTTPResponse
jpayne@68:     default_port = HTTP_PORT
jpayne@68:     auto_open = 1
jpayne@68:     debuglevel = 0
jpayne@68: 
jpayne@68:     @staticmethod
jpayne@68:     def _is_textIO(stream):
jpayne@68:         """Test whether a file-like object is a text or a binary stream.
jpayne@68:         """
jpayne@68:         return isinstance(stream, io.TextIOBase)
jpayne@68: 
jpayne@68:     @staticmethod
jpayne@68:     def _get_content_length(body, method):
jpayne@68:         """Get the content-length based on the body.
jpayne@68: 
jpayne@68:         If the body is None, we set Content-Length: 0 for methods that expect
jpayne@68:         a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
jpayne@68:         any method if the body is a str or bytes-like object and not a file.
jpayne@68:         """
jpayne@68:         if body is None:
jpayne@68:             # do an explicit check for not None here to distinguish
jpayne@68:             # between unset and set but empty
jpayne@68:             if method.upper() in _METHODS_EXPECTING_BODY:
jpayne@68:                 return 0
jpayne@68:             else:
jpayne@68:                 return None
jpayne@68: 
jpayne@68:         if hasattr(body, 'read'):
jpayne@68:             # file-like object.
jpayne@68:             return None
jpayne@68: 
jpayne@68:         try:
jpayne@68:             # does it implement the buffer protocol (bytes, bytearray, array)?
jpayne@68:             mv = memoryview(body)
jpayne@68:             return mv.nbytes
jpayne@68:         except TypeError:
jpayne@68:             pass
jpayne@68: 
jpayne@68:         if isinstance(body, str):
jpayne@68:             return len(body)
jpayne@68: 
jpayne@68:         return None
jpayne@68: 
jpayne@68:     def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
jpayne@68:                  source_address=None, blocksize=8192):
jpayne@68:         self.timeout = timeout
jpayne@68:         self.source_address = source_address
jpayne@68:         self.blocksize = blocksize
jpayne@68:         self.sock = None
jpayne@68:         self._buffer = []
jpayne@68:         self.__response = None
jpayne@68:         self.__state = _CS_IDLE
jpayne@68:         self._method = None
jpayne@68:         self._tunnel_host = None
jpayne@68:         self._tunnel_port = None
jpayne@68:         self._tunnel_headers = {}
jpayne@68: 
jpayne@68:         (self.host, self.port) = self._get_hostport(host, port)
jpayne@68: 
jpayne@68:         # This is stored as an instance variable to allow unit
jpayne@68:         # tests to replace it with a suitable mockup
jpayne@68:         self._create_connection = socket.create_connection
jpayne@68: 
jpayne@68:     def set_tunnel(self, host, port=None, headers=None):
jpayne@68:         """Set up host and port for HTTP CONNECT tunnelling.
jpayne@68: 
jpayne@68:         In a connection that uses HTTP CONNECT tunneling, the host passed to the
jpayne@68:         constructor is used as a proxy server that relays all communication to
jpayne@68:         the endpoint passed to `set_tunnel`. This done by sending an HTTP
jpayne@68:         CONNECT request to the proxy server when the connection is established.
jpayne@68: 
jpayne@68:         This method must be called before the HTML connection has been
jpayne@68:         established.
jpayne@68: 
jpayne@68:         The headers argument should be a mapping of extra HTTP headers to send
jpayne@68:         with the CONNECT request.
jpayne@68:         """
jpayne@68: 
jpayne@68:         if self.sock:
jpayne@68:             raise RuntimeError("Can't set up tunnel for established connection")
jpayne@68: 
jpayne@68:         self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)
jpayne@68:         if headers:
jpayne@68:             self._tunnel_headers = headers
jpayne@68:         else:
jpayne@68:             self._tunnel_headers.clear()
jpayne@68: 
jpayne@68:     def _get_hostport(self, host, port):
jpayne@68:         if port is None:
jpayne@68:             i = host.rfind(':')
jpayne@68:             j = host.rfind(']')         # ipv6 addresses have [...]
jpayne@68:             if i > j:
jpayne@68:                 try:
jpayne@68:                     port = int(host[i+1:])
jpayne@68:                 except ValueError:
jpayne@68:                     if host[i+1:] == "": # http://foo.com:/ == http://foo.com/
jpayne@68:                         port = self.default_port
jpayne@68:                     else:
jpayne@68:                         raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
jpayne@68:                 host = host[:i]
jpayne@68:             else:
jpayne@68:                 port = self.default_port
jpayne@68:             if host and host[0] == '[' and host[-1] == ']':
jpayne@68:                 host = host[1:-1]
jpayne@68: 
jpayne@68:         return (host, port)
jpayne@68: 
jpayne@68:     def set_debuglevel(self, level):
jpayne@68:         self.debuglevel = level
jpayne@68: 
jpayne@68:     def _tunnel(self):
jpayne@68:         connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host,
jpayne@68:             self._tunnel_port)
jpayne@68:         connect_bytes = connect_str.encode("ascii")
jpayne@68:         self.send(connect_bytes)
jpayne@68:         for header, value in self._tunnel_headers.items():
jpayne@68:             header_str = "%s: %s\r\n" % (header, value)
jpayne@68:             header_bytes = header_str.encode("latin-1")
jpayne@68:             self.send(header_bytes)
jpayne@68:         self.send(b'\r\n')
jpayne@68: 
jpayne@68:         response = self.response_class(self.sock, method=self._method)
jpayne@68:         (version, code, message) = response._read_status()
jpayne@68: 
jpayne@68:         if code != http.HTTPStatus.OK:
jpayne@68:             self.close()
jpayne@68:             raise OSError("Tunnel connection failed: %d %s" % (code,
jpayne@68:                                                                message.strip()))
jpayne@68:         while True:
jpayne@68:             line = response.fp.readline(_MAXLINE + 1)
jpayne@68:             if len(line) > _MAXLINE:
jpayne@68:                 raise LineTooLong("header line")
jpayne@68:             if not line:
jpayne@68:                 # for sites which EOF without sending a trailer
jpayne@68:                 break
jpayne@68:             if line in (b'\r\n', b'\n', b''):
jpayne@68:                 break
jpayne@68: 
jpayne@68:             if self.debuglevel > 0:
jpayne@68:                 print('header:', line.decode())
jpayne@68: 
jpayne@68:     def connect(self):
jpayne@68:         """Connect to the host and port specified in __init__."""
jpayne@68:         self.sock = self._create_connection(
jpayne@68:             (self.host,self.port), self.timeout, self.source_address)
jpayne@68:         self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
jpayne@68: 
jpayne@68:         if self._tunnel_host:
jpayne@68:             self._tunnel()
jpayne@68: 
jpayne@68:     def close(self):
jpayne@68:         """Close the connection to the HTTP server."""
jpayne@68:         self.__state = _CS_IDLE
jpayne@68:         try:
jpayne@68:             sock = self.sock
jpayne@68:             if sock:
jpayne@68:                 self.sock = None
jpayne@68:                 sock.close()   # close it manually... there may be other refs
jpayne@68:         finally:
jpayne@68:             response = self.__response
jpayne@68:             if response:
jpayne@68:                 self.__response = None
jpayne@68:                 response.close()
jpayne@68: 
jpayne@68:     def send(self, data):
jpayne@68:         """Send `data' to the server.
jpayne@68:         ``data`` can be a string object, a bytes object, an array object, a
jpayne@68:         file-like object that supports a .read() method, or an iterable object.
jpayne@68:         """
jpayne@68: 
jpayne@68:         if self.sock is None:
jpayne@68:             if self.auto_open:
jpayne@68:                 self.connect()
jpayne@68:             else:
jpayne@68:                 raise NotConnected()
jpayne@68: 
jpayne@68:         if self.debuglevel > 0:
jpayne@68:             print("send:", repr(data))
jpayne@68:         if hasattr(data, "read") :
jpayne@68:             if self.debuglevel > 0:
jpayne@68:                 print("sendIng a read()able")
jpayne@68:             encode = self._is_textIO(data)
jpayne@68:             if encode and self.debuglevel > 0:
jpayne@68:                 print("encoding file using iso-8859-1")
jpayne@68:             while 1:
jpayne@68:                 datablock = data.read(self.blocksize)
jpayne@68:                 if not datablock:
jpayne@68:                     break
jpayne@68:                 if encode:
jpayne@68:                     datablock = datablock.encode("iso-8859-1")
jpayne@68:                 self.sock.sendall(datablock)
jpayne@68:             return
jpayne@68:         try:
jpayne@68:             self.sock.sendall(data)
jpayne@68:         except TypeError:
jpayne@68:             if isinstance(data, collections.abc.Iterable):
jpayne@68:                 for d in data:
jpayne@68:                     self.sock.sendall(d)
jpayne@68:             else:
jpayne@68:                 raise TypeError("data should be a bytes-like object "
jpayne@68:                                 "or an iterable, got %r" % type(data))
jpayne@68: 
jpayne@68:     def _output(self, s):
jpayne@68:         """Add a line of output to the current request buffer.
jpayne@68: 
jpayne@68:         Assumes that the line does *not* end with \\r\\n.
jpayne@68:         """
jpayne@68:         self._buffer.append(s)
jpayne@68: 
jpayne@68:     def _read_readable(self, readable):
jpayne@68:         if self.debuglevel > 0:
jpayne@68:             print("sendIng a read()able")
jpayne@68:         encode = self._is_textIO(readable)
jpayne@68:         if encode and self.debuglevel > 0:
jpayne@68:             print("encoding file using iso-8859-1")
jpayne@68:         while True:
jpayne@68:             datablock = readable.read(self.blocksize)
jpayne@68:             if not datablock:
jpayne@68:                 break
jpayne@68:             if encode:
jpayne@68:                 datablock = datablock.encode("iso-8859-1")
jpayne@68:             yield datablock
jpayne@68: 
jpayne@68:     def _send_output(self, message_body=None, encode_chunked=False):
jpayne@68:         """Send the currently buffered request and clear the buffer.
jpayne@68: 
jpayne@68:         Appends an extra \\r\\n to the buffer.
jpayne@68:         A message_body may be specified, to be appended to the request.
jpayne@68:         """
jpayne@68:         self._buffer.extend((b"", b""))
jpayne@68:         msg = b"\r\n".join(self._buffer)
jpayne@68:         del self._buffer[:]
jpayne@68:         self.send(msg)
jpayne@68: 
jpayne@68:         if message_body is not None:
jpayne@68: 
jpayne@68:             # create a consistent interface to message_body
jpayne@68:             if hasattr(message_body, 'read'):
jpayne@68:                 # Let file-like take precedence over byte-like.  This
jpayne@68:                 # is needed to allow the current position of mmap'ed
jpayne@68:                 # files to be taken into account.
jpayne@68:                 chunks = self._read_readable(message_body)
jpayne@68:             else:
jpayne@68:                 try:
jpayne@68:                     # this is solely to check to see if message_body
jpayne@68:                     # implements the buffer API.  it /would/ be easier
jpayne@68:                     # to capture if PyObject_CheckBuffer was exposed
jpayne@68:                     # to Python.
jpayne@68:                     memoryview(message_body)
jpayne@68:                 except TypeError:
jpayne@68:                     try:
jpayne@68:                         chunks = iter(message_body)
jpayne@68:                     except TypeError:
jpayne@68:                         raise TypeError("message_body should be a bytes-like "
jpayne@68:                                         "object or an iterable, got %r"
jpayne@68:                                         % type(message_body))
jpayne@68:                 else:
jpayne@68:                     # the object implements the buffer interface and
jpayne@68:                     # can be passed directly into socket methods
jpayne@68:                     chunks = (message_body,)
jpayne@68: 
jpayne@68:             for chunk in chunks:
jpayne@68:                 if not chunk:
jpayne@68:                     if self.debuglevel > 0:
jpayne@68:                         print('Zero length chunk ignored')
jpayne@68:                     continue
jpayne@68: 
jpayne@68:                 if encode_chunked and self._http_vsn == 11:
jpayne@68:                     # chunked encoding
jpayne@68:                     chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \
jpayne@68:                         + b'\r\n'
jpayne@68:                 self.send(chunk)
jpayne@68: 
jpayne@68:             if encode_chunked and self._http_vsn == 11:
jpayne@68:                 # end chunked transfer
jpayne@68:                 self.send(b'0\r\n\r\n')
jpayne@68: 
jpayne@68:     def putrequest(self, method, url, skip_host=False,
jpayne@68:                    skip_accept_encoding=False):
jpayne@68:         """Send a request to the server.
jpayne@68: 
jpayne@68:         `method' specifies an HTTP request method, e.g. 'GET'.
jpayne@68:         `url' specifies the object being requested, e.g. '/index.html'.
jpayne@68:         `skip_host' if True does not add automatically a 'Host:' header
jpayne@68:         `skip_accept_encoding' if True does not add automatically an
jpayne@68:            'Accept-Encoding:' header
jpayne@68:         """
jpayne@68: 
jpayne@68:         # if a prior response has been completed, then forget about it.
jpayne@68:         if self.__response and self.__response.isclosed():
jpayne@68:             self.__response = None
jpayne@68: 
jpayne@68: 
jpayne@68:         # in certain cases, we cannot issue another request on this connection.
jpayne@68:         # this occurs when:
jpayne@68:         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
jpayne@68:         #   2) a response to a previous request has signalled that it is going
jpayne@68:         #      to close the connection upon completion.
jpayne@68:         #   3) the headers for the previous response have not been read, thus
jpayne@68:         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
jpayne@68:         #
jpayne@68:         # if there is no prior response, then we can request at will.
jpayne@68:         #
jpayne@68:         # if point (2) is true, then we will have passed the socket to the
jpayne@68:         # response (effectively meaning, "there is no prior response"), and
jpayne@68:         # will open a new one when a new request is made.
jpayne@68:         #
jpayne@68:         # Note: if a prior response exists, then we *can* start a new request.
jpayne@68:         #       We are not allowed to begin fetching the response to this new
jpayne@68:         #       request, however, until that prior response is complete.
jpayne@68:         #
jpayne@68:         if self.__state == _CS_IDLE:
jpayne@68:             self.__state = _CS_REQ_STARTED
jpayne@68:         else:
jpayne@68:             raise CannotSendRequest(self.__state)
jpayne@68: 
jpayne@68:         # Save the method for use later in the response phase
jpayne@68:         self._method = method
jpayne@68: 
jpayne@68:         url = url or '/'
jpayne@68:         self._validate_path(url)
jpayne@68: 
jpayne@68:         request = '%s %s %s' % (method, url, self._http_vsn_str)
jpayne@68: 
jpayne@68:         self._output(self._encode_request(request))
jpayne@68: 
jpayne@68:         if self._http_vsn == 11:
jpayne@68:             # Issue some standard headers for better HTTP/1.1 compliance
jpayne@68: 
jpayne@68:             if not skip_host:
jpayne@68:                 # this header is issued *only* for HTTP/1.1
jpayne@68:                 # connections. more specifically, this means it is
jpayne@68:                 # only issued when the client uses the new
jpayne@68:                 # HTTPConnection() class. backwards-compat clients
jpayne@68:                 # will be using HTTP/1.0 and those clients may be
jpayne@68:                 # issuing this header themselves. we should NOT issue
jpayne@68:                 # it twice; some web servers (such as Apache) barf
jpayne@68:                 # when they see two Host: headers
jpayne@68: 
jpayne@68:                 # If we need a non-standard port,include it in the
jpayne@68:                 # header.  If the request is going through a proxy,
jpayne@68:                 # but the host of the actual URL, not the host of the
jpayne@68:                 # proxy.
jpayne@68: 
jpayne@68:                 netloc = ''
jpayne@68:                 if url.startswith('http'):
jpayne@68:                     nil, netloc, nil, nil, nil = urlsplit(url)
jpayne@68: 
jpayne@68:                 if netloc:
jpayne@68:                     try:
jpayne@68:                         netloc_enc = netloc.encode("ascii")
jpayne@68:                     except UnicodeEncodeError:
jpayne@68:                         netloc_enc = netloc.encode("idna")
jpayne@68:                     self.putheader('Host', netloc_enc)
jpayne@68:                 else:
jpayne@68:                     if self._tunnel_host:
jpayne@68:                         host = self._tunnel_host
jpayne@68:                         port = self._tunnel_port
jpayne@68:                     else:
jpayne@68:                         host = self.host
jpayne@68:                         port = self.port
jpayne@68: 
jpayne@68:                     try:
jpayne@68:                         host_enc = host.encode("ascii")
jpayne@68:                     except UnicodeEncodeError:
jpayne@68:                         host_enc = host.encode("idna")
jpayne@68: 
jpayne@68:                     # As per RFC 273, IPv6 address should be wrapped with []
jpayne@68:                     # when used as Host header
jpayne@68: 
jpayne@68:                     if host.find(':') >= 0:
jpayne@68:                         host_enc = b'[' + host_enc + b']'
jpayne@68: 
jpayne@68:                     if port == self.default_port:
jpayne@68:                         self.putheader('Host', host_enc)
jpayne@68:                     else:
jpayne@68:                         host_enc = host_enc.decode("ascii")
jpayne@68:                         self.putheader('Host', "%s:%s" % (host_enc, port))
jpayne@68: 
jpayne@68:             # note: we are assuming that clients will not attempt to set these
jpayne@68:             #       headers since *this* library must deal with the
jpayne@68:             #       consequences. this also means that when the supporting
jpayne@68:             #       libraries are updated to recognize other forms, then this
jpayne@68:             #       code should be changed (removed or updated).
jpayne@68: 
jpayne@68:             # we only want a Content-Encoding of "identity" since we don't
jpayne@68:             # support encodings such as x-gzip or x-deflate.
jpayne@68:             if not skip_accept_encoding:
jpayne@68:                 self.putheader('Accept-Encoding', 'identity')
jpayne@68: 
jpayne@68:             # we can accept "chunked" Transfer-Encodings, but no others
jpayne@68:             # NOTE: no TE header implies *only* "chunked"
jpayne@68:             #self.putheader('TE', 'chunked')
jpayne@68: 
jpayne@68:             # if TE is supplied in the header, then it must appear in a
jpayne@68:             # Connection header.
jpayne@68:             #self.putheader('Connection', 'TE')
jpayne@68: 
jpayne@68:         else:
jpayne@68:             # For HTTP/1.0, the server will assume "not chunked"
jpayne@68:             pass
jpayne@68: 
jpayne@68:     def _encode_request(self, request):
jpayne@68:         # ASCII also helps prevent CVE-2019-9740.
jpayne@68:         return request.encode('ascii')
jpayne@68: 
jpayne@68:     def _validate_path(self, url):
jpayne@68:         """Validate a url for putrequest."""
jpayne@68:         # Prevent CVE-2019-9740.
jpayne@68:         match = _contains_disallowed_url_pchar_re.search(url)
jpayne@68:         if match:
jpayne@68:             raise InvalidURL(f"URL can't contain control characters. {url!r} "
jpayne@68:                              f"(found at least {match.group()!r})")
jpayne@68: 
jpayne@68:     def putheader(self, header, *values):
jpayne@68:         """Send a request header line to the server.
jpayne@68: 
jpayne@68:         For example: h.putheader('Accept', 'text/html')
jpayne@68:         """
jpayne@68:         if self.__state != _CS_REQ_STARTED:
jpayne@68:             raise CannotSendHeader()
jpayne@68: 
jpayne@68:         if hasattr(header, 'encode'):
jpayne@68:             header = header.encode('ascii')
jpayne@68: 
jpayne@68:         if not _is_legal_header_name(header):
jpayne@68:             raise ValueError('Invalid header name %r' % (header,))
jpayne@68: 
jpayne@68:         values = list(values)
jpayne@68:         for i, one_value in enumerate(values):
jpayne@68:             if hasattr(one_value, 'encode'):
jpayne@68:                 values[i] = one_value.encode('latin-1')
jpayne@68:             elif isinstance(one_value, int):
jpayne@68:                 values[i] = str(one_value).encode('ascii')
jpayne@68: 
jpayne@68:             if _is_illegal_header_value(values[i]):
jpayne@68:                 raise ValueError('Invalid header value %r' % (values[i],))
jpayne@68: 
jpayne@68:         value = b'\r\n\t'.join(values)
jpayne@68:         header = header + b': ' + value
jpayne@68:         self._output(header)
jpayne@68: 
jpayne@68:     def endheaders(self, message_body=None, *, encode_chunked=False):
jpayne@68:         """Indicate that the last header line has been sent to the server.
jpayne@68: 
jpayne@68:         This method sends the request to the server.  The optional message_body
jpayne@68:         argument can be used to pass a message body associated with the
jpayne@68:         request.
jpayne@68:         """
jpayne@68:         if self.__state == _CS_REQ_STARTED:
jpayne@68:             self.__state = _CS_REQ_SENT
jpayne@68:         else:
jpayne@68:             raise CannotSendHeader()
jpayne@68:         self._send_output(message_body, encode_chunked=encode_chunked)
jpayne@68: 
jpayne@68:     def request(self, method, url, body=None, headers={}, *,
jpayne@68:                 encode_chunked=False):
jpayne@68:         """Send a complete request to the server."""
jpayne@68:         self._send_request(method, url, body, headers, encode_chunked)
jpayne@68: 
jpayne@68:     def _send_request(self, method, url, body, headers, encode_chunked):
jpayne@68:         # Honor explicitly requested Host: and Accept-Encoding: headers.
jpayne@68:         header_names = frozenset(k.lower() for k in headers)
jpayne@68:         skips = {}
jpayne@68:         if 'host' in header_names:
jpayne@68:             skips['skip_host'] = 1
jpayne@68:         if 'accept-encoding' in header_names:
jpayne@68:             skips['skip_accept_encoding'] = 1
jpayne@68: 
jpayne@68:         self.putrequest(method, url, **skips)
jpayne@68: 
jpayne@68:         # chunked encoding will happen if HTTP/1.1 is used and either
jpayne@68:         # the caller passes encode_chunked=True or the following
jpayne@68:         # conditions hold:
jpayne@68:         # 1. content-length has not been explicitly set
jpayne@68:         # 2. the body is a file or iterable, but not a str or bytes-like
jpayne@68:         # 3. Transfer-Encoding has NOT been explicitly set by the caller
jpayne@68: 
jpayne@68:         if 'content-length' not in header_names:
jpayne@68:             # only chunk body if not explicitly set for backwards
jpayne@68:             # compatibility, assuming the client code is already handling the
jpayne@68:             # chunking
jpayne@68:             if 'transfer-encoding' not in header_names:
jpayne@68:                 # if content-length cannot be automatically determined, fall
jpayne@68:                 # back to chunked encoding
jpayne@68:                 encode_chunked = False
jpayne@68:                 content_length = self._get_content_length(body, method)
jpayne@68:                 if content_length is None:
jpayne@68:                     if body is not None:
jpayne@68:                         if self.debuglevel > 0:
jpayne@68:                             print('Unable to determine size of %r' % body)
jpayne@68:                         encode_chunked = True
jpayne@68:                         self.putheader('Transfer-Encoding', 'chunked')
jpayne@68:                 else:
jpayne@68:                     self.putheader('Content-Length', str(content_length))
jpayne@68:         else:
jpayne@68:             encode_chunked = False
jpayne@68: 
jpayne@68:         for hdr, value in headers.items():
jpayne@68:             self.putheader(hdr, value)
jpayne@68:         if isinstance(body, str):
jpayne@68:             # RFC 2616 Section 3.7.1 says that text default has a
jpayne@68:             # default charset of iso-8859-1.
jpayne@68:             body = _encode(body, 'body')
jpayne@68:         self.endheaders(body, encode_chunked=encode_chunked)
jpayne@68: 
jpayne@68:     def getresponse(self):
jpayne@68:         """Get the response from the server.
jpayne@68: 
jpayne@68:         If the HTTPConnection is in the correct state, returns an
jpayne@68:         instance of HTTPResponse or of whatever object is returned by
jpayne@68:         the response_class variable.
jpayne@68: 
jpayne@68:         If a request has not been sent or if a previous response has
jpayne@68:         not be handled, ResponseNotReady is raised.  If the HTTP
jpayne@68:         response indicates that the connection should be closed, then
jpayne@68:         it will be closed before the response is returned.  When the
jpayne@68:         connection is closed, the underlying socket is closed.
jpayne@68:         """
jpayne@68: 
jpayne@68:         # if a prior response has been completed, then forget about it.
jpayne@68:         if self.__response and self.__response.isclosed():
jpayne@68:             self.__response = None
jpayne@68: 
jpayne@68:         # if a prior response exists, then it must be completed (otherwise, we
jpayne@68:         # cannot read this response's header to determine the connection-close
jpayne@68:         # behavior)
jpayne@68:         #
jpayne@68:         # note: if a prior response existed, but was connection-close, then the
jpayne@68:         # socket and response were made independent of this HTTPConnection
jpayne@68:         # object since a new request requires that we open a whole new
jpayne@68:         # connection
jpayne@68:         #
jpayne@68:         # this means the prior response had one of two states:
jpayne@68:         #   1) will_close: this connection was reset and the prior socket and
jpayne@68:         #                  response operate independently
jpayne@68:         #   2) persistent: the response was retained and we await its
jpayne@68:         #                  isclosed() status to become true.
jpayne@68:         #
jpayne@68:         if self.__state != _CS_REQ_SENT or self.__response:
jpayne@68:             raise ResponseNotReady(self.__state)
jpayne@68: 
jpayne@68:         if self.debuglevel > 0:
jpayne@68:             response = self.response_class(self.sock, self.debuglevel,
jpayne@68:                                            method=self._method)
jpayne@68:         else:
jpayne@68:             response = self.response_class(self.sock, method=self._method)
jpayne@68: 
jpayne@68:         try:
jpayne@68:             try:
jpayne@68:                 response.begin()
jpayne@68:             except ConnectionError:
jpayne@68:                 self.close()
jpayne@68:                 raise
jpayne@68:             assert response.will_close != _UNKNOWN
jpayne@68:             self.__state = _CS_IDLE
jpayne@68: 
jpayne@68:             if response.will_close:
jpayne@68:                 # this effectively passes the connection to the response
jpayne@68:                 self.close()
jpayne@68:             else:
jpayne@68:                 # remember this, so we can tell when it is complete
jpayne@68:                 self.__response = response
jpayne@68: 
jpayne@68:             return response
jpayne@68:         except:
jpayne@68:             response.close()
jpayne@68:             raise
jpayne@68: 
jpayne@68: try:
jpayne@68:     import ssl
jpayne@68: except ImportError:
jpayne@68:     pass
jpayne@68: else:
jpayne@68:     class HTTPSConnection(HTTPConnection):
jpayne@68:         "This class allows communication via SSL."
jpayne@68: 
jpayne@68:         default_port = HTTPS_PORT
jpayne@68: 
jpayne@68:         # XXX Should key_file and cert_file be deprecated in favour of context?
jpayne@68: 
jpayne@68:         def __init__(self, host, port=None, key_file=None, cert_file=None,
jpayne@68:                      timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
jpayne@68:                      source_address=None, *, context=None,
jpayne@68:                      check_hostname=None, blocksize=8192):
jpayne@68:             super(HTTPSConnection, self).__init__(host, port, timeout,
jpayne@68:                                                   source_address,
jpayne@68:                                                   blocksize=blocksize)
jpayne@68:             if (key_file is not None or cert_file is not None or
jpayne@68:                         check_hostname is not None):
jpayne@68:                 import warnings
jpayne@68:                 warnings.warn("key_file, cert_file and check_hostname are "
jpayne@68:                               "deprecated, use a custom context instead.",
jpayne@68:                               DeprecationWarning, 2)
jpayne@68:             self.key_file = key_file
jpayne@68:             self.cert_file = cert_file
jpayne@68:             if context is None:
jpayne@68:                 context = ssl._create_default_https_context()
jpayne@68:                 # enable PHA for TLS 1.3 connections if available
jpayne@68:                 if context.post_handshake_auth is not None:
jpayne@68:                     context.post_handshake_auth = True
jpayne@68:             will_verify = context.verify_mode != ssl.CERT_NONE
jpayne@68:             if check_hostname is None:
jpayne@68:                 check_hostname = context.check_hostname
jpayne@68:             if check_hostname and not will_verify:
jpayne@68:                 raise ValueError("check_hostname needs a SSL context with "
jpayne@68:                                  "either CERT_OPTIONAL or CERT_REQUIRED")
jpayne@68:             if key_file or cert_file:
jpayne@68:                 context.load_cert_chain(cert_file, key_file)
jpayne@68:                 # cert and key file means the user wants to authenticate.
jpayne@68:                 # enable TLS 1.3 PHA implicitly even for custom contexts.
jpayne@68:                 if context.post_handshake_auth is not None:
jpayne@68:                     context.post_handshake_auth = True
jpayne@68:             self._context = context
jpayne@68:             if check_hostname is not None:
jpayne@68:                 self._context.check_hostname = check_hostname
jpayne@68: 
jpayne@68:         def connect(self):
jpayne@68:             "Connect to a host on a given (SSL) port."
jpayne@68: 
jpayne@68:             super().connect()
jpayne@68: 
jpayne@68:             if self._tunnel_host:
jpayne@68:                 server_hostname = self._tunnel_host
jpayne@68:             else:
jpayne@68:                 server_hostname = self.host
jpayne@68: 
jpayne@68:             self.sock = self._context.wrap_socket(self.sock,
jpayne@68:                                                   server_hostname=server_hostname)
jpayne@68: 
jpayne@68:     __all__.append("HTTPSConnection")
jpayne@68: 
jpayne@68: class HTTPException(Exception):
jpayne@68:     # Subclasses that define an __init__ must call Exception.__init__
jpayne@68:     # or define self.args.  Otherwise, str() will fail.
jpayne@68:     pass
jpayne@68: 
jpayne@68: class NotConnected(HTTPException):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class InvalidURL(HTTPException):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class UnknownProtocol(HTTPException):
jpayne@68:     def __init__(self, version):
jpayne@68:         self.args = version,
jpayne@68:         self.version = version
jpayne@68: 
jpayne@68: class UnknownTransferEncoding(HTTPException):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class UnimplementedFileMode(HTTPException):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class IncompleteRead(HTTPException):
jpayne@68:     def __init__(self, partial, expected=None):
jpayne@68:         self.args = partial,
jpayne@68:         self.partial = partial
jpayne@68:         self.expected = expected
jpayne@68:     def __repr__(self):
jpayne@68:         if self.expected is not None:
jpayne@68:             e = ', %i more expected' % self.expected
jpayne@68:         else:
jpayne@68:             e = ''
jpayne@68:         return '%s(%i bytes read%s)' % (self.__class__.__name__,
jpayne@68:                                         len(self.partial), e)
jpayne@68:     __str__ = object.__str__
jpayne@68: 
jpayne@68: class ImproperConnectionState(HTTPException):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class CannotSendRequest(ImproperConnectionState):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class CannotSendHeader(ImproperConnectionState):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class ResponseNotReady(ImproperConnectionState):
jpayne@68:     pass
jpayne@68: 
jpayne@68: class BadStatusLine(HTTPException):
jpayne@68:     def __init__(self, line):
jpayne@68:         if not line:
jpayne@68:             line = repr(line)
jpayne@68:         self.args = line,
jpayne@68:         self.line = line
jpayne@68: 
jpayne@68: class LineTooLong(HTTPException):
jpayne@68:     def __init__(self, line_type):
jpayne@68:         HTTPException.__init__(self, "got more than %d bytes when reading %s"
jpayne@68:                                      % (_MAXLINE, line_type))
jpayne@68: 
jpayne@68: class RemoteDisconnected(ConnectionResetError, BadStatusLine):
jpayne@68:     def __init__(self, *pos, **kw):
jpayne@68:         BadStatusLine.__init__(self, "")
jpayne@68:         ConnectionResetError.__init__(self, *pos, **kw)
jpayne@68: 
jpayne@68: # for backwards compatibility
jpayne@68: error = HTTPException