jpayne@68: from __future__ import annotations
jpayne@68: 
jpayne@68: import collections
jpayne@68: import io
jpayne@68: import json as _json
jpayne@68: import logging
jpayne@68: import re
jpayne@68: import sys
jpayne@68: import typing
jpayne@68: import warnings
jpayne@68: import zlib
jpayne@68: from contextlib import contextmanager
jpayne@68: from http.client import HTTPMessage as _HttplibHTTPMessage
jpayne@68: from http.client import HTTPResponse as _HttplibHTTPResponse
jpayne@68: from socket import timeout as SocketTimeout
jpayne@68: 
jpayne@68: if typing.TYPE_CHECKING:
jpayne@68:     from ._base_connection import BaseHTTPConnection
jpayne@68: 
jpayne@68: try:
jpayne@68:     try:
jpayne@68:         import brotlicffi as brotli  # type: ignore[import-not-found]
jpayne@68:     except ImportError:
jpayne@68:         import brotli  # type: ignore[import-not-found]
jpayne@68: except ImportError:
jpayne@68:     brotli = None
jpayne@68: 
jpayne@68: try:
jpayne@68:     import zstandard as zstd
jpayne@68: except (AttributeError, ImportError, ValueError):  # Defensive:
jpayne@68:     HAS_ZSTD = False
jpayne@68: else:
jpayne@68:     # The package 'zstandard' added the 'eof' property starting
jpayne@68:     # in v0.18.0 which we require to ensure a complete and
jpayne@68:     # valid zstd stream was fed into the ZstdDecoder.
jpayne@68:     # See: https://github.com/urllib3/urllib3/pull/2624
jpayne@68:     _zstd_version = tuple(
jpayne@68:         map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups())  # type: ignore[union-attr]
jpayne@68:     )
jpayne@68:     if _zstd_version < (0, 18):  # Defensive:
jpayne@68:         HAS_ZSTD = False
jpayne@68:     else:
jpayne@68:         HAS_ZSTD = True
jpayne@68: 
jpayne@68: from . import util
jpayne@68: from ._base_connection import _TYPE_BODY
jpayne@68: from ._collections import HTTPHeaderDict
jpayne@68: from .connection import BaseSSLError, HTTPConnection, HTTPException
jpayne@68: from .exceptions import (
jpayne@68:     BodyNotHttplibCompatible,
jpayne@68:     DecodeError,
jpayne@68:     HTTPError,
jpayne@68:     IncompleteRead,
jpayne@68:     InvalidChunkLength,
jpayne@68:     InvalidHeader,
jpayne@68:     ProtocolError,
jpayne@68:     ReadTimeoutError,
jpayne@68:     ResponseNotChunked,
jpayne@68:     SSLError,
jpayne@68: )
jpayne@68: from .util.response import is_fp_closed, is_response_to_head
jpayne@68: from .util.retry import Retry
jpayne@68: 
jpayne@68: if typing.TYPE_CHECKING:
jpayne@68:     from .connectionpool import HTTPConnectionPool
jpayne@68: 
jpayne@68: log = logging.getLogger(__name__)
jpayne@68: 
jpayne@68: 
jpayne@68: class ContentDecoder:
jpayne@68:     def decompress(self, data: bytes) -> bytes:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def flush(self) -> bytes:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68: 
jpayne@68: class DeflateDecoder(ContentDecoder):
jpayne@68:     def __init__(self) -> None:
jpayne@68:         self._first_try = True
jpayne@68:         self._data = b""
jpayne@68:         self._obj = zlib.decompressobj()
jpayne@68: 
jpayne@68:     def decompress(self, data: bytes) -> bytes:
jpayne@68:         if not data:
jpayne@68:             return data
jpayne@68: 
jpayne@68:         if not self._first_try:
jpayne@68:             return self._obj.decompress(data)
jpayne@68: 
jpayne@68:         self._data += data
jpayne@68:         try:
jpayne@68:             decompressed = self._obj.decompress(data)
jpayne@68:             if decompressed:
jpayne@68:                 self._first_try = False
jpayne@68:                 self._data = None  # type: ignore[assignment]
jpayne@68:             return decompressed
jpayne@68:         except zlib.error:
jpayne@68:             self._first_try = False
jpayne@68:             self._obj = zlib.decompressobj(-zlib.MAX_WBITS)
jpayne@68:             try:
jpayne@68:                 return self.decompress(self._data)
jpayne@68:             finally:
jpayne@68:                 self._data = None  # type: ignore[assignment]
jpayne@68: 
jpayne@68:     def flush(self) -> bytes:
jpayne@68:         return self._obj.flush()
jpayne@68: 
jpayne@68: 
jpayne@68: class GzipDecoderState:
jpayne@68:     FIRST_MEMBER = 0
jpayne@68:     OTHER_MEMBERS = 1
jpayne@68:     SWALLOW_DATA = 2
jpayne@68: 
jpayne@68: 
jpayne@68: class GzipDecoder(ContentDecoder):
jpayne@68:     def __init__(self) -> None:
jpayne@68:         self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
jpayne@68:         self._state = GzipDecoderState.FIRST_MEMBER
jpayne@68: 
jpayne@68:     def decompress(self, data: bytes) -> bytes:
jpayne@68:         ret = bytearray()
jpayne@68:         if self._state == GzipDecoderState.SWALLOW_DATA or not data:
jpayne@68:             return bytes(ret)
jpayne@68:         while True:
jpayne@68:             try:
jpayne@68:                 ret += self._obj.decompress(data)
jpayne@68:             except zlib.error:
jpayne@68:                 previous_state = self._state
jpayne@68:                 # Ignore data after the first error
jpayne@68:                 self._state = GzipDecoderState.SWALLOW_DATA
jpayne@68:                 if previous_state == GzipDecoderState.OTHER_MEMBERS:
jpayne@68:                     # Allow trailing garbage acceptable in other gzip clients
jpayne@68:                     return bytes(ret)
jpayne@68:                 raise
jpayne@68:             data = self._obj.unused_data
jpayne@68:             if not data:
jpayne@68:                 return bytes(ret)
jpayne@68:             self._state = GzipDecoderState.OTHER_MEMBERS
jpayne@68:             self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
jpayne@68: 
jpayne@68:     def flush(self) -> bytes:
jpayne@68:         return self._obj.flush()
jpayne@68: 
jpayne@68: 
jpayne@68: if brotli is not None:
jpayne@68: 
jpayne@68:     class BrotliDecoder(ContentDecoder):
jpayne@68:         # Supports both 'brotlipy' and 'Brotli' packages
jpayne@68:         # since they share an import name. The top branches
jpayne@68:         # are for 'brotlipy' and bottom branches for 'Brotli'
jpayne@68:         def __init__(self) -> None:
jpayne@68:             self._obj = brotli.Decompressor()
jpayne@68:             if hasattr(self._obj, "decompress"):
jpayne@68:                 setattr(self, "decompress", self._obj.decompress)
jpayne@68:             else:
jpayne@68:                 setattr(self, "decompress", self._obj.process)
jpayne@68: 
jpayne@68:         def flush(self) -> bytes:
jpayne@68:             if hasattr(self._obj, "flush"):
jpayne@68:                 return self._obj.flush()  # type: ignore[no-any-return]
jpayne@68:             return b""
jpayne@68: 
jpayne@68: 
jpayne@68: if HAS_ZSTD:
jpayne@68: 
jpayne@68:     class ZstdDecoder(ContentDecoder):
jpayne@68:         def __init__(self) -> None:
jpayne@68:             self._obj = zstd.ZstdDecompressor().decompressobj()
jpayne@68: 
jpayne@68:         def decompress(self, data: bytes) -> bytes:
jpayne@68:             if not data:
jpayne@68:                 return b""
jpayne@68:             data_parts = [self._obj.decompress(data)]
jpayne@68:             while self._obj.eof and self._obj.unused_data:
jpayne@68:                 unused_data = self._obj.unused_data
jpayne@68:                 self._obj = zstd.ZstdDecompressor().decompressobj()
jpayne@68:                 data_parts.append(self._obj.decompress(unused_data))
jpayne@68:             return b"".join(data_parts)
jpayne@68: 
jpayne@68:         def flush(self) -> bytes:
jpayne@68:             ret = self._obj.flush()  # note: this is a no-op
jpayne@68:             if not self._obj.eof:
jpayne@68:                 raise DecodeError("Zstandard data is incomplete")
jpayne@68:             return ret
jpayne@68: 
jpayne@68: 
jpayne@68: class MultiDecoder(ContentDecoder):
jpayne@68:     """
jpayne@68:     From RFC7231:
jpayne@68:         If one or more encodings have been applied to a representation, the
jpayne@68:         sender that applied the encodings MUST generate a Content-Encoding
jpayne@68:         header field that lists the content codings in the order in which
jpayne@68:         they were applied.
jpayne@68:     """
jpayne@68: 
jpayne@68:     def __init__(self, modes: str) -> None:
jpayne@68:         self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
jpayne@68: 
jpayne@68:     def flush(self) -> bytes:
jpayne@68:         return self._decoders[0].flush()
jpayne@68: 
jpayne@68:     def decompress(self, data: bytes) -> bytes:
jpayne@68:         for d in reversed(self._decoders):
jpayne@68:             data = d.decompress(data)
jpayne@68:         return data
jpayne@68: 
jpayne@68: 
jpayne@68: def _get_decoder(mode: str) -> ContentDecoder:
jpayne@68:     if "," in mode:
jpayne@68:         return MultiDecoder(mode)
jpayne@68: 
jpayne@68:     # According to RFC 9110 section 8.4.1.3, recipients should
jpayne@68:     # consider x-gzip equivalent to gzip
jpayne@68:     if mode in ("gzip", "x-gzip"):
jpayne@68:         return GzipDecoder()
jpayne@68: 
jpayne@68:     if brotli is not None and mode == "br":
jpayne@68:         return BrotliDecoder()
jpayne@68: 
jpayne@68:     if HAS_ZSTD and mode == "zstd":
jpayne@68:         return ZstdDecoder()
jpayne@68: 
jpayne@68:     return DeflateDecoder()
jpayne@68: 
jpayne@68: 
jpayne@68: class BytesQueueBuffer:
jpayne@68:     """Memory-efficient bytes buffer
jpayne@68: 
jpayne@68:     To return decoded data in read() and still follow the BufferedIOBase API, we need a
jpayne@68:     buffer to always return the correct amount of bytes.
jpayne@68: 
jpayne@68:     This buffer should be filled using calls to put()
jpayne@68: 
jpayne@68:     Our maximum memory usage is determined by the sum of the size of:
jpayne@68: 
jpayne@68:      * self.buffer, which contains the full data
jpayne@68:      * the largest chunk that we will copy in get()
jpayne@68: 
jpayne@68:     The worst case scenario is a single chunk, in which case we'll make a full copy of
jpayne@68:     the data inside get().
jpayne@68:     """
jpayne@68: 
jpayne@68:     def __init__(self) -> None:
jpayne@68:         self.buffer: typing.Deque[bytes] = collections.deque()
jpayne@68:         self._size: int = 0
jpayne@68: 
jpayne@68:     def __len__(self) -> int:
jpayne@68:         return self._size
jpayne@68: 
jpayne@68:     def put(self, data: bytes) -> None:
jpayne@68:         self.buffer.append(data)
jpayne@68:         self._size += len(data)
jpayne@68: 
jpayne@68:     def get(self, n: int) -> bytes:
jpayne@68:         if n == 0:
jpayne@68:             return b""
jpayne@68:         elif not self.buffer:
jpayne@68:             raise RuntimeError("buffer is empty")
jpayne@68:         elif n < 0:
jpayne@68:             raise ValueError("n should be > 0")
jpayne@68: 
jpayne@68:         fetched = 0
jpayne@68:         ret = io.BytesIO()
jpayne@68:         while fetched < n:
jpayne@68:             remaining = n - fetched
jpayne@68:             chunk = self.buffer.popleft()
jpayne@68:             chunk_length = len(chunk)
jpayne@68:             if remaining < chunk_length:
jpayne@68:                 left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
jpayne@68:                 ret.write(left_chunk)
jpayne@68:                 self.buffer.appendleft(right_chunk)
jpayne@68:                 self._size -= remaining
jpayne@68:                 break
jpayne@68:             else:
jpayne@68:                 ret.write(chunk)
jpayne@68:                 self._size -= chunk_length
jpayne@68:             fetched += chunk_length
jpayne@68: 
jpayne@68:             if not self.buffer:
jpayne@68:                 break
jpayne@68: 
jpayne@68:         return ret.getvalue()
jpayne@68: 
jpayne@68:     def get_all(self) -> bytes:
jpayne@68:         buffer = self.buffer
jpayne@68:         if not buffer:
jpayne@68:             assert self._size == 0
jpayne@68:             return b""
jpayne@68:         if len(buffer) == 1:
jpayne@68:             result = buffer.pop()
jpayne@68:         else:
jpayne@68:             ret = io.BytesIO()
jpayne@68:             ret.writelines(buffer.popleft() for _ in range(len(buffer)))
jpayne@68:             result = ret.getvalue()
jpayne@68:         self._size = 0
jpayne@68:         return result
jpayne@68: 
jpayne@68: 
jpayne@68: class BaseHTTPResponse(io.IOBase):
jpayne@68:     CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"]
jpayne@68:     if brotli is not None:
jpayne@68:         CONTENT_DECODERS += ["br"]
jpayne@68:     if HAS_ZSTD:
jpayne@68:         CONTENT_DECODERS += ["zstd"]
jpayne@68:     REDIRECT_STATUSES = [301, 302, 303, 307, 308]
jpayne@68: 
jpayne@68:     DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error)
jpayne@68:     if brotli is not None:
jpayne@68:         DECODER_ERROR_CLASSES += (brotli.error,)
jpayne@68: 
jpayne@68:     if HAS_ZSTD:
jpayne@68:         DECODER_ERROR_CLASSES += (zstd.ZstdError,)
jpayne@68: 
jpayne@68:     def __init__(
jpayne@68:         self,
jpayne@68:         *,
jpayne@68:         headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
jpayne@68:         status: int,
jpayne@68:         version: int,
jpayne@68:         version_string: str,
jpayne@68:         reason: str | None,
jpayne@68:         decode_content: bool,
jpayne@68:         request_url: str | None,
jpayne@68:         retries: Retry | None = None,
jpayne@68:     ) -> None:
jpayne@68:         if isinstance(headers, HTTPHeaderDict):
jpayne@68:             self.headers = headers
jpayne@68:         else:
jpayne@68:             self.headers = HTTPHeaderDict(headers)  # type: ignore[arg-type]
jpayne@68:         self.status = status
jpayne@68:         self.version = version
jpayne@68:         self.version_string = version_string
jpayne@68:         self.reason = reason
jpayne@68:         self.decode_content = decode_content
jpayne@68:         self._has_decoded_content = False
jpayne@68:         self._request_url: str | None = request_url
jpayne@68:         self.retries = retries
jpayne@68: 
jpayne@68:         self.chunked = False
jpayne@68:         tr_enc = self.headers.get("transfer-encoding", "").lower()
jpayne@68:         # Don't incur the penalty of creating a list and then discarding it
jpayne@68:         encodings = (enc.strip() for enc in tr_enc.split(","))
jpayne@68:         if "chunked" in encodings:
jpayne@68:             self.chunked = True
jpayne@68: 
jpayne@68:         self._decoder: ContentDecoder | None = None
jpayne@68:         self.length_remaining: int | None
jpayne@68: 
jpayne@68:     def get_redirect_location(self) -> str | None | typing.Literal[False]:
jpayne@68:         """
jpayne@68:         Should we redirect and where to?
jpayne@68: 
jpayne@68:         :returns: Truthy redirect location string if we got a redirect status
jpayne@68:             code and valid location. ``None`` if redirect status and no
jpayne@68:             location. ``False`` if not a redirect status code.
jpayne@68:         """
jpayne@68:         if self.status in self.REDIRECT_STATUSES:
jpayne@68:             return self.headers.get("location")
jpayne@68:         return False
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def data(self) -> bytes:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def json(self) -> typing.Any:
jpayne@68:         """
jpayne@68:         Deserializes the body of the HTTP response as a Python object.
jpayne@68: 
jpayne@68:         The body of the HTTP response must be encoded using UTF-8, as per
jpayne@68:         `RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
jpayne@68: 
jpayne@68:         To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
jpayne@68:         your custom decoder instead.
jpayne@68: 
jpayne@68:         If the body of the HTTP response is not decodable to UTF-8, a
jpayne@68:         `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
jpayne@68:         valid JSON document, a `json.JSONDecodeError` will be raised.
jpayne@68: 
jpayne@68:         Read more :ref:`here <json_content>`.
jpayne@68: 
jpayne@68:         :returns: The body of the HTTP response as a Python object.
jpayne@68:         """
jpayne@68:         data = self.data.decode("utf-8")
jpayne@68:         return _json.loads(data)
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def url(self) -> str | None:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     @url.setter
jpayne@68:     def url(self, url: str | None) -> None:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def connection(self) -> BaseHTTPConnection | None:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def retries(self) -> Retry | None:
jpayne@68:         return self._retries
jpayne@68: 
jpayne@68:     @retries.setter
jpayne@68:     def retries(self, retries: Retry | None) -> None:
jpayne@68:         # Override the request_url if retries has a redirect location.
jpayne@68:         if retries is not None and retries.history:
jpayne@68:             self.url = retries.history[-1].redirect_location
jpayne@68:         self._retries = retries
jpayne@68: 
jpayne@68:     def stream(
jpayne@68:         self, amt: int | None = 2**16, decode_content: bool | None = None
jpayne@68:     ) -> typing.Iterator[bytes]:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def read(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         decode_content: bool | None = None,
jpayne@68:         cache_content: bool = False,
jpayne@68:     ) -> bytes:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def read1(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         decode_content: bool | None = None,
jpayne@68:     ) -> bytes:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def read_chunked(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         decode_content: bool | None = None,
jpayne@68:     ) -> typing.Iterator[bytes]:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def release_conn(self) -> None:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def drain_conn(self) -> None:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def close(self) -> None:
jpayne@68:         raise NotImplementedError()
jpayne@68: 
jpayne@68:     def _init_decoder(self) -> None:
jpayne@68:         """
jpayne@68:         Set-up the _decoder attribute if necessary.
jpayne@68:         """
jpayne@68:         # Note: content-encoding value should be case-insensitive, per RFC 7230
jpayne@68:         # Section 3.2
jpayne@68:         content_encoding = self.headers.get("content-encoding", "").lower()
jpayne@68:         if self._decoder is None:
jpayne@68:             if content_encoding in self.CONTENT_DECODERS:
jpayne@68:                 self._decoder = _get_decoder(content_encoding)
jpayne@68:             elif "," in content_encoding:
jpayne@68:                 encodings = [
jpayne@68:                     e.strip()
jpayne@68:                     for e in content_encoding.split(",")
jpayne@68:                     if e.strip() in self.CONTENT_DECODERS
jpayne@68:                 ]
jpayne@68:                 if encodings:
jpayne@68:                     self._decoder = _get_decoder(content_encoding)
jpayne@68: 
jpayne@68:     def _decode(
jpayne@68:         self, data: bytes, decode_content: bool | None, flush_decoder: bool
jpayne@68:     ) -> bytes:
jpayne@68:         """
jpayne@68:         Decode the data passed in and potentially flush the decoder.
jpayne@68:         """
jpayne@68:         if not decode_content:
jpayne@68:             if self._has_decoded_content:
jpayne@68:                 raise RuntimeError(
jpayne@68:                     "Calling read(decode_content=False) is not supported after "
jpayne@68:                     "read(decode_content=True) was called."
jpayne@68:                 )
jpayne@68:             return data
jpayne@68: 
jpayne@68:         try:
jpayne@68:             if self._decoder:
jpayne@68:                 data = self._decoder.decompress(data)
jpayne@68:                 self._has_decoded_content = True
jpayne@68:         except self.DECODER_ERROR_CLASSES as e:
jpayne@68:             content_encoding = self.headers.get("content-encoding", "").lower()
jpayne@68:             raise DecodeError(
jpayne@68:                 "Received response with content-encoding: %s, but "
jpayne@68:                 "failed to decode it." % content_encoding,
jpayne@68:                 e,
jpayne@68:             ) from e
jpayne@68:         if flush_decoder:
jpayne@68:             data += self._flush_decoder()
jpayne@68: 
jpayne@68:         return data
jpayne@68: 
jpayne@68:     def _flush_decoder(self) -> bytes:
jpayne@68:         """
jpayne@68:         Flushes the decoder. Should only be called if the decoder is actually
jpayne@68:         being used.
jpayne@68:         """
jpayne@68:         if self._decoder:
jpayne@68:             return self._decoder.decompress(b"") + self._decoder.flush()
jpayne@68:         return b""
jpayne@68: 
jpayne@68:     # Compatibility methods for `io` module
jpayne@68:     def readinto(self, b: bytearray) -> int:
jpayne@68:         temp = self.read(len(b))
jpayne@68:         if len(temp) == 0:
jpayne@68:             return 0
jpayne@68:         else:
jpayne@68:             b[: len(temp)] = temp
jpayne@68:             return len(temp)
jpayne@68: 
jpayne@68:     # Compatibility methods for http.client.HTTPResponse
jpayne@68:     def getheaders(self) -> HTTPHeaderDict:
jpayne@68:         warnings.warn(
jpayne@68:             "HTTPResponse.getheaders() is deprecated and will be removed "
jpayne@68:             "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.",
jpayne@68:             category=DeprecationWarning,
jpayne@68:             stacklevel=2,
jpayne@68:         )
jpayne@68:         return self.headers
jpayne@68: 
jpayne@68:     def getheader(self, name: str, default: str | None = None) -> str | None:
jpayne@68:         warnings.warn(
jpayne@68:             "HTTPResponse.getheader() is deprecated and will be removed "
jpayne@68:             "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).",
jpayne@68:             category=DeprecationWarning,
jpayne@68:             stacklevel=2,
jpayne@68:         )
jpayne@68:         return self.headers.get(name, default)
jpayne@68: 
jpayne@68:     # Compatibility method for http.cookiejar
jpayne@68:     def info(self) -> HTTPHeaderDict:
jpayne@68:         return self.headers
jpayne@68: 
jpayne@68:     def geturl(self) -> str | None:
jpayne@68:         return self.url
jpayne@68: 
jpayne@68: 
jpayne@68: class HTTPResponse(BaseHTTPResponse):
jpayne@68:     """
jpayne@68:     HTTP Response container.
jpayne@68: 
jpayne@68:     Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is
jpayne@68:     loaded and decoded on-demand when the ``data`` property is accessed.  This
jpayne@68:     class is also compatible with the Python standard library's :mod:`io`
jpayne@68:     module, and can hence be treated as a readable object in the context of that
jpayne@68:     framework.
jpayne@68: 
jpayne@68:     Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`:
jpayne@68: 
jpayne@68:     :param preload_content:
jpayne@68:         If True, the response's body will be preloaded during construction.
jpayne@68: 
jpayne@68:     :param decode_content:
jpayne@68:         If True, will attempt to decode the body based on the
jpayne@68:         'content-encoding' header.
jpayne@68: 
jpayne@68:     :param original_response:
jpayne@68:         When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse`
jpayne@68:         object, it's convenient to include the original for debug purposes. It's
jpayne@68:         otherwise unused.
jpayne@68: 
jpayne@68:     :param retries:
jpayne@68:         The retries contains the last :class:`~urllib3.util.retry.Retry` that
jpayne@68:         was used during the request.
jpayne@68: 
jpayne@68:     :param enforce_content_length:
jpayne@68:         Enforce content length checking. Body returned by server must match
jpayne@68:         value of Content-Length header, if present. Otherwise, raise error.
jpayne@68:     """
jpayne@68: 
jpayne@68:     def __init__(
jpayne@68:         self,
jpayne@68:         body: _TYPE_BODY = "",
jpayne@68:         headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None,
jpayne@68:         status: int = 0,
jpayne@68:         version: int = 0,
jpayne@68:         version_string: str = "HTTP/?",
jpayne@68:         reason: str | None = None,
jpayne@68:         preload_content: bool = True,
jpayne@68:         decode_content: bool = True,
jpayne@68:         original_response: _HttplibHTTPResponse | None = None,
jpayne@68:         pool: HTTPConnectionPool | None = None,
jpayne@68:         connection: HTTPConnection | None = None,
jpayne@68:         msg: _HttplibHTTPMessage | None = None,
jpayne@68:         retries: Retry | None = None,
jpayne@68:         enforce_content_length: bool = True,
jpayne@68:         request_method: str | None = None,
jpayne@68:         request_url: str | None = None,
jpayne@68:         auto_close: bool = True,
jpayne@68:     ) -> None:
jpayne@68:         super().__init__(
jpayne@68:             headers=headers,
jpayne@68:             status=status,
jpayne@68:             version=version,
jpayne@68:             version_string=version_string,
jpayne@68:             reason=reason,
jpayne@68:             decode_content=decode_content,
jpayne@68:             request_url=request_url,
jpayne@68:             retries=retries,
jpayne@68:         )
jpayne@68: 
jpayne@68:         self.enforce_content_length = enforce_content_length
jpayne@68:         self.auto_close = auto_close
jpayne@68: 
jpayne@68:         self._body = None
jpayne@68:         self._fp: _HttplibHTTPResponse | None = None
jpayne@68:         self._original_response = original_response
jpayne@68:         self._fp_bytes_read = 0
jpayne@68:         self.msg = msg
jpayne@68: 
jpayne@68:         if body and isinstance(body, (str, bytes)):
jpayne@68:             self._body = body
jpayne@68: 
jpayne@68:         self._pool = pool
jpayne@68:         self._connection = connection
jpayne@68: 
jpayne@68:         if hasattr(body, "read"):
jpayne@68:             self._fp = body  # type: ignore[assignment]
jpayne@68: 
jpayne@68:         # Are we using the chunked-style of transfer encoding?
jpayne@68:         self.chunk_left: int | None = None
jpayne@68: 
jpayne@68:         # Determine length of response
jpayne@68:         self.length_remaining = self._init_length(request_method)
jpayne@68: 
jpayne@68:         # Used to return the correct amount of bytes for partial read()s
jpayne@68:         self._decoded_buffer = BytesQueueBuffer()
jpayne@68: 
jpayne@68:         # If requested, preload the body.
jpayne@68:         if preload_content and not self._body:
jpayne@68:             self._body = self.read(decode_content=decode_content)
jpayne@68: 
jpayne@68:     def release_conn(self) -> None:
jpayne@68:         if not self._pool or not self._connection:
jpayne@68:             return None
jpayne@68: 
jpayne@68:         self._pool._put_conn(self._connection)
jpayne@68:         self._connection = None
jpayne@68: 
jpayne@68:     def drain_conn(self) -> None:
jpayne@68:         """
jpayne@68:         Read and discard any remaining HTTP response data in the response connection.
jpayne@68: 
jpayne@68:         Unread data in the HTTPResponse connection blocks the connection from being released back to the pool.
jpayne@68:         """
jpayne@68:         try:
jpayne@68:             self.read()
jpayne@68:         except (HTTPError, OSError, BaseSSLError, HTTPException):
jpayne@68:             pass
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def data(self) -> bytes:
jpayne@68:         # For backwards-compat with earlier urllib3 0.4 and earlier.
jpayne@68:         if self._body:
jpayne@68:             return self._body  # type: ignore[return-value]
jpayne@68: 
jpayne@68:         if self._fp:
jpayne@68:             return self.read(cache_content=True)
jpayne@68: 
jpayne@68:         return None  # type: ignore[return-value]
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def connection(self) -> HTTPConnection | None:
jpayne@68:         return self._connection
jpayne@68: 
jpayne@68:     def isclosed(self) -> bool:
jpayne@68:         return is_fp_closed(self._fp)
jpayne@68: 
jpayne@68:     def tell(self) -> int:
jpayne@68:         """
jpayne@68:         Obtain the number of bytes pulled over the wire so far. May differ from
jpayne@68:         the amount of content returned by :meth:``urllib3.response.HTTPResponse.read``
jpayne@68:         if bytes are encoded on the wire (e.g, compressed).
jpayne@68:         """
jpayne@68:         return self._fp_bytes_read
jpayne@68: 
jpayne@68:     def _init_length(self, request_method: str | None) -> int | None:
jpayne@68:         """
jpayne@68:         Set initial length value for Response content if available.
jpayne@68:         """
jpayne@68:         length: int | None
jpayne@68:         content_length: str | None = self.headers.get("content-length")
jpayne@68: 
jpayne@68:         if content_length is not None:
jpayne@68:             if self.chunked:
jpayne@68:                 # This Response will fail with an IncompleteRead if it can't be
jpayne@68:                 # received as chunked. This method falls back to attempt reading
jpayne@68:                 # the response before raising an exception.
jpayne@68:                 log.warning(
jpayne@68:                     "Received response with both Content-Length and "
jpayne@68:                     "Transfer-Encoding set. This is expressly forbidden "
jpayne@68:                     "by RFC 7230 sec 3.3.2. Ignoring Content-Length and "
jpayne@68:                     "attempting to process response as Transfer-Encoding: "
jpayne@68:                     "chunked."
jpayne@68:                 )
jpayne@68:                 return None
jpayne@68: 
jpayne@68:             try:
jpayne@68:                 # RFC 7230 section 3.3.2 specifies multiple content lengths can
jpayne@68:                 # be sent in a single Content-Length header
jpayne@68:                 # (e.g. Content-Length: 42, 42). This line ensures the values
jpayne@68:                 # are all valid ints and that as long as the `set` length is 1,
jpayne@68:                 # all values are the same. Otherwise, the header is invalid.
jpayne@68:                 lengths = {int(val) for val in content_length.split(",")}
jpayne@68:                 if len(lengths) > 1:
jpayne@68:                     raise InvalidHeader(
jpayne@68:                         "Content-Length contained multiple "
jpayne@68:                         "unmatching values (%s)" % content_length
jpayne@68:                     )
jpayne@68:                 length = lengths.pop()
jpayne@68:             except ValueError:
jpayne@68:                 length = None
jpayne@68:             else:
jpayne@68:                 if length < 0:
jpayne@68:                     length = None
jpayne@68: 
jpayne@68:         else:  # if content_length is None
jpayne@68:             length = None
jpayne@68: 
jpayne@68:         # Convert status to int for comparison
jpayne@68:         # In some cases, httplib returns a status of "_UNKNOWN"
jpayne@68:         try:
jpayne@68:             status = int(self.status)
jpayne@68:         except ValueError:
jpayne@68:             status = 0
jpayne@68: 
jpayne@68:         # Check for responses that shouldn't include a body
jpayne@68:         if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD":
jpayne@68:             length = 0
jpayne@68: 
jpayne@68:         return length
jpayne@68: 
jpayne@68:     @contextmanager
jpayne@68:     def _error_catcher(self) -> typing.Generator[None, None, None]:
jpayne@68:         """
jpayne@68:         Catch low-level python exceptions, instead re-raising urllib3
jpayne@68:         variants, so that low-level exceptions are not leaked in the
jpayne@68:         high-level api.
jpayne@68: 
jpayne@68:         On exit, release the connection back to the pool.
jpayne@68:         """
jpayne@68:         clean_exit = False
jpayne@68: 
jpayne@68:         try:
jpayne@68:             try:
jpayne@68:                 yield
jpayne@68: 
jpayne@68:             except SocketTimeout as e:
jpayne@68:                 # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
jpayne@68:                 # there is yet no clean way to get at it from this context.
jpayne@68:                 raise ReadTimeoutError(self._pool, None, "Read timed out.") from e  # type: ignore[arg-type]
jpayne@68: 
jpayne@68:             except BaseSSLError as e:
jpayne@68:                 # FIXME: Is there a better way to differentiate between SSLErrors?
jpayne@68:                 if "read operation timed out" not in str(e):
jpayne@68:                     # SSL errors related to framing/MAC get wrapped and reraised here
jpayne@68:                     raise SSLError(e) from e
jpayne@68: 
jpayne@68:                 raise ReadTimeoutError(self._pool, None, "Read timed out.") from e  # type: ignore[arg-type]
jpayne@68: 
jpayne@68:             except IncompleteRead as e:
jpayne@68:                 if (
jpayne@68:                     e.expected is not None
jpayne@68:                     and e.partial is not None
jpayne@68:                     and e.expected == -e.partial
jpayne@68:                 ):
jpayne@68:                     arg = "Response may not contain content."
jpayne@68:                 else:
jpayne@68:                     arg = f"Connection broken: {e!r}"
jpayne@68:                 raise ProtocolError(arg, e) from e
jpayne@68: 
jpayne@68:             except (HTTPException, OSError) as e:
jpayne@68:                 raise ProtocolError(f"Connection broken: {e!r}", e) from e
jpayne@68: 
jpayne@68:             # If no exception is thrown, we should avoid cleaning up
jpayne@68:             # unnecessarily.
jpayne@68:             clean_exit = True
jpayne@68:         finally:
jpayne@68:             # If we didn't terminate cleanly, we need to throw away our
jpayne@68:             # connection.
jpayne@68:             if not clean_exit:
jpayne@68:                 # The response may not be closed but we're not going to use it
jpayne@68:                 # anymore so close it now to ensure that the connection is
jpayne@68:                 # released back to the pool.
jpayne@68:                 if self._original_response:
jpayne@68:                     self._original_response.close()
jpayne@68: 
jpayne@68:                 # Closing the response may not actually be sufficient to close
jpayne@68:                 # everything, so if we have a hold of the connection close that
jpayne@68:                 # too.
jpayne@68:                 if self._connection:
jpayne@68:                     self._connection.close()
jpayne@68: 
jpayne@68:             # If we hold the original response but it's closed now, we should
jpayne@68:             # return the connection back to the pool.
jpayne@68:             if self._original_response and self._original_response.isclosed():
jpayne@68:                 self.release_conn()
jpayne@68: 
jpayne@68:     def _fp_read(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         *,
jpayne@68:         read1: bool = False,
jpayne@68:     ) -> bytes:
jpayne@68:         """
jpayne@68:         Read a response with the thought that reading the number of bytes
jpayne@68:         larger than can fit in a 32-bit int at a time via SSL in some
jpayne@68:         known cases leads to an overflow error that has to be prevented
jpayne@68:         if `amt` or `self.length_remaining` indicate that a problem may
jpayne@68:         happen.
jpayne@68: 
jpayne@68:         The known cases:
jpayne@68:           * 3.8 <= CPython < 3.9.7 because of a bug
jpayne@68:             https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900.
jpayne@68:           * urllib3 injected with pyOpenSSL-backed SSL-support.
jpayne@68:           * CPython < 3.10 only when `amt` does not fit 32-bit int.
jpayne@68:         """
jpayne@68:         assert self._fp
jpayne@68:         c_int_max = 2**31 - 1
jpayne@68:         if (
jpayne@68:             (amt and amt > c_int_max)
jpayne@68:             or (
jpayne@68:                 amt is None
jpayne@68:                 and self.length_remaining
jpayne@68:                 and self.length_remaining > c_int_max
jpayne@68:             )
jpayne@68:         ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)):
jpayne@68:             if read1:
jpayne@68:                 return self._fp.read1(c_int_max)
jpayne@68:             buffer = io.BytesIO()
jpayne@68:             # Besides `max_chunk_amt` being a maximum chunk size, it
jpayne@68:             # affects memory overhead of reading a response by this
jpayne@68:             # method in CPython.
jpayne@68:             # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum
jpayne@68:             # chunk size that does not lead to an overflow error, but
jpayne@68:             # 256 MiB is a compromise.
jpayne@68:             max_chunk_amt = 2**28
jpayne@68:             while amt is None or amt != 0:
jpayne@68:                 if amt is not None:
jpayne@68:                     chunk_amt = min(amt, max_chunk_amt)
jpayne@68:                     amt -= chunk_amt
jpayne@68:                 else:
jpayne@68:                     chunk_amt = max_chunk_amt
jpayne@68:                 data = self._fp.read(chunk_amt)
jpayne@68:                 if not data:
jpayne@68:                     break
jpayne@68:                 buffer.write(data)
jpayne@68:                 del data  # to reduce peak memory usage by `max_chunk_amt`.
jpayne@68:             return buffer.getvalue()
jpayne@68:         elif read1:
jpayne@68:             return self._fp.read1(amt) if amt is not None else self._fp.read1()
jpayne@68:         else:
jpayne@68:             # StringIO doesn't like amt=None
jpayne@68:             return self._fp.read(amt) if amt is not None else self._fp.read()
jpayne@68: 
jpayne@68:     def _raw_read(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         *,
jpayne@68:         read1: bool = False,
jpayne@68:     ) -> bytes:
jpayne@68:         """
jpayne@68:         Reads `amt` of bytes from the socket.
jpayne@68:         """
jpayne@68:         if self._fp is None:
jpayne@68:             return None  # type: ignore[return-value]
jpayne@68: 
jpayne@68:         fp_closed = getattr(self._fp, "closed", False)
jpayne@68: 
jpayne@68:         with self._error_catcher():
jpayne@68:             data = self._fp_read(amt, read1=read1) if not fp_closed else b""
jpayne@68:             if amt is not None and amt != 0 and not data:
jpayne@68:                 # Platform-specific: Buggy versions of Python.
jpayne@68:                 # Close the connection when no data is returned
jpayne@68:                 #
jpayne@68:                 # This is redundant to what httplib/http.client _should_
jpayne@68:                 # already do.  However, versions of python released before
jpayne@68:                 # December 15, 2012 (http://bugs.python.org/issue16298) do
jpayne@68:                 # not properly close the connection in all cases. There is
jpayne@68:                 # no harm in redundantly calling close.
jpayne@68:                 self._fp.close()
jpayne@68:                 if (
jpayne@68:                     self.enforce_content_length
jpayne@68:                     and self.length_remaining is not None
jpayne@68:                     and self.length_remaining != 0
jpayne@68:                 ):
jpayne@68:                     # This is an edge case that httplib failed to cover due
jpayne@68:                     # to concerns of backward compatibility. We're
jpayne@68:                     # addressing it here to make sure IncompleteRead is
jpayne@68:                     # raised during streaming, so all calls with incorrect
jpayne@68:                     # Content-Length are caught.
jpayne@68:                     raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
jpayne@68:             elif read1 and (
jpayne@68:                 (amt != 0 and not data) or self.length_remaining == len(data)
jpayne@68:             ):
jpayne@68:                 # All data has been read, but `self._fp.read1` in
jpayne@68:                 # CPython 3.12 and older doesn't always close
jpayne@68:                 # `http.client.HTTPResponse`, so we close it here.
jpayne@68:                 # See https://github.com/python/cpython/issues/113199
jpayne@68:                 self._fp.close()
jpayne@68: 
jpayne@68:         if data:
jpayne@68:             self._fp_bytes_read += len(data)
jpayne@68:             if self.length_remaining is not None:
jpayne@68:                 self.length_remaining -= len(data)
jpayne@68:         return data
jpayne@68: 
jpayne@68:     def read(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         decode_content: bool | None = None,
jpayne@68:         cache_content: bool = False,
jpayne@68:     ) -> bytes:
jpayne@68:         """
jpayne@68:         Similar to :meth:`http.client.HTTPResponse.read`, but with two additional
jpayne@68:         parameters: ``decode_content`` and ``cache_content``.
jpayne@68: 
jpayne@68:         :param amt:
jpayne@68:             How much of the content to read. If specified, caching is skipped
jpayne@68:             because it doesn't make sense to cache partial content as the full
jpayne@68:             response.
jpayne@68: 
jpayne@68:         :param decode_content:
jpayne@68:             If True, will attempt to decode the body based on the
jpayne@68:             'content-encoding' header.
jpayne@68: 
jpayne@68:         :param cache_content:
jpayne@68:             If True, will save the returned data such that the same result is
jpayne@68:             returned despite of the state of the underlying file object. This
jpayne@68:             is useful if you want the ``.data`` property to continue working
jpayne@68:             after having ``.read()`` the file object. (Overridden if ``amt`` is
jpayne@68:             set.)
jpayne@68:         """
jpayne@68:         self._init_decoder()
jpayne@68:         if decode_content is None:
jpayne@68:             decode_content = self.decode_content
jpayne@68: 
jpayne@68:         if amt and amt < 0:
jpayne@68:             # Negative numbers and `None` should be treated the same.
jpayne@68:             amt = None
jpayne@68:         elif amt is not None:
jpayne@68:             cache_content = False
jpayne@68: 
jpayne@68:             if len(self._decoded_buffer) >= amt:
jpayne@68:                 return self._decoded_buffer.get(amt)
jpayne@68: 
jpayne@68:         data = self._raw_read(amt)
jpayne@68: 
jpayne@68:         flush_decoder = amt is None or (amt != 0 and not data)
jpayne@68: 
jpayne@68:         if not data and len(self._decoded_buffer) == 0:
jpayne@68:             return data
jpayne@68: 
jpayne@68:         if amt is None:
jpayne@68:             data = self._decode(data, decode_content, flush_decoder)
jpayne@68:             if cache_content:
jpayne@68:                 self._body = data
jpayne@68:         else:
jpayne@68:             # do not waste memory on buffer when not decoding
jpayne@68:             if not decode_content:
jpayne@68:                 if self._has_decoded_content:
jpayne@68:                     raise RuntimeError(
jpayne@68:                         "Calling read(decode_content=False) is not supported after "
jpayne@68:                         "read(decode_content=True) was called."
jpayne@68:                     )
jpayne@68:                 return data
jpayne@68: 
jpayne@68:             decoded_data = self._decode(data, decode_content, flush_decoder)
jpayne@68:             self._decoded_buffer.put(decoded_data)
jpayne@68: 
jpayne@68:             while len(self._decoded_buffer) < amt and data:
jpayne@68:                 # TODO make sure to initially read enough data to get past the headers
jpayne@68:                 # For example, the GZ file header takes 10 bytes, we don't want to read
jpayne@68:                 # it one byte at a time
jpayne@68:                 data = self._raw_read(amt)
jpayne@68:                 decoded_data = self._decode(data, decode_content, flush_decoder)
jpayne@68:                 self._decoded_buffer.put(decoded_data)
jpayne@68:             data = self._decoded_buffer.get(amt)
jpayne@68: 
jpayne@68:         return data
jpayne@68: 
jpayne@68:     def read1(
jpayne@68:         self,
jpayne@68:         amt: int | None = None,
jpayne@68:         decode_content: bool | None = None,
jpayne@68:     ) -> bytes:
jpayne@68:         """
jpayne@68:         Similar to ``http.client.HTTPResponse.read1`` and documented
jpayne@68:         in :meth:`io.BufferedReader.read1`, but with an additional parameter:
jpayne@68:         ``decode_content``.
jpayne@68: 
jpayne@68:         :param amt:
jpayne@68:             How much of the content to read.
jpayne@68: 
jpayne@68:         :param decode_content:
jpayne@68:             If True, will attempt to decode the body based on the
jpayne@68:             'content-encoding' header.
jpayne@68:         """
jpayne@68:         if decode_content is None:
jpayne@68:             decode_content = self.decode_content
jpayne@68:         if amt and amt < 0:
jpayne@68:             # Negative numbers and `None` should be treated the same.
jpayne@68:             amt = None
jpayne@68:         # try and respond without going to the network
jpayne@68:         if self._has_decoded_content:
jpayne@68:             if not decode_content:
jpayne@68:                 raise RuntimeError(
jpayne@68:                     "Calling read1(decode_content=False) is not supported after "
jpayne@68:                     "read1(decode_content=True) was called."
jpayne@68:                 )
jpayne@68:             if len(self._decoded_buffer) > 0:
jpayne@68:                 if amt is None:
jpayne@68:                     return self._decoded_buffer.get_all()
jpayne@68:                 return self._decoded_buffer.get(amt)
jpayne@68:         if amt == 0:
jpayne@68:             return b""
jpayne@68: 
jpayne@68:         # FIXME, this method's type doesn't say returning None is possible
jpayne@68:         data = self._raw_read(amt, read1=True)
jpayne@68:         if not decode_content or data is None:
jpayne@68:             return data
jpayne@68: 
jpayne@68:         self._init_decoder()
jpayne@68:         while True:
jpayne@68:             flush_decoder = not data
jpayne@68:             decoded_data = self._decode(data, decode_content, flush_decoder)
jpayne@68:             self._decoded_buffer.put(decoded_data)
jpayne@68:             if decoded_data or flush_decoder:
jpayne@68:                 break
jpayne@68:             data = self._raw_read(8192, read1=True)
jpayne@68: 
jpayne@68:         if amt is None:
jpayne@68:             return self._decoded_buffer.get_all()
jpayne@68:         return self._decoded_buffer.get(amt)
jpayne@68: 
jpayne@68:     def stream(
jpayne@68:         self, amt: int | None = 2**16, decode_content: bool | None = None
jpayne@68:     ) -> typing.Generator[bytes, None, None]:
jpayne@68:         """
jpayne@68:         A generator wrapper for the read() method. A call will block until
jpayne@68:         ``amt`` bytes have been read from the connection or until the
jpayne@68:         connection is closed.
jpayne@68: 
jpayne@68:         :param amt:
jpayne@68:             How much of the content to read. The generator will return up to
jpayne@68:             much data per iteration, but may return less. This is particularly
jpayne@68:             likely when using compressed data. However, the empty string will
jpayne@68:             never be returned.
jpayne@68: 
jpayne@68:         :param decode_content:
jpayne@68:             If True, will attempt to decode the body based on the
jpayne@68:             'content-encoding' header.
jpayne@68:         """
jpayne@68:         if self.chunked and self.supports_chunked_reads():
jpayne@68:             yield from self.read_chunked(amt, decode_content=decode_content)
jpayne@68:         else:
jpayne@68:             while not is_fp_closed(self._fp) or len(self._decoded_buffer) > 0:
jpayne@68:                 data = self.read(amt=amt, decode_content=decode_content)
jpayne@68: 
jpayne@68:                 if data:
jpayne@68:                     yield data
jpayne@68: 
jpayne@68:     # Overrides from io.IOBase
jpayne@68:     def readable(self) -> bool:
jpayne@68:         return True
jpayne@68: 
jpayne@68:     def close(self) -> None:
jpayne@68:         if not self.closed and self._fp:
jpayne@68:             self._fp.close()
jpayne@68: 
jpayne@68:         if self._connection:
jpayne@68:             self._connection.close()
jpayne@68: 
jpayne@68:         if not self.auto_close:
jpayne@68:             io.IOBase.close(self)
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def closed(self) -> bool:
jpayne@68:         if not self.auto_close:
jpayne@68:             return io.IOBase.closed.__get__(self)  # type: ignore[no-any-return]
jpayne@68:         elif self._fp is None:
jpayne@68:             return True
jpayne@68:         elif hasattr(self._fp, "isclosed"):
jpayne@68:             return self._fp.isclosed()
jpayne@68:         elif hasattr(self._fp, "closed"):
jpayne@68:             return self._fp.closed
jpayne@68:         else:
jpayne@68:             return True
jpayne@68: 
jpayne@68:     def fileno(self) -> int:
jpayne@68:         if self._fp is None:
jpayne@68:             raise OSError("HTTPResponse has no file to get a fileno from")
jpayne@68:         elif hasattr(self._fp, "fileno"):
jpayne@68:             return self._fp.fileno()
jpayne@68:         else:
jpayne@68:             raise OSError(
jpayne@68:                 "The file-like object this HTTPResponse is wrapped "
jpayne@68:                 "around has no file descriptor"
jpayne@68:             )
jpayne@68: 
jpayne@68:     def flush(self) -> None:
jpayne@68:         if (
jpayne@68:             self._fp is not None
jpayne@68:             and hasattr(self._fp, "flush")
jpayne@68:             and not getattr(self._fp, "closed", False)
jpayne@68:         ):
jpayne@68:             return self._fp.flush()
jpayne@68: 
jpayne@68:     def supports_chunked_reads(self) -> bool:
jpayne@68:         """
jpayne@68:         Checks if the underlying file-like object looks like a
jpayne@68:         :class:`http.client.HTTPResponse` object. We do this by testing for
jpayne@68:         the fp attribute. If it is present we assume it returns raw chunks as
jpayne@68:         processed by read_chunked().
jpayne@68:         """
jpayne@68:         return hasattr(self._fp, "fp")
jpayne@68: 
jpayne@68:     def _update_chunk_length(self) -> None:
jpayne@68:         # First, we'll figure out length of a chunk and then
jpayne@68:         # we'll try to read it from socket.
jpayne@68:         if self.chunk_left is not None:
jpayne@68:             return None
jpayne@68:         line = self._fp.fp.readline()  # type: ignore[union-attr]
jpayne@68:         line = line.split(b";", 1)[0]
jpayne@68:         try:
jpayne@68:             self.chunk_left = int(line, 16)
jpayne@68:         except ValueError:
jpayne@68:             self.close()
jpayne@68:             if line:
jpayne@68:                 # Invalid chunked protocol response, abort.
jpayne@68:                 raise InvalidChunkLength(self, line) from None
jpayne@68:             else:
jpayne@68:                 # Truncated at start of next chunk
jpayne@68:                 raise ProtocolError("Response ended prematurely") from None
jpayne@68: 
jpayne@68:     def _handle_chunk(self, amt: int | None) -> bytes:
jpayne@68:         returned_chunk = None
jpayne@68:         if amt is None:
jpayne@68:             chunk = self._fp._safe_read(self.chunk_left)  # type: ignore[union-attr]
jpayne@68:             returned_chunk = chunk
jpayne@68:             self._fp._safe_read(2)  # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
jpayne@68:             self.chunk_left = None
jpayne@68:         elif self.chunk_left is not None and amt < self.chunk_left:
jpayne@68:             value = self._fp._safe_read(amt)  # type: ignore[union-attr]
jpayne@68:             self.chunk_left = self.chunk_left - amt
jpayne@68:             returned_chunk = value
jpayne@68:         elif amt == self.chunk_left:
jpayne@68:             value = self._fp._safe_read(amt)  # type: ignore[union-attr]
jpayne@68:             self._fp._safe_read(2)  # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
jpayne@68:             self.chunk_left = None
jpayne@68:             returned_chunk = value
jpayne@68:         else:  # amt > self.chunk_left
jpayne@68:             returned_chunk = self._fp._safe_read(self.chunk_left)  # type: ignore[union-attr]
jpayne@68:             self._fp._safe_read(2)  # type: ignore[union-attr] # Toss the CRLF at the end of the chunk.
jpayne@68:             self.chunk_left = None
jpayne@68:         return returned_chunk  # type: ignore[no-any-return]
jpayne@68: 
jpayne@68:     def read_chunked(
jpayne@68:         self, amt: int | None = None, decode_content: bool | None = None
jpayne@68:     ) -> typing.Generator[bytes, None, None]:
jpayne@68:         """
jpayne@68:         Similar to :meth:`HTTPResponse.read`, but with an additional
jpayne@68:         parameter: ``decode_content``.
jpayne@68: 
jpayne@68:         :param amt:
jpayne@68:             How much of the content to read. If specified, caching is skipped
jpayne@68:             because it doesn't make sense to cache partial content as the full
jpayne@68:             response.
jpayne@68: 
jpayne@68:         :param decode_content:
jpayne@68:             If True, will attempt to decode the body based on the
jpayne@68:             'content-encoding' header.
jpayne@68:         """
jpayne@68:         self._init_decoder()
jpayne@68:         # FIXME: Rewrite this method and make it a class with a better structured logic.
jpayne@68:         if not self.chunked:
jpayne@68:             raise ResponseNotChunked(
jpayne@68:                 "Response is not chunked. "
jpayne@68:                 "Header 'transfer-encoding: chunked' is missing."
jpayne@68:             )
jpayne@68:         if not self.supports_chunked_reads():
jpayne@68:             raise BodyNotHttplibCompatible(
jpayne@68:                 "Body should be http.client.HTTPResponse like. "
jpayne@68:                 "It should have have an fp attribute which returns raw chunks."
jpayne@68:             )
jpayne@68: 
jpayne@68:         with self._error_catcher():
jpayne@68:             # Don't bother reading the body of a HEAD request.
jpayne@68:             if self._original_response and is_response_to_head(self._original_response):
jpayne@68:                 self._original_response.close()
jpayne@68:                 return None
jpayne@68: 
jpayne@68:             # If a response is already read and closed
jpayne@68:             # then return immediately.
jpayne@68:             if self._fp.fp is None:  # type: ignore[union-attr]
jpayne@68:                 return None
jpayne@68: 
jpayne@68:             if amt and amt < 0:
jpayne@68:                 # Negative numbers and `None` should be treated the same,
jpayne@68:                 # but httplib handles only `None` correctly.
jpayne@68:                 amt = None
jpayne@68: 
jpayne@68:             while True:
jpayne@68:                 self._update_chunk_length()
jpayne@68:                 if self.chunk_left == 0:
jpayne@68:                     break
jpayne@68:                 chunk = self._handle_chunk(amt)
jpayne@68:                 decoded = self._decode(
jpayne@68:                     chunk, decode_content=decode_content, flush_decoder=False
jpayne@68:                 )
jpayne@68:                 if decoded:
jpayne@68:                     yield decoded
jpayne@68: 
jpayne@68:             if decode_content:
jpayne@68:                 # On CPython and PyPy, we should never need to flush the
jpayne@68:                 # decoder. However, on Jython we *might* need to, so
jpayne@68:                 # lets defensively do it anyway.
jpayne@68:                 decoded = self._flush_decoder()
jpayne@68:                 if decoded:  # Platform-specific: Jython.
jpayne@68:                     yield decoded
jpayne@68: 
jpayne@68:             # Chunk content ends with \r\n: discard it.
jpayne@68:             while self._fp is not None:
jpayne@68:                 line = self._fp.fp.readline()
jpayne@68:                 if not line:
jpayne@68:                     # Some sites may not end with '\r\n'.
jpayne@68:                     break
jpayne@68:                 if line == b"\r\n":
jpayne@68:                     break
jpayne@68: 
jpayne@68:             # We read everything; close the "file".
jpayne@68:             if self._original_response:
jpayne@68:                 self._original_response.close()
jpayne@68: 
jpayne@68:     @property
jpayne@68:     def url(self) -> str | None:
jpayne@68:         """
jpayne@68:         Returns the URL that was the source of this response.
jpayne@68:         If the request that generated this response redirected, this method
jpayne@68:         will return the final redirect location.
jpayne@68:         """
jpayne@68:         return self._request_url
jpayne@68: 
jpayne@68:     @url.setter
jpayne@68:     def url(self, url: str) -> None:
jpayne@68:         self._request_url = url
jpayne@68: 
jpayne@68:     def __iter__(self) -> typing.Iterator[bytes]:
jpayne@68:         buffer: list[bytes] = []
jpayne@68:         for chunk in self.stream(decode_content=True):
jpayne@68:             if b"\n" in chunk:
jpayne@68:                 chunks = chunk.split(b"\n")
jpayne@68:                 yield b"".join(buffer) + chunks[0] + b"\n"
jpayne@68:                 for x in chunks[1:-1]:
jpayne@68:                     yield x + b"\n"
jpayne@68:                 if chunks[-1]:
jpayne@68:                     buffer = [chunks[-1]]
jpayne@68:                 else:
jpayne@68:                     buffer = []
jpayne@68:             else:
jpayne@68:                 buffer.append(chunk)
jpayne@68:         if buffer:
jpayne@68:             yield b"".join(buffer)