jpayne@69: from __future__ import annotations jpayne@69: jpayne@69: import collections jpayne@69: import io jpayne@69: import json as _json jpayne@69: import logging jpayne@69: import re jpayne@69: import sys jpayne@69: import typing jpayne@69: import warnings jpayne@69: import zlib jpayne@69: from contextlib import contextmanager jpayne@69: from http.client import HTTPMessage as _HttplibHTTPMessage jpayne@69: from http.client import HTTPResponse as _HttplibHTTPResponse jpayne@69: from socket import timeout as SocketTimeout jpayne@69: jpayne@69: if typing.TYPE_CHECKING: jpayne@69: from ._base_connection import BaseHTTPConnection jpayne@69: jpayne@69: try: jpayne@69: try: jpayne@69: import brotlicffi as brotli # type: ignore[import-not-found] jpayne@69: except ImportError: jpayne@69: import brotli # type: ignore[import-not-found] jpayne@69: except ImportError: jpayne@69: brotli = None jpayne@69: jpayne@69: try: jpayne@69: import zstandard as zstd jpayne@69: except (AttributeError, ImportError, ValueError): # Defensive: jpayne@69: HAS_ZSTD = False jpayne@69: else: jpayne@69: # The package 'zstandard' added the 'eof' property starting jpayne@69: # in v0.18.0 which we require to ensure a complete and jpayne@69: # valid zstd stream was fed into the ZstdDecoder. jpayne@69: # See: https://github.com/urllib3/urllib3/pull/2624 jpayne@69: _zstd_version = tuple( jpayne@69: map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr] jpayne@69: ) jpayne@69: if _zstd_version < (0, 18): # Defensive: jpayne@69: HAS_ZSTD = False jpayne@69: else: jpayne@69: HAS_ZSTD = True jpayne@69: jpayne@69: from . import util jpayne@69: from ._base_connection import _TYPE_BODY jpayne@69: from ._collections import HTTPHeaderDict jpayne@69: from .connection import BaseSSLError, HTTPConnection, HTTPException jpayne@69: from .exceptions import ( jpayne@69: BodyNotHttplibCompatible, jpayne@69: DecodeError, jpayne@69: HTTPError, jpayne@69: IncompleteRead, jpayne@69: InvalidChunkLength, jpayne@69: InvalidHeader, jpayne@69: ProtocolError, jpayne@69: ReadTimeoutError, jpayne@69: ResponseNotChunked, jpayne@69: SSLError, jpayne@69: ) jpayne@69: from .util.response import is_fp_closed, is_response_to_head jpayne@69: from .util.retry import Retry jpayne@69: jpayne@69: if typing.TYPE_CHECKING: jpayne@69: from .connectionpool import HTTPConnectionPool jpayne@69: jpayne@69: log = logging.getLogger(__name__) jpayne@69: jpayne@69: jpayne@69: class ContentDecoder: jpayne@69: def decompress(self, data: bytes) -> bytes: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def flush(self) -> bytes: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: jpayne@69: class DeflateDecoder(ContentDecoder): jpayne@69: def __init__(self) -> None: jpayne@69: self._first_try = True jpayne@69: self._data = b"" jpayne@69: self._obj = zlib.decompressobj() jpayne@69: jpayne@69: def decompress(self, data: bytes) -> bytes: jpayne@69: if not data: jpayne@69: return data jpayne@69: jpayne@69: if not self._first_try: jpayne@69: return self._obj.decompress(data) jpayne@69: jpayne@69: self._data += data jpayne@69: try: jpayne@69: decompressed = self._obj.decompress(data) jpayne@69: if decompressed: jpayne@69: self._first_try = False jpayne@69: self._data = None # type: ignore[assignment] jpayne@69: return decompressed jpayne@69: except zlib.error: jpayne@69: self._first_try = False jpayne@69: self._obj = zlib.decompressobj(-zlib.MAX_WBITS) jpayne@69: try: jpayne@69: return self.decompress(self._data) jpayne@69: finally: jpayne@69: self._data = None # type: ignore[assignment] jpayne@69: jpayne@69: def flush(self) -> bytes: jpayne@69: return self._obj.flush() jpayne@69: jpayne@69: jpayne@69: class GzipDecoderState: jpayne@69: FIRST_MEMBER = 0 jpayne@69: OTHER_MEMBERS = 1 jpayne@69: SWALLOW_DATA = 2 jpayne@69: jpayne@69: jpayne@69: class GzipDecoder(ContentDecoder): jpayne@69: def __init__(self) -> None: jpayne@69: self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) jpayne@69: self._state = GzipDecoderState.FIRST_MEMBER jpayne@69: jpayne@69: def decompress(self, data: bytes) -> bytes: jpayne@69: ret = bytearray() jpayne@69: if self._state == GzipDecoderState.SWALLOW_DATA or not data: jpayne@69: return bytes(ret) jpayne@69: while True: jpayne@69: try: jpayne@69: ret += self._obj.decompress(data) jpayne@69: except zlib.error: jpayne@69: previous_state = self._state jpayne@69: # Ignore data after the first error jpayne@69: self._state = GzipDecoderState.SWALLOW_DATA jpayne@69: if previous_state == GzipDecoderState.OTHER_MEMBERS: jpayne@69: # Allow trailing garbage acceptable in other gzip clients jpayne@69: return bytes(ret) jpayne@69: raise jpayne@69: data = self._obj.unused_data jpayne@69: if not data: jpayne@69: return bytes(ret) jpayne@69: self._state = GzipDecoderState.OTHER_MEMBERS jpayne@69: self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) jpayne@69: jpayne@69: def flush(self) -> bytes: jpayne@69: return self._obj.flush() jpayne@69: jpayne@69: jpayne@69: if brotli is not None: jpayne@69: jpayne@69: class BrotliDecoder(ContentDecoder): jpayne@69: # Supports both 'brotlipy' and 'Brotli' packages jpayne@69: # since they share an import name. The top branches jpayne@69: # are for 'brotlipy' and bottom branches for 'Brotli' jpayne@69: def __init__(self) -> None: jpayne@69: self._obj = brotli.Decompressor() jpayne@69: if hasattr(self._obj, "decompress"): jpayne@69: setattr(self, "decompress", self._obj.decompress) jpayne@69: else: jpayne@69: setattr(self, "decompress", self._obj.process) jpayne@69: jpayne@69: def flush(self) -> bytes: jpayne@69: if hasattr(self._obj, "flush"): jpayne@69: return self._obj.flush() # type: ignore[no-any-return] jpayne@69: return b"" jpayne@69: jpayne@69: jpayne@69: if HAS_ZSTD: jpayne@69: jpayne@69: class ZstdDecoder(ContentDecoder): jpayne@69: def __init__(self) -> None: jpayne@69: self._obj = zstd.ZstdDecompressor().decompressobj() jpayne@69: jpayne@69: def decompress(self, data: bytes) -> bytes: jpayne@69: if not data: jpayne@69: return b"" jpayne@69: data_parts = [self._obj.decompress(data)] jpayne@69: while self._obj.eof and self._obj.unused_data: jpayne@69: unused_data = self._obj.unused_data jpayne@69: self._obj = zstd.ZstdDecompressor().decompressobj() jpayne@69: data_parts.append(self._obj.decompress(unused_data)) jpayne@69: return b"".join(data_parts) jpayne@69: jpayne@69: def flush(self) -> bytes: jpayne@69: ret = self._obj.flush() # note: this is a no-op jpayne@69: if not self._obj.eof: jpayne@69: raise DecodeError("Zstandard data is incomplete") jpayne@69: return ret jpayne@69: jpayne@69: jpayne@69: class MultiDecoder(ContentDecoder): jpayne@69: """ jpayne@69: From RFC7231: jpayne@69: If one or more encodings have been applied to a representation, the jpayne@69: sender that applied the encodings MUST generate a Content-Encoding jpayne@69: header field that lists the content codings in the order in which jpayne@69: they were applied. jpayne@69: """ jpayne@69: jpayne@69: def __init__(self, modes: str) -> None: jpayne@69: self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] jpayne@69: jpayne@69: def flush(self) -> bytes: jpayne@69: return self._decoders[0].flush() jpayne@69: jpayne@69: def decompress(self, data: bytes) -> bytes: jpayne@69: for d in reversed(self._decoders): jpayne@69: data = d.decompress(data) jpayne@69: return data jpayne@69: jpayne@69: jpayne@69: def _get_decoder(mode: str) -> ContentDecoder: jpayne@69: if "," in mode: jpayne@69: return MultiDecoder(mode) jpayne@69: jpayne@69: # According to RFC 9110 section 8.4.1.3, recipients should jpayne@69: # consider x-gzip equivalent to gzip jpayne@69: if mode in ("gzip", "x-gzip"): jpayne@69: return GzipDecoder() jpayne@69: jpayne@69: if brotli is not None and mode == "br": jpayne@69: return BrotliDecoder() jpayne@69: jpayne@69: if HAS_ZSTD and mode == "zstd": jpayne@69: return ZstdDecoder() jpayne@69: jpayne@69: return DeflateDecoder() jpayne@69: jpayne@69: jpayne@69: class BytesQueueBuffer: jpayne@69: """Memory-efficient bytes buffer jpayne@69: jpayne@69: To return decoded data in read() and still follow the BufferedIOBase API, we need a jpayne@69: buffer to always return the correct amount of bytes. jpayne@69: jpayne@69: This buffer should be filled using calls to put() jpayne@69: jpayne@69: Our maximum memory usage is determined by the sum of the size of: jpayne@69: jpayne@69: * self.buffer, which contains the full data jpayne@69: * the largest chunk that we will copy in get() jpayne@69: jpayne@69: The worst case scenario is a single chunk, in which case we'll make a full copy of jpayne@69: the data inside get(). jpayne@69: """ jpayne@69: jpayne@69: def __init__(self) -> None: jpayne@69: self.buffer: typing.Deque[bytes] = collections.deque() jpayne@69: self._size: int = 0 jpayne@69: jpayne@69: def __len__(self) -> int: jpayne@69: return self._size jpayne@69: jpayne@69: def put(self, data: bytes) -> None: jpayne@69: self.buffer.append(data) jpayne@69: self._size += len(data) jpayne@69: jpayne@69: def get(self, n: int) -> bytes: jpayne@69: if n == 0: jpayne@69: return b"" jpayne@69: elif not self.buffer: jpayne@69: raise RuntimeError("buffer is empty") jpayne@69: elif n < 0: jpayne@69: raise ValueError("n should be > 0") jpayne@69: jpayne@69: fetched = 0 jpayne@69: ret = io.BytesIO() jpayne@69: while fetched < n: jpayne@69: remaining = n - fetched jpayne@69: chunk = self.buffer.popleft() jpayne@69: chunk_length = len(chunk) jpayne@69: if remaining < chunk_length: jpayne@69: left_chunk, right_chunk = chunk[:remaining], chunk[remaining:] jpayne@69: ret.write(left_chunk) jpayne@69: self.buffer.appendleft(right_chunk) jpayne@69: self._size -= remaining jpayne@69: break jpayne@69: else: jpayne@69: ret.write(chunk) jpayne@69: self._size -= chunk_length jpayne@69: fetched += chunk_length jpayne@69: jpayne@69: if not self.buffer: jpayne@69: break jpayne@69: jpayne@69: return ret.getvalue() jpayne@69: jpayne@69: def get_all(self) -> bytes: jpayne@69: buffer = self.buffer jpayne@69: if not buffer: jpayne@69: assert self._size == 0 jpayne@69: return b"" jpayne@69: if len(buffer) == 1: jpayne@69: result = buffer.pop() jpayne@69: else: jpayne@69: ret = io.BytesIO() jpayne@69: ret.writelines(buffer.popleft() for _ in range(len(buffer))) jpayne@69: result = ret.getvalue() jpayne@69: self._size = 0 jpayne@69: return result jpayne@69: jpayne@69: jpayne@69: class BaseHTTPResponse(io.IOBase): jpayne@69: CONTENT_DECODERS = ["gzip", "x-gzip", "deflate"] jpayne@69: if brotli is not None: jpayne@69: CONTENT_DECODERS += ["br"] jpayne@69: if HAS_ZSTD: jpayne@69: CONTENT_DECODERS += ["zstd"] jpayne@69: REDIRECT_STATUSES = [301, 302, 303, 307, 308] jpayne@69: jpayne@69: DECODER_ERROR_CLASSES: tuple[type[Exception], ...] = (IOError, zlib.error) jpayne@69: if brotli is not None: jpayne@69: DECODER_ERROR_CLASSES += (brotli.error,) jpayne@69: jpayne@69: if HAS_ZSTD: jpayne@69: DECODER_ERROR_CLASSES += (zstd.ZstdError,) jpayne@69: jpayne@69: def __init__( jpayne@69: self, jpayne@69: *, jpayne@69: headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, jpayne@69: status: int, jpayne@69: version: int, jpayne@69: version_string: str, jpayne@69: reason: str | None, jpayne@69: decode_content: bool, jpayne@69: request_url: str | None, jpayne@69: retries: Retry | None = None, jpayne@69: ) -> None: jpayne@69: if isinstance(headers, HTTPHeaderDict): jpayne@69: self.headers = headers jpayne@69: else: jpayne@69: self.headers = HTTPHeaderDict(headers) # type: ignore[arg-type] jpayne@69: self.status = status jpayne@69: self.version = version jpayne@69: self.version_string = version_string jpayne@69: self.reason = reason jpayne@69: self.decode_content = decode_content jpayne@69: self._has_decoded_content = False jpayne@69: self._request_url: str | None = request_url jpayne@69: self.retries = retries jpayne@69: jpayne@69: self.chunked = False jpayne@69: tr_enc = self.headers.get("transfer-encoding", "").lower() jpayne@69: # Don't incur the penalty of creating a list and then discarding it jpayne@69: encodings = (enc.strip() for enc in tr_enc.split(",")) jpayne@69: if "chunked" in encodings: jpayne@69: self.chunked = True jpayne@69: jpayne@69: self._decoder: ContentDecoder | None = None jpayne@69: self.length_remaining: int | None jpayne@69: jpayne@69: def get_redirect_location(self) -> str | None | typing.Literal[False]: jpayne@69: """ jpayne@69: Should we redirect and where to? jpayne@69: jpayne@69: :returns: Truthy redirect location string if we got a redirect status jpayne@69: code and valid location. ``None`` if redirect status and no jpayne@69: location. ``False`` if not a redirect status code. jpayne@69: """ jpayne@69: if self.status in self.REDIRECT_STATUSES: jpayne@69: return self.headers.get("location") jpayne@69: return False jpayne@69: jpayne@69: @property jpayne@69: def data(self) -> bytes: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def json(self) -> typing.Any: jpayne@69: """ jpayne@69: Deserializes the body of the HTTP response as a Python object. jpayne@69: jpayne@69: The body of the HTTP response must be encoded using UTF-8, as per jpayne@69: `RFC 8529 Section 8.1 `_. jpayne@69: jpayne@69: To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to jpayne@69: your custom decoder instead. jpayne@69: jpayne@69: If the body of the HTTP response is not decodable to UTF-8, a jpayne@69: `UnicodeDecodeError` will be raised. If the body of the HTTP response is not a jpayne@69: valid JSON document, a `json.JSONDecodeError` will be raised. jpayne@69: jpayne@69: Read more :ref:`here `. jpayne@69: jpayne@69: :returns: The body of the HTTP response as a Python object. jpayne@69: """ jpayne@69: data = self.data.decode("utf-8") jpayne@69: return _json.loads(data) jpayne@69: jpayne@69: @property jpayne@69: def url(self) -> str | None: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: @url.setter jpayne@69: def url(self, url: str | None) -> None: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: @property jpayne@69: def connection(self) -> BaseHTTPConnection | None: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: @property jpayne@69: def retries(self) -> Retry | None: jpayne@69: return self._retries jpayne@69: jpayne@69: @retries.setter jpayne@69: def retries(self, retries: Retry | None) -> None: jpayne@69: # Override the request_url if retries has a redirect location. jpayne@69: if retries is not None and retries.history: jpayne@69: self.url = retries.history[-1].redirect_location jpayne@69: self._retries = retries jpayne@69: jpayne@69: def stream( jpayne@69: self, amt: int | None = 2**16, decode_content: bool | None = None jpayne@69: ) -> typing.Iterator[bytes]: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def read( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: decode_content: bool | None = None, jpayne@69: cache_content: bool = False, jpayne@69: ) -> bytes: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def read1( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: decode_content: bool | None = None, jpayne@69: ) -> bytes: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def read_chunked( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: decode_content: bool | None = None, jpayne@69: ) -> typing.Iterator[bytes]: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def release_conn(self) -> None: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def drain_conn(self) -> None: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def close(self) -> None: jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def _init_decoder(self) -> None: jpayne@69: """ jpayne@69: Set-up the _decoder attribute if necessary. jpayne@69: """ jpayne@69: # Note: content-encoding value should be case-insensitive, per RFC 7230 jpayne@69: # Section 3.2 jpayne@69: content_encoding = self.headers.get("content-encoding", "").lower() jpayne@69: if self._decoder is None: jpayne@69: if content_encoding in self.CONTENT_DECODERS: jpayne@69: self._decoder = _get_decoder(content_encoding) jpayne@69: elif "," in content_encoding: jpayne@69: encodings = [ jpayne@69: e.strip() jpayne@69: for e in content_encoding.split(",") jpayne@69: if e.strip() in self.CONTENT_DECODERS jpayne@69: ] jpayne@69: if encodings: jpayne@69: self._decoder = _get_decoder(content_encoding) jpayne@69: jpayne@69: def _decode( jpayne@69: self, data: bytes, decode_content: bool | None, flush_decoder: bool jpayne@69: ) -> bytes: jpayne@69: """ jpayne@69: Decode the data passed in and potentially flush the decoder. jpayne@69: """ jpayne@69: if not decode_content: jpayne@69: if self._has_decoded_content: jpayne@69: raise RuntimeError( jpayne@69: "Calling read(decode_content=False) is not supported after " jpayne@69: "read(decode_content=True) was called." jpayne@69: ) jpayne@69: return data jpayne@69: jpayne@69: try: jpayne@69: if self._decoder: jpayne@69: data = self._decoder.decompress(data) jpayne@69: self._has_decoded_content = True jpayne@69: except self.DECODER_ERROR_CLASSES as e: jpayne@69: content_encoding = self.headers.get("content-encoding", "").lower() jpayne@69: raise DecodeError( jpayne@69: "Received response with content-encoding: %s, but " jpayne@69: "failed to decode it." % content_encoding, jpayne@69: e, jpayne@69: ) from e jpayne@69: if flush_decoder: jpayne@69: data += self._flush_decoder() jpayne@69: jpayne@69: return data jpayne@69: jpayne@69: def _flush_decoder(self) -> bytes: jpayne@69: """ jpayne@69: Flushes the decoder. Should only be called if the decoder is actually jpayne@69: being used. jpayne@69: """ jpayne@69: if self._decoder: jpayne@69: return self._decoder.decompress(b"") + self._decoder.flush() jpayne@69: return b"" jpayne@69: jpayne@69: # Compatibility methods for `io` module jpayne@69: def readinto(self, b: bytearray) -> int: jpayne@69: temp = self.read(len(b)) jpayne@69: if len(temp) == 0: jpayne@69: return 0 jpayne@69: else: jpayne@69: b[: len(temp)] = temp jpayne@69: return len(temp) jpayne@69: jpayne@69: # Compatibility methods for http.client.HTTPResponse jpayne@69: def getheaders(self) -> HTTPHeaderDict: jpayne@69: warnings.warn( jpayne@69: "HTTPResponse.getheaders() is deprecated and will be removed " jpayne@69: "in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.", jpayne@69: category=DeprecationWarning, jpayne@69: stacklevel=2, jpayne@69: ) jpayne@69: return self.headers jpayne@69: jpayne@69: def getheader(self, name: str, default: str | None = None) -> str | None: jpayne@69: warnings.warn( jpayne@69: "HTTPResponse.getheader() is deprecated and will be removed " jpayne@69: "in urllib3 v2.1.0. Instead use HTTPResponse.headers.get(name, default).", jpayne@69: category=DeprecationWarning, jpayne@69: stacklevel=2, jpayne@69: ) jpayne@69: return self.headers.get(name, default) jpayne@69: jpayne@69: # Compatibility method for http.cookiejar jpayne@69: def info(self) -> HTTPHeaderDict: jpayne@69: return self.headers jpayne@69: jpayne@69: def geturl(self) -> str | None: jpayne@69: return self.url jpayne@69: jpayne@69: jpayne@69: class HTTPResponse(BaseHTTPResponse): jpayne@69: """ jpayne@69: HTTP Response container. jpayne@69: jpayne@69: Backwards-compatible with :class:`http.client.HTTPResponse` but the response ``body`` is jpayne@69: loaded and decoded on-demand when the ``data`` property is accessed. This jpayne@69: class is also compatible with the Python standard library's :mod:`io` jpayne@69: module, and can hence be treated as a readable object in the context of that jpayne@69: framework. jpayne@69: jpayne@69: Extra parameters for behaviour not present in :class:`http.client.HTTPResponse`: jpayne@69: jpayne@69: :param preload_content: jpayne@69: If True, the response's body will be preloaded during construction. jpayne@69: jpayne@69: :param decode_content: jpayne@69: If True, will attempt to decode the body based on the jpayne@69: 'content-encoding' header. jpayne@69: jpayne@69: :param original_response: jpayne@69: When this HTTPResponse wrapper is generated from an :class:`http.client.HTTPResponse` jpayne@69: object, it's convenient to include the original for debug purposes. It's jpayne@69: otherwise unused. jpayne@69: jpayne@69: :param retries: jpayne@69: The retries contains the last :class:`~urllib3.util.retry.Retry` that jpayne@69: was used during the request. jpayne@69: jpayne@69: :param enforce_content_length: jpayne@69: Enforce content length checking. Body returned by server must match jpayne@69: value of Content-Length header, if present. Otherwise, raise error. jpayne@69: """ jpayne@69: jpayne@69: def __init__( jpayne@69: self, jpayne@69: body: _TYPE_BODY = "", jpayne@69: headers: typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | None = None, jpayne@69: status: int = 0, jpayne@69: version: int = 0, jpayne@69: version_string: str = "HTTP/?", jpayne@69: reason: str | None = None, jpayne@69: preload_content: bool = True, jpayne@69: decode_content: bool = True, jpayne@69: original_response: _HttplibHTTPResponse | None = None, jpayne@69: pool: HTTPConnectionPool | None = None, jpayne@69: connection: HTTPConnection | None = None, jpayne@69: msg: _HttplibHTTPMessage | None = None, jpayne@69: retries: Retry | None = None, jpayne@69: enforce_content_length: bool = True, jpayne@69: request_method: str | None = None, jpayne@69: request_url: str | None = None, jpayne@69: auto_close: bool = True, jpayne@69: ) -> None: jpayne@69: super().__init__( jpayne@69: headers=headers, jpayne@69: status=status, jpayne@69: version=version, jpayne@69: version_string=version_string, jpayne@69: reason=reason, jpayne@69: decode_content=decode_content, jpayne@69: request_url=request_url, jpayne@69: retries=retries, jpayne@69: ) jpayne@69: jpayne@69: self.enforce_content_length = enforce_content_length jpayne@69: self.auto_close = auto_close jpayne@69: jpayne@69: self._body = None jpayne@69: self._fp: _HttplibHTTPResponse | None = None jpayne@69: self._original_response = original_response jpayne@69: self._fp_bytes_read = 0 jpayne@69: self.msg = msg jpayne@69: jpayne@69: if body and isinstance(body, (str, bytes)): jpayne@69: self._body = body jpayne@69: jpayne@69: self._pool = pool jpayne@69: self._connection = connection jpayne@69: jpayne@69: if hasattr(body, "read"): jpayne@69: self._fp = body # type: ignore[assignment] jpayne@69: jpayne@69: # Are we using the chunked-style of transfer encoding? jpayne@69: self.chunk_left: int | None = None jpayne@69: jpayne@69: # Determine length of response jpayne@69: self.length_remaining = self._init_length(request_method) jpayne@69: jpayne@69: # Used to return the correct amount of bytes for partial read()s jpayne@69: self._decoded_buffer = BytesQueueBuffer() jpayne@69: jpayne@69: # If requested, preload the body. jpayne@69: if preload_content and not self._body: jpayne@69: self._body = self.read(decode_content=decode_content) jpayne@69: jpayne@69: def release_conn(self) -> None: jpayne@69: if not self._pool or not self._connection: jpayne@69: return None jpayne@69: jpayne@69: self._pool._put_conn(self._connection) jpayne@69: self._connection = None jpayne@69: jpayne@69: def drain_conn(self) -> None: jpayne@69: """ jpayne@69: Read and discard any remaining HTTP response data in the response connection. jpayne@69: jpayne@69: Unread data in the HTTPResponse connection blocks the connection from being released back to the pool. jpayne@69: """ jpayne@69: try: jpayne@69: self.read() jpayne@69: except (HTTPError, OSError, BaseSSLError, HTTPException): jpayne@69: pass jpayne@69: jpayne@69: @property jpayne@69: def data(self) -> bytes: jpayne@69: # For backwards-compat with earlier urllib3 0.4 and earlier. jpayne@69: if self._body: jpayne@69: return self._body # type: ignore[return-value] jpayne@69: jpayne@69: if self._fp: jpayne@69: return self.read(cache_content=True) jpayne@69: jpayne@69: return None # type: ignore[return-value] jpayne@69: jpayne@69: @property jpayne@69: def connection(self) -> HTTPConnection | None: jpayne@69: return self._connection jpayne@69: jpayne@69: def isclosed(self) -> bool: jpayne@69: return is_fp_closed(self._fp) jpayne@69: jpayne@69: def tell(self) -> int: jpayne@69: """ jpayne@69: Obtain the number of bytes pulled over the wire so far. May differ from jpayne@69: the amount of content returned by :meth:``urllib3.response.HTTPResponse.read`` jpayne@69: if bytes are encoded on the wire (e.g, compressed). jpayne@69: """ jpayne@69: return self._fp_bytes_read jpayne@69: jpayne@69: def _init_length(self, request_method: str | None) -> int | None: jpayne@69: """ jpayne@69: Set initial length value for Response content if available. jpayne@69: """ jpayne@69: length: int | None jpayne@69: content_length: str | None = self.headers.get("content-length") jpayne@69: jpayne@69: if content_length is not None: jpayne@69: if self.chunked: jpayne@69: # This Response will fail with an IncompleteRead if it can't be jpayne@69: # received as chunked. This method falls back to attempt reading jpayne@69: # the response before raising an exception. jpayne@69: log.warning( jpayne@69: "Received response with both Content-Length and " jpayne@69: "Transfer-Encoding set. This is expressly forbidden " jpayne@69: "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " jpayne@69: "attempting to process response as Transfer-Encoding: " jpayne@69: "chunked." jpayne@69: ) jpayne@69: return None jpayne@69: jpayne@69: try: jpayne@69: # RFC 7230 section 3.3.2 specifies multiple content lengths can jpayne@69: # be sent in a single Content-Length header jpayne@69: # (e.g. Content-Length: 42, 42). This line ensures the values jpayne@69: # are all valid ints and that as long as the `set` length is 1, jpayne@69: # all values are the same. Otherwise, the header is invalid. jpayne@69: lengths = {int(val) for val in content_length.split(",")} jpayne@69: if len(lengths) > 1: jpayne@69: raise InvalidHeader( jpayne@69: "Content-Length contained multiple " jpayne@69: "unmatching values (%s)" % content_length jpayne@69: ) jpayne@69: length = lengths.pop() jpayne@69: except ValueError: jpayne@69: length = None jpayne@69: else: jpayne@69: if length < 0: jpayne@69: length = None jpayne@69: jpayne@69: else: # if content_length is None jpayne@69: length = None jpayne@69: jpayne@69: # Convert status to int for comparison jpayne@69: # In some cases, httplib returns a status of "_UNKNOWN" jpayne@69: try: jpayne@69: status = int(self.status) jpayne@69: except ValueError: jpayne@69: status = 0 jpayne@69: jpayne@69: # Check for responses that shouldn't include a body jpayne@69: if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD": jpayne@69: length = 0 jpayne@69: jpayne@69: return length jpayne@69: jpayne@69: @contextmanager jpayne@69: def _error_catcher(self) -> typing.Generator[None, None, None]: jpayne@69: """ jpayne@69: Catch low-level python exceptions, instead re-raising urllib3 jpayne@69: variants, so that low-level exceptions are not leaked in the jpayne@69: high-level api. jpayne@69: jpayne@69: On exit, release the connection back to the pool. jpayne@69: """ jpayne@69: clean_exit = False jpayne@69: jpayne@69: try: jpayne@69: try: jpayne@69: yield jpayne@69: jpayne@69: except SocketTimeout as e: jpayne@69: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but jpayne@69: # there is yet no clean way to get at it from this context. jpayne@69: raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] jpayne@69: jpayne@69: except BaseSSLError as e: jpayne@69: # FIXME: Is there a better way to differentiate between SSLErrors? jpayne@69: if "read operation timed out" not in str(e): jpayne@69: # SSL errors related to framing/MAC get wrapped and reraised here jpayne@69: raise SSLError(e) from e jpayne@69: jpayne@69: raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] jpayne@69: jpayne@69: except IncompleteRead as e: jpayne@69: if ( jpayne@69: e.expected is not None jpayne@69: and e.partial is not None jpayne@69: and e.expected == -e.partial jpayne@69: ): jpayne@69: arg = "Response may not contain content." jpayne@69: else: jpayne@69: arg = f"Connection broken: {e!r}" jpayne@69: raise ProtocolError(arg, e) from e jpayne@69: jpayne@69: except (HTTPException, OSError) as e: jpayne@69: raise ProtocolError(f"Connection broken: {e!r}", e) from e jpayne@69: jpayne@69: # If no exception is thrown, we should avoid cleaning up jpayne@69: # unnecessarily. jpayne@69: clean_exit = True jpayne@69: finally: jpayne@69: # If we didn't terminate cleanly, we need to throw away our jpayne@69: # connection. jpayne@69: if not clean_exit: jpayne@69: # The response may not be closed but we're not going to use it jpayne@69: # anymore so close it now to ensure that the connection is jpayne@69: # released back to the pool. jpayne@69: if self._original_response: jpayne@69: self._original_response.close() jpayne@69: jpayne@69: # Closing the response may not actually be sufficient to close jpayne@69: # everything, so if we have a hold of the connection close that jpayne@69: # too. jpayne@69: if self._connection: jpayne@69: self._connection.close() jpayne@69: jpayne@69: # If we hold the original response but it's closed now, we should jpayne@69: # return the connection back to the pool. jpayne@69: if self._original_response and self._original_response.isclosed(): jpayne@69: self.release_conn() jpayne@69: jpayne@69: def _fp_read( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: *, jpayne@69: read1: bool = False, jpayne@69: ) -> bytes: jpayne@69: """ jpayne@69: Read a response with the thought that reading the number of bytes jpayne@69: larger than can fit in a 32-bit int at a time via SSL in some jpayne@69: known cases leads to an overflow error that has to be prevented jpayne@69: if `amt` or `self.length_remaining` indicate that a problem may jpayne@69: happen. jpayne@69: jpayne@69: The known cases: jpayne@69: * 3.8 <= CPython < 3.9.7 because of a bug jpayne@69: https://github.com/urllib3/urllib3/issues/2513#issuecomment-1152559900. jpayne@69: * urllib3 injected with pyOpenSSL-backed SSL-support. jpayne@69: * CPython < 3.10 only when `amt` does not fit 32-bit int. jpayne@69: """ jpayne@69: assert self._fp jpayne@69: c_int_max = 2**31 - 1 jpayne@69: if ( jpayne@69: (amt and amt > c_int_max) jpayne@69: or ( jpayne@69: amt is None jpayne@69: and self.length_remaining jpayne@69: and self.length_remaining > c_int_max jpayne@69: ) jpayne@69: ) and (util.IS_PYOPENSSL or sys.version_info < (3, 10)): jpayne@69: if read1: jpayne@69: return self._fp.read1(c_int_max) jpayne@69: buffer = io.BytesIO() jpayne@69: # Besides `max_chunk_amt` being a maximum chunk size, it jpayne@69: # affects memory overhead of reading a response by this jpayne@69: # method in CPython. jpayne@69: # `c_int_max` equal to 2 GiB - 1 byte is the actual maximum jpayne@69: # chunk size that does not lead to an overflow error, but jpayne@69: # 256 MiB is a compromise. jpayne@69: max_chunk_amt = 2**28 jpayne@69: while amt is None or amt != 0: jpayne@69: if amt is not None: jpayne@69: chunk_amt = min(amt, max_chunk_amt) jpayne@69: amt -= chunk_amt jpayne@69: else: jpayne@69: chunk_amt = max_chunk_amt jpayne@69: data = self._fp.read(chunk_amt) jpayne@69: if not data: jpayne@69: break jpayne@69: buffer.write(data) jpayne@69: del data # to reduce peak memory usage by `max_chunk_amt`. jpayne@69: return buffer.getvalue() jpayne@69: elif read1: jpayne@69: return self._fp.read1(amt) if amt is not None else self._fp.read1() jpayne@69: else: jpayne@69: # StringIO doesn't like amt=None jpayne@69: return self._fp.read(amt) if amt is not None else self._fp.read() jpayne@69: jpayne@69: def _raw_read( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: *, jpayne@69: read1: bool = False, jpayne@69: ) -> bytes: jpayne@69: """ jpayne@69: Reads `amt` of bytes from the socket. jpayne@69: """ jpayne@69: if self._fp is None: jpayne@69: return None # type: ignore[return-value] jpayne@69: jpayne@69: fp_closed = getattr(self._fp, "closed", False) jpayne@69: jpayne@69: with self._error_catcher(): jpayne@69: data = self._fp_read(amt, read1=read1) if not fp_closed else b"" jpayne@69: if amt is not None and amt != 0 and not data: jpayne@69: # Platform-specific: Buggy versions of Python. jpayne@69: # Close the connection when no data is returned jpayne@69: # jpayne@69: # This is redundant to what httplib/http.client _should_ jpayne@69: # already do. However, versions of python released before jpayne@69: # December 15, 2012 (http://bugs.python.org/issue16298) do jpayne@69: # not properly close the connection in all cases. There is jpayne@69: # no harm in redundantly calling close. jpayne@69: self._fp.close() jpayne@69: if ( jpayne@69: self.enforce_content_length jpayne@69: and self.length_remaining is not None jpayne@69: and self.length_remaining != 0 jpayne@69: ): jpayne@69: # This is an edge case that httplib failed to cover due jpayne@69: # to concerns of backward compatibility. We're jpayne@69: # addressing it here to make sure IncompleteRead is jpayne@69: # raised during streaming, so all calls with incorrect jpayne@69: # Content-Length are caught. jpayne@69: raise IncompleteRead(self._fp_bytes_read, self.length_remaining) jpayne@69: elif read1 and ( jpayne@69: (amt != 0 and not data) or self.length_remaining == len(data) jpayne@69: ): jpayne@69: # All data has been read, but `self._fp.read1` in jpayne@69: # CPython 3.12 and older doesn't always close jpayne@69: # `http.client.HTTPResponse`, so we close it here. jpayne@69: # See https://github.com/python/cpython/issues/113199 jpayne@69: self._fp.close() jpayne@69: jpayne@69: if data: jpayne@69: self._fp_bytes_read += len(data) jpayne@69: if self.length_remaining is not None: jpayne@69: self.length_remaining -= len(data) jpayne@69: return data jpayne@69: jpayne@69: def read( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: decode_content: bool | None = None, jpayne@69: cache_content: bool = False, jpayne@69: ) -> bytes: jpayne@69: """ jpayne@69: Similar to :meth:`http.client.HTTPResponse.read`, but with two additional jpayne@69: parameters: ``decode_content`` and ``cache_content``. jpayne@69: jpayne@69: :param amt: jpayne@69: How much of the content to read. If specified, caching is skipped jpayne@69: because it doesn't make sense to cache partial content as the full jpayne@69: response. jpayne@69: jpayne@69: :param decode_content: jpayne@69: If True, will attempt to decode the body based on the jpayne@69: 'content-encoding' header. jpayne@69: jpayne@69: :param cache_content: jpayne@69: If True, will save the returned data such that the same result is jpayne@69: returned despite of the state of the underlying file object. This jpayne@69: is useful if you want the ``.data`` property to continue working jpayne@69: after having ``.read()`` the file object. (Overridden if ``amt`` is jpayne@69: set.) jpayne@69: """ jpayne@69: self._init_decoder() jpayne@69: if decode_content is None: jpayne@69: decode_content = self.decode_content jpayne@69: jpayne@69: if amt and amt < 0: jpayne@69: # Negative numbers and `None` should be treated the same. jpayne@69: amt = None jpayne@69: elif amt is not None: jpayne@69: cache_content = False jpayne@69: jpayne@69: if len(self._decoded_buffer) >= amt: jpayne@69: return self._decoded_buffer.get(amt) jpayne@69: jpayne@69: data = self._raw_read(amt) jpayne@69: jpayne@69: flush_decoder = amt is None or (amt != 0 and not data) jpayne@69: jpayne@69: if not data and len(self._decoded_buffer) == 0: jpayne@69: return data jpayne@69: jpayne@69: if amt is None: jpayne@69: data = self._decode(data, decode_content, flush_decoder) jpayne@69: if cache_content: jpayne@69: self._body = data jpayne@69: else: jpayne@69: # do not waste memory on buffer when not decoding jpayne@69: if not decode_content: jpayne@69: if self._has_decoded_content: jpayne@69: raise RuntimeError( jpayne@69: "Calling read(decode_content=False) is not supported after " jpayne@69: "read(decode_content=True) was called." jpayne@69: ) jpayne@69: return data jpayne@69: jpayne@69: decoded_data = self._decode(data, decode_content, flush_decoder) jpayne@69: self._decoded_buffer.put(decoded_data) jpayne@69: jpayne@69: while len(self._decoded_buffer) < amt and data: jpayne@69: # TODO make sure to initially read enough data to get past the headers jpayne@69: # For example, the GZ file header takes 10 bytes, we don't want to read jpayne@69: # it one byte at a time jpayne@69: data = self._raw_read(amt) jpayne@69: decoded_data = self._decode(data, decode_content, flush_decoder) jpayne@69: self._decoded_buffer.put(decoded_data) jpayne@69: data = self._decoded_buffer.get(amt) jpayne@69: jpayne@69: return data jpayne@69: jpayne@69: def read1( jpayne@69: self, jpayne@69: amt: int | None = None, jpayne@69: decode_content: bool | None = None, jpayne@69: ) -> bytes: jpayne@69: """ jpayne@69: Similar to ``http.client.HTTPResponse.read1`` and documented jpayne@69: in :meth:`io.BufferedReader.read1`, but with an additional parameter: jpayne@69: ``decode_content``. jpayne@69: jpayne@69: :param amt: jpayne@69: How much of the content to read. jpayne@69: jpayne@69: :param decode_content: jpayne@69: If True, will attempt to decode the body based on the jpayne@69: 'content-encoding' header. jpayne@69: """ jpayne@69: if decode_content is None: jpayne@69: decode_content = self.decode_content jpayne@69: if amt and amt < 0: jpayne@69: # Negative numbers and `None` should be treated the same. jpayne@69: amt = None jpayne@69: # try and respond without going to the network jpayne@69: if self._has_decoded_content: jpayne@69: if not decode_content: jpayne@69: raise RuntimeError( jpayne@69: "Calling read1(decode_content=False) is not supported after " jpayne@69: "read1(decode_content=True) was called." jpayne@69: ) jpayne@69: if len(self._decoded_buffer) > 0: jpayne@69: if amt is None: jpayne@69: return self._decoded_buffer.get_all() jpayne@69: return self._decoded_buffer.get(amt) jpayne@69: if amt == 0: jpayne@69: return b"" jpayne@69: jpayne@69: # FIXME, this method's type doesn't say returning None is possible jpayne@69: data = self._raw_read(amt, read1=True) jpayne@69: if not decode_content or data is None: jpayne@69: return data jpayne@69: jpayne@69: self._init_decoder() jpayne@69: while True: jpayne@69: flush_decoder = not data jpayne@69: decoded_data = self._decode(data, decode_content, flush_decoder) jpayne@69: self._decoded_buffer.put(decoded_data) jpayne@69: if decoded_data or flush_decoder: jpayne@69: break jpayne@69: data = self._raw_read(8192, read1=True) jpayne@69: jpayne@69: if amt is None: jpayne@69: return self._decoded_buffer.get_all() jpayne@69: return self._decoded_buffer.get(amt) jpayne@69: jpayne@69: def stream( jpayne@69: self, amt: int | None = 2**16, decode_content: bool | None = None jpayne@69: ) -> typing.Generator[bytes, None, None]: jpayne@69: """ jpayne@69: A generator wrapper for the read() method. A call will block until jpayne@69: ``amt`` bytes have been read from the connection or until the jpayne@69: connection is closed. jpayne@69: jpayne@69: :param amt: jpayne@69: How much of the content to read. The generator will return up to jpayne@69: much data per iteration, but may return less. This is particularly jpayne@69: likely when using compressed data. However, the empty string will jpayne@69: never be returned. jpayne@69: jpayne@69: :param decode_content: jpayne@69: If True, will attempt to decode the body based on the jpayne@69: 'content-encoding' header. jpayne@69: """ jpayne@69: if self.chunked and self.supports_chunked_reads(): jpayne@69: yield from self.read_chunked(amt, decode_content=decode_content) jpayne@69: else: jpayne@69: while not is_fp_closed(self._fp) or len(self._decoded_buffer) > 0: jpayne@69: data = self.read(amt=amt, decode_content=decode_content) jpayne@69: jpayne@69: if data: jpayne@69: yield data jpayne@69: jpayne@69: # Overrides from io.IOBase jpayne@69: def readable(self) -> bool: jpayne@69: return True jpayne@69: jpayne@69: def close(self) -> None: jpayne@69: if not self.closed and self._fp: jpayne@69: self._fp.close() jpayne@69: jpayne@69: if self._connection: jpayne@69: self._connection.close() jpayne@69: jpayne@69: if not self.auto_close: jpayne@69: io.IOBase.close(self) jpayne@69: jpayne@69: @property jpayne@69: def closed(self) -> bool: jpayne@69: if not self.auto_close: jpayne@69: return io.IOBase.closed.__get__(self) # type: ignore[no-any-return] jpayne@69: elif self._fp is None: jpayne@69: return True jpayne@69: elif hasattr(self._fp, "isclosed"): jpayne@69: return self._fp.isclosed() jpayne@69: elif hasattr(self._fp, "closed"): jpayne@69: return self._fp.closed jpayne@69: else: jpayne@69: return True jpayne@69: jpayne@69: def fileno(self) -> int: jpayne@69: if self._fp is None: jpayne@69: raise OSError("HTTPResponse has no file to get a fileno from") jpayne@69: elif hasattr(self._fp, "fileno"): jpayne@69: return self._fp.fileno() jpayne@69: else: jpayne@69: raise OSError( jpayne@69: "The file-like object this HTTPResponse is wrapped " jpayne@69: "around has no file descriptor" jpayne@69: ) jpayne@69: jpayne@69: def flush(self) -> None: jpayne@69: if ( jpayne@69: self._fp is not None jpayne@69: and hasattr(self._fp, "flush") jpayne@69: and not getattr(self._fp, "closed", False) jpayne@69: ): jpayne@69: return self._fp.flush() jpayne@69: jpayne@69: def supports_chunked_reads(self) -> bool: jpayne@69: """ jpayne@69: Checks if the underlying file-like object looks like a jpayne@69: :class:`http.client.HTTPResponse` object. We do this by testing for jpayne@69: the fp attribute. If it is present we assume it returns raw chunks as jpayne@69: processed by read_chunked(). jpayne@69: """ jpayne@69: return hasattr(self._fp, "fp") jpayne@69: jpayne@69: def _update_chunk_length(self) -> None: jpayne@69: # First, we'll figure out length of a chunk and then jpayne@69: # we'll try to read it from socket. jpayne@69: if self.chunk_left is not None: jpayne@69: return None jpayne@69: line = self._fp.fp.readline() # type: ignore[union-attr] jpayne@69: line = line.split(b";", 1)[0] jpayne@69: try: jpayne@69: self.chunk_left = int(line, 16) jpayne@69: except ValueError: jpayne@69: self.close() jpayne@69: if line: jpayne@69: # Invalid chunked protocol response, abort. jpayne@69: raise InvalidChunkLength(self, line) from None jpayne@69: else: jpayne@69: # Truncated at start of next chunk jpayne@69: raise ProtocolError("Response ended prematurely") from None jpayne@69: jpayne@69: def _handle_chunk(self, amt: int | None) -> bytes: jpayne@69: returned_chunk = None jpayne@69: if amt is None: jpayne@69: chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr] jpayne@69: returned_chunk = chunk jpayne@69: self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. jpayne@69: self.chunk_left = None jpayne@69: elif self.chunk_left is not None and amt < self.chunk_left: jpayne@69: value = self._fp._safe_read(amt) # type: ignore[union-attr] jpayne@69: self.chunk_left = self.chunk_left - amt jpayne@69: returned_chunk = value jpayne@69: elif amt == self.chunk_left: jpayne@69: value = self._fp._safe_read(amt) # type: ignore[union-attr] jpayne@69: self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. jpayne@69: self.chunk_left = None jpayne@69: returned_chunk = value jpayne@69: else: # amt > self.chunk_left jpayne@69: returned_chunk = self._fp._safe_read(self.chunk_left) # type: ignore[union-attr] jpayne@69: self._fp._safe_read(2) # type: ignore[union-attr] # Toss the CRLF at the end of the chunk. jpayne@69: self.chunk_left = None jpayne@69: return returned_chunk # type: ignore[no-any-return] jpayne@69: jpayne@69: def read_chunked( jpayne@69: self, amt: int | None = None, decode_content: bool | None = None jpayne@69: ) -> typing.Generator[bytes, None, None]: jpayne@69: """ jpayne@69: Similar to :meth:`HTTPResponse.read`, but with an additional jpayne@69: parameter: ``decode_content``. jpayne@69: jpayne@69: :param amt: jpayne@69: How much of the content to read. If specified, caching is skipped jpayne@69: because it doesn't make sense to cache partial content as the full jpayne@69: response. jpayne@69: jpayne@69: :param decode_content: jpayne@69: If True, will attempt to decode the body based on the jpayne@69: 'content-encoding' header. jpayne@69: """ jpayne@69: self._init_decoder() jpayne@69: # FIXME: Rewrite this method and make it a class with a better structured logic. jpayne@69: if not self.chunked: jpayne@69: raise ResponseNotChunked( jpayne@69: "Response is not chunked. " jpayne@69: "Header 'transfer-encoding: chunked' is missing." jpayne@69: ) jpayne@69: if not self.supports_chunked_reads(): jpayne@69: raise BodyNotHttplibCompatible( jpayne@69: "Body should be http.client.HTTPResponse like. " jpayne@69: "It should have have an fp attribute which returns raw chunks." jpayne@69: ) jpayne@69: jpayne@69: with self._error_catcher(): jpayne@69: # Don't bother reading the body of a HEAD request. jpayne@69: if self._original_response and is_response_to_head(self._original_response): jpayne@69: self._original_response.close() jpayne@69: return None jpayne@69: jpayne@69: # If a response is already read and closed jpayne@69: # then return immediately. jpayne@69: if self._fp.fp is None: # type: ignore[union-attr] jpayne@69: return None jpayne@69: jpayne@69: if amt and amt < 0: jpayne@69: # Negative numbers and `None` should be treated the same, jpayne@69: # but httplib handles only `None` correctly. jpayne@69: amt = None jpayne@69: jpayne@69: while True: jpayne@69: self._update_chunk_length() jpayne@69: if self.chunk_left == 0: jpayne@69: break jpayne@69: chunk = self._handle_chunk(amt) jpayne@69: decoded = self._decode( jpayne@69: chunk, decode_content=decode_content, flush_decoder=False jpayne@69: ) jpayne@69: if decoded: jpayne@69: yield decoded jpayne@69: jpayne@69: if decode_content: jpayne@69: # On CPython and PyPy, we should never need to flush the jpayne@69: # decoder. However, on Jython we *might* need to, so jpayne@69: # lets defensively do it anyway. jpayne@69: decoded = self._flush_decoder() jpayne@69: if decoded: # Platform-specific: Jython. jpayne@69: yield decoded jpayne@69: jpayne@69: # Chunk content ends with \r\n: discard it. jpayne@69: while self._fp is not None: jpayne@69: line = self._fp.fp.readline() jpayne@69: if not line: jpayne@69: # Some sites may not end with '\r\n'. jpayne@69: break jpayne@69: if line == b"\r\n": jpayne@69: break jpayne@69: jpayne@69: # We read everything; close the "file". jpayne@69: if self._original_response: jpayne@69: self._original_response.close() jpayne@69: jpayne@69: @property jpayne@69: def url(self) -> str | None: jpayne@69: """ jpayne@69: Returns the URL that was the source of this response. jpayne@69: If the request that generated this response redirected, this method jpayne@69: will return the final redirect location. jpayne@69: """ jpayne@69: return self._request_url jpayne@69: jpayne@69: @url.setter jpayne@69: def url(self, url: str) -> None: jpayne@69: self._request_url = url jpayne@69: jpayne@69: def __iter__(self) -> typing.Iterator[bytes]: jpayne@69: buffer: list[bytes] = [] jpayne@69: for chunk in self.stream(decode_content=True): jpayne@69: if b"\n" in chunk: jpayne@69: chunks = chunk.split(b"\n") jpayne@69: yield b"".join(buffer) + chunks[0] + b"\n" jpayne@69: for x in chunks[1:-1]: jpayne@69: yield x + b"\n" jpayne@69: if chunks[-1]: jpayne@69: buffer = [chunks[-1]] jpayne@69: else: jpayne@69: buffer = [] jpayne@69: else: jpayne@69: buffer.append(chunk) jpayne@69: if buffer: jpayne@69: yield b"".join(buffer)