jpayne@69: """ jpayne@69: requests.cookies jpayne@69: ~~~~~~~~~~~~~~~~ jpayne@69: jpayne@69: Compatibility code to be able to use `http.cookiejar.CookieJar` with requests. jpayne@69: jpayne@69: requests.utils imports from here, so be careful with imports. jpayne@69: """ jpayne@69: jpayne@69: import calendar jpayne@69: import copy jpayne@69: import time jpayne@69: jpayne@69: from ._internal_utils import to_native_string jpayne@69: from .compat import Morsel, MutableMapping, cookielib, urlparse, urlunparse jpayne@69: jpayne@69: try: jpayne@69: import threading jpayne@69: except ImportError: jpayne@69: import dummy_threading as threading jpayne@69: jpayne@69: jpayne@69: class MockRequest: jpayne@69: """Wraps a `requests.Request` to mimic a `urllib2.Request`. jpayne@69: jpayne@69: The code in `http.cookiejar.CookieJar` expects this interface in order to correctly jpayne@69: manage cookie policies, i.e., determine whether a cookie can be set, given the jpayne@69: domains of the request and the cookie. jpayne@69: jpayne@69: The original request object is read-only. The client is responsible for collecting jpayne@69: the new headers via `get_new_headers()` and interpreting them appropriately. You jpayne@69: probably want `get_cookie_header`, defined below. jpayne@69: """ jpayne@69: jpayne@69: def __init__(self, request): jpayne@69: self._r = request jpayne@69: self._new_headers = {} jpayne@69: self.type = urlparse(self._r.url).scheme jpayne@69: jpayne@69: def get_type(self): jpayne@69: return self.type jpayne@69: jpayne@69: def get_host(self): jpayne@69: return urlparse(self._r.url).netloc jpayne@69: jpayne@69: def get_origin_req_host(self): jpayne@69: return self.get_host() jpayne@69: jpayne@69: def get_full_url(self): jpayne@69: # Only return the response's URL if the user hadn't set the Host jpayne@69: # header jpayne@69: if not self._r.headers.get("Host"): jpayne@69: return self._r.url jpayne@69: # If they did set it, retrieve it and reconstruct the expected domain jpayne@69: host = to_native_string(self._r.headers["Host"], encoding="utf-8") jpayne@69: parsed = urlparse(self._r.url) jpayne@69: # Reconstruct the URL as we expect it jpayne@69: return urlunparse( jpayne@69: [ jpayne@69: parsed.scheme, jpayne@69: host, jpayne@69: parsed.path, jpayne@69: parsed.params, jpayne@69: parsed.query, jpayne@69: parsed.fragment, jpayne@69: ] jpayne@69: ) jpayne@69: jpayne@69: def is_unverifiable(self): jpayne@69: return True jpayne@69: jpayne@69: def has_header(self, name): jpayne@69: return name in self._r.headers or name in self._new_headers jpayne@69: jpayne@69: def get_header(self, name, default=None): jpayne@69: return self._r.headers.get(name, self._new_headers.get(name, default)) jpayne@69: jpayne@69: def add_header(self, key, val): jpayne@69: """cookiejar has no legitimate use for this method; add it back if you find one.""" jpayne@69: raise NotImplementedError( jpayne@69: "Cookie headers should be added with add_unredirected_header()" jpayne@69: ) jpayne@69: jpayne@69: def add_unredirected_header(self, name, value): jpayne@69: self._new_headers[name] = value jpayne@69: jpayne@69: def get_new_headers(self): jpayne@69: return self._new_headers jpayne@69: jpayne@69: @property jpayne@69: def unverifiable(self): jpayne@69: return self.is_unverifiable() jpayne@69: jpayne@69: @property jpayne@69: def origin_req_host(self): jpayne@69: return self.get_origin_req_host() jpayne@69: jpayne@69: @property jpayne@69: def host(self): jpayne@69: return self.get_host() jpayne@69: jpayne@69: jpayne@69: class MockResponse: jpayne@69: """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. jpayne@69: jpayne@69: ...what? Basically, expose the parsed HTTP headers from the server response jpayne@69: the way `http.cookiejar` expects to see them. jpayne@69: """ jpayne@69: jpayne@69: def __init__(self, headers): jpayne@69: """Make a MockResponse for `cookiejar` to read. jpayne@69: jpayne@69: :param headers: a httplib.HTTPMessage or analogous carrying the headers jpayne@69: """ jpayne@69: self._headers = headers jpayne@69: jpayne@69: def info(self): jpayne@69: return self._headers jpayne@69: jpayne@69: def getheaders(self, name): jpayne@69: self._headers.getheaders(name) jpayne@69: jpayne@69: jpayne@69: def extract_cookies_to_jar(jar, request, response): jpayne@69: """Extract the cookies from the response into a CookieJar. jpayne@69: jpayne@69: :param jar: http.cookiejar.CookieJar (not necessarily a RequestsCookieJar) jpayne@69: :param request: our own requests.Request object jpayne@69: :param response: urllib3.HTTPResponse object jpayne@69: """ jpayne@69: if not (hasattr(response, "_original_response") and response._original_response): jpayne@69: return jpayne@69: # the _original_response field is the wrapped httplib.HTTPResponse object, jpayne@69: req = MockRequest(request) jpayne@69: # pull out the HTTPMessage with the headers and put it in the mock: jpayne@69: res = MockResponse(response._original_response.msg) jpayne@69: jar.extract_cookies(res, req) jpayne@69: jpayne@69: jpayne@69: def get_cookie_header(jar, request): jpayne@69: """ jpayne@69: Produce an appropriate Cookie header string to be sent with `request`, or None. jpayne@69: jpayne@69: :rtype: str jpayne@69: """ jpayne@69: r = MockRequest(request) jpayne@69: jar.add_cookie_header(r) jpayne@69: return r.get_new_headers().get("Cookie") jpayne@69: jpayne@69: jpayne@69: def remove_cookie_by_name(cookiejar, name, domain=None, path=None): jpayne@69: """Unsets a cookie by name, by default over all domains and paths. jpayne@69: jpayne@69: Wraps CookieJar.clear(), is O(n). jpayne@69: """ jpayne@69: clearables = [] jpayne@69: for cookie in cookiejar: jpayne@69: if cookie.name != name: jpayne@69: continue jpayne@69: if domain is not None and domain != cookie.domain: jpayne@69: continue jpayne@69: if path is not None and path != cookie.path: jpayne@69: continue jpayne@69: clearables.append((cookie.domain, cookie.path, cookie.name)) jpayne@69: jpayne@69: for domain, path, name in clearables: jpayne@69: cookiejar.clear(domain, path, name) jpayne@69: jpayne@69: jpayne@69: class CookieConflictError(RuntimeError): jpayne@69: """There are two cookies that meet the criteria specified in the cookie jar. jpayne@69: Use .get and .set and include domain and path args in order to be more specific. jpayne@69: """ jpayne@69: jpayne@69: jpayne@69: class RequestsCookieJar(cookielib.CookieJar, MutableMapping): jpayne@69: """Compatibility class; is a http.cookiejar.CookieJar, but exposes a dict jpayne@69: interface. jpayne@69: jpayne@69: This is the CookieJar we create by default for requests and sessions that jpayne@69: don't specify one, since some clients may expect response.cookies and jpayne@69: session.cookies to support dict operations. jpayne@69: jpayne@69: Requests does not use the dict interface internally; it's just for jpayne@69: compatibility with external client code. All requests code should work jpayne@69: out of the box with externally provided instances of ``CookieJar``, e.g. jpayne@69: ``LWPCookieJar`` and ``FileCookieJar``. jpayne@69: jpayne@69: Unlike a regular CookieJar, this class is pickleable. jpayne@69: jpayne@69: .. warning:: dictionary operations that are normally O(1) may be O(n). jpayne@69: """ jpayne@69: jpayne@69: def get(self, name, default=None, domain=None, path=None): jpayne@69: """Dict-like get() that also supports optional domain and path args in jpayne@69: order to resolve naming collisions from using one cookie jar over jpayne@69: multiple domains. jpayne@69: jpayne@69: .. warning:: operation is O(n), not O(1). jpayne@69: """ jpayne@69: try: jpayne@69: return self._find_no_duplicates(name, domain, path) jpayne@69: except KeyError: jpayne@69: return default jpayne@69: jpayne@69: def set(self, name, value, **kwargs): jpayne@69: """Dict-like set() that also supports optional domain and path args in jpayne@69: order to resolve naming collisions from using one cookie jar over jpayne@69: multiple domains. jpayne@69: """ jpayne@69: # support client code that unsets cookies by assignment of a None value: jpayne@69: if value is None: jpayne@69: remove_cookie_by_name( jpayne@69: self, name, domain=kwargs.get("domain"), path=kwargs.get("path") jpayne@69: ) jpayne@69: return jpayne@69: jpayne@69: if isinstance(value, Morsel): jpayne@69: c = morsel_to_cookie(value) jpayne@69: else: jpayne@69: c = create_cookie(name, value, **kwargs) jpayne@69: self.set_cookie(c) jpayne@69: return c jpayne@69: jpayne@69: def iterkeys(self): jpayne@69: """Dict-like iterkeys() that returns an iterator of names of cookies jpayne@69: from the jar. jpayne@69: jpayne@69: .. seealso:: itervalues() and iteritems(). jpayne@69: """ jpayne@69: for cookie in iter(self): jpayne@69: yield cookie.name jpayne@69: jpayne@69: def keys(self): jpayne@69: """Dict-like keys() that returns a list of names of cookies from the jpayne@69: jar. jpayne@69: jpayne@69: .. seealso:: values() and items(). jpayne@69: """ jpayne@69: return list(self.iterkeys()) jpayne@69: jpayne@69: def itervalues(self): jpayne@69: """Dict-like itervalues() that returns an iterator of values of cookies jpayne@69: from the jar. jpayne@69: jpayne@69: .. seealso:: iterkeys() and iteritems(). jpayne@69: """ jpayne@69: for cookie in iter(self): jpayne@69: yield cookie.value jpayne@69: jpayne@69: def values(self): jpayne@69: """Dict-like values() that returns a list of values of cookies from the jpayne@69: jar. jpayne@69: jpayne@69: .. seealso:: keys() and items(). jpayne@69: """ jpayne@69: return list(self.itervalues()) jpayne@69: jpayne@69: def iteritems(self): jpayne@69: """Dict-like iteritems() that returns an iterator of name-value tuples jpayne@69: from the jar. jpayne@69: jpayne@69: .. seealso:: iterkeys() and itervalues(). jpayne@69: """ jpayne@69: for cookie in iter(self): jpayne@69: yield cookie.name, cookie.value jpayne@69: jpayne@69: def items(self): jpayne@69: """Dict-like items() that returns a list of name-value tuples from the jpayne@69: jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a jpayne@69: vanilla python dict of key value pairs. jpayne@69: jpayne@69: .. seealso:: keys() and values(). jpayne@69: """ jpayne@69: return list(self.iteritems()) jpayne@69: jpayne@69: def list_domains(self): jpayne@69: """Utility method to list all the domains in the jar.""" jpayne@69: domains = [] jpayne@69: for cookie in iter(self): jpayne@69: if cookie.domain not in domains: jpayne@69: domains.append(cookie.domain) jpayne@69: return domains jpayne@69: jpayne@69: def list_paths(self): jpayne@69: """Utility method to list all the paths in the jar.""" jpayne@69: paths = [] jpayne@69: for cookie in iter(self): jpayne@69: if cookie.path not in paths: jpayne@69: paths.append(cookie.path) jpayne@69: return paths jpayne@69: jpayne@69: def multiple_domains(self): jpayne@69: """Returns True if there are multiple domains in the jar. jpayne@69: Returns False otherwise. jpayne@69: jpayne@69: :rtype: bool jpayne@69: """ jpayne@69: domains = [] jpayne@69: for cookie in iter(self): jpayne@69: if cookie.domain is not None and cookie.domain in domains: jpayne@69: return True jpayne@69: domains.append(cookie.domain) jpayne@69: return False # there is only one domain in jar jpayne@69: jpayne@69: def get_dict(self, domain=None, path=None): jpayne@69: """Takes as an argument an optional domain and path and returns a plain jpayne@69: old Python dict of name-value pairs of cookies that meet the jpayne@69: requirements. jpayne@69: jpayne@69: :rtype: dict jpayne@69: """ jpayne@69: dictionary = {} jpayne@69: for cookie in iter(self): jpayne@69: if (domain is None or cookie.domain == domain) and ( jpayne@69: path is None or cookie.path == path jpayne@69: ): jpayne@69: dictionary[cookie.name] = cookie.value jpayne@69: return dictionary jpayne@69: jpayne@69: def __contains__(self, name): jpayne@69: try: jpayne@69: return super().__contains__(name) jpayne@69: except CookieConflictError: jpayne@69: return True jpayne@69: jpayne@69: def __getitem__(self, name): jpayne@69: """Dict-like __getitem__() for compatibility with client code. Throws jpayne@69: exception if there are more than one cookie with name. In that case, jpayne@69: use the more explicit get() method instead. jpayne@69: jpayne@69: .. warning:: operation is O(n), not O(1). jpayne@69: """ jpayne@69: return self._find_no_duplicates(name) jpayne@69: jpayne@69: def __setitem__(self, name, value): jpayne@69: """Dict-like __setitem__ for compatibility with client code. Throws jpayne@69: exception if there is already a cookie of that name in the jar. In that jpayne@69: case, use the more explicit set() method instead. jpayne@69: """ jpayne@69: self.set(name, value) jpayne@69: jpayne@69: def __delitem__(self, name): jpayne@69: """Deletes a cookie given a name. Wraps ``http.cookiejar.CookieJar``'s jpayne@69: ``remove_cookie_by_name()``. jpayne@69: """ jpayne@69: remove_cookie_by_name(self, name) jpayne@69: jpayne@69: def set_cookie(self, cookie, *args, **kwargs): jpayne@69: if ( jpayne@69: hasattr(cookie.value, "startswith") jpayne@69: and cookie.value.startswith('"') jpayne@69: and cookie.value.endswith('"') jpayne@69: ): jpayne@69: cookie.value = cookie.value.replace('\\"', "") jpayne@69: return super().set_cookie(cookie, *args, **kwargs) jpayne@69: jpayne@69: def update(self, other): jpayne@69: """Updates this jar with cookies from another CookieJar or dict-like""" jpayne@69: if isinstance(other, cookielib.CookieJar): jpayne@69: for cookie in other: jpayne@69: self.set_cookie(copy.copy(cookie)) jpayne@69: else: jpayne@69: super().update(other) jpayne@69: jpayne@69: def _find(self, name, domain=None, path=None): jpayne@69: """Requests uses this method internally to get cookie values. jpayne@69: jpayne@69: If there are conflicting cookies, _find arbitrarily chooses one. jpayne@69: See _find_no_duplicates if you want an exception thrown if there are jpayne@69: conflicting cookies. jpayne@69: jpayne@69: :param name: a string containing name of cookie jpayne@69: :param domain: (optional) string containing domain of cookie jpayne@69: :param path: (optional) string containing path of cookie jpayne@69: :return: cookie.value jpayne@69: """ jpayne@69: for cookie in iter(self): jpayne@69: if cookie.name == name: jpayne@69: if domain is None or cookie.domain == domain: jpayne@69: if path is None or cookie.path == path: jpayne@69: return cookie.value jpayne@69: jpayne@69: raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") jpayne@69: jpayne@69: def _find_no_duplicates(self, name, domain=None, path=None): jpayne@69: """Both ``__get_item__`` and ``get`` call this function: it's never jpayne@69: used elsewhere in Requests. jpayne@69: jpayne@69: :param name: a string containing name of cookie jpayne@69: :param domain: (optional) string containing domain of cookie jpayne@69: :param path: (optional) string containing path of cookie jpayne@69: :raises KeyError: if cookie is not found jpayne@69: :raises CookieConflictError: if there are multiple cookies jpayne@69: that match name and optionally domain and path jpayne@69: :return: cookie.value jpayne@69: """ jpayne@69: toReturn = None jpayne@69: for cookie in iter(self): jpayne@69: if cookie.name == name: jpayne@69: if domain is None or cookie.domain == domain: jpayne@69: if path is None or cookie.path == path: jpayne@69: if toReturn is not None: jpayne@69: # if there are multiple cookies that meet passed in criteria jpayne@69: raise CookieConflictError( jpayne@69: f"There are multiple cookies with name, {name!r}" jpayne@69: ) jpayne@69: # we will eventually return this as long as no cookie conflict jpayne@69: toReturn = cookie.value jpayne@69: jpayne@69: if toReturn: jpayne@69: return toReturn jpayne@69: raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") jpayne@69: jpayne@69: def __getstate__(self): jpayne@69: """Unlike a normal CookieJar, this class is pickleable.""" jpayne@69: state = self.__dict__.copy() jpayne@69: # remove the unpickleable RLock object jpayne@69: state.pop("_cookies_lock") jpayne@69: return state jpayne@69: jpayne@69: def __setstate__(self, state): jpayne@69: """Unlike a normal CookieJar, this class is pickleable.""" jpayne@69: self.__dict__.update(state) jpayne@69: if "_cookies_lock" not in self.__dict__: jpayne@69: self._cookies_lock = threading.RLock() jpayne@69: jpayne@69: def copy(self): jpayne@69: """Return a copy of this RequestsCookieJar.""" jpayne@69: new_cj = RequestsCookieJar() jpayne@69: new_cj.set_policy(self.get_policy()) jpayne@69: new_cj.update(self) jpayne@69: return new_cj jpayne@69: jpayne@69: def get_policy(self): jpayne@69: """Return the CookiePolicy instance used.""" jpayne@69: return self._policy jpayne@69: jpayne@69: jpayne@69: def _copy_cookie_jar(jar): jpayne@69: if jar is None: jpayne@69: return None jpayne@69: jpayne@69: if hasattr(jar, "copy"): jpayne@69: # We're dealing with an instance of RequestsCookieJar jpayne@69: return jar.copy() jpayne@69: # We're dealing with a generic CookieJar instance jpayne@69: new_jar = copy.copy(jar) jpayne@69: new_jar.clear() jpayne@69: for cookie in jar: jpayne@69: new_jar.set_cookie(copy.copy(cookie)) jpayne@69: return new_jar jpayne@69: jpayne@69: jpayne@69: def create_cookie(name, value, **kwargs): jpayne@69: """Make a cookie from underspecified parameters. jpayne@69: jpayne@69: By default, the pair of `name` and `value` will be set for the domain '' jpayne@69: and sent on every request (this is sometimes called a "supercookie"). jpayne@69: """ jpayne@69: result = { jpayne@69: "version": 0, jpayne@69: "name": name, jpayne@69: "value": value, jpayne@69: "port": None, jpayne@69: "domain": "", jpayne@69: "path": "/", jpayne@69: "secure": False, jpayne@69: "expires": None, jpayne@69: "discard": True, jpayne@69: "comment": None, jpayne@69: "comment_url": None, jpayne@69: "rest": {"HttpOnly": None}, jpayne@69: "rfc2109": False, jpayne@69: } jpayne@69: jpayne@69: badargs = set(kwargs) - set(result) jpayne@69: if badargs: jpayne@69: raise TypeError( jpayne@69: f"create_cookie() got unexpected keyword arguments: {list(badargs)}" jpayne@69: ) jpayne@69: jpayne@69: result.update(kwargs) jpayne@69: result["port_specified"] = bool(result["port"]) jpayne@69: result["domain_specified"] = bool(result["domain"]) jpayne@69: result["domain_initial_dot"] = result["domain"].startswith(".") jpayne@69: result["path_specified"] = bool(result["path"]) jpayne@69: jpayne@69: return cookielib.Cookie(**result) jpayne@69: jpayne@69: jpayne@69: def morsel_to_cookie(morsel): jpayne@69: """Convert a Morsel object into a Cookie containing the one k/v pair.""" jpayne@69: jpayne@69: expires = None jpayne@69: if morsel["max-age"]: jpayne@69: try: jpayne@69: expires = int(time.time() + int(morsel["max-age"])) jpayne@69: except ValueError: jpayne@69: raise TypeError(f"max-age: {morsel['max-age']} must be integer") jpayne@69: elif morsel["expires"]: jpayne@69: time_template = "%a, %d-%b-%Y %H:%M:%S GMT" jpayne@69: expires = calendar.timegm(time.strptime(morsel["expires"], time_template)) jpayne@69: return create_cookie( jpayne@69: comment=morsel["comment"], jpayne@69: comment_url=bool(morsel["comment"]), jpayne@69: discard=False, jpayne@69: domain=morsel["domain"], jpayne@69: expires=expires, jpayne@69: name=morsel.key, jpayne@69: path=morsel["path"], jpayne@69: port=None, jpayne@69: rest={"HttpOnly": morsel["httponly"]}, jpayne@69: rfc2109=False, jpayne@69: secure=bool(morsel["secure"]), jpayne@69: value=morsel.value, jpayne@69: version=morsel["version"] or 0, jpayne@69: ) jpayne@69: jpayne@69: jpayne@69: def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): jpayne@69: """Returns a CookieJar from a key/value dictionary. jpayne@69: jpayne@69: :param cookie_dict: Dict of key/values to insert into CookieJar. jpayne@69: :param cookiejar: (optional) A cookiejar to add the cookies to. jpayne@69: :param overwrite: (optional) If False, will not replace cookies jpayne@69: already in the jar with new ones. jpayne@69: :rtype: CookieJar jpayne@69: """ jpayne@69: if cookiejar is None: jpayne@69: cookiejar = RequestsCookieJar() jpayne@69: jpayne@69: if cookie_dict is not None: jpayne@69: names_from_jar = [cookie.name for cookie in cookiejar] jpayne@69: for name in cookie_dict: jpayne@69: if overwrite or (name not in names_from_jar): jpayne@69: cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) jpayne@69: jpayne@69: return cookiejar jpayne@69: jpayne@69: jpayne@69: def merge_cookies(cookiejar, cookies): jpayne@69: """Add cookies to cookiejar and returns a merged CookieJar. jpayne@69: jpayne@69: :param cookiejar: CookieJar object to add the cookies to. jpayne@69: :param cookies: Dictionary or CookieJar object to be added. jpayne@69: :rtype: CookieJar jpayne@69: """ jpayne@69: if not isinstance(cookiejar, cookielib.CookieJar): jpayne@69: raise ValueError("You can only merge into CookieJar") jpayne@69: jpayne@69: if isinstance(cookies, dict): jpayne@69: cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False) jpayne@69: elif isinstance(cookies, cookielib.CookieJar): jpayne@69: try: jpayne@69: cookiejar.update(cookies) jpayne@69: except AttributeError: jpayne@69: for cookie_in_jar in cookies: jpayne@69: cookiejar.set_cookie(cookie_in_jar) jpayne@69: jpayne@69: return cookiejar