jpayne@7: """ jpayne@7: requests.cookies jpayne@7: ~~~~~~~~~~~~~~~~ jpayne@7: jpayne@7: Compatibility code to be able to use `cookielib.CookieJar` with requests. jpayne@7: jpayne@7: requests.utils imports from here, so be careful with imports. jpayne@7: """ jpayne@7: jpayne@7: import calendar jpayne@7: import copy jpayne@7: import time jpayne@7: jpayne@7: from ._internal_utils import to_native_string jpayne@7: from .compat import Morsel, MutableMapping, cookielib, urlparse, urlunparse jpayne@7: jpayne@7: try: jpayne@7: import threading jpayne@7: except ImportError: jpayne@7: import dummy_threading as threading jpayne@7: jpayne@7: jpayne@7: class MockRequest: jpayne@7: """Wraps a `requests.Request` to mimic a `urllib2.Request`. jpayne@7: jpayne@7: The code in `cookielib.CookieJar` expects this interface in order to correctly jpayne@7: manage cookie policies, i.e., determine whether a cookie can be set, given the jpayne@7: domains of the request and the cookie. jpayne@7: jpayne@7: The original request object is read-only. The client is responsible for collecting jpayne@7: the new headers via `get_new_headers()` and interpreting them appropriately. You jpayne@7: probably want `get_cookie_header`, defined below. jpayne@7: """ jpayne@7: jpayne@7: def __init__(self, request): jpayne@7: self._r = request jpayne@7: self._new_headers = {} jpayne@7: self.type = urlparse(self._r.url).scheme jpayne@7: jpayne@7: def get_type(self): jpayne@7: return self.type jpayne@7: jpayne@7: def get_host(self): jpayne@7: return urlparse(self._r.url).netloc jpayne@7: jpayne@7: def get_origin_req_host(self): jpayne@7: return self.get_host() jpayne@7: jpayne@7: def get_full_url(self): jpayne@7: # Only return the response's URL if the user hadn't set the Host jpayne@7: # header jpayne@7: if not self._r.headers.get("Host"): jpayne@7: return self._r.url jpayne@7: # If they did set it, retrieve it and reconstruct the expected domain jpayne@7: host = to_native_string(self._r.headers["Host"], encoding="utf-8") jpayne@7: parsed = urlparse(self._r.url) jpayne@7: # Reconstruct the URL as we expect it jpayne@7: return urlunparse( jpayne@7: [ jpayne@7: parsed.scheme, jpayne@7: host, jpayne@7: parsed.path, jpayne@7: parsed.params, jpayne@7: parsed.query, jpayne@7: parsed.fragment, jpayne@7: ] jpayne@7: ) jpayne@7: jpayne@7: def is_unverifiable(self): jpayne@7: return True jpayne@7: jpayne@7: def has_header(self, name): jpayne@7: return name in self._r.headers or name in self._new_headers jpayne@7: jpayne@7: def get_header(self, name, default=None): jpayne@7: return self._r.headers.get(name, self._new_headers.get(name, default)) jpayne@7: jpayne@7: def add_header(self, key, val): jpayne@7: """cookielib has no legitimate use for this method; add it back if you find one.""" jpayne@7: raise NotImplementedError( jpayne@7: "Cookie headers should be added with add_unredirected_header()" jpayne@7: ) jpayne@7: jpayne@7: def add_unredirected_header(self, name, value): jpayne@7: self._new_headers[name] = value jpayne@7: jpayne@7: def get_new_headers(self): jpayne@7: return self._new_headers jpayne@7: jpayne@7: @property jpayne@7: def unverifiable(self): jpayne@7: return self.is_unverifiable() jpayne@7: jpayne@7: @property jpayne@7: def origin_req_host(self): jpayne@7: return self.get_origin_req_host() jpayne@7: jpayne@7: @property jpayne@7: def host(self): jpayne@7: return self.get_host() jpayne@7: jpayne@7: jpayne@7: class MockResponse: jpayne@7: """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. jpayne@7: jpayne@7: ...what? Basically, expose the parsed HTTP headers from the server response jpayne@7: the way `cookielib` expects to see them. jpayne@7: """ jpayne@7: jpayne@7: def __init__(self, headers): jpayne@7: """Make a MockResponse for `cookielib` to read. jpayne@7: jpayne@7: :param headers: a httplib.HTTPMessage or analogous carrying the headers jpayne@7: """ jpayne@7: self._headers = headers jpayne@7: jpayne@7: def info(self): jpayne@7: return self._headers jpayne@7: jpayne@7: def getheaders(self, name): jpayne@7: self._headers.getheaders(name) jpayne@7: jpayne@7: jpayne@7: def extract_cookies_to_jar(jar, request, response): jpayne@7: """Extract the cookies from the response into a CookieJar. jpayne@7: jpayne@7: :param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar) jpayne@7: :param request: our own requests.Request object jpayne@7: :param response: urllib3.HTTPResponse object jpayne@7: """ jpayne@7: if not (hasattr(response, "_original_response") and response._original_response): jpayne@7: return jpayne@7: # the _original_response field is the wrapped httplib.HTTPResponse object, jpayne@7: req = MockRequest(request) jpayne@7: # pull out the HTTPMessage with the headers and put it in the mock: jpayne@7: res = MockResponse(response._original_response.msg) jpayne@7: jar.extract_cookies(res, req) jpayne@7: jpayne@7: jpayne@7: def get_cookie_header(jar, request): jpayne@7: """ jpayne@7: Produce an appropriate Cookie header string to be sent with `request`, or None. jpayne@7: jpayne@7: :rtype: str jpayne@7: """ jpayne@7: r = MockRequest(request) jpayne@7: jar.add_cookie_header(r) jpayne@7: return r.get_new_headers().get("Cookie") jpayne@7: jpayne@7: jpayne@7: def remove_cookie_by_name(cookiejar, name, domain=None, path=None): jpayne@7: """Unsets a cookie by name, by default over all domains and paths. jpayne@7: jpayne@7: Wraps CookieJar.clear(), is O(n). jpayne@7: """ jpayne@7: clearables = [] jpayne@7: for cookie in cookiejar: jpayne@7: if cookie.name != name: jpayne@7: continue jpayne@7: if domain is not None and domain != cookie.domain: jpayne@7: continue jpayne@7: if path is not None and path != cookie.path: jpayne@7: continue jpayne@7: clearables.append((cookie.domain, cookie.path, cookie.name)) jpayne@7: jpayne@7: for domain, path, name in clearables: jpayne@7: cookiejar.clear(domain, path, name) jpayne@7: jpayne@7: jpayne@7: class CookieConflictError(RuntimeError): jpayne@7: """There are two cookies that meet the criteria specified in the cookie jar. jpayne@7: Use .get and .set and include domain and path args in order to be more specific. jpayne@7: """ jpayne@7: jpayne@7: jpayne@7: class RequestsCookieJar(cookielib.CookieJar, MutableMapping): jpayne@7: """Compatibility class; is a cookielib.CookieJar, but exposes a dict jpayne@7: interface. jpayne@7: jpayne@7: This is the CookieJar we create by default for requests and sessions that jpayne@7: don't specify one, since some clients may expect response.cookies and jpayne@7: session.cookies to support dict operations. jpayne@7: jpayne@7: Requests does not use the dict interface internally; it's just for jpayne@7: compatibility with external client code. All requests code should work jpayne@7: out of the box with externally provided instances of ``CookieJar``, e.g. jpayne@7: ``LWPCookieJar`` and ``FileCookieJar``. jpayne@7: jpayne@7: Unlike a regular CookieJar, this class is pickleable. jpayne@7: jpayne@7: .. warning:: dictionary operations that are normally O(1) may be O(n). jpayne@7: """ jpayne@7: jpayne@7: def get(self, name, default=None, domain=None, path=None): jpayne@7: """Dict-like get() that also supports optional domain and path args in jpayne@7: order to resolve naming collisions from using one cookie jar over jpayne@7: multiple domains. jpayne@7: jpayne@7: .. warning:: operation is O(n), not O(1). jpayne@7: """ jpayne@7: try: jpayne@7: return self._find_no_duplicates(name, domain, path) jpayne@7: except KeyError: jpayne@7: return default jpayne@7: jpayne@7: def set(self, name, value, **kwargs): jpayne@7: """Dict-like set() that also supports optional domain and path args in jpayne@7: order to resolve naming collisions from using one cookie jar over jpayne@7: multiple domains. jpayne@7: """ jpayne@7: # support client code that unsets cookies by assignment of a None value: jpayne@7: if value is None: jpayne@7: remove_cookie_by_name( jpayne@7: self, name, domain=kwargs.get("domain"), path=kwargs.get("path") jpayne@7: ) jpayne@7: return jpayne@7: jpayne@7: if isinstance(value, Morsel): jpayne@7: c = morsel_to_cookie(value) jpayne@7: else: jpayne@7: c = create_cookie(name, value, **kwargs) jpayne@7: self.set_cookie(c) jpayne@7: return c jpayne@7: jpayne@7: def iterkeys(self): jpayne@7: """Dict-like iterkeys() that returns an iterator of names of cookies jpayne@7: from the jar. jpayne@7: jpayne@7: .. seealso:: itervalues() and iteritems(). jpayne@7: """ jpayne@7: for cookie in iter(self): jpayne@7: yield cookie.name jpayne@7: jpayne@7: def keys(self): jpayne@7: """Dict-like keys() that returns a list of names of cookies from the jpayne@7: jar. jpayne@7: jpayne@7: .. seealso:: values() and items(). jpayne@7: """ jpayne@7: return list(self.iterkeys()) jpayne@7: jpayne@7: def itervalues(self): jpayne@7: """Dict-like itervalues() that returns an iterator of values of cookies jpayne@7: from the jar. jpayne@7: jpayne@7: .. seealso:: iterkeys() and iteritems(). jpayne@7: """ jpayne@7: for cookie in iter(self): jpayne@7: yield cookie.value jpayne@7: jpayne@7: def values(self): jpayne@7: """Dict-like values() that returns a list of values of cookies from the jpayne@7: jar. jpayne@7: jpayne@7: .. seealso:: keys() and items(). jpayne@7: """ jpayne@7: return list(self.itervalues()) jpayne@7: jpayne@7: def iteritems(self): jpayne@7: """Dict-like iteritems() that returns an iterator of name-value tuples jpayne@7: from the jar. jpayne@7: jpayne@7: .. seealso:: iterkeys() and itervalues(). jpayne@7: """ jpayne@7: for cookie in iter(self): jpayne@7: yield cookie.name, cookie.value jpayne@7: jpayne@7: def items(self): jpayne@7: """Dict-like items() that returns a list of name-value tuples from the jpayne@7: jar. Allows client-code to call ``dict(RequestsCookieJar)`` and get a jpayne@7: vanilla python dict of key value pairs. jpayne@7: jpayne@7: .. seealso:: keys() and values(). jpayne@7: """ jpayne@7: return list(self.iteritems()) jpayne@7: jpayne@7: def list_domains(self): jpayne@7: """Utility method to list all the domains in the jar.""" jpayne@7: domains = [] jpayne@7: for cookie in iter(self): jpayne@7: if cookie.domain not in domains: jpayne@7: domains.append(cookie.domain) jpayne@7: return domains jpayne@7: jpayne@7: def list_paths(self): jpayne@7: """Utility method to list all the paths in the jar.""" jpayne@7: paths = [] jpayne@7: for cookie in iter(self): jpayne@7: if cookie.path not in paths: jpayne@7: paths.append(cookie.path) jpayne@7: return paths jpayne@7: jpayne@7: def multiple_domains(self): jpayne@7: """Returns True if there are multiple domains in the jar. jpayne@7: Returns False otherwise. jpayne@7: jpayne@7: :rtype: bool jpayne@7: """ jpayne@7: domains = [] jpayne@7: for cookie in iter(self): jpayne@7: if cookie.domain is not None and cookie.domain in domains: jpayne@7: return True jpayne@7: domains.append(cookie.domain) jpayne@7: return False # there is only one domain in jar jpayne@7: jpayne@7: def get_dict(self, domain=None, path=None): jpayne@7: """Takes as an argument an optional domain and path and returns a plain jpayne@7: old Python dict of name-value pairs of cookies that meet the jpayne@7: requirements. jpayne@7: jpayne@7: :rtype: dict jpayne@7: """ jpayne@7: dictionary = {} jpayne@7: for cookie in iter(self): jpayne@7: if (domain is None or cookie.domain == domain) and ( jpayne@7: path is None or cookie.path == path jpayne@7: ): jpayne@7: dictionary[cookie.name] = cookie.value jpayne@7: return dictionary jpayne@7: jpayne@7: def __contains__(self, name): jpayne@7: try: jpayne@7: return super().__contains__(name) jpayne@7: except CookieConflictError: jpayne@7: return True jpayne@7: jpayne@7: def __getitem__(self, name): jpayne@7: """Dict-like __getitem__() for compatibility with client code. Throws jpayne@7: exception if there are more than one cookie with name. In that case, jpayne@7: use the more explicit get() method instead. jpayne@7: jpayne@7: .. warning:: operation is O(n), not O(1). jpayne@7: """ jpayne@7: return self._find_no_duplicates(name) jpayne@7: jpayne@7: def __setitem__(self, name, value): jpayne@7: """Dict-like __setitem__ for compatibility with client code. Throws jpayne@7: exception if there is already a cookie of that name in the jar. In that jpayne@7: case, use the more explicit set() method instead. jpayne@7: """ jpayne@7: self.set(name, value) jpayne@7: jpayne@7: def __delitem__(self, name): jpayne@7: """Deletes a cookie given a name. Wraps ``cookielib.CookieJar``'s jpayne@7: ``remove_cookie_by_name()``. jpayne@7: """ jpayne@7: remove_cookie_by_name(self, name) jpayne@7: jpayne@7: def set_cookie(self, cookie, *args, **kwargs): jpayne@7: if ( jpayne@7: hasattr(cookie.value, "startswith") jpayne@7: and cookie.value.startswith('"') jpayne@7: and cookie.value.endswith('"') jpayne@7: ): jpayne@7: cookie.value = cookie.value.replace('\\"', "") jpayne@7: return super().set_cookie(cookie, *args, **kwargs) jpayne@7: jpayne@7: def update(self, other): jpayne@7: """Updates this jar with cookies from another CookieJar or dict-like""" jpayne@7: if isinstance(other, cookielib.CookieJar): jpayne@7: for cookie in other: jpayne@7: self.set_cookie(copy.copy(cookie)) jpayne@7: else: jpayne@7: super().update(other) jpayne@7: jpayne@7: def _find(self, name, domain=None, path=None): jpayne@7: """Requests uses this method internally to get cookie values. jpayne@7: jpayne@7: If there are conflicting cookies, _find arbitrarily chooses one. jpayne@7: See _find_no_duplicates if you want an exception thrown if there are jpayne@7: conflicting cookies. jpayne@7: jpayne@7: :param name: a string containing name of cookie jpayne@7: :param domain: (optional) string containing domain of cookie jpayne@7: :param path: (optional) string containing path of cookie jpayne@7: :return: cookie.value jpayne@7: """ jpayne@7: for cookie in iter(self): jpayne@7: if cookie.name == name: jpayne@7: if domain is None or cookie.domain == domain: jpayne@7: if path is None or cookie.path == path: jpayne@7: return cookie.value jpayne@7: jpayne@7: raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") jpayne@7: jpayne@7: def _find_no_duplicates(self, name, domain=None, path=None): jpayne@7: """Both ``__get_item__`` and ``get`` call this function: it's never jpayne@7: used elsewhere in Requests. jpayne@7: jpayne@7: :param name: a string containing name of cookie jpayne@7: :param domain: (optional) string containing domain of cookie jpayne@7: :param path: (optional) string containing path of cookie jpayne@7: :raises KeyError: if cookie is not found jpayne@7: :raises CookieConflictError: if there are multiple cookies jpayne@7: that match name and optionally domain and path jpayne@7: :return: cookie.value jpayne@7: """ jpayne@7: toReturn = None jpayne@7: for cookie in iter(self): jpayne@7: if cookie.name == name: jpayne@7: if domain is None or cookie.domain == domain: jpayne@7: if path is None or cookie.path == path: jpayne@7: if toReturn is not None: jpayne@7: # if there are multiple cookies that meet passed in criteria jpayne@7: raise CookieConflictError( jpayne@7: f"There are multiple cookies with name, {name!r}" jpayne@7: ) jpayne@7: # we will eventually return this as long as no cookie conflict jpayne@7: toReturn = cookie.value jpayne@7: jpayne@7: if toReturn: jpayne@7: return toReturn jpayne@7: raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") jpayne@7: jpayne@7: def __getstate__(self): jpayne@7: """Unlike a normal CookieJar, this class is pickleable.""" jpayne@7: state = self.__dict__.copy() jpayne@7: # remove the unpickleable RLock object jpayne@7: state.pop("_cookies_lock") jpayne@7: return state jpayne@7: jpayne@7: def __setstate__(self, state): jpayne@7: """Unlike a normal CookieJar, this class is pickleable.""" jpayne@7: self.__dict__.update(state) jpayne@7: if "_cookies_lock" not in self.__dict__: jpayne@7: self._cookies_lock = threading.RLock() jpayne@7: jpayne@7: def copy(self): jpayne@7: """Return a copy of this RequestsCookieJar.""" jpayne@7: new_cj = RequestsCookieJar() jpayne@7: new_cj.set_policy(self.get_policy()) jpayne@7: new_cj.update(self) jpayne@7: return new_cj jpayne@7: jpayne@7: def get_policy(self): jpayne@7: """Return the CookiePolicy instance used.""" jpayne@7: return self._policy jpayne@7: jpayne@7: jpayne@7: def _copy_cookie_jar(jar): jpayne@7: if jar is None: jpayne@7: return None jpayne@7: jpayne@7: if hasattr(jar, "copy"): jpayne@7: # We're dealing with an instance of RequestsCookieJar jpayne@7: return jar.copy() jpayne@7: # We're dealing with a generic CookieJar instance jpayne@7: new_jar = copy.copy(jar) jpayne@7: new_jar.clear() jpayne@7: for cookie in jar: jpayne@7: new_jar.set_cookie(copy.copy(cookie)) jpayne@7: return new_jar jpayne@7: jpayne@7: jpayne@7: def create_cookie(name, value, **kwargs): jpayne@7: """Make a cookie from underspecified parameters. jpayne@7: jpayne@7: By default, the pair of `name` and `value` will be set for the domain '' jpayne@7: and sent on every request (this is sometimes called a "supercookie"). jpayne@7: """ jpayne@7: result = { jpayne@7: "version": 0, jpayne@7: "name": name, jpayne@7: "value": value, jpayne@7: "port": None, jpayne@7: "domain": "", jpayne@7: "path": "/", jpayne@7: "secure": False, jpayne@7: "expires": None, jpayne@7: "discard": True, jpayne@7: "comment": None, jpayne@7: "comment_url": None, jpayne@7: "rest": {"HttpOnly": None}, jpayne@7: "rfc2109": False, jpayne@7: } jpayne@7: jpayne@7: badargs = set(kwargs) - set(result) jpayne@7: if badargs: jpayne@7: raise TypeError( jpayne@7: f"create_cookie() got unexpected keyword arguments: {list(badargs)}" jpayne@7: ) jpayne@7: jpayne@7: result.update(kwargs) jpayne@7: result["port_specified"] = bool(result["port"]) jpayne@7: result["domain_specified"] = bool(result["domain"]) jpayne@7: result["domain_initial_dot"] = result["domain"].startswith(".") jpayne@7: result["path_specified"] = bool(result["path"]) jpayne@7: jpayne@7: return cookielib.Cookie(**result) jpayne@7: jpayne@7: jpayne@7: def morsel_to_cookie(morsel): jpayne@7: """Convert a Morsel object into a Cookie containing the one k/v pair.""" jpayne@7: jpayne@7: expires = None jpayne@7: if morsel["max-age"]: jpayne@7: try: jpayne@7: expires = int(time.time() + int(morsel["max-age"])) jpayne@7: except ValueError: jpayne@7: raise TypeError(f"max-age: {morsel['max-age']} must be integer") jpayne@7: elif morsel["expires"]: jpayne@7: time_template = "%a, %d-%b-%Y %H:%M:%S GMT" jpayne@7: expires = calendar.timegm(time.strptime(morsel["expires"], time_template)) jpayne@7: return create_cookie( jpayne@7: comment=morsel["comment"], jpayne@7: comment_url=bool(morsel["comment"]), jpayne@7: discard=False, jpayne@7: domain=morsel["domain"], jpayne@7: expires=expires, jpayne@7: name=morsel.key, jpayne@7: path=morsel["path"], jpayne@7: port=None, jpayne@7: rest={"HttpOnly": morsel["httponly"]}, jpayne@7: rfc2109=False, jpayne@7: secure=bool(morsel["secure"]), jpayne@7: value=morsel.value, jpayne@7: version=morsel["version"] or 0, jpayne@7: ) jpayne@7: jpayne@7: jpayne@7: def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): jpayne@7: """Returns a CookieJar from a key/value dictionary. jpayne@7: jpayne@7: :param cookie_dict: Dict of key/values to insert into CookieJar. jpayne@7: :param cookiejar: (optional) A cookiejar to add the cookies to. jpayne@7: :param overwrite: (optional) If False, will not replace cookies jpayne@7: already in the jar with new ones. jpayne@7: :rtype: CookieJar jpayne@7: """ jpayne@7: if cookiejar is None: jpayne@7: cookiejar = RequestsCookieJar() jpayne@7: jpayne@7: if cookie_dict is not None: jpayne@7: names_from_jar = [cookie.name for cookie in cookiejar] jpayne@7: for name in cookie_dict: jpayne@7: if overwrite or (name not in names_from_jar): jpayne@7: cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) jpayne@7: jpayne@7: return cookiejar jpayne@7: jpayne@7: jpayne@7: def merge_cookies(cookiejar, cookies): jpayne@7: """Add cookies to cookiejar and returns a merged CookieJar. jpayne@7: jpayne@7: :param cookiejar: CookieJar object to add the cookies to. jpayne@7: :param cookies: Dictionary or CookieJar object to be added. jpayne@7: :rtype: CookieJar jpayne@7: """ jpayne@7: if not isinstance(cookiejar, cookielib.CookieJar): jpayne@7: raise ValueError("You can only merge into CookieJar") jpayne@7: jpayne@7: if isinstance(cookies, dict): jpayne@7: cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False) jpayne@7: elif isinstance(cookies, cookielib.CookieJar): jpayne@7: try: jpayne@7: cookiejar.update(cookies) jpayne@7: except AttributeError: jpayne@7: for cookie_in_jar in cookies: jpayne@7: cookiejar.set_cookie(cookie_in_jar) jpayne@7: jpayne@7: return cookiejar