jpayne@69: r"""HTTP cookie handling for web clients. jpayne@69: jpayne@69: This module has (now fairly distant) origins in Gisle Aas' Perl module jpayne@69: HTTP::Cookies, from the libwww-perl library. jpayne@69: jpayne@69: Docstrings, comments and debug strings in this code refer to the jpayne@69: attributes of the HTTP cookie system as cookie-attributes, to distinguish jpayne@69: them clearly from Python attributes. jpayne@69: jpayne@69: Class diagram (note that BSDDBCookieJar and the MSIE* classes are not jpayne@69: distributed with the Python standard library, but are available from jpayne@69: http://wwwsearch.sf.net/): jpayne@69: jpayne@69: CookieJar____ jpayne@69: / \ \ jpayne@69: FileCookieJar \ \ jpayne@69: / | \ \ \ jpayne@69: MozillaCookieJar | LWPCookieJar \ \ jpayne@69: | | \ jpayne@69: | ---MSIEBase | \ jpayne@69: | / | | \ jpayne@69: | / MSIEDBCookieJar BSDDBCookieJar jpayne@69: |/ jpayne@69: MSIECookieJar jpayne@69: jpayne@69: """ jpayne@69: jpayne@69: __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy', jpayne@69: 'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar'] jpayne@69: jpayne@69: import os jpayne@69: import copy jpayne@69: import datetime jpayne@69: import re jpayne@69: import time jpayne@69: import urllib.parse, urllib.request jpayne@69: import threading as _threading jpayne@69: import http.client # only for the default HTTP port jpayne@69: from calendar import timegm jpayne@69: jpayne@69: debug = False # set to True to enable debugging via the logging module jpayne@69: logger = None jpayne@69: jpayne@69: def _debug(*args): jpayne@69: if not debug: jpayne@69: return jpayne@69: global logger jpayne@69: if not logger: jpayne@69: import logging jpayne@69: logger = logging.getLogger("http.cookiejar") jpayne@69: return logger.debug(*args) jpayne@69: jpayne@69: jpayne@69: DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT) jpayne@69: MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar " jpayne@69: "instance initialised with one)") jpayne@69: jpayne@69: def _warn_unhandled_exception(): jpayne@69: # There are a few catch-all except: statements in this module, for jpayne@69: # catching input that's bad in unexpected ways. Warn if any jpayne@69: # exceptions are caught there. jpayne@69: import io, warnings, traceback jpayne@69: f = io.StringIO() jpayne@69: traceback.print_exc(None, f) jpayne@69: msg = f.getvalue() jpayne@69: warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2) jpayne@69: jpayne@69: jpayne@69: # Date/time conversion jpayne@69: # ----------------------------------------------------------------------------- jpayne@69: jpayne@69: EPOCH_YEAR = 1970 jpayne@69: def _timegm(tt): jpayne@69: year, month, mday, hour, min, sec = tt[:6] jpayne@69: if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and jpayne@69: (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)): jpayne@69: return timegm(tt) jpayne@69: else: jpayne@69: return None jpayne@69: jpayne@69: DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] jpayne@69: MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", jpayne@69: "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] jpayne@69: MONTHS_LOWER = [] jpayne@69: for month in MONTHS: MONTHS_LOWER.append(month.lower()) jpayne@69: jpayne@69: def time2isoz(t=None): jpayne@69: """Return a string representing time in seconds since epoch, t. jpayne@69: jpayne@69: If the function is called without an argument, it will use the current jpayne@69: time. jpayne@69: jpayne@69: The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ", jpayne@69: representing Universal Time (UTC, aka GMT). An example of this format is: jpayne@69: jpayne@69: 1994-11-24 08:49:37Z jpayne@69: jpayne@69: """ jpayne@69: if t is None: jpayne@69: dt = datetime.datetime.utcnow() jpayne@69: else: jpayne@69: dt = datetime.datetime.utcfromtimestamp(t) jpayne@69: return "%04d-%02d-%02d %02d:%02d:%02dZ" % ( jpayne@69: dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) jpayne@69: jpayne@69: def time2netscape(t=None): jpayne@69: """Return a string representing time in seconds since epoch, t. jpayne@69: jpayne@69: If the function is called without an argument, it will use the current jpayne@69: time. jpayne@69: jpayne@69: The format of the returned string is like this: jpayne@69: jpayne@69: Wed, DD-Mon-YYYY HH:MM:SS GMT jpayne@69: jpayne@69: """ jpayne@69: if t is None: jpayne@69: dt = datetime.datetime.utcnow() jpayne@69: else: jpayne@69: dt = datetime.datetime.utcfromtimestamp(t) jpayne@69: return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % ( jpayne@69: DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1], jpayne@69: dt.year, dt.hour, dt.minute, dt.second) jpayne@69: jpayne@69: jpayne@69: UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None} jpayne@69: jpayne@69: TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII) jpayne@69: def offset_from_tz_string(tz): jpayne@69: offset = None jpayne@69: if tz in UTC_ZONES: jpayne@69: offset = 0 jpayne@69: else: jpayne@69: m = TIMEZONE_RE.search(tz) jpayne@69: if m: jpayne@69: offset = 3600 * int(m.group(2)) jpayne@69: if m.group(3): jpayne@69: offset = offset + 60 * int(m.group(3)) jpayne@69: if m.group(1) == '-': jpayne@69: offset = -offset jpayne@69: return offset jpayne@69: jpayne@69: def _str2time(day, mon, yr, hr, min, sec, tz): jpayne@69: yr = int(yr) jpayne@69: if yr > datetime.MAXYEAR: jpayne@69: return None jpayne@69: jpayne@69: # translate month name to number jpayne@69: # month numbers start with 1 (January) jpayne@69: try: jpayne@69: mon = MONTHS_LOWER.index(mon.lower())+1 jpayne@69: except ValueError: jpayne@69: # maybe it's already a number jpayne@69: try: jpayne@69: imon = int(mon) jpayne@69: except ValueError: jpayne@69: return None jpayne@69: if 1 <= imon <= 12: jpayne@69: mon = imon jpayne@69: else: jpayne@69: return None jpayne@69: jpayne@69: # make sure clock elements are defined jpayne@69: if hr is None: hr = 0 jpayne@69: if min is None: min = 0 jpayne@69: if sec is None: sec = 0 jpayne@69: jpayne@69: day = int(day) jpayne@69: hr = int(hr) jpayne@69: min = int(min) jpayne@69: sec = int(sec) jpayne@69: jpayne@69: if yr < 1000: jpayne@69: # find "obvious" year jpayne@69: cur_yr = time.localtime(time.time())[0] jpayne@69: m = cur_yr % 100 jpayne@69: tmp = yr jpayne@69: yr = yr + cur_yr - m jpayne@69: m = m - tmp jpayne@69: if abs(m) > 50: jpayne@69: if m > 0: yr = yr + 100 jpayne@69: else: yr = yr - 100 jpayne@69: jpayne@69: # convert UTC time tuple to seconds since epoch (not timezone-adjusted) jpayne@69: t = _timegm((yr, mon, day, hr, min, sec, tz)) jpayne@69: jpayne@69: if t is not None: jpayne@69: # adjust time using timezone string, to get absolute time since epoch jpayne@69: if tz is None: jpayne@69: tz = "UTC" jpayne@69: tz = tz.upper() jpayne@69: offset = offset_from_tz_string(tz) jpayne@69: if offset is None: jpayne@69: return None jpayne@69: t = t - offset jpayne@69: jpayne@69: return t jpayne@69: jpayne@69: STRICT_DATE_RE = re.compile( jpayne@69: r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) " jpayne@69: r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII) jpayne@69: WEEKDAY_RE = re.compile( jpayne@69: r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII) jpayne@69: LOOSE_HTTP_DATE_RE = re.compile( jpayne@69: r"""^ jpayne@69: (\d\d?) # day jpayne@69: (?:\s+|[-\/]) jpayne@69: (\w+) # month jpayne@69: (?:\s+|[-\/]) jpayne@69: (\d+) # year jpayne@69: (?: jpayne@69: (?:\s+|:) # separator before clock jpayne@69: (\d\d?):(\d\d) # hour:min jpayne@69: (?::(\d\d))? # optional seconds jpayne@69: )? # optional clock jpayne@69: \s* jpayne@69: (?: jpayne@69: ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone jpayne@69: \s* jpayne@69: )? jpayne@69: (?: jpayne@69: \(\w+\) # ASCII representation of timezone in parens. jpayne@69: \s* jpayne@69: )?$""", re.X | re.ASCII) jpayne@69: def http2time(text): jpayne@69: """Returns time in seconds since epoch of time represented by a string. jpayne@69: jpayne@69: Return value is an integer. jpayne@69: jpayne@69: None is returned if the format of str is unrecognized, the time is outside jpayne@69: the representable range, or the timezone string is not recognized. If the jpayne@69: string contains no timezone, UTC is assumed. jpayne@69: jpayne@69: The timezone in the string may be numerical (like "-0800" or "+0100") or a jpayne@69: string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the jpayne@69: timezone strings equivalent to UTC (zero offset) are known to the function. jpayne@69: jpayne@69: The function loosely parses the following formats: jpayne@69: jpayne@69: Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format jpayne@69: Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format jpayne@69: Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format jpayne@69: 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday) jpayne@69: 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday) jpayne@69: 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday) jpayne@69: jpayne@69: The parser ignores leading and trailing whitespace. The time may be jpayne@69: absent. jpayne@69: jpayne@69: If the year is given with only 2 digits, the function will select the jpayne@69: century that makes the year closest to the current date. jpayne@69: jpayne@69: """ jpayne@69: # fast exit for strictly conforming string jpayne@69: m = STRICT_DATE_RE.search(text) jpayne@69: if m: jpayne@69: g = m.groups() jpayne@69: mon = MONTHS_LOWER.index(g[1].lower()) + 1 jpayne@69: tt = (int(g[2]), mon, int(g[0]), jpayne@69: int(g[3]), int(g[4]), float(g[5])) jpayne@69: return _timegm(tt) jpayne@69: jpayne@69: # No, we need some messy parsing... jpayne@69: jpayne@69: # clean up jpayne@69: text = text.lstrip() jpayne@69: text = WEEKDAY_RE.sub("", text, 1) # Useless weekday jpayne@69: jpayne@69: # tz is time zone specifier string jpayne@69: day, mon, yr, hr, min, sec, tz = [None]*7 jpayne@69: jpayne@69: # loose regexp parse jpayne@69: m = LOOSE_HTTP_DATE_RE.search(text) jpayne@69: if m is not None: jpayne@69: day, mon, yr, hr, min, sec, tz = m.groups() jpayne@69: else: jpayne@69: return None # bad format jpayne@69: jpayne@69: return _str2time(day, mon, yr, hr, min, sec, tz) jpayne@69: jpayne@69: ISO_DATE_RE = re.compile( jpayne@69: r"""^ jpayne@69: (\d{4}) # year jpayne@69: [-\/]? jpayne@69: (\d\d?) # numerical month jpayne@69: [-\/]? jpayne@69: (\d\d?) # day jpayne@69: (?: jpayne@69: (?:\s+|[-:Tt]) # separator before clock jpayne@69: (\d\d?):?(\d\d) # hour:min jpayne@69: (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) jpayne@69: )? # optional clock jpayne@69: \s* jpayne@69: (?: jpayne@69: ([-+]?\d\d?:?(:?\d\d)? jpayne@69: |Z|z) # timezone (Z is "zero meridian", i.e. GMT) jpayne@69: \s* jpayne@69: )?$""", re.X | re. ASCII) jpayne@69: def iso2time(text): jpayne@69: """ jpayne@69: As for http2time, but parses the ISO 8601 formats: jpayne@69: jpayne@69: 1994-02-03 14:15:29 -0100 -- ISO 8601 format jpayne@69: 1994-02-03 14:15:29 -- zone is optional jpayne@69: 1994-02-03 -- only date jpayne@69: 1994-02-03T14:15:29 -- Use T as separator jpayne@69: 19940203T141529Z -- ISO 8601 compact format jpayne@69: 19940203 -- only date jpayne@69: jpayne@69: """ jpayne@69: # clean up jpayne@69: text = text.lstrip() jpayne@69: jpayne@69: # tz is time zone specifier string jpayne@69: day, mon, yr, hr, min, sec, tz = [None]*7 jpayne@69: jpayne@69: # loose regexp parse jpayne@69: m = ISO_DATE_RE.search(text) jpayne@69: if m is not None: jpayne@69: # XXX there's an extra bit of the timezone I'm ignoring here: is jpayne@69: # this the right thing to do? jpayne@69: yr, mon, day, hr, min, sec, tz, _ = m.groups() jpayne@69: else: jpayne@69: return None # bad format jpayne@69: jpayne@69: return _str2time(day, mon, yr, hr, min, sec, tz) jpayne@69: jpayne@69: jpayne@69: # Header parsing jpayne@69: # ----------------------------------------------------------------------------- jpayne@69: jpayne@69: def unmatched(match): jpayne@69: """Return unmatched part of re.Match object.""" jpayne@69: start, end = match.span(0) jpayne@69: return match.string[:start]+match.string[end:] jpayne@69: jpayne@69: HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)") jpayne@69: HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"") jpayne@69: HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)") jpayne@69: HEADER_ESCAPE_RE = re.compile(r"\\(.)") jpayne@69: def split_header_words(header_values): jpayne@69: r"""Parse header values into a list of lists containing key,value pairs. jpayne@69: jpayne@69: The function knows how to deal with ",", ";" and "=" as well as quoted jpayne@69: values after "=". A list of space separated tokens are parsed as if they jpayne@69: were separated by ";". jpayne@69: jpayne@69: If the header_values passed as argument contains multiple values, then they jpayne@69: are treated as if they were a single value separated by comma ",". jpayne@69: jpayne@69: This means that this function is useful for parsing header fields that jpayne@69: follow this syntax (BNF as from the HTTP/1.1 specification, but we relax jpayne@69: the requirement for tokens). jpayne@69: jpayne@69: headers = #header jpayne@69: header = (token | parameter) *( [";"] (token | parameter)) jpayne@69: jpayne@69: token = 1* jpayne@69: separators = "(" | ")" | "<" | ">" | "@" jpayne@69: | "," | ";" | ":" | "\" | <"> jpayne@69: | "/" | "[" | "]" | "?" | "=" jpayne@69: | "{" | "}" | SP | HT jpayne@69: jpayne@69: quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) jpayne@69: qdtext = > jpayne@69: quoted-pair = "\" CHAR jpayne@69: jpayne@69: parameter = attribute "=" value jpayne@69: attribute = token jpayne@69: value = token | quoted-string jpayne@69: jpayne@69: Each header is represented by a list of key/value pairs. The value for a jpayne@69: simple token (not part of a parameter) is None. Syntactically incorrect jpayne@69: headers will not necessarily be parsed as you would want. jpayne@69: jpayne@69: This is easier to describe with some examples: jpayne@69: jpayne@69: >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz']) jpayne@69: [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]] jpayne@69: >>> split_header_words(['text/html; charset="iso-8859-1"']) jpayne@69: [[('text/html', None), ('charset', 'iso-8859-1')]] jpayne@69: >>> split_header_words([r'Basic realm="\"foo\bar\""']) jpayne@69: [[('Basic', None), ('realm', '"foobar"')]] jpayne@69: jpayne@69: """ jpayne@69: assert not isinstance(header_values, str) jpayne@69: result = [] jpayne@69: for text in header_values: jpayne@69: orig_text = text jpayne@69: pairs = [] jpayne@69: while text: jpayne@69: m = HEADER_TOKEN_RE.search(text) jpayne@69: if m: jpayne@69: text = unmatched(m) jpayne@69: name = m.group(1) jpayne@69: m = HEADER_QUOTED_VALUE_RE.search(text) jpayne@69: if m: # quoted value jpayne@69: text = unmatched(m) jpayne@69: value = m.group(1) jpayne@69: value = HEADER_ESCAPE_RE.sub(r"\1", value) jpayne@69: else: jpayne@69: m = HEADER_VALUE_RE.search(text) jpayne@69: if m: # unquoted value jpayne@69: text = unmatched(m) jpayne@69: value = m.group(1) jpayne@69: value = value.rstrip() jpayne@69: else: jpayne@69: # no value, a lone token jpayne@69: value = None jpayne@69: pairs.append((name, value)) jpayne@69: elif text.lstrip().startswith(","): jpayne@69: # concatenated headers, as per RFC 2616 section 4.2 jpayne@69: text = text.lstrip()[1:] jpayne@69: if pairs: result.append(pairs) jpayne@69: pairs = [] jpayne@69: else: jpayne@69: # skip junk jpayne@69: non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text) jpayne@69: assert nr_junk_chars > 0, ( jpayne@69: "split_header_words bug: '%s', '%s', %s" % jpayne@69: (orig_text, text, pairs)) jpayne@69: text = non_junk jpayne@69: if pairs: result.append(pairs) jpayne@69: return result jpayne@69: jpayne@69: HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])") jpayne@69: def join_header_words(lists): jpayne@69: """Do the inverse (almost) of the conversion done by split_header_words. jpayne@69: jpayne@69: Takes a list of lists of (key, value) pairs and produces a single header jpayne@69: value. Attribute values are quoted if needed. jpayne@69: jpayne@69: >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]]) jpayne@69: 'text/plain; charset="iso-8859-1"' jpayne@69: >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]]) jpayne@69: 'text/plain, charset="iso-8859-1"' jpayne@69: jpayne@69: """ jpayne@69: headers = [] jpayne@69: for pairs in lists: jpayne@69: attr = [] jpayne@69: for k, v in pairs: jpayne@69: if v is not None: jpayne@69: if not re.search(r"^\w+$", v): jpayne@69: v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \ jpayne@69: v = '"%s"' % v jpayne@69: k = "%s=%s" % (k, v) jpayne@69: attr.append(k) jpayne@69: if attr: headers.append("; ".join(attr)) jpayne@69: return ", ".join(headers) jpayne@69: jpayne@69: def strip_quotes(text): jpayne@69: if text.startswith('"'): jpayne@69: text = text[1:] jpayne@69: if text.endswith('"'): jpayne@69: text = text[:-1] jpayne@69: return text jpayne@69: jpayne@69: def parse_ns_headers(ns_headers): jpayne@69: """Ad-hoc parser for Netscape protocol cookie-attributes. jpayne@69: jpayne@69: The old Netscape cookie format for Set-Cookie can for instance contain jpayne@69: an unquoted "," in the expires field, so we have to use this ad-hoc jpayne@69: parser instead of split_header_words. jpayne@69: jpayne@69: XXX This may not make the best possible effort to parse all the crap jpayne@69: that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient jpayne@69: parser is probably better, so could do worse than following that if jpayne@69: this ever gives any trouble. jpayne@69: jpayne@69: Currently, this is also used for parsing RFC 2109 cookies. jpayne@69: jpayne@69: """ jpayne@69: known_attrs = ("expires", "domain", "path", "secure", jpayne@69: # RFC 2109 attrs (may turn up in Netscape cookies, too) jpayne@69: "version", "port", "max-age") jpayne@69: jpayne@69: result = [] jpayne@69: for ns_header in ns_headers: jpayne@69: pairs = [] jpayne@69: version_set = False jpayne@69: jpayne@69: # XXX: The following does not strictly adhere to RFCs in that empty jpayne@69: # names and values are legal (the former will only appear once and will jpayne@69: # be overwritten if multiple occurrences are present). This is jpayne@69: # mostly to deal with backwards compatibility. jpayne@69: for ii, param in enumerate(ns_header.split(';')): jpayne@69: param = param.strip() jpayne@69: jpayne@69: key, sep, val = param.partition('=') jpayne@69: key = key.strip() jpayne@69: jpayne@69: if not key: jpayne@69: if ii == 0: jpayne@69: break jpayne@69: else: jpayne@69: continue jpayne@69: jpayne@69: # allow for a distinction between present and empty and missing jpayne@69: # altogether jpayne@69: val = val.strip() if sep else None jpayne@69: jpayne@69: if ii != 0: jpayne@69: lc = key.lower() jpayne@69: if lc in known_attrs: jpayne@69: key = lc jpayne@69: jpayne@69: if key == "version": jpayne@69: # This is an RFC 2109 cookie. jpayne@69: if val is not None: jpayne@69: val = strip_quotes(val) jpayne@69: version_set = True jpayne@69: elif key == "expires": jpayne@69: # convert expires date to seconds since epoch jpayne@69: if val is not None: jpayne@69: val = http2time(strip_quotes(val)) # None if invalid jpayne@69: pairs.append((key, val)) jpayne@69: jpayne@69: if pairs: jpayne@69: if not version_set: jpayne@69: pairs.append(("version", "0")) jpayne@69: result.append(pairs) jpayne@69: jpayne@69: return result jpayne@69: jpayne@69: jpayne@69: IPV4_RE = re.compile(r"\.\d+$", re.ASCII) jpayne@69: def is_HDN(text): jpayne@69: """Return True if text is a host domain name.""" jpayne@69: # XXX jpayne@69: # This may well be wrong. Which RFC is HDN defined in, if any (for jpayne@69: # the purposes of RFC 2965)? jpayne@69: # For the current implementation, what about IPv6? Remember to look jpayne@69: # at other uses of IPV4_RE also, if change this. jpayne@69: if IPV4_RE.search(text): jpayne@69: return False jpayne@69: if text == "": jpayne@69: return False jpayne@69: if text[0] == "." or text[-1] == ".": jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def domain_match(A, B): jpayne@69: """Return True if domain A domain-matches domain B, according to RFC 2965. jpayne@69: jpayne@69: A and B may be host domain names or IP addresses. jpayne@69: jpayne@69: RFC 2965, section 1: jpayne@69: jpayne@69: Host names can be specified either as an IP address or a HDN string. jpayne@69: Sometimes we compare one host name with another. (Such comparisons SHALL jpayne@69: be case-insensitive.) Host A's name domain-matches host B's if jpayne@69: jpayne@69: * their host name strings string-compare equal; or jpayne@69: jpayne@69: * A is a HDN string and has the form NB, where N is a non-empty jpayne@69: name string, B has the form .B', and B' is a HDN string. (So, jpayne@69: x.y.com domain-matches .Y.com but not Y.com.) jpayne@69: jpayne@69: Note that domain-match is not a commutative operation: a.b.c.com jpayne@69: domain-matches .c.com, but not the reverse. jpayne@69: jpayne@69: """ jpayne@69: # Note that, if A or B are IP addresses, the only relevant part of the jpayne@69: # definition of the domain-match algorithm is the direct string-compare. jpayne@69: A = A.lower() jpayne@69: B = B.lower() jpayne@69: if A == B: jpayne@69: return True jpayne@69: if not is_HDN(A): jpayne@69: return False jpayne@69: i = A.rfind(B) jpayne@69: if i == -1 or i == 0: jpayne@69: # A does not have form NB, or N is the empty string jpayne@69: return False jpayne@69: if not B.startswith("."): jpayne@69: return False jpayne@69: if not is_HDN(B[1:]): jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def liberal_is_HDN(text): jpayne@69: """Return True if text is a sort-of-like a host domain name. jpayne@69: jpayne@69: For accepting/blocking domains. jpayne@69: jpayne@69: """ jpayne@69: if IPV4_RE.search(text): jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def user_domain_match(A, B): jpayne@69: """For blocking/accepting domains. jpayne@69: jpayne@69: A and B may be host domain names or IP addresses. jpayne@69: jpayne@69: """ jpayne@69: A = A.lower() jpayne@69: B = B.lower() jpayne@69: if not (liberal_is_HDN(A) and liberal_is_HDN(B)): jpayne@69: if A == B: jpayne@69: # equal IP addresses jpayne@69: return True jpayne@69: return False jpayne@69: initial_dot = B.startswith(".") jpayne@69: if initial_dot and A.endswith(B): jpayne@69: return True jpayne@69: if not initial_dot and A == B: jpayne@69: return True jpayne@69: return False jpayne@69: jpayne@69: cut_port_re = re.compile(r":\d+$", re.ASCII) jpayne@69: def request_host(request): jpayne@69: """Return request-host, as defined by RFC 2965. jpayne@69: jpayne@69: Variation from RFC: returned value is lowercased, for convenient jpayne@69: comparison. jpayne@69: jpayne@69: """ jpayne@69: url = request.get_full_url() jpayne@69: host = urllib.parse.urlparse(url)[1] jpayne@69: if host == "": jpayne@69: host = request.get_header("Host", "") jpayne@69: jpayne@69: # remove port, if present jpayne@69: host = cut_port_re.sub("", host, 1) jpayne@69: return host.lower() jpayne@69: jpayne@69: def eff_request_host(request): jpayne@69: """Return a tuple (request-host, effective request-host name). jpayne@69: jpayne@69: As defined by RFC 2965, except both are lowercased. jpayne@69: jpayne@69: """ jpayne@69: erhn = req_host = request_host(request) jpayne@69: if req_host.find(".") == -1 and not IPV4_RE.search(req_host): jpayne@69: erhn = req_host + ".local" jpayne@69: return req_host, erhn jpayne@69: jpayne@69: def request_path(request): jpayne@69: """Path component of request-URI, as defined by RFC 2965.""" jpayne@69: url = request.get_full_url() jpayne@69: parts = urllib.parse.urlsplit(url) jpayne@69: path = escape_path(parts.path) jpayne@69: if not path.startswith("/"): jpayne@69: # fix bad RFC 2396 absoluteURI jpayne@69: path = "/" + path jpayne@69: return path jpayne@69: jpayne@69: def request_port(request): jpayne@69: host = request.host jpayne@69: i = host.find(':') jpayne@69: if i >= 0: jpayne@69: port = host[i+1:] jpayne@69: try: jpayne@69: int(port) jpayne@69: except ValueError: jpayne@69: _debug("nonnumeric port: '%s'", port) jpayne@69: return None jpayne@69: else: jpayne@69: port = DEFAULT_HTTP_PORT jpayne@69: return port jpayne@69: jpayne@69: # Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't jpayne@69: # need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738). jpayne@69: HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()" jpayne@69: ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])") jpayne@69: def uppercase_escaped_char(match): jpayne@69: return "%%%s" % match.group(1).upper() jpayne@69: def escape_path(path): jpayne@69: """Escape any invalid characters in HTTP URL, and uppercase all escapes.""" jpayne@69: # There's no knowing what character encoding was used to create URLs jpayne@69: # containing %-escapes, but since we have to pick one to escape invalid jpayne@69: # path characters, we pick UTF-8, as recommended in the HTML 4.0 jpayne@69: # specification: jpayne@69: # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1 jpayne@69: # And here, kind of: draft-fielding-uri-rfc2396bis-03 jpayne@69: # (And in draft IRI specification: draft-duerst-iri-05) jpayne@69: # (And here, for new URI schemes: RFC 2718) jpayne@69: path = urllib.parse.quote(path, HTTP_PATH_SAFE) jpayne@69: path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path) jpayne@69: return path jpayne@69: jpayne@69: def reach(h): jpayne@69: """Return reach of host h, as defined by RFC 2965, section 1. jpayne@69: jpayne@69: The reach R of a host name H is defined as follows: jpayne@69: jpayne@69: * If jpayne@69: jpayne@69: - H is the host domain name of a host; and, jpayne@69: jpayne@69: - H has the form A.B; and jpayne@69: jpayne@69: - A has no embedded (that is, interior) dots; and jpayne@69: jpayne@69: - B has at least one embedded dot, or B is the string "local". jpayne@69: then the reach of H is .B. jpayne@69: jpayne@69: * Otherwise, the reach of H is H. jpayne@69: jpayne@69: >>> reach("www.acme.com") jpayne@69: '.acme.com' jpayne@69: >>> reach("acme.com") jpayne@69: 'acme.com' jpayne@69: >>> reach("acme.local") jpayne@69: '.local' jpayne@69: jpayne@69: """ jpayne@69: i = h.find(".") jpayne@69: if i >= 0: jpayne@69: #a = h[:i] # this line is only here to show what a is jpayne@69: b = h[i+1:] jpayne@69: i = b.find(".") jpayne@69: if is_HDN(h) and (i >= 0 or b == "local"): jpayne@69: return "."+b jpayne@69: return h jpayne@69: jpayne@69: def is_third_party(request): jpayne@69: """ jpayne@69: jpayne@69: RFC 2965, section 3.3.6: jpayne@69: jpayne@69: An unverifiable transaction is to a third-party host if its request- jpayne@69: host U does not domain-match the reach R of the request-host O in the jpayne@69: origin transaction. jpayne@69: jpayne@69: """ jpayne@69: req_host = request_host(request) jpayne@69: if not domain_match(req_host, reach(request.origin_req_host)): jpayne@69: return True jpayne@69: else: jpayne@69: return False jpayne@69: jpayne@69: jpayne@69: class Cookie: jpayne@69: """HTTP Cookie. jpayne@69: jpayne@69: This class represents both Netscape and RFC 2965 cookies. jpayne@69: jpayne@69: This is deliberately a very simple class. It just holds attributes. It's jpayne@69: possible to construct Cookie instances that don't comply with the cookie jpayne@69: standards. CookieJar.make_cookies is the factory function for Cookie jpayne@69: objects -- it deals with cookie parsing, supplying defaults, and jpayne@69: normalising to the representation used in this class. CookiePolicy is jpayne@69: responsible for checking them to see whether they should be accepted from jpayne@69: and returned to the server. jpayne@69: jpayne@69: Note that the port may be present in the headers, but unspecified ("Port" jpayne@69: rather than"Port=80", for example); if this is the case, port is None. jpayne@69: jpayne@69: """ jpayne@69: jpayne@69: def __init__(self, version, name, value, jpayne@69: port, port_specified, jpayne@69: domain, domain_specified, domain_initial_dot, jpayne@69: path, path_specified, jpayne@69: secure, jpayne@69: expires, jpayne@69: discard, jpayne@69: comment, jpayne@69: comment_url, jpayne@69: rest, jpayne@69: rfc2109=False, jpayne@69: ): jpayne@69: jpayne@69: if version is not None: version = int(version) jpayne@69: if expires is not None: expires = int(float(expires)) jpayne@69: if port is None and port_specified is True: jpayne@69: raise ValueError("if port is None, port_specified must be false") jpayne@69: jpayne@69: self.version = version jpayne@69: self.name = name jpayne@69: self.value = value jpayne@69: self.port = port jpayne@69: self.port_specified = port_specified jpayne@69: # normalise case, as per RFC 2965 section 3.3.3 jpayne@69: self.domain = domain.lower() jpayne@69: self.domain_specified = domain_specified jpayne@69: # Sigh. We need to know whether the domain given in the jpayne@69: # cookie-attribute had an initial dot, in order to follow RFC 2965 jpayne@69: # (as clarified in draft errata). Needed for the returned $Domain jpayne@69: # value. jpayne@69: self.domain_initial_dot = domain_initial_dot jpayne@69: self.path = path jpayne@69: self.path_specified = path_specified jpayne@69: self.secure = secure jpayne@69: self.expires = expires jpayne@69: self.discard = discard jpayne@69: self.comment = comment jpayne@69: self.comment_url = comment_url jpayne@69: self.rfc2109 = rfc2109 jpayne@69: jpayne@69: self._rest = copy.copy(rest) jpayne@69: jpayne@69: def has_nonstandard_attr(self, name): jpayne@69: return name in self._rest jpayne@69: def get_nonstandard_attr(self, name, default=None): jpayne@69: return self._rest.get(name, default) jpayne@69: def set_nonstandard_attr(self, name, value): jpayne@69: self._rest[name] = value jpayne@69: jpayne@69: def is_expired(self, now=None): jpayne@69: if now is None: now = time.time() jpayne@69: if (self.expires is not None) and (self.expires <= now): jpayne@69: return True jpayne@69: return False jpayne@69: jpayne@69: def __str__(self): jpayne@69: if self.port is None: p = "" jpayne@69: else: p = ":"+self.port jpayne@69: limit = self.domain + p + self.path jpayne@69: if self.value is not None: jpayne@69: namevalue = "%s=%s" % (self.name, self.value) jpayne@69: else: jpayne@69: namevalue = self.name jpayne@69: return "" % (namevalue, limit) jpayne@69: jpayne@69: def __repr__(self): jpayne@69: args = [] jpayne@69: for name in ("version", "name", "value", jpayne@69: "port", "port_specified", jpayne@69: "domain", "domain_specified", "domain_initial_dot", jpayne@69: "path", "path_specified", jpayne@69: "secure", "expires", "discard", "comment", "comment_url", jpayne@69: ): jpayne@69: attr = getattr(self, name) jpayne@69: args.append("%s=%s" % (name, repr(attr))) jpayne@69: args.append("rest=%s" % repr(self._rest)) jpayne@69: args.append("rfc2109=%s" % repr(self.rfc2109)) jpayne@69: return "%s(%s)" % (self.__class__.__name__, ", ".join(args)) jpayne@69: jpayne@69: jpayne@69: class CookiePolicy: jpayne@69: """Defines which cookies get accepted from and returned to server. jpayne@69: jpayne@69: May also modify cookies, though this is probably a bad idea. jpayne@69: jpayne@69: The subclass DefaultCookiePolicy defines the standard rules for Netscape jpayne@69: and RFC 2965 cookies -- override that if you want a customized policy. jpayne@69: jpayne@69: """ jpayne@69: def set_ok(self, cookie, request): jpayne@69: """Return true if (and only if) cookie should be accepted from server. jpayne@69: jpayne@69: Currently, pre-expired cookies never get this far -- the CookieJar jpayne@69: class deletes such cookies itself. jpayne@69: jpayne@69: """ jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def return_ok(self, cookie, request): jpayne@69: """Return true if (and only if) cookie should be returned to server.""" jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def domain_return_ok(self, domain, request): jpayne@69: """Return false if cookies should not be returned, given cookie domain. jpayne@69: """ jpayne@69: return True jpayne@69: jpayne@69: def path_return_ok(self, path, request): jpayne@69: """Return false if cookies should not be returned, given cookie path. jpayne@69: """ jpayne@69: return True jpayne@69: jpayne@69: jpayne@69: class DefaultCookiePolicy(CookiePolicy): jpayne@69: """Implements the standard rules for accepting and returning cookies.""" jpayne@69: jpayne@69: DomainStrictNoDots = 1 jpayne@69: DomainStrictNonDomain = 2 jpayne@69: DomainRFC2965Match = 4 jpayne@69: jpayne@69: DomainLiberal = 0 jpayne@69: DomainStrict = DomainStrictNoDots|DomainStrictNonDomain jpayne@69: jpayne@69: def __init__(self, jpayne@69: blocked_domains=None, allowed_domains=None, jpayne@69: netscape=True, rfc2965=False, jpayne@69: rfc2109_as_netscape=None, jpayne@69: hide_cookie2=False, jpayne@69: strict_domain=False, jpayne@69: strict_rfc2965_unverifiable=True, jpayne@69: strict_ns_unverifiable=False, jpayne@69: strict_ns_domain=DomainLiberal, jpayne@69: strict_ns_set_initial_dollar=False, jpayne@69: strict_ns_set_path=False, jpayne@69: secure_protocols=("https", "wss") jpayne@69: ): jpayne@69: """Constructor arguments should be passed as keyword arguments only.""" jpayne@69: self.netscape = netscape jpayne@69: self.rfc2965 = rfc2965 jpayne@69: self.rfc2109_as_netscape = rfc2109_as_netscape jpayne@69: self.hide_cookie2 = hide_cookie2 jpayne@69: self.strict_domain = strict_domain jpayne@69: self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable jpayne@69: self.strict_ns_unverifiable = strict_ns_unverifiable jpayne@69: self.strict_ns_domain = strict_ns_domain jpayne@69: self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar jpayne@69: self.strict_ns_set_path = strict_ns_set_path jpayne@69: self.secure_protocols = secure_protocols jpayne@69: jpayne@69: if blocked_domains is not None: jpayne@69: self._blocked_domains = tuple(blocked_domains) jpayne@69: else: jpayne@69: self._blocked_domains = () jpayne@69: jpayne@69: if allowed_domains is not None: jpayne@69: allowed_domains = tuple(allowed_domains) jpayne@69: self._allowed_domains = allowed_domains jpayne@69: jpayne@69: def blocked_domains(self): jpayne@69: """Return the sequence of blocked domains (as a tuple).""" jpayne@69: return self._blocked_domains jpayne@69: def set_blocked_domains(self, blocked_domains): jpayne@69: """Set the sequence of blocked domains.""" jpayne@69: self._blocked_domains = tuple(blocked_domains) jpayne@69: jpayne@69: def is_blocked(self, domain): jpayne@69: for blocked_domain in self._blocked_domains: jpayne@69: if user_domain_match(domain, blocked_domain): jpayne@69: return True jpayne@69: return False jpayne@69: jpayne@69: def allowed_domains(self): jpayne@69: """Return None, or the sequence of allowed domains (as a tuple).""" jpayne@69: return self._allowed_domains jpayne@69: def set_allowed_domains(self, allowed_domains): jpayne@69: """Set the sequence of allowed domains, or None.""" jpayne@69: if allowed_domains is not None: jpayne@69: allowed_domains = tuple(allowed_domains) jpayne@69: self._allowed_domains = allowed_domains jpayne@69: jpayne@69: def is_not_allowed(self, domain): jpayne@69: if self._allowed_domains is None: jpayne@69: return False jpayne@69: for allowed_domain in self._allowed_domains: jpayne@69: if user_domain_match(domain, allowed_domain): jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def set_ok(self, cookie, request): jpayne@69: """ jpayne@69: If you override .set_ok(), be sure to call this method. If it returns jpayne@69: false, so should your subclass (assuming your subclass wants to be more jpayne@69: strict about which cookies to accept). jpayne@69: jpayne@69: """ jpayne@69: _debug(" - checking cookie %s=%s", cookie.name, cookie.value) jpayne@69: jpayne@69: assert cookie.name is not None jpayne@69: jpayne@69: for n in "version", "verifiability", "name", "path", "domain", "port": jpayne@69: fn_name = "set_ok_"+n jpayne@69: fn = getattr(self, fn_name) jpayne@69: if not fn(cookie, request): jpayne@69: return False jpayne@69: jpayne@69: return True jpayne@69: jpayne@69: def set_ok_version(self, cookie, request): jpayne@69: if cookie.version is None: jpayne@69: # Version is always set to 0 by parse_ns_headers if it's a Netscape jpayne@69: # cookie, so this must be an invalid RFC 2965 cookie. jpayne@69: _debug(" Set-Cookie2 without version attribute (%s=%s)", jpayne@69: cookie.name, cookie.value) jpayne@69: return False jpayne@69: if cookie.version > 0 and not self.rfc2965: jpayne@69: _debug(" RFC 2965 cookies are switched off") jpayne@69: return False jpayne@69: elif cookie.version == 0 and not self.netscape: jpayne@69: _debug(" Netscape cookies are switched off") jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def set_ok_verifiability(self, cookie, request): jpayne@69: if request.unverifiable and is_third_party(request): jpayne@69: if cookie.version > 0 and self.strict_rfc2965_unverifiable: jpayne@69: _debug(" third-party RFC 2965 cookie during " jpayne@69: "unverifiable transaction") jpayne@69: return False jpayne@69: elif cookie.version == 0 and self.strict_ns_unverifiable: jpayne@69: _debug(" third-party Netscape cookie during " jpayne@69: "unverifiable transaction") jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def set_ok_name(self, cookie, request): jpayne@69: # Try and stop servers setting V0 cookies designed to hack other jpayne@69: # servers that know both V0 and V1 protocols. jpayne@69: if (cookie.version == 0 and self.strict_ns_set_initial_dollar and jpayne@69: cookie.name.startswith("$")): jpayne@69: _debug(" illegal name (starts with '$'): '%s'", cookie.name) jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def set_ok_path(self, cookie, request): jpayne@69: if cookie.path_specified: jpayne@69: req_path = request_path(request) jpayne@69: if ((cookie.version > 0 or jpayne@69: (cookie.version == 0 and self.strict_ns_set_path)) and jpayne@69: not self.path_return_ok(cookie.path, request)): jpayne@69: _debug(" path attribute %s is not a prefix of request " jpayne@69: "path %s", cookie.path, req_path) jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def set_ok_domain(self, cookie, request): jpayne@69: if self.is_blocked(cookie.domain): jpayne@69: _debug(" domain %s is in user block-list", cookie.domain) jpayne@69: return False jpayne@69: if self.is_not_allowed(cookie.domain): jpayne@69: _debug(" domain %s is not in user allow-list", cookie.domain) jpayne@69: return False jpayne@69: if cookie.domain_specified: jpayne@69: req_host, erhn = eff_request_host(request) jpayne@69: domain = cookie.domain jpayne@69: if self.strict_domain and (domain.count(".") >= 2): jpayne@69: # XXX This should probably be compared with the Konqueror jpayne@69: # (kcookiejar.cpp) and Mozilla implementations, but it's a jpayne@69: # losing battle. jpayne@69: i = domain.rfind(".") jpayne@69: j = domain.rfind(".", 0, i) jpayne@69: if j == 0: # domain like .foo.bar jpayne@69: tld = domain[i+1:] jpayne@69: sld = domain[j+1:i] jpayne@69: if sld.lower() in ("co", "ac", "com", "edu", "org", "net", jpayne@69: "gov", "mil", "int", "aero", "biz", "cat", "coop", jpayne@69: "info", "jobs", "mobi", "museum", "name", "pro", jpayne@69: "travel", "eu") and len(tld) == 2: jpayne@69: # domain like .co.uk jpayne@69: _debug(" country-code second level domain %s", domain) jpayne@69: return False jpayne@69: if domain.startswith("."): jpayne@69: undotted_domain = domain[1:] jpayne@69: else: jpayne@69: undotted_domain = domain jpayne@69: embedded_dots = (undotted_domain.find(".") >= 0) jpayne@69: if not embedded_dots and domain != ".local": jpayne@69: _debug(" non-local domain %s contains no embedded dot", jpayne@69: domain) jpayne@69: return False jpayne@69: if cookie.version == 0: jpayne@69: if (not erhn.endswith(domain) and jpayne@69: (not erhn.startswith(".") and jpayne@69: not ("."+erhn).endswith(domain))): jpayne@69: _debug(" effective request-host %s (even with added " jpayne@69: "initial dot) does not end with %s", jpayne@69: erhn, domain) jpayne@69: return False jpayne@69: if (cookie.version > 0 or jpayne@69: (self.strict_ns_domain & self.DomainRFC2965Match)): jpayne@69: if not domain_match(erhn, domain): jpayne@69: _debug(" effective request-host %s does not domain-match " jpayne@69: "%s", erhn, domain) jpayne@69: return False jpayne@69: if (cookie.version > 0 or jpayne@69: (self.strict_ns_domain & self.DomainStrictNoDots)): jpayne@69: host_prefix = req_host[:-len(domain)] jpayne@69: if (host_prefix.find(".") >= 0 and jpayne@69: not IPV4_RE.search(req_host)): jpayne@69: _debug(" host prefix %s for domain %s contains a dot", jpayne@69: host_prefix, domain) jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def set_ok_port(self, cookie, request): jpayne@69: if cookie.port_specified: jpayne@69: req_port = request_port(request) jpayne@69: if req_port is None: jpayne@69: req_port = "80" jpayne@69: else: jpayne@69: req_port = str(req_port) jpayne@69: for p in cookie.port.split(","): jpayne@69: try: jpayne@69: int(p) jpayne@69: except ValueError: jpayne@69: _debug(" bad port %s (not numeric)", p) jpayne@69: return False jpayne@69: if p == req_port: jpayne@69: break jpayne@69: else: jpayne@69: _debug(" request port (%s) not found in %s", jpayne@69: req_port, cookie.port) jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok(self, cookie, request): jpayne@69: """ jpayne@69: If you override .return_ok(), be sure to call this method. If it jpayne@69: returns false, so should your subclass (assuming your subclass wants to jpayne@69: be more strict about which cookies to return). jpayne@69: jpayne@69: """ jpayne@69: # Path has already been checked by .path_return_ok(), and domain jpayne@69: # blocking done by .domain_return_ok(). jpayne@69: _debug(" - checking cookie %s=%s", cookie.name, cookie.value) jpayne@69: jpayne@69: for n in "version", "verifiability", "secure", "expires", "port", "domain": jpayne@69: fn_name = "return_ok_"+n jpayne@69: fn = getattr(self, fn_name) jpayne@69: if not fn(cookie, request): jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok_version(self, cookie, request): jpayne@69: if cookie.version > 0 and not self.rfc2965: jpayne@69: _debug(" RFC 2965 cookies are switched off") jpayne@69: return False jpayne@69: elif cookie.version == 0 and not self.netscape: jpayne@69: _debug(" Netscape cookies are switched off") jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok_verifiability(self, cookie, request): jpayne@69: if request.unverifiable and is_third_party(request): jpayne@69: if cookie.version > 0 and self.strict_rfc2965_unverifiable: jpayne@69: _debug(" third-party RFC 2965 cookie during unverifiable " jpayne@69: "transaction") jpayne@69: return False jpayne@69: elif cookie.version == 0 and self.strict_ns_unverifiable: jpayne@69: _debug(" third-party Netscape cookie during unverifiable " jpayne@69: "transaction") jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok_secure(self, cookie, request): jpayne@69: if cookie.secure and request.type not in self.secure_protocols: jpayne@69: _debug(" secure cookie with non-secure request") jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok_expires(self, cookie, request): jpayne@69: if cookie.is_expired(self._now): jpayne@69: _debug(" cookie expired") jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok_port(self, cookie, request): jpayne@69: if cookie.port: jpayne@69: req_port = request_port(request) jpayne@69: if req_port is None: jpayne@69: req_port = "80" jpayne@69: for p in cookie.port.split(","): jpayne@69: if p == req_port: jpayne@69: break jpayne@69: else: jpayne@69: _debug(" request port %s does not match cookie port %s", jpayne@69: req_port, cookie.port) jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def return_ok_domain(self, cookie, request): jpayne@69: req_host, erhn = eff_request_host(request) jpayne@69: domain = cookie.domain jpayne@69: jpayne@69: if domain and not domain.startswith("."): jpayne@69: dotdomain = "." + domain jpayne@69: else: jpayne@69: dotdomain = domain jpayne@69: jpayne@69: # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't jpayne@69: if (cookie.version == 0 and jpayne@69: (self.strict_ns_domain & self.DomainStrictNonDomain) and jpayne@69: not cookie.domain_specified and domain != erhn): jpayne@69: _debug(" cookie with unspecified domain does not string-compare " jpayne@69: "equal to request domain") jpayne@69: return False jpayne@69: jpayne@69: if cookie.version > 0 and not domain_match(erhn, domain): jpayne@69: _debug(" effective request-host name %s does not domain-match " jpayne@69: "RFC 2965 cookie domain %s", erhn, domain) jpayne@69: return False jpayne@69: if cookie.version == 0 and not ("."+erhn).endswith(dotdomain): jpayne@69: _debug(" request-host %s does not match Netscape cookie domain " jpayne@69: "%s", req_host, domain) jpayne@69: return False jpayne@69: return True jpayne@69: jpayne@69: def domain_return_ok(self, domain, request): jpayne@69: # Liberal check of. This is here as an optimization to avoid jpayne@69: # having to load lots of MSIE cookie files unless necessary. jpayne@69: req_host, erhn = eff_request_host(request) jpayne@69: if not req_host.startswith("."): jpayne@69: req_host = "."+req_host jpayne@69: if not erhn.startswith("."): jpayne@69: erhn = "."+erhn jpayne@69: if domain and not domain.startswith("."): jpayne@69: dotdomain = "." + domain jpayne@69: else: jpayne@69: dotdomain = domain jpayne@69: if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)): jpayne@69: #_debug(" request domain %s does not match cookie domain %s", jpayne@69: # req_host, domain) jpayne@69: return False jpayne@69: jpayne@69: if self.is_blocked(domain): jpayne@69: _debug(" domain %s is in user block-list", domain) jpayne@69: return False jpayne@69: if self.is_not_allowed(domain): jpayne@69: _debug(" domain %s is not in user allow-list", domain) jpayne@69: return False jpayne@69: jpayne@69: return True jpayne@69: jpayne@69: def path_return_ok(self, path, request): jpayne@69: _debug("- checking cookie path=%s", path) jpayne@69: req_path = request_path(request) jpayne@69: pathlen = len(path) jpayne@69: if req_path == path: jpayne@69: return True jpayne@69: elif (req_path.startswith(path) and jpayne@69: (path.endswith("/") or req_path[pathlen:pathlen+1] == "/")): jpayne@69: return True jpayne@69: jpayne@69: _debug(" %s does not path-match %s", req_path, path) jpayne@69: return False jpayne@69: jpayne@69: def vals_sorted_by_key(adict): jpayne@69: keys = sorted(adict.keys()) jpayne@69: return map(adict.get, keys) jpayne@69: jpayne@69: def deepvalues(mapping): jpayne@69: """Iterates over nested mapping, depth-first, in sorted order by key.""" jpayne@69: values = vals_sorted_by_key(mapping) jpayne@69: for obj in values: jpayne@69: mapping = False jpayne@69: try: jpayne@69: obj.items jpayne@69: except AttributeError: jpayne@69: pass jpayne@69: else: jpayne@69: mapping = True jpayne@69: yield from deepvalues(obj) jpayne@69: if not mapping: jpayne@69: yield obj jpayne@69: jpayne@69: jpayne@69: # Used as second parameter to dict.get() method, to distinguish absent jpayne@69: # dict key from one with a None value. jpayne@69: class Absent: pass jpayne@69: jpayne@69: class CookieJar: jpayne@69: """Collection of HTTP cookies. jpayne@69: jpayne@69: You may not need to know about this class: try jpayne@69: urllib.request.build_opener(HTTPCookieProcessor).open(url). jpayne@69: """ jpayne@69: jpayne@69: non_word_re = re.compile(r"\W") jpayne@69: quote_re = re.compile(r"([\"\\])") jpayne@69: strict_domain_re = re.compile(r"\.?[^.]*") jpayne@69: domain_re = re.compile(r"[^.]*") jpayne@69: dots_re = re.compile(r"^\.+") jpayne@69: jpayne@69: magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII) jpayne@69: jpayne@69: def __init__(self, policy=None): jpayne@69: if policy is None: jpayne@69: policy = DefaultCookiePolicy() jpayne@69: self._policy = policy jpayne@69: jpayne@69: self._cookies_lock = _threading.RLock() jpayne@69: self._cookies = {} jpayne@69: jpayne@69: def set_policy(self, policy): jpayne@69: self._policy = policy jpayne@69: jpayne@69: def _cookies_for_domain(self, domain, request): jpayne@69: cookies = [] jpayne@69: if not self._policy.domain_return_ok(domain, request): jpayne@69: return [] jpayne@69: _debug("Checking %s for cookies to return", domain) jpayne@69: cookies_by_path = self._cookies[domain] jpayne@69: for path in cookies_by_path.keys(): jpayne@69: if not self._policy.path_return_ok(path, request): jpayne@69: continue jpayne@69: cookies_by_name = cookies_by_path[path] jpayne@69: for cookie in cookies_by_name.values(): jpayne@69: if not self._policy.return_ok(cookie, request): jpayne@69: _debug(" not returning cookie") jpayne@69: continue jpayne@69: _debug(" it's a match") jpayne@69: cookies.append(cookie) jpayne@69: return cookies jpayne@69: jpayne@69: def _cookies_for_request(self, request): jpayne@69: """Return a list of cookies to be returned to server.""" jpayne@69: cookies = [] jpayne@69: for domain in self._cookies.keys(): jpayne@69: cookies.extend(self._cookies_for_domain(domain, request)) jpayne@69: return cookies jpayne@69: jpayne@69: def _cookie_attrs(self, cookies): jpayne@69: """Return a list of cookie-attributes to be returned to server. jpayne@69: jpayne@69: like ['foo="bar"; $Path="/"', ...] jpayne@69: jpayne@69: The $Version attribute is also added when appropriate (currently only jpayne@69: once per request). jpayne@69: jpayne@69: """ jpayne@69: # add cookies in order of most specific (ie. longest) path first jpayne@69: cookies.sort(key=lambda a: len(a.path), reverse=True) jpayne@69: jpayne@69: version_set = False jpayne@69: jpayne@69: attrs = [] jpayne@69: for cookie in cookies: jpayne@69: # set version of Cookie header jpayne@69: # XXX jpayne@69: # What should it be if multiple matching Set-Cookie headers have jpayne@69: # different versions themselves? jpayne@69: # Answer: there is no answer; was supposed to be settled by jpayne@69: # RFC 2965 errata, but that may never appear... jpayne@69: version = cookie.version jpayne@69: if not version_set: jpayne@69: version_set = True jpayne@69: if version > 0: jpayne@69: attrs.append("$Version=%s" % version) jpayne@69: jpayne@69: # quote cookie value if necessary jpayne@69: # (not for Netscape protocol, which already has any quotes jpayne@69: # intact, due to the poorly-specified Netscape Cookie: syntax) jpayne@69: if ((cookie.value is not None) and jpayne@69: self.non_word_re.search(cookie.value) and version > 0): jpayne@69: value = self.quote_re.sub(r"\\\1", cookie.value) jpayne@69: else: jpayne@69: value = cookie.value jpayne@69: jpayne@69: # add cookie-attributes to be returned in Cookie header jpayne@69: if cookie.value is None: jpayne@69: attrs.append(cookie.name) jpayne@69: else: jpayne@69: attrs.append("%s=%s" % (cookie.name, value)) jpayne@69: if version > 0: jpayne@69: if cookie.path_specified: jpayne@69: attrs.append('$Path="%s"' % cookie.path) jpayne@69: if cookie.domain.startswith("."): jpayne@69: domain = cookie.domain jpayne@69: if (not cookie.domain_initial_dot and jpayne@69: domain.startswith(".")): jpayne@69: domain = domain[1:] jpayne@69: attrs.append('$Domain="%s"' % domain) jpayne@69: if cookie.port is not None: jpayne@69: p = "$Port" jpayne@69: if cookie.port_specified: jpayne@69: p = p + ('="%s"' % cookie.port) jpayne@69: attrs.append(p) jpayne@69: jpayne@69: return attrs jpayne@69: jpayne@69: def add_cookie_header(self, request): jpayne@69: """Add correct Cookie: header to request (urllib.request.Request object). jpayne@69: jpayne@69: The Cookie2 header is also added unless policy.hide_cookie2 is true. jpayne@69: jpayne@69: """ jpayne@69: _debug("add_cookie_header") jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: jpayne@69: self._policy._now = self._now = int(time.time()) jpayne@69: jpayne@69: cookies = self._cookies_for_request(request) jpayne@69: jpayne@69: attrs = self._cookie_attrs(cookies) jpayne@69: if attrs: jpayne@69: if not request.has_header("Cookie"): jpayne@69: request.add_unredirected_header( jpayne@69: "Cookie", "; ".join(attrs)) jpayne@69: jpayne@69: # if necessary, advertise that we know RFC 2965 jpayne@69: if (self._policy.rfc2965 and not self._policy.hide_cookie2 and jpayne@69: not request.has_header("Cookie2")): jpayne@69: for cookie in cookies: jpayne@69: if cookie.version != 1: jpayne@69: request.add_unredirected_header("Cookie2", '$Version="1"') jpayne@69: break jpayne@69: jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: self.clear_expired_cookies() jpayne@69: jpayne@69: def _normalized_cookie_tuples(self, attrs_set): jpayne@69: """Return list of tuples containing normalised cookie information. jpayne@69: jpayne@69: attrs_set is the list of lists of key,value pairs extracted from jpayne@69: the Set-Cookie or Set-Cookie2 headers. jpayne@69: jpayne@69: Tuples are name, value, standard, rest, where name and value are the jpayne@69: cookie name and value, standard is a dictionary containing the standard jpayne@69: cookie-attributes (discard, secure, version, expires or max-age, jpayne@69: domain, path and port) and rest is a dictionary containing the rest of jpayne@69: the cookie-attributes. jpayne@69: jpayne@69: """ jpayne@69: cookie_tuples = [] jpayne@69: jpayne@69: boolean_attrs = "discard", "secure" jpayne@69: value_attrs = ("version", jpayne@69: "expires", "max-age", jpayne@69: "domain", "path", "port", jpayne@69: "comment", "commenturl") jpayne@69: jpayne@69: for cookie_attrs in attrs_set: jpayne@69: name, value = cookie_attrs[0] jpayne@69: jpayne@69: # Build dictionary of standard cookie-attributes (standard) and jpayne@69: # dictionary of other cookie-attributes (rest). jpayne@69: jpayne@69: # Note: expiry time is normalised to seconds since epoch. V0 jpayne@69: # cookies should have the Expires cookie-attribute, and V1 cookies jpayne@69: # should have Max-Age, but since V1 includes RFC 2109 cookies (and jpayne@69: # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we jpayne@69: # accept either (but prefer Max-Age). jpayne@69: max_age_set = False jpayne@69: jpayne@69: bad_cookie = False jpayne@69: jpayne@69: standard = {} jpayne@69: rest = {} jpayne@69: for k, v in cookie_attrs[1:]: jpayne@69: lc = k.lower() jpayne@69: # don't lose case distinction for unknown fields jpayne@69: if lc in value_attrs or lc in boolean_attrs: jpayne@69: k = lc jpayne@69: if k in boolean_attrs and v is None: jpayne@69: # boolean cookie-attribute is present, but has no value jpayne@69: # (like "discard", rather than "port=80") jpayne@69: v = True jpayne@69: if k in standard: jpayne@69: # only first value is significant jpayne@69: continue jpayne@69: if k == "domain": jpayne@69: if v is None: jpayne@69: _debug(" missing value for domain attribute") jpayne@69: bad_cookie = True jpayne@69: break jpayne@69: # RFC 2965 section 3.3.3 jpayne@69: v = v.lower() jpayne@69: if k == "expires": jpayne@69: if max_age_set: jpayne@69: # Prefer max-age to expires (like Mozilla) jpayne@69: continue jpayne@69: if v is None: jpayne@69: _debug(" missing or invalid value for expires " jpayne@69: "attribute: treating as session cookie") jpayne@69: continue jpayne@69: if k == "max-age": jpayne@69: max_age_set = True jpayne@69: try: jpayne@69: v = int(v) jpayne@69: except ValueError: jpayne@69: _debug(" missing or invalid (non-numeric) value for " jpayne@69: "max-age attribute") jpayne@69: bad_cookie = True jpayne@69: break jpayne@69: # convert RFC 2965 Max-Age to seconds since epoch jpayne@69: # XXX Strictly you're supposed to follow RFC 2616 jpayne@69: # age-calculation rules. Remember that zero Max-Age jpayne@69: # is a request to discard (old and new) cookie, though. jpayne@69: k = "expires" jpayne@69: v = self._now + v jpayne@69: if (k in value_attrs) or (k in boolean_attrs): jpayne@69: if (v is None and jpayne@69: k not in ("port", "comment", "commenturl")): jpayne@69: _debug(" missing value for %s attribute" % k) jpayne@69: bad_cookie = True jpayne@69: break jpayne@69: standard[k] = v jpayne@69: else: jpayne@69: rest[k] = v jpayne@69: jpayne@69: if bad_cookie: jpayne@69: continue jpayne@69: jpayne@69: cookie_tuples.append((name, value, standard, rest)) jpayne@69: jpayne@69: return cookie_tuples jpayne@69: jpayne@69: def _cookie_from_cookie_tuple(self, tup, request): jpayne@69: # standard is dict of standard cookie-attributes, rest is dict of the jpayne@69: # rest of them jpayne@69: name, value, standard, rest = tup jpayne@69: jpayne@69: domain = standard.get("domain", Absent) jpayne@69: path = standard.get("path", Absent) jpayne@69: port = standard.get("port", Absent) jpayne@69: expires = standard.get("expires", Absent) jpayne@69: jpayne@69: # set the easy defaults jpayne@69: version = standard.get("version", None) jpayne@69: if version is not None: jpayne@69: try: jpayne@69: version = int(version) jpayne@69: except ValueError: jpayne@69: return None # invalid version, ignore cookie jpayne@69: secure = standard.get("secure", False) jpayne@69: # (discard is also set if expires is Absent) jpayne@69: discard = standard.get("discard", False) jpayne@69: comment = standard.get("comment", None) jpayne@69: comment_url = standard.get("commenturl", None) jpayne@69: jpayne@69: # set default path jpayne@69: if path is not Absent and path != "": jpayne@69: path_specified = True jpayne@69: path = escape_path(path) jpayne@69: else: jpayne@69: path_specified = False jpayne@69: path = request_path(request) jpayne@69: i = path.rfind("/") jpayne@69: if i != -1: jpayne@69: if version == 0: jpayne@69: # Netscape spec parts company from reality here jpayne@69: path = path[:i] jpayne@69: else: jpayne@69: path = path[:i+1] jpayne@69: if len(path) == 0: path = "/" jpayne@69: jpayne@69: # set default domain jpayne@69: domain_specified = domain is not Absent jpayne@69: # but first we have to remember whether it starts with a dot jpayne@69: domain_initial_dot = False jpayne@69: if domain_specified: jpayne@69: domain_initial_dot = bool(domain.startswith(".")) jpayne@69: if domain is Absent: jpayne@69: req_host, erhn = eff_request_host(request) jpayne@69: domain = erhn jpayne@69: elif not domain.startswith("."): jpayne@69: domain = "."+domain jpayne@69: jpayne@69: # set default port jpayne@69: port_specified = False jpayne@69: if port is not Absent: jpayne@69: if port is None: jpayne@69: # Port attr present, but has no value: default to request port. jpayne@69: # Cookie should then only be sent back on that port. jpayne@69: port = request_port(request) jpayne@69: else: jpayne@69: port_specified = True jpayne@69: port = re.sub(r"\s+", "", port) jpayne@69: else: jpayne@69: # No port attr present. Cookie can be sent back on any port. jpayne@69: port = None jpayne@69: jpayne@69: # set default expires and discard jpayne@69: if expires is Absent: jpayne@69: expires = None jpayne@69: discard = True jpayne@69: elif expires <= self._now: jpayne@69: # Expiry date in past is request to delete cookie. This can't be jpayne@69: # in DefaultCookiePolicy, because can't delete cookies there. jpayne@69: try: jpayne@69: self.clear(domain, path, name) jpayne@69: except KeyError: jpayne@69: pass jpayne@69: _debug("Expiring cookie, domain='%s', path='%s', name='%s'", jpayne@69: domain, path, name) jpayne@69: return None jpayne@69: jpayne@69: return Cookie(version, jpayne@69: name, value, jpayne@69: port, port_specified, jpayne@69: domain, domain_specified, domain_initial_dot, jpayne@69: path, path_specified, jpayne@69: secure, jpayne@69: expires, jpayne@69: discard, jpayne@69: comment, jpayne@69: comment_url, jpayne@69: rest) jpayne@69: jpayne@69: def _cookies_from_attrs_set(self, attrs_set, request): jpayne@69: cookie_tuples = self._normalized_cookie_tuples(attrs_set) jpayne@69: jpayne@69: cookies = [] jpayne@69: for tup in cookie_tuples: jpayne@69: cookie = self._cookie_from_cookie_tuple(tup, request) jpayne@69: if cookie: cookies.append(cookie) jpayne@69: return cookies jpayne@69: jpayne@69: def _process_rfc2109_cookies(self, cookies): jpayne@69: rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None) jpayne@69: if rfc2109_as_ns is None: jpayne@69: rfc2109_as_ns = not self._policy.rfc2965 jpayne@69: for cookie in cookies: jpayne@69: if cookie.version == 1: jpayne@69: cookie.rfc2109 = True jpayne@69: if rfc2109_as_ns: jpayne@69: # treat 2109 cookies as Netscape cookies rather than jpayne@69: # as RFC2965 cookies jpayne@69: cookie.version = 0 jpayne@69: jpayne@69: def make_cookies(self, response, request): jpayne@69: """Return sequence of Cookie objects extracted from response object.""" jpayne@69: # get cookie-attributes for RFC 2965 and Netscape protocols jpayne@69: headers = response.info() jpayne@69: rfc2965_hdrs = headers.get_all("Set-Cookie2", []) jpayne@69: ns_hdrs = headers.get_all("Set-Cookie", []) jpayne@69: self._policy._now = self._now = int(time.time()) jpayne@69: jpayne@69: rfc2965 = self._policy.rfc2965 jpayne@69: netscape = self._policy.netscape jpayne@69: jpayne@69: if ((not rfc2965_hdrs and not ns_hdrs) or jpayne@69: (not ns_hdrs and not rfc2965) or jpayne@69: (not rfc2965_hdrs and not netscape) or jpayne@69: (not netscape and not rfc2965)): jpayne@69: return [] # no relevant cookie headers: quick exit jpayne@69: jpayne@69: try: jpayne@69: cookies = self._cookies_from_attrs_set( jpayne@69: split_header_words(rfc2965_hdrs), request) jpayne@69: except Exception: jpayne@69: _warn_unhandled_exception() jpayne@69: cookies = [] jpayne@69: jpayne@69: if ns_hdrs and netscape: jpayne@69: try: jpayne@69: # RFC 2109 and Netscape cookies jpayne@69: ns_cookies = self._cookies_from_attrs_set( jpayne@69: parse_ns_headers(ns_hdrs), request) jpayne@69: except Exception: jpayne@69: _warn_unhandled_exception() jpayne@69: ns_cookies = [] jpayne@69: self._process_rfc2109_cookies(ns_cookies) jpayne@69: jpayne@69: # Look for Netscape cookies (from Set-Cookie headers) that match jpayne@69: # corresponding RFC 2965 cookies (from Set-Cookie2 headers). jpayne@69: # For each match, keep the RFC 2965 cookie and ignore the Netscape jpayne@69: # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are jpayne@69: # bundled in with the Netscape cookies for this purpose, which is jpayne@69: # reasonable behaviour. jpayne@69: if rfc2965: jpayne@69: lookup = {} jpayne@69: for cookie in cookies: jpayne@69: lookup[(cookie.domain, cookie.path, cookie.name)] = None jpayne@69: jpayne@69: def no_matching_rfc2965(ns_cookie, lookup=lookup): jpayne@69: key = ns_cookie.domain, ns_cookie.path, ns_cookie.name jpayne@69: return key not in lookup jpayne@69: ns_cookies = filter(no_matching_rfc2965, ns_cookies) jpayne@69: jpayne@69: if ns_cookies: jpayne@69: cookies.extend(ns_cookies) jpayne@69: jpayne@69: return cookies jpayne@69: jpayne@69: def set_cookie_if_ok(self, cookie, request): jpayne@69: """Set a cookie if policy says it's OK to do so.""" jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: self._policy._now = self._now = int(time.time()) jpayne@69: jpayne@69: if self._policy.set_ok(cookie, request): jpayne@69: self.set_cookie(cookie) jpayne@69: jpayne@69: jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: def set_cookie(self, cookie): jpayne@69: """Set a cookie, without checking whether or not it should be set.""" jpayne@69: c = self._cookies jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: if cookie.domain not in c: c[cookie.domain] = {} jpayne@69: c2 = c[cookie.domain] jpayne@69: if cookie.path not in c2: c2[cookie.path] = {} jpayne@69: c3 = c2[cookie.path] jpayne@69: c3[cookie.name] = cookie jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: def extract_cookies(self, response, request): jpayne@69: """Extract cookies from response, where allowable given the request.""" jpayne@69: _debug("extract_cookies: %s", response.info()) jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: for cookie in self.make_cookies(response, request): jpayne@69: if self._policy.set_ok(cookie, request): jpayne@69: _debug(" setting cookie: %s", cookie) jpayne@69: self.set_cookie(cookie) jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: def clear(self, domain=None, path=None, name=None): jpayne@69: """Clear some cookies. jpayne@69: jpayne@69: Invoking this method without arguments will clear all cookies. If jpayne@69: given a single argument, only cookies belonging to that domain will be jpayne@69: removed. If given two arguments, cookies belonging to the specified jpayne@69: path within that domain are removed. If given three arguments, then jpayne@69: the cookie with the specified name, path and domain is removed. jpayne@69: jpayne@69: Raises KeyError if no matching cookie exists. jpayne@69: jpayne@69: """ jpayne@69: if name is not None: jpayne@69: if (domain is None) or (path is None): jpayne@69: raise ValueError( jpayne@69: "domain and path must be given to remove a cookie by name") jpayne@69: del self._cookies[domain][path][name] jpayne@69: elif path is not None: jpayne@69: if domain is None: jpayne@69: raise ValueError( jpayne@69: "domain must be given to remove cookies by path") jpayne@69: del self._cookies[domain][path] jpayne@69: elif domain is not None: jpayne@69: del self._cookies[domain] jpayne@69: else: jpayne@69: self._cookies = {} jpayne@69: jpayne@69: def clear_session_cookies(self): jpayne@69: """Discard all session cookies. jpayne@69: jpayne@69: Note that the .save() method won't save session cookies anyway, unless jpayne@69: you ask otherwise by passing a true ignore_discard argument. jpayne@69: jpayne@69: """ jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: for cookie in self: jpayne@69: if cookie.discard: jpayne@69: self.clear(cookie.domain, cookie.path, cookie.name) jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: def clear_expired_cookies(self): jpayne@69: """Discard all expired cookies. jpayne@69: jpayne@69: You probably don't need to call this method: expired cookies are never jpayne@69: sent back to the server (provided you're using DefaultCookiePolicy), jpayne@69: this method is called by CookieJar itself every so often, and the jpayne@69: .save() method won't save expired cookies anyway (unless you ask jpayne@69: otherwise by passing a true ignore_expires argument). jpayne@69: jpayne@69: """ jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: now = time.time() jpayne@69: for cookie in self: jpayne@69: if cookie.is_expired(now): jpayne@69: self.clear(cookie.domain, cookie.path, cookie.name) jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: def __iter__(self): jpayne@69: return deepvalues(self._cookies) jpayne@69: jpayne@69: def __len__(self): jpayne@69: """Return number of contained cookies.""" jpayne@69: i = 0 jpayne@69: for cookie in self: i = i + 1 jpayne@69: return i jpayne@69: jpayne@69: def __repr__(self): jpayne@69: r = [] jpayne@69: for cookie in self: r.append(repr(cookie)) jpayne@69: return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) jpayne@69: jpayne@69: def __str__(self): jpayne@69: r = [] jpayne@69: for cookie in self: r.append(str(cookie)) jpayne@69: return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r)) jpayne@69: jpayne@69: jpayne@69: # derives from OSError for backwards-compatibility with Python 2.4.0 jpayne@69: class LoadError(OSError): pass jpayne@69: jpayne@69: class FileCookieJar(CookieJar): jpayne@69: """CookieJar that can be loaded from and saved to a file.""" jpayne@69: jpayne@69: def __init__(self, filename=None, delayload=False, policy=None): jpayne@69: """ jpayne@69: Cookies are NOT loaded from the named file until either the .load() or jpayne@69: .revert() method is called. jpayne@69: jpayne@69: """ jpayne@69: CookieJar.__init__(self, policy) jpayne@69: if filename is not None: jpayne@69: filename = os.fspath(filename) jpayne@69: self.filename = filename jpayne@69: self.delayload = bool(delayload) jpayne@69: jpayne@69: def save(self, filename=None, ignore_discard=False, ignore_expires=False): jpayne@69: """Save cookies to a file.""" jpayne@69: raise NotImplementedError() jpayne@69: jpayne@69: def load(self, filename=None, ignore_discard=False, ignore_expires=False): jpayne@69: """Load cookies from a file.""" jpayne@69: if filename is None: jpayne@69: if self.filename is not None: filename = self.filename jpayne@69: else: raise ValueError(MISSING_FILENAME_TEXT) jpayne@69: jpayne@69: with open(filename) as f: jpayne@69: self._really_load(f, filename, ignore_discard, ignore_expires) jpayne@69: jpayne@69: def revert(self, filename=None, jpayne@69: ignore_discard=False, ignore_expires=False): jpayne@69: """Clear all cookies and reload cookies from a saved file. jpayne@69: jpayne@69: Raises LoadError (or OSError) if reversion is not successful; the jpayne@69: object's state will not be altered if this happens. jpayne@69: jpayne@69: """ jpayne@69: if filename is None: jpayne@69: if self.filename is not None: filename = self.filename jpayne@69: else: raise ValueError(MISSING_FILENAME_TEXT) jpayne@69: jpayne@69: self._cookies_lock.acquire() jpayne@69: try: jpayne@69: jpayne@69: old_state = copy.deepcopy(self._cookies) jpayne@69: self._cookies = {} jpayne@69: try: jpayne@69: self.load(filename, ignore_discard, ignore_expires) jpayne@69: except OSError: jpayne@69: self._cookies = old_state jpayne@69: raise jpayne@69: jpayne@69: finally: jpayne@69: self._cookies_lock.release() jpayne@69: jpayne@69: jpayne@69: def lwp_cookie_str(cookie): jpayne@69: """Return string representation of Cookie in the LWP cookie file format. jpayne@69: jpayne@69: Actually, the format is extended a bit -- see module docstring. jpayne@69: jpayne@69: """ jpayne@69: h = [(cookie.name, cookie.value), jpayne@69: ("path", cookie.path), jpayne@69: ("domain", cookie.domain)] jpayne@69: if cookie.port is not None: h.append(("port", cookie.port)) jpayne@69: if cookie.path_specified: h.append(("path_spec", None)) jpayne@69: if cookie.port_specified: h.append(("port_spec", None)) jpayne@69: if cookie.domain_initial_dot: h.append(("domain_dot", None)) jpayne@69: if cookie.secure: h.append(("secure", None)) jpayne@69: if cookie.expires: h.append(("expires", jpayne@69: time2isoz(float(cookie.expires)))) jpayne@69: if cookie.discard: h.append(("discard", None)) jpayne@69: if cookie.comment: h.append(("comment", cookie.comment)) jpayne@69: if cookie.comment_url: h.append(("commenturl", cookie.comment_url)) jpayne@69: jpayne@69: keys = sorted(cookie._rest.keys()) jpayne@69: for k in keys: jpayne@69: h.append((k, str(cookie._rest[k]))) jpayne@69: jpayne@69: h.append(("version", str(cookie.version))) jpayne@69: jpayne@69: return join_header_words([h]) jpayne@69: jpayne@69: class LWPCookieJar(FileCookieJar): jpayne@69: """ jpayne@69: The LWPCookieJar saves a sequence of "Set-Cookie3" lines. jpayne@69: "Set-Cookie3" is the format used by the libwww-perl library, not known jpayne@69: to be compatible with any browser, but which is easy to read and jpayne@69: doesn't lose information about RFC 2965 cookies. jpayne@69: jpayne@69: Additional methods jpayne@69: jpayne@69: as_lwp_str(ignore_discard=True, ignore_expired=True) jpayne@69: jpayne@69: """ jpayne@69: jpayne@69: def as_lwp_str(self, ignore_discard=True, ignore_expires=True): jpayne@69: """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers. jpayne@69: jpayne@69: ignore_discard and ignore_expires: see docstring for FileCookieJar.save jpayne@69: jpayne@69: """ jpayne@69: now = time.time() jpayne@69: r = [] jpayne@69: for cookie in self: jpayne@69: if not ignore_discard and cookie.discard: jpayne@69: continue jpayne@69: if not ignore_expires and cookie.is_expired(now): jpayne@69: continue jpayne@69: r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie)) jpayne@69: return "\n".join(r+[""]) jpayne@69: jpayne@69: def save(self, filename=None, ignore_discard=False, ignore_expires=False): jpayne@69: if filename is None: jpayne@69: if self.filename is not None: filename = self.filename jpayne@69: else: raise ValueError(MISSING_FILENAME_TEXT) jpayne@69: jpayne@69: with open(filename, "w") as f: jpayne@69: # There really isn't an LWP Cookies 2.0 format, but this indicates jpayne@69: # that there is extra information in here (domain_dot and jpayne@69: # port_spec) while still being compatible with libwww-perl, I hope. jpayne@69: f.write("#LWP-Cookies-2.0\n") jpayne@69: f.write(self.as_lwp_str(ignore_discard, ignore_expires)) jpayne@69: jpayne@69: def _really_load(self, f, filename, ignore_discard, ignore_expires): jpayne@69: magic = f.readline() jpayne@69: if not self.magic_re.search(magic): jpayne@69: msg = ("%r does not look like a Set-Cookie3 (LWP) format " jpayne@69: "file" % filename) jpayne@69: raise LoadError(msg) jpayne@69: jpayne@69: now = time.time() jpayne@69: jpayne@69: header = "Set-Cookie3:" jpayne@69: boolean_attrs = ("port_spec", "path_spec", "domain_dot", jpayne@69: "secure", "discard") jpayne@69: value_attrs = ("version", jpayne@69: "port", "path", "domain", jpayne@69: "expires", jpayne@69: "comment", "commenturl") jpayne@69: jpayne@69: try: jpayne@69: while 1: jpayne@69: line = f.readline() jpayne@69: if line == "": break jpayne@69: if not line.startswith(header): jpayne@69: continue jpayne@69: line = line[len(header):].strip() jpayne@69: jpayne@69: for data in split_header_words([line]): jpayne@69: name, value = data[0] jpayne@69: standard = {} jpayne@69: rest = {} jpayne@69: for k in boolean_attrs: jpayne@69: standard[k] = False jpayne@69: for k, v in data[1:]: jpayne@69: if k is not None: jpayne@69: lc = k.lower() jpayne@69: else: jpayne@69: lc = None jpayne@69: # don't lose case distinction for unknown fields jpayne@69: if (lc in value_attrs) or (lc in boolean_attrs): jpayne@69: k = lc jpayne@69: if k in boolean_attrs: jpayne@69: if v is None: v = True jpayne@69: standard[k] = v jpayne@69: elif k in value_attrs: jpayne@69: standard[k] = v jpayne@69: else: jpayne@69: rest[k] = v jpayne@69: jpayne@69: h = standard.get jpayne@69: expires = h("expires") jpayne@69: discard = h("discard") jpayne@69: if expires is not None: jpayne@69: expires = iso2time(expires) jpayne@69: if expires is None: jpayne@69: discard = True jpayne@69: domain = h("domain") jpayne@69: domain_specified = domain.startswith(".") jpayne@69: c = Cookie(h("version"), name, value, jpayne@69: h("port"), h("port_spec"), jpayne@69: domain, domain_specified, h("domain_dot"), jpayne@69: h("path"), h("path_spec"), jpayne@69: h("secure"), jpayne@69: expires, jpayne@69: discard, jpayne@69: h("comment"), jpayne@69: h("commenturl"), jpayne@69: rest) jpayne@69: if not ignore_discard and c.discard: jpayne@69: continue jpayne@69: if not ignore_expires and c.is_expired(now): jpayne@69: continue jpayne@69: self.set_cookie(c) jpayne@69: except OSError: jpayne@69: raise jpayne@69: except Exception: jpayne@69: _warn_unhandled_exception() jpayne@69: raise LoadError("invalid Set-Cookie3 format file %r: %r" % jpayne@69: (filename, line)) jpayne@69: jpayne@69: jpayne@69: class MozillaCookieJar(FileCookieJar): jpayne@69: """ jpayne@69: jpayne@69: WARNING: you may want to backup your browser's cookies file if you use jpayne@69: this class to save cookies. I *think* it works, but there have been jpayne@69: bugs in the past! jpayne@69: jpayne@69: This class differs from CookieJar only in the format it uses to save and jpayne@69: load cookies to and from a file. This class uses the Mozilla/Netscape jpayne@69: `cookies.txt' format. lynx uses this file format, too. jpayne@69: jpayne@69: Don't expect cookies saved while the browser is running to be noticed by jpayne@69: the browser (in fact, Mozilla on unix will overwrite your saved cookies if jpayne@69: you change them on disk while it's running; on Windows, you probably can't jpayne@69: save at all while the browser is running). jpayne@69: jpayne@69: Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to jpayne@69: Netscape cookies on saving. jpayne@69: jpayne@69: In particular, the cookie version and port number information is lost, jpayne@69: together with information about whether or not Path, Port and Discard were jpayne@69: specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the jpayne@69: domain as set in the HTTP header started with a dot (yes, I'm aware some jpayne@69: domains in Netscape files start with a dot and some don't -- trust me, you jpayne@69: really don't want to know any more about this). jpayne@69: jpayne@69: Note that though Mozilla and Netscape use the same format, they use jpayne@69: slightly different headers. The class saves cookies using the Netscape jpayne@69: header by default (Mozilla can cope with that). jpayne@69: jpayne@69: """ jpayne@69: magic_re = re.compile("#( Netscape)? HTTP Cookie File") jpayne@69: header = """\ jpayne@69: # Netscape HTTP Cookie File jpayne@69: # http://curl.haxx.se/rfc/cookie_spec.html jpayne@69: # This is a generated file! Do not edit. jpayne@69: jpayne@69: """ jpayne@69: jpayne@69: def _really_load(self, f, filename, ignore_discard, ignore_expires): jpayne@69: now = time.time() jpayne@69: jpayne@69: magic = f.readline() jpayne@69: if not self.magic_re.search(magic): jpayne@69: raise LoadError( jpayne@69: "%r does not look like a Netscape format cookies file" % jpayne@69: filename) jpayne@69: jpayne@69: try: jpayne@69: while 1: jpayne@69: line = f.readline() jpayne@69: if line == "": break jpayne@69: jpayne@69: # last field may be absent, so keep any trailing tab jpayne@69: if line.endswith("\n"): line = line[:-1] jpayne@69: jpayne@69: # skip comments and blank lines XXX what is $ for? jpayne@69: if (line.strip().startswith(("#", "$")) or jpayne@69: line.strip() == ""): jpayne@69: continue jpayne@69: jpayne@69: domain, domain_specified, path, secure, expires, name, value = \ jpayne@69: line.split("\t") jpayne@69: secure = (secure == "TRUE") jpayne@69: domain_specified = (domain_specified == "TRUE") jpayne@69: if name == "": jpayne@69: # cookies.txt regards 'Set-Cookie: foo' as a cookie jpayne@69: # with no name, whereas http.cookiejar regards it as a jpayne@69: # cookie with no value. jpayne@69: name = value jpayne@69: value = None jpayne@69: jpayne@69: initial_dot = domain.startswith(".") jpayne@69: assert domain_specified == initial_dot jpayne@69: jpayne@69: discard = False jpayne@69: if expires == "": jpayne@69: expires = None jpayne@69: discard = True jpayne@69: jpayne@69: # assume path_specified is false jpayne@69: c = Cookie(0, name, value, jpayne@69: None, False, jpayne@69: domain, domain_specified, initial_dot, jpayne@69: path, False, jpayne@69: secure, jpayne@69: expires, jpayne@69: discard, jpayne@69: None, jpayne@69: None, jpayne@69: {}) jpayne@69: if not ignore_discard and c.discard: jpayne@69: continue jpayne@69: if not ignore_expires and c.is_expired(now): jpayne@69: continue jpayne@69: self.set_cookie(c) jpayne@69: jpayne@69: except OSError: jpayne@69: raise jpayne@69: except Exception: jpayne@69: _warn_unhandled_exception() jpayne@69: raise LoadError("invalid Netscape format cookies file %r: %r" % jpayne@69: (filename, line)) jpayne@69: jpayne@69: def save(self, filename=None, ignore_discard=False, ignore_expires=False): jpayne@69: if filename is None: jpayne@69: if self.filename is not None: filename = self.filename jpayne@69: else: raise ValueError(MISSING_FILENAME_TEXT) jpayne@69: jpayne@69: with open(filename, "w") as f: jpayne@69: f.write(self.header) jpayne@69: now = time.time() jpayne@69: for cookie in self: jpayne@69: if not ignore_discard and cookie.discard: jpayne@69: continue jpayne@69: if not ignore_expires and cookie.is_expired(now): jpayne@69: continue jpayne@69: if cookie.secure: secure = "TRUE" jpayne@69: else: secure = "FALSE" jpayne@69: if cookie.domain.startswith("."): initial_dot = "TRUE" jpayne@69: else: initial_dot = "FALSE" jpayne@69: if cookie.expires is not None: jpayne@69: expires = str(cookie.expires) jpayne@69: else: jpayne@69: expires = "" jpayne@69: if cookie.value is None: jpayne@69: # cookies.txt regards 'Set-Cookie: foo' as a cookie jpayne@69: # with no name, whereas http.cookiejar regards it as a jpayne@69: # cookie with no value. jpayne@69: name = "" jpayne@69: value = cookie.name jpayne@69: else: jpayne@69: name = cookie.name jpayne@69: value = cookie.value jpayne@69: f.write( jpayne@69: "\t".join([cookie.domain, initial_dot, cookie.path, jpayne@69: secure, expires, name, value])+ jpayne@69: "\n")