Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/http/cookiejar.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 r"""HTTP cookie handling for web clients.
jpayne@69 2
jpayne@69 3 This module has (now fairly distant) origins in Gisle Aas' Perl module
jpayne@69 4 HTTP::Cookies, from the libwww-perl library.
jpayne@69 5
jpayne@69 6 Docstrings, comments and debug strings in this code refer to the
jpayne@69 7 attributes of the HTTP cookie system as cookie-attributes, to distinguish
jpayne@69 8 them clearly from Python attributes.
jpayne@69 9
jpayne@69 10 Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
jpayne@69 11 distributed with the Python standard library, but are available from
jpayne@69 12 http://wwwsearch.sf.net/):
jpayne@69 13
jpayne@69 14 CookieJar____
jpayne@69 15 / \ \
jpayne@69 16 FileCookieJar \ \
jpayne@69 17 / | \ \ \
jpayne@69 18 MozillaCookieJar | LWPCookieJar \ \
jpayne@69 19 | | \
jpayne@69 20 | ---MSIEBase | \
jpayne@69 21 | / | | \
jpayne@69 22 | / MSIEDBCookieJar BSDDBCookieJar
jpayne@69 23 |/
jpayne@69 24 MSIECookieJar
jpayne@69 25
jpayne@69 26 """
jpayne@69 27
jpayne@69 28 __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
jpayne@69 29 'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
jpayne@69 30
jpayne@69 31 import os
jpayne@69 32 import copy
jpayne@69 33 import datetime
jpayne@69 34 import re
jpayne@69 35 import time
jpayne@69 36 import urllib.parse, urllib.request
jpayne@69 37 import threading as _threading
jpayne@69 38 import http.client # only for the default HTTP port
jpayne@69 39 from calendar import timegm
jpayne@69 40
jpayne@69 41 debug = False # set to True to enable debugging via the logging module
jpayne@69 42 logger = None
jpayne@69 43
jpayne@69 44 def _debug(*args):
jpayne@69 45 if not debug:
jpayne@69 46 return
jpayne@69 47 global logger
jpayne@69 48 if not logger:
jpayne@69 49 import logging
jpayne@69 50 logger = logging.getLogger("http.cookiejar")
jpayne@69 51 return logger.debug(*args)
jpayne@69 52
jpayne@69 53
jpayne@69 54 DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
jpayne@69 55 MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
jpayne@69 56 "instance initialised with one)")
jpayne@69 57
jpayne@69 58 def _warn_unhandled_exception():
jpayne@69 59 # There are a few catch-all except: statements in this module, for
jpayne@69 60 # catching input that's bad in unexpected ways. Warn if any
jpayne@69 61 # exceptions are caught there.
jpayne@69 62 import io, warnings, traceback
jpayne@69 63 f = io.StringIO()
jpayne@69 64 traceback.print_exc(None, f)
jpayne@69 65 msg = f.getvalue()
jpayne@69 66 warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2)
jpayne@69 67
jpayne@69 68
jpayne@69 69 # Date/time conversion
jpayne@69 70 # -----------------------------------------------------------------------------
jpayne@69 71
jpayne@69 72 EPOCH_YEAR = 1970
jpayne@69 73 def _timegm(tt):
jpayne@69 74 year, month, mday, hour, min, sec = tt[:6]
jpayne@69 75 if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and
jpayne@69 76 (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
jpayne@69 77 return timegm(tt)
jpayne@69 78 else:
jpayne@69 79 return None
jpayne@69 80
jpayne@69 81 DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
jpayne@69 82 MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
jpayne@69 83 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
jpayne@69 84 MONTHS_LOWER = []
jpayne@69 85 for month in MONTHS: MONTHS_LOWER.append(month.lower())
jpayne@69 86
jpayne@69 87 def time2isoz(t=None):
jpayne@69 88 """Return a string representing time in seconds since epoch, t.
jpayne@69 89
jpayne@69 90 If the function is called without an argument, it will use the current
jpayne@69 91 time.
jpayne@69 92
jpayne@69 93 The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
jpayne@69 94 representing Universal Time (UTC, aka GMT). An example of this format is:
jpayne@69 95
jpayne@69 96 1994-11-24 08:49:37Z
jpayne@69 97
jpayne@69 98 """
jpayne@69 99 if t is None:
jpayne@69 100 dt = datetime.datetime.utcnow()
jpayne@69 101 else:
jpayne@69 102 dt = datetime.datetime.utcfromtimestamp(t)
jpayne@69 103 return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
jpayne@69 104 dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
jpayne@69 105
jpayne@69 106 def time2netscape(t=None):
jpayne@69 107 """Return a string representing time in seconds since epoch, t.
jpayne@69 108
jpayne@69 109 If the function is called without an argument, it will use the current
jpayne@69 110 time.
jpayne@69 111
jpayne@69 112 The format of the returned string is like this:
jpayne@69 113
jpayne@69 114 Wed, DD-Mon-YYYY HH:MM:SS GMT
jpayne@69 115
jpayne@69 116 """
jpayne@69 117 if t is None:
jpayne@69 118 dt = datetime.datetime.utcnow()
jpayne@69 119 else:
jpayne@69 120 dt = datetime.datetime.utcfromtimestamp(t)
jpayne@69 121 return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
jpayne@69 122 DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1],
jpayne@69 123 dt.year, dt.hour, dt.minute, dt.second)
jpayne@69 124
jpayne@69 125
jpayne@69 126 UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
jpayne@69 127
jpayne@69 128 TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII)
jpayne@69 129 def offset_from_tz_string(tz):
jpayne@69 130 offset = None
jpayne@69 131 if tz in UTC_ZONES:
jpayne@69 132 offset = 0
jpayne@69 133 else:
jpayne@69 134 m = TIMEZONE_RE.search(tz)
jpayne@69 135 if m:
jpayne@69 136 offset = 3600 * int(m.group(2))
jpayne@69 137 if m.group(3):
jpayne@69 138 offset = offset + 60 * int(m.group(3))
jpayne@69 139 if m.group(1) == '-':
jpayne@69 140 offset = -offset
jpayne@69 141 return offset
jpayne@69 142
jpayne@69 143 def _str2time(day, mon, yr, hr, min, sec, tz):
jpayne@69 144 yr = int(yr)
jpayne@69 145 if yr > datetime.MAXYEAR:
jpayne@69 146 return None
jpayne@69 147
jpayne@69 148 # translate month name to number
jpayne@69 149 # month numbers start with 1 (January)
jpayne@69 150 try:
jpayne@69 151 mon = MONTHS_LOWER.index(mon.lower())+1
jpayne@69 152 except ValueError:
jpayne@69 153 # maybe it's already a number
jpayne@69 154 try:
jpayne@69 155 imon = int(mon)
jpayne@69 156 except ValueError:
jpayne@69 157 return None
jpayne@69 158 if 1 <= imon <= 12:
jpayne@69 159 mon = imon
jpayne@69 160 else:
jpayne@69 161 return None
jpayne@69 162
jpayne@69 163 # make sure clock elements are defined
jpayne@69 164 if hr is None: hr = 0
jpayne@69 165 if min is None: min = 0
jpayne@69 166 if sec is None: sec = 0
jpayne@69 167
jpayne@69 168 day = int(day)
jpayne@69 169 hr = int(hr)
jpayne@69 170 min = int(min)
jpayne@69 171 sec = int(sec)
jpayne@69 172
jpayne@69 173 if yr < 1000:
jpayne@69 174 # find "obvious" year
jpayne@69 175 cur_yr = time.localtime(time.time())[0]
jpayne@69 176 m = cur_yr % 100
jpayne@69 177 tmp = yr
jpayne@69 178 yr = yr + cur_yr - m
jpayne@69 179 m = m - tmp
jpayne@69 180 if abs(m) > 50:
jpayne@69 181 if m > 0: yr = yr + 100
jpayne@69 182 else: yr = yr - 100
jpayne@69 183
jpayne@69 184 # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
jpayne@69 185 t = _timegm((yr, mon, day, hr, min, sec, tz))
jpayne@69 186
jpayne@69 187 if t is not None:
jpayne@69 188 # adjust time using timezone string, to get absolute time since epoch
jpayne@69 189 if tz is None:
jpayne@69 190 tz = "UTC"
jpayne@69 191 tz = tz.upper()
jpayne@69 192 offset = offset_from_tz_string(tz)
jpayne@69 193 if offset is None:
jpayne@69 194 return None
jpayne@69 195 t = t - offset
jpayne@69 196
jpayne@69 197 return t
jpayne@69 198
jpayne@69 199 STRICT_DATE_RE = re.compile(
jpayne@69 200 r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
jpayne@69 201 r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII)
jpayne@69 202 WEEKDAY_RE = re.compile(
jpayne@69 203 r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII)
jpayne@69 204 LOOSE_HTTP_DATE_RE = re.compile(
jpayne@69 205 r"""^
jpayne@69 206 (\d\d?) # day
jpayne@69 207 (?:\s+|[-\/])
jpayne@69 208 (\w+) # month
jpayne@69 209 (?:\s+|[-\/])
jpayne@69 210 (\d+) # year
jpayne@69 211 (?:
jpayne@69 212 (?:\s+|:) # separator before clock
jpayne@69 213 (\d\d?):(\d\d) # hour:min
jpayne@69 214 (?::(\d\d))? # optional seconds
jpayne@69 215 )? # optional clock
jpayne@69 216 \s*
jpayne@69 217 (?:
jpayne@69 218 ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone
jpayne@69 219 \s*
jpayne@69 220 )?
jpayne@69 221 (?:
jpayne@69 222 \(\w+\) # ASCII representation of timezone in parens.
jpayne@69 223 \s*
jpayne@69 224 )?$""", re.X | re.ASCII)
jpayne@69 225 def http2time(text):
jpayne@69 226 """Returns time in seconds since epoch of time represented by a string.
jpayne@69 227
jpayne@69 228 Return value is an integer.
jpayne@69 229
jpayne@69 230 None is returned if the format of str is unrecognized, the time is outside
jpayne@69 231 the representable range, or the timezone string is not recognized. If the
jpayne@69 232 string contains no timezone, UTC is assumed.
jpayne@69 233
jpayne@69 234 The timezone in the string may be numerical (like "-0800" or "+0100") or a
jpayne@69 235 string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
jpayne@69 236 timezone strings equivalent to UTC (zero offset) are known to the function.
jpayne@69 237
jpayne@69 238 The function loosely parses the following formats:
jpayne@69 239
jpayne@69 240 Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
jpayne@69 241 Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
jpayne@69 242 Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
jpayne@69 243 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
jpayne@69 244 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
jpayne@69 245 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
jpayne@69 246
jpayne@69 247 The parser ignores leading and trailing whitespace. The time may be
jpayne@69 248 absent.
jpayne@69 249
jpayne@69 250 If the year is given with only 2 digits, the function will select the
jpayne@69 251 century that makes the year closest to the current date.
jpayne@69 252
jpayne@69 253 """
jpayne@69 254 # fast exit for strictly conforming string
jpayne@69 255 m = STRICT_DATE_RE.search(text)
jpayne@69 256 if m:
jpayne@69 257 g = m.groups()
jpayne@69 258 mon = MONTHS_LOWER.index(g[1].lower()) + 1
jpayne@69 259 tt = (int(g[2]), mon, int(g[0]),
jpayne@69 260 int(g[3]), int(g[4]), float(g[5]))
jpayne@69 261 return _timegm(tt)
jpayne@69 262
jpayne@69 263 # No, we need some messy parsing...
jpayne@69 264
jpayne@69 265 # clean up
jpayne@69 266 text = text.lstrip()
jpayne@69 267 text = WEEKDAY_RE.sub("", text, 1) # Useless weekday
jpayne@69 268
jpayne@69 269 # tz is time zone specifier string
jpayne@69 270 day, mon, yr, hr, min, sec, tz = [None]*7
jpayne@69 271
jpayne@69 272 # loose regexp parse
jpayne@69 273 m = LOOSE_HTTP_DATE_RE.search(text)
jpayne@69 274 if m is not None:
jpayne@69 275 day, mon, yr, hr, min, sec, tz = m.groups()
jpayne@69 276 else:
jpayne@69 277 return None # bad format
jpayne@69 278
jpayne@69 279 return _str2time(day, mon, yr, hr, min, sec, tz)
jpayne@69 280
jpayne@69 281 ISO_DATE_RE = re.compile(
jpayne@69 282 r"""^
jpayne@69 283 (\d{4}) # year
jpayne@69 284 [-\/]?
jpayne@69 285 (\d\d?) # numerical month
jpayne@69 286 [-\/]?
jpayne@69 287 (\d\d?) # day
jpayne@69 288 (?:
jpayne@69 289 (?:\s+|[-:Tt]) # separator before clock
jpayne@69 290 (\d\d?):?(\d\d) # hour:min
jpayne@69 291 (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
jpayne@69 292 )? # optional clock
jpayne@69 293 \s*
jpayne@69 294 (?:
jpayne@69 295 ([-+]?\d\d?:?(:?\d\d)?
jpayne@69 296 |Z|z) # timezone (Z is "zero meridian", i.e. GMT)
jpayne@69 297 \s*
jpayne@69 298 )?$""", re.X | re. ASCII)
jpayne@69 299 def iso2time(text):
jpayne@69 300 """
jpayne@69 301 As for http2time, but parses the ISO 8601 formats:
jpayne@69 302
jpayne@69 303 1994-02-03 14:15:29 -0100 -- ISO 8601 format
jpayne@69 304 1994-02-03 14:15:29 -- zone is optional
jpayne@69 305 1994-02-03 -- only date
jpayne@69 306 1994-02-03T14:15:29 -- Use T as separator
jpayne@69 307 19940203T141529Z -- ISO 8601 compact format
jpayne@69 308 19940203 -- only date
jpayne@69 309
jpayne@69 310 """
jpayne@69 311 # clean up
jpayne@69 312 text = text.lstrip()
jpayne@69 313
jpayne@69 314 # tz is time zone specifier string
jpayne@69 315 day, mon, yr, hr, min, sec, tz = [None]*7
jpayne@69 316
jpayne@69 317 # loose regexp parse
jpayne@69 318 m = ISO_DATE_RE.search(text)
jpayne@69 319 if m is not None:
jpayne@69 320 # XXX there's an extra bit of the timezone I'm ignoring here: is
jpayne@69 321 # this the right thing to do?
jpayne@69 322 yr, mon, day, hr, min, sec, tz, _ = m.groups()
jpayne@69 323 else:
jpayne@69 324 return None # bad format
jpayne@69 325
jpayne@69 326 return _str2time(day, mon, yr, hr, min, sec, tz)
jpayne@69 327
jpayne@69 328
jpayne@69 329 # Header parsing
jpayne@69 330 # -----------------------------------------------------------------------------
jpayne@69 331
jpayne@69 332 def unmatched(match):
jpayne@69 333 """Return unmatched part of re.Match object."""
jpayne@69 334 start, end = match.span(0)
jpayne@69 335 return match.string[:start]+match.string[end:]
jpayne@69 336
jpayne@69 337 HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)")
jpayne@69 338 HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")
jpayne@69 339 HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)")
jpayne@69 340 HEADER_ESCAPE_RE = re.compile(r"\\(.)")
jpayne@69 341 def split_header_words(header_values):
jpayne@69 342 r"""Parse header values into a list of lists containing key,value pairs.
jpayne@69 343
jpayne@69 344 The function knows how to deal with ",", ";" and "=" as well as quoted
jpayne@69 345 values after "=". A list of space separated tokens are parsed as if they
jpayne@69 346 were separated by ";".
jpayne@69 347
jpayne@69 348 If the header_values passed as argument contains multiple values, then they
jpayne@69 349 are treated as if they were a single value separated by comma ",".
jpayne@69 350
jpayne@69 351 This means that this function is useful for parsing header fields that
jpayne@69 352 follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
jpayne@69 353 the requirement for tokens).
jpayne@69 354
jpayne@69 355 headers = #header
jpayne@69 356 header = (token | parameter) *( [";"] (token | parameter))
jpayne@69 357
jpayne@69 358 token = 1*<any CHAR except CTLs or separators>
jpayne@69 359 separators = "(" | ")" | "<" | ">" | "@"
jpayne@69 360 | "," | ";" | ":" | "\" | <">
jpayne@69 361 | "/" | "[" | "]" | "?" | "="
jpayne@69 362 | "{" | "}" | SP | HT
jpayne@69 363
jpayne@69 364 quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
jpayne@69 365 qdtext = <any TEXT except <">>
jpayne@69 366 quoted-pair = "\" CHAR
jpayne@69 367
jpayne@69 368 parameter = attribute "=" value
jpayne@69 369 attribute = token
jpayne@69 370 value = token | quoted-string
jpayne@69 371
jpayne@69 372 Each header is represented by a list of key/value pairs. The value for a
jpayne@69 373 simple token (not part of a parameter) is None. Syntactically incorrect
jpayne@69 374 headers will not necessarily be parsed as you would want.
jpayne@69 375
jpayne@69 376 This is easier to describe with some examples:
jpayne@69 377
jpayne@69 378 >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
jpayne@69 379 [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
jpayne@69 380 >>> split_header_words(['text/html; charset="iso-8859-1"'])
jpayne@69 381 [[('text/html', None), ('charset', 'iso-8859-1')]]
jpayne@69 382 >>> split_header_words([r'Basic realm="\"foo\bar\""'])
jpayne@69 383 [[('Basic', None), ('realm', '"foobar"')]]
jpayne@69 384
jpayne@69 385 """
jpayne@69 386 assert not isinstance(header_values, str)
jpayne@69 387 result = []
jpayne@69 388 for text in header_values:
jpayne@69 389 orig_text = text
jpayne@69 390 pairs = []
jpayne@69 391 while text:
jpayne@69 392 m = HEADER_TOKEN_RE.search(text)
jpayne@69 393 if m:
jpayne@69 394 text = unmatched(m)
jpayne@69 395 name = m.group(1)
jpayne@69 396 m = HEADER_QUOTED_VALUE_RE.search(text)
jpayne@69 397 if m: # quoted value
jpayne@69 398 text = unmatched(m)
jpayne@69 399 value = m.group(1)
jpayne@69 400 value = HEADER_ESCAPE_RE.sub(r"\1", value)
jpayne@69 401 else:
jpayne@69 402 m = HEADER_VALUE_RE.search(text)
jpayne@69 403 if m: # unquoted value
jpayne@69 404 text = unmatched(m)
jpayne@69 405 value = m.group(1)
jpayne@69 406 value = value.rstrip()
jpayne@69 407 else:
jpayne@69 408 # no value, a lone token
jpayne@69 409 value = None
jpayne@69 410 pairs.append((name, value))
jpayne@69 411 elif text.lstrip().startswith(","):
jpayne@69 412 # concatenated headers, as per RFC 2616 section 4.2
jpayne@69 413 text = text.lstrip()[1:]
jpayne@69 414 if pairs: result.append(pairs)
jpayne@69 415 pairs = []
jpayne@69 416 else:
jpayne@69 417 # skip junk
jpayne@69 418 non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text)
jpayne@69 419 assert nr_junk_chars > 0, (
jpayne@69 420 "split_header_words bug: '%s', '%s', %s" %
jpayne@69 421 (orig_text, text, pairs))
jpayne@69 422 text = non_junk
jpayne@69 423 if pairs: result.append(pairs)
jpayne@69 424 return result
jpayne@69 425
jpayne@69 426 HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")
jpayne@69 427 def join_header_words(lists):
jpayne@69 428 """Do the inverse (almost) of the conversion done by split_header_words.
jpayne@69 429
jpayne@69 430 Takes a list of lists of (key, value) pairs and produces a single header
jpayne@69 431 value. Attribute values are quoted if needed.
jpayne@69 432
jpayne@69 433 >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
jpayne@69 434 'text/plain; charset="iso-8859-1"'
jpayne@69 435 >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
jpayne@69 436 'text/plain, charset="iso-8859-1"'
jpayne@69 437
jpayne@69 438 """
jpayne@69 439 headers = []
jpayne@69 440 for pairs in lists:
jpayne@69 441 attr = []
jpayne@69 442 for k, v in pairs:
jpayne@69 443 if v is not None:
jpayne@69 444 if not re.search(r"^\w+$", v):
jpayne@69 445 v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \
jpayne@69 446 v = '"%s"' % v
jpayne@69 447 k = "%s=%s" % (k, v)
jpayne@69 448 attr.append(k)
jpayne@69 449 if attr: headers.append("; ".join(attr))
jpayne@69 450 return ", ".join(headers)
jpayne@69 451
jpayne@69 452 def strip_quotes(text):
jpayne@69 453 if text.startswith('"'):
jpayne@69 454 text = text[1:]
jpayne@69 455 if text.endswith('"'):
jpayne@69 456 text = text[:-1]
jpayne@69 457 return text
jpayne@69 458
jpayne@69 459 def parse_ns_headers(ns_headers):
jpayne@69 460 """Ad-hoc parser for Netscape protocol cookie-attributes.
jpayne@69 461
jpayne@69 462 The old Netscape cookie format for Set-Cookie can for instance contain
jpayne@69 463 an unquoted "," in the expires field, so we have to use this ad-hoc
jpayne@69 464 parser instead of split_header_words.
jpayne@69 465
jpayne@69 466 XXX This may not make the best possible effort to parse all the crap
jpayne@69 467 that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
jpayne@69 468 parser is probably better, so could do worse than following that if
jpayne@69 469 this ever gives any trouble.
jpayne@69 470
jpayne@69 471 Currently, this is also used for parsing RFC 2109 cookies.
jpayne@69 472
jpayne@69 473 """
jpayne@69 474 known_attrs = ("expires", "domain", "path", "secure",
jpayne@69 475 # RFC 2109 attrs (may turn up in Netscape cookies, too)
jpayne@69 476 "version", "port", "max-age")
jpayne@69 477
jpayne@69 478 result = []
jpayne@69 479 for ns_header in ns_headers:
jpayne@69 480 pairs = []
jpayne@69 481 version_set = False
jpayne@69 482
jpayne@69 483 # XXX: The following does not strictly adhere to RFCs in that empty
jpayne@69 484 # names and values are legal (the former will only appear once and will
jpayne@69 485 # be overwritten if multiple occurrences are present). This is
jpayne@69 486 # mostly to deal with backwards compatibility.
jpayne@69 487 for ii, param in enumerate(ns_header.split(';')):
jpayne@69 488 param = param.strip()
jpayne@69 489
jpayne@69 490 key, sep, val = param.partition('=')
jpayne@69 491 key = key.strip()
jpayne@69 492
jpayne@69 493 if not key:
jpayne@69 494 if ii == 0:
jpayne@69 495 break
jpayne@69 496 else:
jpayne@69 497 continue
jpayne@69 498
jpayne@69 499 # allow for a distinction between present and empty and missing
jpayne@69 500 # altogether
jpayne@69 501 val = val.strip() if sep else None
jpayne@69 502
jpayne@69 503 if ii != 0:
jpayne@69 504 lc = key.lower()
jpayne@69 505 if lc in known_attrs:
jpayne@69 506 key = lc
jpayne@69 507
jpayne@69 508 if key == "version":
jpayne@69 509 # This is an RFC 2109 cookie.
jpayne@69 510 if val is not None:
jpayne@69 511 val = strip_quotes(val)
jpayne@69 512 version_set = True
jpayne@69 513 elif key == "expires":
jpayne@69 514 # convert expires date to seconds since epoch
jpayne@69 515 if val is not None:
jpayne@69 516 val = http2time(strip_quotes(val)) # None if invalid
jpayne@69 517 pairs.append((key, val))
jpayne@69 518
jpayne@69 519 if pairs:
jpayne@69 520 if not version_set:
jpayne@69 521 pairs.append(("version", "0"))
jpayne@69 522 result.append(pairs)
jpayne@69 523
jpayne@69 524 return result
jpayne@69 525
jpayne@69 526
jpayne@69 527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
jpayne@69 528 def is_HDN(text):
jpayne@69 529 """Return True if text is a host domain name."""
jpayne@69 530 # XXX
jpayne@69 531 # This may well be wrong. Which RFC is HDN defined in, if any (for
jpayne@69 532 # the purposes of RFC 2965)?
jpayne@69 533 # For the current implementation, what about IPv6? Remember to look
jpayne@69 534 # at other uses of IPV4_RE also, if change this.
jpayne@69 535 if IPV4_RE.search(text):
jpayne@69 536 return False
jpayne@69 537 if text == "":
jpayne@69 538 return False
jpayne@69 539 if text[0] == "." or text[-1] == ".":
jpayne@69 540 return False
jpayne@69 541 return True
jpayne@69 542
jpayne@69 543 def domain_match(A, B):
jpayne@69 544 """Return True if domain A domain-matches domain B, according to RFC 2965.
jpayne@69 545
jpayne@69 546 A and B may be host domain names or IP addresses.
jpayne@69 547
jpayne@69 548 RFC 2965, section 1:
jpayne@69 549
jpayne@69 550 Host names can be specified either as an IP address or a HDN string.
jpayne@69 551 Sometimes we compare one host name with another. (Such comparisons SHALL
jpayne@69 552 be case-insensitive.) Host A's name domain-matches host B's if
jpayne@69 553
jpayne@69 554 * their host name strings string-compare equal; or
jpayne@69 555
jpayne@69 556 * A is a HDN string and has the form NB, where N is a non-empty
jpayne@69 557 name string, B has the form .B', and B' is a HDN string. (So,
jpayne@69 558 x.y.com domain-matches .Y.com but not Y.com.)
jpayne@69 559
jpayne@69 560 Note that domain-match is not a commutative operation: a.b.c.com
jpayne@69 561 domain-matches .c.com, but not the reverse.
jpayne@69 562
jpayne@69 563 """
jpayne@69 564 # Note that, if A or B are IP addresses, the only relevant part of the
jpayne@69 565 # definition of the domain-match algorithm is the direct string-compare.
jpayne@69 566 A = A.lower()
jpayne@69 567 B = B.lower()
jpayne@69 568 if A == B:
jpayne@69 569 return True
jpayne@69 570 if not is_HDN(A):
jpayne@69 571 return False
jpayne@69 572 i = A.rfind(B)
jpayne@69 573 if i == -1 or i == 0:
jpayne@69 574 # A does not have form NB, or N is the empty string
jpayne@69 575 return False
jpayne@69 576 if not B.startswith("."):
jpayne@69 577 return False
jpayne@69 578 if not is_HDN(B[1:]):
jpayne@69 579 return False
jpayne@69 580 return True
jpayne@69 581
jpayne@69 582 def liberal_is_HDN(text):
jpayne@69 583 """Return True if text is a sort-of-like a host domain name.
jpayne@69 584
jpayne@69 585 For accepting/blocking domains.
jpayne@69 586
jpayne@69 587 """
jpayne@69 588 if IPV4_RE.search(text):
jpayne@69 589 return False
jpayne@69 590 return True
jpayne@69 591
jpayne@69 592 def user_domain_match(A, B):
jpayne@69 593 """For blocking/accepting domains.
jpayne@69 594
jpayne@69 595 A and B may be host domain names or IP addresses.
jpayne@69 596
jpayne@69 597 """
jpayne@69 598 A = A.lower()
jpayne@69 599 B = B.lower()
jpayne@69 600 if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
jpayne@69 601 if A == B:
jpayne@69 602 # equal IP addresses
jpayne@69 603 return True
jpayne@69 604 return False
jpayne@69 605 initial_dot = B.startswith(".")
jpayne@69 606 if initial_dot and A.endswith(B):
jpayne@69 607 return True
jpayne@69 608 if not initial_dot and A == B:
jpayne@69 609 return True
jpayne@69 610 return False
jpayne@69 611
jpayne@69 612 cut_port_re = re.compile(r":\d+$", re.ASCII)
jpayne@69 613 def request_host(request):
jpayne@69 614 """Return request-host, as defined by RFC 2965.
jpayne@69 615
jpayne@69 616 Variation from RFC: returned value is lowercased, for convenient
jpayne@69 617 comparison.
jpayne@69 618
jpayne@69 619 """
jpayne@69 620 url = request.get_full_url()
jpayne@69 621 host = urllib.parse.urlparse(url)[1]
jpayne@69 622 if host == "":
jpayne@69 623 host = request.get_header("Host", "")
jpayne@69 624
jpayne@69 625 # remove port, if present
jpayne@69 626 host = cut_port_re.sub("", host, 1)
jpayne@69 627 return host.lower()
jpayne@69 628
jpayne@69 629 def eff_request_host(request):
jpayne@69 630 """Return a tuple (request-host, effective request-host name).
jpayne@69 631
jpayne@69 632 As defined by RFC 2965, except both are lowercased.
jpayne@69 633
jpayne@69 634 """
jpayne@69 635 erhn = req_host = request_host(request)
jpayne@69 636 if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
jpayne@69 637 erhn = req_host + ".local"
jpayne@69 638 return req_host, erhn
jpayne@69 639
jpayne@69 640 def request_path(request):
jpayne@69 641 """Path component of request-URI, as defined by RFC 2965."""
jpayne@69 642 url = request.get_full_url()
jpayne@69 643 parts = urllib.parse.urlsplit(url)
jpayne@69 644 path = escape_path(parts.path)
jpayne@69 645 if not path.startswith("/"):
jpayne@69 646 # fix bad RFC 2396 absoluteURI
jpayne@69 647 path = "/" + path
jpayne@69 648 return path
jpayne@69 649
jpayne@69 650 def request_port(request):
jpayne@69 651 host = request.host
jpayne@69 652 i = host.find(':')
jpayne@69 653 if i >= 0:
jpayne@69 654 port = host[i+1:]
jpayne@69 655 try:
jpayne@69 656 int(port)
jpayne@69 657 except ValueError:
jpayne@69 658 _debug("nonnumeric port: '%s'", port)
jpayne@69 659 return None
jpayne@69 660 else:
jpayne@69 661 port = DEFAULT_HTTP_PORT
jpayne@69 662 return port
jpayne@69 663
jpayne@69 664 # Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
jpayne@69 665 # need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
jpayne@69 666 HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
jpayne@69 667 ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
jpayne@69 668 def uppercase_escaped_char(match):
jpayne@69 669 return "%%%s" % match.group(1).upper()
jpayne@69 670 def escape_path(path):
jpayne@69 671 """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
jpayne@69 672 # There's no knowing what character encoding was used to create URLs
jpayne@69 673 # containing %-escapes, but since we have to pick one to escape invalid
jpayne@69 674 # path characters, we pick UTF-8, as recommended in the HTML 4.0
jpayne@69 675 # specification:
jpayne@69 676 # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
jpayne@69 677 # And here, kind of: draft-fielding-uri-rfc2396bis-03
jpayne@69 678 # (And in draft IRI specification: draft-duerst-iri-05)
jpayne@69 679 # (And here, for new URI schemes: RFC 2718)
jpayne@69 680 path = urllib.parse.quote(path, HTTP_PATH_SAFE)
jpayne@69 681 path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
jpayne@69 682 return path
jpayne@69 683
jpayne@69 684 def reach(h):
jpayne@69 685 """Return reach of host h, as defined by RFC 2965, section 1.
jpayne@69 686
jpayne@69 687 The reach R of a host name H is defined as follows:
jpayne@69 688
jpayne@69 689 * If
jpayne@69 690
jpayne@69 691 - H is the host domain name of a host; and,
jpayne@69 692
jpayne@69 693 - H has the form A.B; and
jpayne@69 694
jpayne@69 695 - A has no embedded (that is, interior) dots; and
jpayne@69 696
jpayne@69 697 - B has at least one embedded dot, or B is the string "local".
jpayne@69 698 then the reach of H is .B.
jpayne@69 699
jpayne@69 700 * Otherwise, the reach of H is H.
jpayne@69 701
jpayne@69 702 >>> reach("www.acme.com")
jpayne@69 703 '.acme.com'
jpayne@69 704 >>> reach("acme.com")
jpayne@69 705 'acme.com'
jpayne@69 706 >>> reach("acme.local")
jpayne@69 707 '.local'
jpayne@69 708
jpayne@69 709 """
jpayne@69 710 i = h.find(".")
jpayne@69 711 if i >= 0:
jpayne@69 712 #a = h[:i] # this line is only here to show what a is
jpayne@69 713 b = h[i+1:]
jpayne@69 714 i = b.find(".")
jpayne@69 715 if is_HDN(h) and (i >= 0 or b == "local"):
jpayne@69 716 return "."+b
jpayne@69 717 return h
jpayne@69 718
jpayne@69 719 def is_third_party(request):
jpayne@69 720 """
jpayne@69 721
jpayne@69 722 RFC 2965, section 3.3.6:
jpayne@69 723
jpayne@69 724 An unverifiable transaction is to a third-party host if its request-
jpayne@69 725 host U does not domain-match the reach R of the request-host O in the
jpayne@69 726 origin transaction.
jpayne@69 727
jpayne@69 728 """
jpayne@69 729 req_host = request_host(request)
jpayne@69 730 if not domain_match(req_host, reach(request.origin_req_host)):
jpayne@69 731 return True
jpayne@69 732 else:
jpayne@69 733 return False
jpayne@69 734
jpayne@69 735
jpayne@69 736 class Cookie:
jpayne@69 737 """HTTP Cookie.
jpayne@69 738
jpayne@69 739 This class represents both Netscape and RFC 2965 cookies.
jpayne@69 740
jpayne@69 741 This is deliberately a very simple class. It just holds attributes. It's
jpayne@69 742 possible to construct Cookie instances that don't comply with the cookie
jpayne@69 743 standards. CookieJar.make_cookies is the factory function for Cookie
jpayne@69 744 objects -- it deals with cookie parsing, supplying defaults, and
jpayne@69 745 normalising to the representation used in this class. CookiePolicy is
jpayne@69 746 responsible for checking them to see whether they should be accepted from
jpayne@69 747 and returned to the server.
jpayne@69 748
jpayne@69 749 Note that the port may be present in the headers, but unspecified ("Port"
jpayne@69 750 rather than"Port=80", for example); if this is the case, port is None.
jpayne@69 751
jpayne@69 752 """
jpayne@69 753
jpayne@69 754 def __init__(self, version, name, value,
jpayne@69 755 port, port_specified,
jpayne@69 756 domain, domain_specified, domain_initial_dot,
jpayne@69 757 path, path_specified,
jpayne@69 758 secure,
jpayne@69 759 expires,
jpayne@69 760 discard,
jpayne@69 761 comment,
jpayne@69 762 comment_url,
jpayne@69 763 rest,
jpayne@69 764 rfc2109=False,
jpayne@69 765 ):
jpayne@69 766
jpayne@69 767 if version is not None: version = int(version)
jpayne@69 768 if expires is not None: expires = int(float(expires))
jpayne@69 769 if port is None and port_specified is True:
jpayne@69 770 raise ValueError("if port is None, port_specified must be false")
jpayne@69 771
jpayne@69 772 self.version = version
jpayne@69 773 self.name = name
jpayne@69 774 self.value = value
jpayne@69 775 self.port = port
jpayne@69 776 self.port_specified = port_specified
jpayne@69 777 # normalise case, as per RFC 2965 section 3.3.3
jpayne@69 778 self.domain = domain.lower()
jpayne@69 779 self.domain_specified = domain_specified
jpayne@69 780 # Sigh. We need to know whether the domain given in the
jpayne@69 781 # cookie-attribute had an initial dot, in order to follow RFC 2965
jpayne@69 782 # (as clarified in draft errata). Needed for the returned $Domain
jpayne@69 783 # value.
jpayne@69 784 self.domain_initial_dot = domain_initial_dot
jpayne@69 785 self.path = path
jpayne@69 786 self.path_specified = path_specified
jpayne@69 787 self.secure = secure
jpayne@69 788 self.expires = expires
jpayne@69 789 self.discard = discard
jpayne@69 790 self.comment = comment
jpayne@69 791 self.comment_url = comment_url
jpayne@69 792 self.rfc2109 = rfc2109
jpayne@69 793
jpayne@69 794 self._rest = copy.copy(rest)
jpayne@69 795
jpayne@69 796 def has_nonstandard_attr(self, name):
jpayne@69 797 return name in self._rest
jpayne@69 798 def get_nonstandard_attr(self, name, default=None):
jpayne@69 799 return self._rest.get(name, default)
jpayne@69 800 def set_nonstandard_attr(self, name, value):
jpayne@69 801 self._rest[name] = value
jpayne@69 802
jpayne@69 803 def is_expired(self, now=None):
jpayne@69 804 if now is None: now = time.time()
jpayne@69 805 if (self.expires is not None) and (self.expires <= now):
jpayne@69 806 return True
jpayne@69 807 return False
jpayne@69 808
jpayne@69 809 def __str__(self):
jpayne@69 810 if self.port is None: p = ""
jpayne@69 811 else: p = ":"+self.port
jpayne@69 812 limit = self.domain + p + self.path
jpayne@69 813 if self.value is not None:
jpayne@69 814 namevalue = "%s=%s" % (self.name, self.value)
jpayne@69 815 else:
jpayne@69 816 namevalue = self.name
jpayne@69 817 return "<Cookie %s for %s>" % (namevalue, limit)
jpayne@69 818
jpayne@69 819 def __repr__(self):
jpayne@69 820 args = []
jpayne@69 821 for name in ("version", "name", "value",
jpayne@69 822 "port", "port_specified",
jpayne@69 823 "domain", "domain_specified", "domain_initial_dot",
jpayne@69 824 "path", "path_specified",
jpayne@69 825 "secure", "expires", "discard", "comment", "comment_url",
jpayne@69 826 ):
jpayne@69 827 attr = getattr(self, name)
jpayne@69 828 args.append("%s=%s" % (name, repr(attr)))
jpayne@69 829 args.append("rest=%s" % repr(self._rest))
jpayne@69 830 args.append("rfc2109=%s" % repr(self.rfc2109))
jpayne@69 831 return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
jpayne@69 832
jpayne@69 833
jpayne@69 834 class CookiePolicy:
jpayne@69 835 """Defines which cookies get accepted from and returned to server.
jpayne@69 836
jpayne@69 837 May also modify cookies, though this is probably a bad idea.
jpayne@69 838
jpayne@69 839 The subclass DefaultCookiePolicy defines the standard rules for Netscape
jpayne@69 840 and RFC 2965 cookies -- override that if you want a customized policy.
jpayne@69 841
jpayne@69 842 """
jpayne@69 843 def set_ok(self, cookie, request):
jpayne@69 844 """Return true if (and only if) cookie should be accepted from server.
jpayne@69 845
jpayne@69 846 Currently, pre-expired cookies never get this far -- the CookieJar
jpayne@69 847 class deletes such cookies itself.
jpayne@69 848
jpayne@69 849 """
jpayne@69 850 raise NotImplementedError()
jpayne@69 851
jpayne@69 852 def return_ok(self, cookie, request):
jpayne@69 853 """Return true if (and only if) cookie should be returned to server."""
jpayne@69 854 raise NotImplementedError()
jpayne@69 855
jpayne@69 856 def domain_return_ok(self, domain, request):
jpayne@69 857 """Return false if cookies should not be returned, given cookie domain.
jpayne@69 858 """
jpayne@69 859 return True
jpayne@69 860
jpayne@69 861 def path_return_ok(self, path, request):
jpayne@69 862 """Return false if cookies should not be returned, given cookie path.
jpayne@69 863 """
jpayne@69 864 return True
jpayne@69 865
jpayne@69 866
jpayne@69 867 class DefaultCookiePolicy(CookiePolicy):
jpayne@69 868 """Implements the standard rules for accepting and returning cookies."""
jpayne@69 869
jpayne@69 870 DomainStrictNoDots = 1
jpayne@69 871 DomainStrictNonDomain = 2
jpayne@69 872 DomainRFC2965Match = 4
jpayne@69 873
jpayne@69 874 DomainLiberal = 0
jpayne@69 875 DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
jpayne@69 876
jpayne@69 877 def __init__(self,
jpayne@69 878 blocked_domains=None, allowed_domains=None,
jpayne@69 879 netscape=True, rfc2965=False,
jpayne@69 880 rfc2109_as_netscape=None,
jpayne@69 881 hide_cookie2=False,
jpayne@69 882 strict_domain=False,
jpayne@69 883 strict_rfc2965_unverifiable=True,
jpayne@69 884 strict_ns_unverifiable=False,
jpayne@69 885 strict_ns_domain=DomainLiberal,
jpayne@69 886 strict_ns_set_initial_dollar=False,
jpayne@69 887 strict_ns_set_path=False,
jpayne@69 888 secure_protocols=("https", "wss")
jpayne@69 889 ):
jpayne@69 890 """Constructor arguments should be passed as keyword arguments only."""
jpayne@69 891 self.netscape = netscape
jpayne@69 892 self.rfc2965 = rfc2965
jpayne@69 893 self.rfc2109_as_netscape = rfc2109_as_netscape
jpayne@69 894 self.hide_cookie2 = hide_cookie2
jpayne@69 895 self.strict_domain = strict_domain
jpayne@69 896 self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
jpayne@69 897 self.strict_ns_unverifiable = strict_ns_unverifiable
jpayne@69 898 self.strict_ns_domain = strict_ns_domain
jpayne@69 899 self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
jpayne@69 900 self.strict_ns_set_path = strict_ns_set_path
jpayne@69 901 self.secure_protocols = secure_protocols
jpayne@69 902
jpayne@69 903 if blocked_domains is not None:
jpayne@69 904 self._blocked_domains = tuple(blocked_domains)
jpayne@69 905 else:
jpayne@69 906 self._blocked_domains = ()
jpayne@69 907
jpayne@69 908 if allowed_domains is not None:
jpayne@69 909 allowed_domains = tuple(allowed_domains)
jpayne@69 910 self._allowed_domains = allowed_domains
jpayne@69 911
jpayne@69 912 def blocked_domains(self):
jpayne@69 913 """Return the sequence of blocked domains (as a tuple)."""
jpayne@69 914 return self._blocked_domains
jpayne@69 915 def set_blocked_domains(self, blocked_domains):
jpayne@69 916 """Set the sequence of blocked domains."""
jpayne@69 917 self._blocked_domains = tuple(blocked_domains)
jpayne@69 918
jpayne@69 919 def is_blocked(self, domain):
jpayne@69 920 for blocked_domain in self._blocked_domains:
jpayne@69 921 if user_domain_match(domain, blocked_domain):
jpayne@69 922 return True
jpayne@69 923 return False
jpayne@69 924
jpayne@69 925 def allowed_domains(self):
jpayne@69 926 """Return None, or the sequence of allowed domains (as a tuple)."""
jpayne@69 927 return self._allowed_domains
jpayne@69 928 def set_allowed_domains(self, allowed_domains):
jpayne@69 929 """Set the sequence of allowed domains, or None."""
jpayne@69 930 if allowed_domains is not None:
jpayne@69 931 allowed_domains = tuple(allowed_domains)
jpayne@69 932 self._allowed_domains = allowed_domains
jpayne@69 933
jpayne@69 934 def is_not_allowed(self, domain):
jpayne@69 935 if self._allowed_domains is None:
jpayne@69 936 return False
jpayne@69 937 for allowed_domain in self._allowed_domains:
jpayne@69 938 if user_domain_match(domain, allowed_domain):
jpayne@69 939 return False
jpayne@69 940 return True
jpayne@69 941
jpayne@69 942 def set_ok(self, cookie, request):
jpayne@69 943 """
jpayne@69 944 If you override .set_ok(), be sure to call this method. If it returns
jpayne@69 945 false, so should your subclass (assuming your subclass wants to be more
jpayne@69 946 strict about which cookies to accept).
jpayne@69 947
jpayne@69 948 """
jpayne@69 949 _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
jpayne@69 950
jpayne@69 951 assert cookie.name is not None
jpayne@69 952
jpayne@69 953 for n in "version", "verifiability", "name", "path", "domain", "port":
jpayne@69 954 fn_name = "set_ok_"+n
jpayne@69 955 fn = getattr(self, fn_name)
jpayne@69 956 if not fn(cookie, request):
jpayne@69 957 return False
jpayne@69 958
jpayne@69 959 return True
jpayne@69 960
jpayne@69 961 def set_ok_version(self, cookie, request):
jpayne@69 962 if cookie.version is None:
jpayne@69 963 # Version is always set to 0 by parse_ns_headers if it's a Netscape
jpayne@69 964 # cookie, so this must be an invalid RFC 2965 cookie.
jpayne@69 965 _debug(" Set-Cookie2 without version attribute (%s=%s)",
jpayne@69 966 cookie.name, cookie.value)
jpayne@69 967 return False
jpayne@69 968 if cookie.version > 0 and not self.rfc2965:
jpayne@69 969 _debug(" RFC 2965 cookies are switched off")
jpayne@69 970 return False
jpayne@69 971 elif cookie.version == 0 and not self.netscape:
jpayne@69 972 _debug(" Netscape cookies are switched off")
jpayne@69 973 return False
jpayne@69 974 return True
jpayne@69 975
jpayne@69 976 def set_ok_verifiability(self, cookie, request):
jpayne@69 977 if request.unverifiable and is_third_party(request):
jpayne@69 978 if cookie.version > 0 and self.strict_rfc2965_unverifiable:
jpayne@69 979 _debug(" third-party RFC 2965 cookie during "
jpayne@69 980 "unverifiable transaction")
jpayne@69 981 return False
jpayne@69 982 elif cookie.version == 0 and self.strict_ns_unverifiable:
jpayne@69 983 _debug(" third-party Netscape cookie during "
jpayne@69 984 "unverifiable transaction")
jpayne@69 985 return False
jpayne@69 986 return True
jpayne@69 987
jpayne@69 988 def set_ok_name(self, cookie, request):
jpayne@69 989 # Try and stop servers setting V0 cookies designed to hack other
jpayne@69 990 # servers that know both V0 and V1 protocols.
jpayne@69 991 if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
jpayne@69 992 cookie.name.startswith("$")):
jpayne@69 993 _debug(" illegal name (starts with '$'): '%s'", cookie.name)
jpayne@69 994 return False
jpayne@69 995 return True
jpayne@69 996
jpayne@69 997 def set_ok_path(self, cookie, request):
jpayne@69 998 if cookie.path_specified:
jpayne@69 999 req_path = request_path(request)
jpayne@69 1000 if ((cookie.version > 0 or
jpayne@69 1001 (cookie.version == 0 and self.strict_ns_set_path)) and
jpayne@69 1002 not self.path_return_ok(cookie.path, request)):
jpayne@69 1003 _debug(" path attribute %s is not a prefix of request "
jpayne@69 1004 "path %s", cookie.path, req_path)
jpayne@69 1005 return False
jpayne@69 1006 return True
jpayne@69 1007
jpayne@69 1008 def set_ok_domain(self, cookie, request):
jpayne@69 1009 if self.is_blocked(cookie.domain):
jpayne@69 1010 _debug(" domain %s is in user block-list", cookie.domain)
jpayne@69 1011 return False
jpayne@69 1012 if self.is_not_allowed(cookie.domain):
jpayne@69 1013 _debug(" domain %s is not in user allow-list", cookie.domain)
jpayne@69 1014 return False
jpayne@69 1015 if cookie.domain_specified:
jpayne@69 1016 req_host, erhn = eff_request_host(request)
jpayne@69 1017 domain = cookie.domain
jpayne@69 1018 if self.strict_domain and (domain.count(".") >= 2):
jpayne@69 1019 # XXX This should probably be compared with the Konqueror
jpayne@69 1020 # (kcookiejar.cpp) and Mozilla implementations, but it's a
jpayne@69 1021 # losing battle.
jpayne@69 1022 i = domain.rfind(".")
jpayne@69 1023 j = domain.rfind(".", 0, i)
jpayne@69 1024 if j == 0: # domain like .foo.bar
jpayne@69 1025 tld = domain[i+1:]
jpayne@69 1026 sld = domain[j+1:i]
jpayne@69 1027 if sld.lower() in ("co", "ac", "com", "edu", "org", "net",
jpayne@69 1028 "gov", "mil", "int", "aero", "biz", "cat", "coop",
jpayne@69 1029 "info", "jobs", "mobi", "museum", "name", "pro",
jpayne@69 1030 "travel", "eu") and len(tld) == 2:
jpayne@69 1031 # domain like .co.uk
jpayne@69 1032 _debug(" country-code second level domain %s", domain)
jpayne@69 1033 return False
jpayne@69 1034 if domain.startswith("."):
jpayne@69 1035 undotted_domain = domain[1:]
jpayne@69 1036 else:
jpayne@69 1037 undotted_domain = domain
jpayne@69 1038 embedded_dots = (undotted_domain.find(".") >= 0)
jpayne@69 1039 if not embedded_dots and domain != ".local":
jpayne@69 1040 _debug(" non-local domain %s contains no embedded dot",
jpayne@69 1041 domain)
jpayne@69 1042 return False
jpayne@69 1043 if cookie.version == 0:
jpayne@69 1044 if (not erhn.endswith(domain) and
jpayne@69 1045 (not erhn.startswith(".") and
jpayne@69 1046 not ("."+erhn).endswith(domain))):
jpayne@69 1047 _debug(" effective request-host %s (even with added "
jpayne@69 1048 "initial dot) does not end with %s",
jpayne@69 1049 erhn, domain)
jpayne@69 1050 return False
jpayne@69 1051 if (cookie.version > 0 or
jpayne@69 1052 (self.strict_ns_domain & self.DomainRFC2965Match)):
jpayne@69 1053 if not domain_match(erhn, domain):
jpayne@69 1054 _debug(" effective request-host %s does not domain-match "
jpayne@69 1055 "%s", erhn, domain)
jpayne@69 1056 return False
jpayne@69 1057 if (cookie.version > 0 or
jpayne@69 1058 (self.strict_ns_domain & self.DomainStrictNoDots)):
jpayne@69 1059 host_prefix = req_host[:-len(domain)]
jpayne@69 1060 if (host_prefix.find(".") >= 0 and
jpayne@69 1061 not IPV4_RE.search(req_host)):
jpayne@69 1062 _debug(" host prefix %s for domain %s contains a dot",
jpayne@69 1063 host_prefix, domain)
jpayne@69 1064 return False
jpayne@69 1065 return True
jpayne@69 1066
jpayne@69 1067 def set_ok_port(self, cookie, request):
jpayne@69 1068 if cookie.port_specified:
jpayne@69 1069 req_port = request_port(request)
jpayne@69 1070 if req_port is None:
jpayne@69 1071 req_port = "80"
jpayne@69 1072 else:
jpayne@69 1073 req_port = str(req_port)
jpayne@69 1074 for p in cookie.port.split(","):
jpayne@69 1075 try:
jpayne@69 1076 int(p)
jpayne@69 1077 except ValueError:
jpayne@69 1078 _debug(" bad port %s (not numeric)", p)
jpayne@69 1079 return False
jpayne@69 1080 if p == req_port:
jpayne@69 1081 break
jpayne@69 1082 else:
jpayne@69 1083 _debug(" request port (%s) not found in %s",
jpayne@69 1084 req_port, cookie.port)
jpayne@69 1085 return False
jpayne@69 1086 return True
jpayne@69 1087
jpayne@69 1088 def return_ok(self, cookie, request):
jpayne@69 1089 """
jpayne@69 1090 If you override .return_ok(), be sure to call this method. If it
jpayne@69 1091 returns false, so should your subclass (assuming your subclass wants to
jpayne@69 1092 be more strict about which cookies to return).
jpayne@69 1093
jpayne@69 1094 """
jpayne@69 1095 # Path has already been checked by .path_return_ok(), and domain
jpayne@69 1096 # blocking done by .domain_return_ok().
jpayne@69 1097 _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
jpayne@69 1098
jpayne@69 1099 for n in "version", "verifiability", "secure", "expires", "port", "domain":
jpayne@69 1100 fn_name = "return_ok_"+n
jpayne@69 1101 fn = getattr(self, fn_name)
jpayne@69 1102 if not fn(cookie, request):
jpayne@69 1103 return False
jpayne@69 1104 return True
jpayne@69 1105
jpayne@69 1106 def return_ok_version(self, cookie, request):
jpayne@69 1107 if cookie.version > 0 and not self.rfc2965:
jpayne@69 1108 _debug(" RFC 2965 cookies are switched off")
jpayne@69 1109 return False
jpayne@69 1110 elif cookie.version == 0 and not self.netscape:
jpayne@69 1111 _debug(" Netscape cookies are switched off")
jpayne@69 1112 return False
jpayne@69 1113 return True
jpayne@69 1114
jpayne@69 1115 def return_ok_verifiability(self, cookie, request):
jpayne@69 1116 if request.unverifiable and is_third_party(request):
jpayne@69 1117 if cookie.version > 0 and self.strict_rfc2965_unverifiable:
jpayne@69 1118 _debug(" third-party RFC 2965 cookie during unverifiable "
jpayne@69 1119 "transaction")
jpayne@69 1120 return False
jpayne@69 1121 elif cookie.version == 0 and self.strict_ns_unverifiable:
jpayne@69 1122 _debug(" third-party Netscape cookie during unverifiable "
jpayne@69 1123 "transaction")
jpayne@69 1124 return False
jpayne@69 1125 return True
jpayne@69 1126
jpayne@69 1127 def return_ok_secure(self, cookie, request):
jpayne@69 1128 if cookie.secure and request.type not in self.secure_protocols:
jpayne@69 1129 _debug(" secure cookie with non-secure request")
jpayne@69 1130 return False
jpayne@69 1131 return True
jpayne@69 1132
jpayne@69 1133 def return_ok_expires(self, cookie, request):
jpayne@69 1134 if cookie.is_expired(self._now):
jpayne@69 1135 _debug(" cookie expired")
jpayne@69 1136 return False
jpayne@69 1137 return True
jpayne@69 1138
jpayne@69 1139 def return_ok_port(self, cookie, request):
jpayne@69 1140 if cookie.port:
jpayne@69 1141 req_port = request_port(request)
jpayne@69 1142 if req_port is None:
jpayne@69 1143 req_port = "80"
jpayne@69 1144 for p in cookie.port.split(","):
jpayne@69 1145 if p == req_port:
jpayne@69 1146 break
jpayne@69 1147 else:
jpayne@69 1148 _debug(" request port %s does not match cookie port %s",
jpayne@69 1149 req_port, cookie.port)
jpayne@69 1150 return False
jpayne@69 1151 return True
jpayne@69 1152
jpayne@69 1153 def return_ok_domain(self, cookie, request):
jpayne@69 1154 req_host, erhn = eff_request_host(request)
jpayne@69 1155 domain = cookie.domain
jpayne@69 1156
jpayne@69 1157 if domain and not domain.startswith("."):
jpayne@69 1158 dotdomain = "." + domain
jpayne@69 1159 else:
jpayne@69 1160 dotdomain = domain
jpayne@69 1161
jpayne@69 1162 # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
jpayne@69 1163 if (cookie.version == 0 and
jpayne@69 1164 (self.strict_ns_domain & self.DomainStrictNonDomain) and
jpayne@69 1165 not cookie.domain_specified and domain != erhn):
jpayne@69 1166 _debug(" cookie with unspecified domain does not string-compare "
jpayne@69 1167 "equal to request domain")
jpayne@69 1168 return False
jpayne@69 1169
jpayne@69 1170 if cookie.version > 0 and not domain_match(erhn, domain):
jpayne@69 1171 _debug(" effective request-host name %s does not domain-match "
jpayne@69 1172 "RFC 2965 cookie domain %s", erhn, domain)
jpayne@69 1173 return False
jpayne@69 1174 if cookie.version == 0 and not ("."+erhn).endswith(dotdomain):
jpayne@69 1175 _debug(" request-host %s does not match Netscape cookie domain "
jpayne@69 1176 "%s", req_host, domain)
jpayne@69 1177 return False
jpayne@69 1178 return True
jpayne@69 1179
jpayne@69 1180 def domain_return_ok(self, domain, request):
jpayne@69 1181 # Liberal check of. This is here as an optimization to avoid
jpayne@69 1182 # having to load lots of MSIE cookie files unless necessary.
jpayne@69 1183 req_host, erhn = eff_request_host(request)
jpayne@69 1184 if not req_host.startswith("."):
jpayne@69 1185 req_host = "."+req_host
jpayne@69 1186 if not erhn.startswith("."):
jpayne@69 1187 erhn = "."+erhn
jpayne@69 1188 if domain and not domain.startswith("."):
jpayne@69 1189 dotdomain = "." + domain
jpayne@69 1190 else:
jpayne@69 1191 dotdomain = domain
jpayne@69 1192 if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)):
jpayne@69 1193 #_debug(" request domain %s does not match cookie domain %s",
jpayne@69 1194 # req_host, domain)
jpayne@69 1195 return False
jpayne@69 1196
jpayne@69 1197 if self.is_blocked(domain):
jpayne@69 1198 _debug(" domain %s is in user block-list", domain)
jpayne@69 1199 return False
jpayne@69 1200 if self.is_not_allowed(domain):
jpayne@69 1201 _debug(" domain %s is not in user allow-list", domain)
jpayne@69 1202 return False
jpayne@69 1203
jpayne@69 1204 return True
jpayne@69 1205
jpayne@69 1206 def path_return_ok(self, path, request):
jpayne@69 1207 _debug("- checking cookie path=%s", path)
jpayne@69 1208 req_path = request_path(request)
jpayne@69 1209 pathlen = len(path)
jpayne@69 1210 if req_path == path:
jpayne@69 1211 return True
jpayne@69 1212 elif (req_path.startswith(path) and
jpayne@69 1213 (path.endswith("/") or req_path[pathlen:pathlen+1] == "/")):
jpayne@69 1214 return True
jpayne@69 1215
jpayne@69 1216 _debug(" %s does not path-match %s", req_path, path)
jpayne@69 1217 return False
jpayne@69 1218
jpayne@69 1219 def vals_sorted_by_key(adict):
jpayne@69 1220 keys = sorted(adict.keys())
jpayne@69 1221 return map(adict.get, keys)
jpayne@69 1222
jpayne@69 1223 def deepvalues(mapping):
jpayne@69 1224 """Iterates over nested mapping, depth-first, in sorted order by key."""
jpayne@69 1225 values = vals_sorted_by_key(mapping)
jpayne@69 1226 for obj in values:
jpayne@69 1227 mapping = False
jpayne@69 1228 try:
jpayne@69 1229 obj.items
jpayne@69 1230 except AttributeError:
jpayne@69 1231 pass
jpayne@69 1232 else:
jpayne@69 1233 mapping = True
jpayne@69 1234 yield from deepvalues(obj)
jpayne@69 1235 if not mapping:
jpayne@69 1236 yield obj
jpayne@69 1237
jpayne@69 1238
jpayne@69 1239 # Used as second parameter to dict.get() method, to distinguish absent
jpayne@69 1240 # dict key from one with a None value.
jpayne@69 1241 class Absent: pass
jpayne@69 1242
jpayne@69 1243 class CookieJar:
jpayne@69 1244 """Collection of HTTP cookies.
jpayne@69 1245
jpayne@69 1246 You may not need to know about this class: try
jpayne@69 1247 urllib.request.build_opener(HTTPCookieProcessor).open(url).
jpayne@69 1248 """
jpayne@69 1249
jpayne@69 1250 non_word_re = re.compile(r"\W")
jpayne@69 1251 quote_re = re.compile(r"([\"\\])")
jpayne@69 1252 strict_domain_re = re.compile(r"\.?[^.]*")
jpayne@69 1253 domain_re = re.compile(r"[^.]*")
jpayne@69 1254 dots_re = re.compile(r"^\.+")
jpayne@69 1255
jpayne@69 1256 magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII)
jpayne@69 1257
jpayne@69 1258 def __init__(self, policy=None):
jpayne@69 1259 if policy is None:
jpayne@69 1260 policy = DefaultCookiePolicy()
jpayne@69 1261 self._policy = policy
jpayne@69 1262
jpayne@69 1263 self._cookies_lock = _threading.RLock()
jpayne@69 1264 self._cookies = {}
jpayne@69 1265
jpayne@69 1266 def set_policy(self, policy):
jpayne@69 1267 self._policy = policy
jpayne@69 1268
jpayne@69 1269 def _cookies_for_domain(self, domain, request):
jpayne@69 1270 cookies = []
jpayne@69 1271 if not self._policy.domain_return_ok(domain, request):
jpayne@69 1272 return []
jpayne@69 1273 _debug("Checking %s for cookies to return", domain)
jpayne@69 1274 cookies_by_path = self._cookies[domain]
jpayne@69 1275 for path in cookies_by_path.keys():
jpayne@69 1276 if not self._policy.path_return_ok(path, request):
jpayne@69 1277 continue
jpayne@69 1278 cookies_by_name = cookies_by_path[path]
jpayne@69 1279 for cookie in cookies_by_name.values():
jpayne@69 1280 if not self._policy.return_ok(cookie, request):
jpayne@69 1281 _debug(" not returning cookie")
jpayne@69 1282 continue
jpayne@69 1283 _debug(" it's a match")
jpayne@69 1284 cookies.append(cookie)
jpayne@69 1285 return cookies
jpayne@69 1286
jpayne@69 1287 def _cookies_for_request(self, request):
jpayne@69 1288 """Return a list of cookies to be returned to server."""
jpayne@69 1289 cookies = []
jpayne@69 1290 for domain in self._cookies.keys():
jpayne@69 1291 cookies.extend(self._cookies_for_domain(domain, request))
jpayne@69 1292 return cookies
jpayne@69 1293
jpayne@69 1294 def _cookie_attrs(self, cookies):
jpayne@69 1295 """Return a list of cookie-attributes to be returned to server.
jpayne@69 1296
jpayne@69 1297 like ['foo="bar"; $Path="/"', ...]
jpayne@69 1298
jpayne@69 1299 The $Version attribute is also added when appropriate (currently only
jpayne@69 1300 once per request).
jpayne@69 1301
jpayne@69 1302 """
jpayne@69 1303 # add cookies in order of most specific (ie. longest) path first
jpayne@69 1304 cookies.sort(key=lambda a: len(a.path), reverse=True)
jpayne@69 1305
jpayne@69 1306 version_set = False
jpayne@69 1307
jpayne@69 1308 attrs = []
jpayne@69 1309 for cookie in cookies:
jpayne@69 1310 # set version of Cookie header
jpayne@69 1311 # XXX
jpayne@69 1312 # What should it be if multiple matching Set-Cookie headers have
jpayne@69 1313 # different versions themselves?
jpayne@69 1314 # Answer: there is no answer; was supposed to be settled by
jpayne@69 1315 # RFC 2965 errata, but that may never appear...
jpayne@69 1316 version = cookie.version
jpayne@69 1317 if not version_set:
jpayne@69 1318 version_set = True
jpayne@69 1319 if version > 0:
jpayne@69 1320 attrs.append("$Version=%s" % version)
jpayne@69 1321
jpayne@69 1322 # quote cookie value if necessary
jpayne@69 1323 # (not for Netscape protocol, which already has any quotes
jpayne@69 1324 # intact, due to the poorly-specified Netscape Cookie: syntax)
jpayne@69 1325 if ((cookie.value is not None) and
jpayne@69 1326 self.non_word_re.search(cookie.value) and version > 0):
jpayne@69 1327 value = self.quote_re.sub(r"\\\1", cookie.value)
jpayne@69 1328 else:
jpayne@69 1329 value = cookie.value
jpayne@69 1330
jpayne@69 1331 # add cookie-attributes to be returned in Cookie header
jpayne@69 1332 if cookie.value is None:
jpayne@69 1333 attrs.append(cookie.name)
jpayne@69 1334 else:
jpayne@69 1335 attrs.append("%s=%s" % (cookie.name, value))
jpayne@69 1336 if version > 0:
jpayne@69 1337 if cookie.path_specified:
jpayne@69 1338 attrs.append('$Path="%s"' % cookie.path)
jpayne@69 1339 if cookie.domain.startswith("."):
jpayne@69 1340 domain = cookie.domain
jpayne@69 1341 if (not cookie.domain_initial_dot and
jpayne@69 1342 domain.startswith(".")):
jpayne@69 1343 domain = domain[1:]
jpayne@69 1344 attrs.append('$Domain="%s"' % domain)
jpayne@69 1345 if cookie.port is not None:
jpayne@69 1346 p = "$Port"
jpayne@69 1347 if cookie.port_specified:
jpayne@69 1348 p = p + ('="%s"' % cookie.port)
jpayne@69 1349 attrs.append(p)
jpayne@69 1350
jpayne@69 1351 return attrs
jpayne@69 1352
jpayne@69 1353 def add_cookie_header(self, request):
jpayne@69 1354 """Add correct Cookie: header to request (urllib.request.Request object).
jpayne@69 1355
jpayne@69 1356 The Cookie2 header is also added unless policy.hide_cookie2 is true.
jpayne@69 1357
jpayne@69 1358 """
jpayne@69 1359 _debug("add_cookie_header")
jpayne@69 1360 self._cookies_lock.acquire()
jpayne@69 1361 try:
jpayne@69 1362
jpayne@69 1363 self._policy._now = self._now = int(time.time())
jpayne@69 1364
jpayne@69 1365 cookies = self._cookies_for_request(request)
jpayne@69 1366
jpayne@69 1367 attrs = self._cookie_attrs(cookies)
jpayne@69 1368 if attrs:
jpayne@69 1369 if not request.has_header("Cookie"):
jpayne@69 1370 request.add_unredirected_header(
jpayne@69 1371 "Cookie", "; ".join(attrs))
jpayne@69 1372
jpayne@69 1373 # if necessary, advertise that we know RFC 2965
jpayne@69 1374 if (self._policy.rfc2965 and not self._policy.hide_cookie2 and
jpayne@69 1375 not request.has_header("Cookie2")):
jpayne@69 1376 for cookie in cookies:
jpayne@69 1377 if cookie.version != 1:
jpayne@69 1378 request.add_unredirected_header("Cookie2", '$Version="1"')
jpayne@69 1379 break
jpayne@69 1380
jpayne@69 1381 finally:
jpayne@69 1382 self._cookies_lock.release()
jpayne@69 1383
jpayne@69 1384 self.clear_expired_cookies()
jpayne@69 1385
jpayne@69 1386 def _normalized_cookie_tuples(self, attrs_set):
jpayne@69 1387 """Return list of tuples containing normalised cookie information.
jpayne@69 1388
jpayne@69 1389 attrs_set is the list of lists of key,value pairs extracted from
jpayne@69 1390 the Set-Cookie or Set-Cookie2 headers.
jpayne@69 1391
jpayne@69 1392 Tuples are name, value, standard, rest, where name and value are the
jpayne@69 1393 cookie name and value, standard is a dictionary containing the standard
jpayne@69 1394 cookie-attributes (discard, secure, version, expires or max-age,
jpayne@69 1395 domain, path and port) and rest is a dictionary containing the rest of
jpayne@69 1396 the cookie-attributes.
jpayne@69 1397
jpayne@69 1398 """
jpayne@69 1399 cookie_tuples = []
jpayne@69 1400
jpayne@69 1401 boolean_attrs = "discard", "secure"
jpayne@69 1402 value_attrs = ("version",
jpayne@69 1403 "expires", "max-age",
jpayne@69 1404 "domain", "path", "port",
jpayne@69 1405 "comment", "commenturl")
jpayne@69 1406
jpayne@69 1407 for cookie_attrs in attrs_set:
jpayne@69 1408 name, value = cookie_attrs[0]
jpayne@69 1409
jpayne@69 1410 # Build dictionary of standard cookie-attributes (standard) and
jpayne@69 1411 # dictionary of other cookie-attributes (rest).
jpayne@69 1412
jpayne@69 1413 # Note: expiry time is normalised to seconds since epoch. V0
jpayne@69 1414 # cookies should have the Expires cookie-attribute, and V1 cookies
jpayne@69 1415 # should have Max-Age, but since V1 includes RFC 2109 cookies (and
jpayne@69 1416 # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
jpayne@69 1417 # accept either (but prefer Max-Age).
jpayne@69 1418 max_age_set = False
jpayne@69 1419
jpayne@69 1420 bad_cookie = False
jpayne@69 1421
jpayne@69 1422 standard = {}
jpayne@69 1423 rest = {}
jpayne@69 1424 for k, v in cookie_attrs[1:]:
jpayne@69 1425 lc = k.lower()
jpayne@69 1426 # don't lose case distinction for unknown fields
jpayne@69 1427 if lc in value_attrs or lc in boolean_attrs:
jpayne@69 1428 k = lc
jpayne@69 1429 if k in boolean_attrs and v is None:
jpayne@69 1430 # boolean cookie-attribute is present, but has no value
jpayne@69 1431 # (like "discard", rather than "port=80")
jpayne@69 1432 v = True
jpayne@69 1433 if k in standard:
jpayne@69 1434 # only first value is significant
jpayne@69 1435 continue
jpayne@69 1436 if k == "domain":
jpayne@69 1437 if v is None:
jpayne@69 1438 _debug(" missing value for domain attribute")
jpayne@69 1439 bad_cookie = True
jpayne@69 1440 break
jpayne@69 1441 # RFC 2965 section 3.3.3
jpayne@69 1442 v = v.lower()
jpayne@69 1443 if k == "expires":
jpayne@69 1444 if max_age_set:
jpayne@69 1445 # Prefer max-age to expires (like Mozilla)
jpayne@69 1446 continue
jpayne@69 1447 if v is None:
jpayne@69 1448 _debug(" missing or invalid value for expires "
jpayne@69 1449 "attribute: treating as session cookie")
jpayne@69 1450 continue
jpayne@69 1451 if k == "max-age":
jpayne@69 1452 max_age_set = True
jpayne@69 1453 try:
jpayne@69 1454 v = int(v)
jpayne@69 1455 except ValueError:
jpayne@69 1456 _debug(" missing or invalid (non-numeric) value for "
jpayne@69 1457 "max-age attribute")
jpayne@69 1458 bad_cookie = True
jpayne@69 1459 break
jpayne@69 1460 # convert RFC 2965 Max-Age to seconds since epoch
jpayne@69 1461 # XXX Strictly you're supposed to follow RFC 2616
jpayne@69 1462 # age-calculation rules. Remember that zero Max-Age
jpayne@69 1463 # is a request to discard (old and new) cookie, though.
jpayne@69 1464 k = "expires"
jpayne@69 1465 v = self._now + v
jpayne@69 1466 if (k in value_attrs) or (k in boolean_attrs):
jpayne@69 1467 if (v is None and
jpayne@69 1468 k not in ("port", "comment", "commenturl")):
jpayne@69 1469 _debug(" missing value for %s attribute" % k)
jpayne@69 1470 bad_cookie = True
jpayne@69 1471 break
jpayne@69 1472 standard[k] = v
jpayne@69 1473 else:
jpayne@69 1474 rest[k] = v
jpayne@69 1475
jpayne@69 1476 if bad_cookie:
jpayne@69 1477 continue
jpayne@69 1478
jpayne@69 1479 cookie_tuples.append((name, value, standard, rest))
jpayne@69 1480
jpayne@69 1481 return cookie_tuples
jpayne@69 1482
jpayne@69 1483 def _cookie_from_cookie_tuple(self, tup, request):
jpayne@69 1484 # standard is dict of standard cookie-attributes, rest is dict of the
jpayne@69 1485 # rest of them
jpayne@69 1486 name, value, standard, rest = tup
jpayne@69 1487
jpayne@69 1488 domain = standard.get("domain", Absent)
jpayne@69 1489 path = standard.get("path", Absent)
jpayne@69 1490 port = standard.get("port", Absent)
jpayne@69 1491 expires = standard.get("expires", Absent)
jpayne@69 1492
jpayne@69 1493 # set the easy defaults
jpayne@69 1494 version = standard.get("version", None)
jpayne@69 1495 if version is not None:
jpayne@69 1496 try:
jpayne@69 1497 version = int(version)
jpayne@69 1498 except ValueError:
jpayne@69 1499 return None # invalid version, ignore cookie
jpayne@69 1500 secure = standard.get("secure", False)
jpayne@69 1501 # (discard is also set if expires is Absent)
jpayne@69 1502 discard = standard.get("discard", False)
jpayne@69 1503 comment = standard.get("comment", None)
jpayne@69 1504 comment_url = standard.get("commenturl", None)
jpayne@69 1505
jpayne@69 1506 # set default path
jpayne@69 1507 if path is not Absent and path != "":
jpayne@69 1508 path_specified = True
jpayne@69 1509 path = escape_path(path)
jpayne@69 1510 else:
jpayne@69 1511 path_specified = False
jpayne@69 1512 path = request_path(request)
jpayne@69 1513 i = path.rfind("/")
jpayne@69 1514 if i != -1:
jpayne@69 1515 if version == 0:
jpayne@69 1516 # Netscape spec parts company from reality here
jpayne@69 1517 path = path[:i]
jpayne@69 1518 else:
jpayne@69 1519 path = path[:i+1]
jpayne@69 1520 if len(path) == 0: path = "/"
jpayne@69 1521
jpayne@69 1522 # set default domain
jpayne@69 1523 domain_specified = domain is not Absent
jpayne@69 1524 # but first we have to remember whether it starts with a dot
jpayne@69 1525 domain_initial_dot = False
jpayne@69 1526 if domain_specified:
jpayne@69 1527 domain_initial_dot = bool(domain.startswith("."))
jpayne@69 1528 if domain is Absent:
jpayne@69 1529 req_host, erhn = eff_request_host(request)
jpayne@69 1530 domain = erhn
jpayne@69 1531 elif not domain.startswith("."):
jpayne@69 1532 domain = "."+domain
jpayne@69 1533
jpayne@69 1534 # set default port
jpayne@69 1535 port_specified = False
jpayne@69 1536 if port is not Absent:
jpayne@69 1537 if port is None:
jpayne@69 1538 # Port attr present, but has no value: default to request port.
jpayne@69 1539 # Cookie should then only be sent back on that port.
jpayne@69 1540 port = request_port(request)
jpayne@69 1541 else:
jpayne@69 1542 port_specified = True
jpayne@69 1543 port = re.sub(r"\s+", "", port)
jpayne@69 1544 else:
jpayne@69 1545 # No port attr present. Cookie can be sent back on any port.
jpayne@69 1546 port = None
jpayne@69 1547
jpayne@69 1548 # set default expires and discard
jpayne@69 1549 if expires is Absent:
jpayne@69 1550 expires = None
jpayne@69 1551 discard = True
jpayne@69 1552 elif expires <= self._now:
jpayne@69 1553 # Expiry date in past is request to delete cookie. This can't be
jpayne@69 1554 # in DefaultCookiePolicy, because can't delete cookies there.
jpayne@69 1555 try:
jpayne@69 1556 self.clear(domain, path, name)
jpayne@69 1557 except KeyError:
jpayne@69 1558 pass
jpayne@69 1559 _debug("Expiring cookie, domain='%s', path='%s', name='%s'",
jpayne@69 1560 domain, path, name)
jpayne@69 1561 return None
jpayne@69 1562
jpayne@69 1563 return Cookie(version,
jpayne@69 1564 name, value,
jpayne@69 1565 port, port_specified,
jpayne@69 1566 domain, domain_specified, domain_initial_dot,
jpayne@69 1567 path, path_specified,
jpayne@69 1568 secure,
jpayne@69 1569 expires,
jpayne@69 1570 discard,
jpayne@69 1571 comment,
jpayne@69 1572 comment_url,
jpayne@69 1573 rest)
jpayne@69 1574
jpayne@69 1575 def _cookies_from_attrs_set(self, attrs_set, request):
jpayne@69 1576 cookie_tuples = self._normalized_cookie_tuples(attrs_set)
jpayne@69 1577
jpayne@69 1578 cookies = []
jpayne@69 1579 for tup in cookie_tuples:
jpayne@69 1580 cookie = self._cookie_from_cookie_tuple(tup, request)
jpayne@69 1581 if cookie: cookies.append(cookie)
jpayne@69 1582 return cookies
jpayne@69 1583
jpayne@69 1584 def _process_rfc2109_cookies(self, cookies):
jpayne@69 1585 rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
jpayne@69 1586 if rfc2109_as_ns is None:
jpayne@69 1587 rfc2109_as_ns = not self._policy.rfc2965
jpayne@69 1588 for cookie in cookies:
jpayne@69 1589 if cookie.version == 1:
jpayne@69 1590 cookie.rfc2109 = True
jpayne@69 1591 if rfc2109_as_ns:
jpayne@69 1592 # treat 2109 cookies as Netscape cookies rather than
jpayne@69 1593 # as RFC2965 cookies
jpayne@69 1594 cookie.version = 0
jpayne@69 1595
jpayne@69 1596 def make_cookies(self, response, request):
jpayne@69 1597 """Return sequence of Cookie objects extracted from response object."""
jpayne@69 1598 # get cookie-attributes for RFC 2965 and Netscape protocols
jpayne@69 1599 headers = response.info()
jpayne@69 1600 rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
jpayne@69 1601 ns_hdrs = headers.get_all("Set-Cookie", [])
jpayne@69 1602 self._policy._now = self._now = int(time.time())
jpayne@69 1603
jpayne@69 1604 rfc2965 = self._policy.rfc2965
jpayne@69 1605 netscape = self._policy.netscape
jpayne@69 1606
jpayne@69 1607 if ((not rfc2965_hdrs and not ns_hdrs) or
jpayne@69 1608 (not ns_hdrs and not rfc2965) or
jpayne@69 1609 (not rfc2965_hdrs and not netscape) or
jpayne@69 1610 (not netscape and not rfc2965)):
jpayne@69 1611 return [] # no relevant cookie headers: quick exit
jpayne@69 1612
jpayne@69 1613 try:
jpayne@69 1614 cookies = self._cookies_from_attrs_set(
jpayne@69 1615 split_header_words(rfc2965_hdrs), request)
jpayne@69 1616 except Exception:
jpayne@69 1617 _warn_unhandled_exception()
jpayne@69 1618 cookies = []
jpayne@69 1619
jpayne@69 1620 if ns_hdrs and netscape:
jpayne@69 1621 try:
jpayne@69 1622 # RFC 2109 and Netscape cookies
jpayne@69 1623 ns_cookies = self._cookies_from_attrs_set(
jpayne@69 1624 parse_ns_headers(ns_hdrs), request)
jpayne@69 1625 except Exception:
jpayne@69 1626 _warn_unhandled_exception()
jpayne@69 1627 ns_cookies = []
jpayne@69 1628 self._process_rfc2109_cookies(ns_cookies)
jpayne@69 1629
jpayne@69 1630 # Look for Netscape cookies (from Set-Cookie headers) that match
jpayne@69 1631 # corresponding RFC 2965 cookies (from Set-Cookie2 headers).
jpayne@69 1632 # For each match, keep the RFC 2965 cookie and ignore the Netscape
jpayne@69 1633 # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
jpayne@69 1634 # bundled in with the Netscape cookies for this purpose, which is
jpayne@69 1635 # reasonable behaviour.
jpayne@69 1636 if rfc2965:
jpayne@69 1637 lookup = {}
jpayne@69 1638 for cookie in cookies:
jpayne@69 1639 lookup[(cookie.domain, cookie.path, cookie.name)] = None
jpayne@69 1640
jpayne@69 1641 def no_matching_rfc2965(ns_cookie, lookup=lookup):
jpayne@69 1642 key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
jpayne@69 1643 return key not in lookup
jpayne@69 1644 ns_cookies = filter(no_matching_rfc2965, ns_cookies)
jpayne@69 1645
jpayne@69 1646 if ns_cookies:
jpayne@69 1647 cookies.extend(ns_cookies)
jpayne@69 1648
jpayne@69 1649 return cookies
jpayne@69 1650
jpayne@69 1651 def set_cookie_if_ok(self, cookie, request):
jpayne@69 1652 """Set a cookie if policy says it's OK to do so."""
jpayne@69 1653 self._cookies_lock.acquire()
jpayne@69 1654 try:
jpayne@69 1655 self._policy._now = self._now = int(time.time())
jpayne@69 1656
jpayne@69 1657 if self._policy.set_ok(cookie, request):
jpayne@69 1658 self.set_cookie(cookie)
jpayne@69 1659
jpayne@69 1660
jpayne@69 1661 finally:
jpayne@69 1662 self._cookies_lock.release()
jpayne@69 1663
jpayne@69 1664 def set_cookie(self, cookie):
jpayne@69 1665 """Set a cookie, without checking whether or not it should be set."""
jpayne@69 1666 c = self._cookies
jpayne@69 1667 self._cookies_lock.acquire()
jpayne@69 1668 try:
jpayne@69 1669 if cookie.domain not in c: c[cookie.domain] = {}
jpayne@69 1670 c2 = c[cookie.domain]
jpayne@69 1671 if cookie.path not in c2: c2[cookie.path] = {}
jpayne@69 1672 c3 = c2[cookie.path]
jpayne@69 1673 c3[cookie.name] = cookie
jpayne@69 1674 finally:
jpayne@69 1675 self._cookies_lock.release()
jpayne@69 1676
jpayne@69 1677 def extract_cookies(self, response, request):
jpayne@69 1678 """Extract cookies from response, where allowable given the request."""
jpayne@69 1679 _debug("extract_cookies: %s", response.info())
jpayne@69 1680 self._cookies_lock.acquire()
jpayne@69 1681 try:
jpayne@69 1682 for cookie in self.make_cookies(response, request):
jpayne@69 1683 if self._policy.set_ok(cookie, request):
jpayne@69 1684 _debug(" setting cookie: %s", cookie)
jpayne@69 1685 self.set_cookie(cookie)
jpayne@69 1686 finally:
jpayne@69 1687 self._cookies_lock.release()
jpayne@69 1688
jpayne@69 1689 def clear(self, domain=None, path=None, name=None):
jpayne@69 1690 """Clear some cookies.
jpayne@69 1691
jpayne@69 1692 Invoking this method without arguments will clear all cookies. If
jpayne@69 1693 given a single argument, only cookies belonging to that domain will be
jpayne@69 1694 removed. If given two arguments, cookies belonging to the specified
jpayne@69 1695 path within that domain are removed. If given three arguments, then
jpayne@69 1696 the cookie with the specified name, path and domain is removed.
jpayne@69 1697
jpayne@69 1698 Raises KeyError if no matching cookie exists.
jpayne@69 1699
jpayne@69 1700 """
jpayne@69 1701 if name is not None:
jpayne@69 1702 if (domain is None) or (path is None):
jpayne@69 1703 raise ValueError(
jpayne@69 1704 "domain and path must be given to remove a cookie by name")
jpayne@69 1705 del self._cookies[domain][path][name]
jpayne@69 1706 elif path is not None:
jpayne@69 1707 if domain is None:
jpayne@69 1708 raise ValueError(
jpayne@69 1709 "domain must be given to remove cookies by path")
jpayne@69 1710 del self._cookies[domain][path]
jpayne@69 1711 elif domain is not None:
jpayne@69 1712 del self._cookies[domain]
jpayne@69 1713 else:
jpayne@69 1714 self._cookies = {}
jpayne@69 1715
jpayne@69 1716 def clear_session_cookies(self):
jpayne@69 1717 """Discard all session cookies.
jpayne@69 1718
jpayne@69 1719 Note that the .save() method won't save session cookies anyway, unless
jpayne@69 1720 you ask otherwise by passing a true ignore_discard argument.
jpayne@69 1721
jpayne@69 1722 """
jpayne@69 1723 self._cookies_lock.acquire()
jpayne@69 1724 try:
jpayne@69 1725 for cookie in self:
jpayne@69 1726 if cookie.discard:
jpayne@69 1727 self.clear(cookie.domain, cookie.path, cookie.name)
jpayne@69 1728 finally:
jpayne@69 1729 self._cookies_lock.release()
jpayne@69 1730
jpayne@69 1731 def clear_expired_cookies(self):
jpayne@69 1732 """Discard all expired cookies.
jpayne@69 1733
jpayne@69 1734 You probably don't need to call this method: expired cookies are never
jpayne@69 1735 sent back to the server (provided you're using DefaultCookiePolicy),
jpayne@69 1736 this method is called by CookieJar itself every so often, and the
jpayne@69 1737 .save() method won't save expired cookies anyway (unless you ask
jpayne@69 1738 otherwise by passing a true ignore_expires argument).
jpayne@69 1739
jpayne@69 1740 """
jpayne@69 1741 self._cookies_lock.acquire()
jpayne@69 1742 try:
jpayne@69 1743 now = time.time()
jpayne@69 1744 for cookie in self:
jpayne@69 1745 if cookie.is_expired(now):
jpayne@69 1746 self.clear(cookie.domain, cookie.path, cookie.name)
jpayne@69 1747 finally:
jpayne@69 1748 self._cookies_lock.release()
jpayne@69 1749
jpayne@69 1750 def __iter__(self):
jpayne@69 1751 return deepvalues(self._cookies)
jpayne@69 1752
jpayne@69 1753 def __len__(self):
jpayne@69 1754 """Return number of contained cookies."""
jpayne@69 1755 i = 0
jpayne@69 1756 for cookie in self: i = i + 1
jpayne@69 1757 return i
jpayne@69 1758
jpayne@69 1759 def __repr__(self):
jpayne@69 1760 r = []
jpayne@69 1761 for cookie in self: r.append(repr(cookie))
jpayne@69 1762 return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
jpayne@69 1763
jpayne@69 1764 def __str__(self):
jpayne@69 1765 r = []
jpayne@69 1766 for cookie in self: r.append(str(cookie))
jpayne@69 1767 return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
jpayne@69 1768
jpayne@69 1769
jpayne@69 1770 # derives from OSError for backwards-compatibility with Python 2.4.0
jpayne@69 1771 class LoadError(OSError): pass
jpayne@69 1772
jpayne@69 1773 class FileCookieJar(CookieJar):
jpayne@69 1774 """CookieJar that can be loaded from and saved to a file."""
jpayne@69 1775
jpayne@69 1776 def __init__(self, filename=None, delayload=False, policy=None):
jpayne@69 1777 """
jpayne@69 1778 Cookies are NOT loaded from the named file until either the .load() or
jpayne@69 1779 .revert() method is called.
jpayne@69 1780
jpayne@69 1781 """
jpayne@69 1782 CookieJar.__init__(self, policy)
jpayne@69 1783 if filename is not None:
jpayne@69 1784 filename = os.fspath(filename)
jpayne@69 1785 self.filename = filename
jpayne@69 1786 self.delayload = bool(delayload)
jpayne@69 1787
jpayne@69 1788 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@69 1789 """Save cookies to a file."""
jpayne@69 1790 raise NotImplementedError()
jpayne@69 1791
jpayne@69 1792 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@69 1793 """Load cookies from a file."""
jpayne@69 1794 if filename is None:
jpayne@69 1795 if self.filename is not None: filename = self.filename
jpayne@69 1796 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@69 1797
jpayne@69 1798 with open(filename) as f:
jpayne@69 1799 self._really_load(f, filename, ignore_discard, ignore_expires)
jpayne@69 1800
jpayne@69 1801 def revert(self, filename=None,
jpayne@69 1802 ignore_discard=False, ignore_expires=False):
jpayne@69 1803 """Clear all cookies and reload cookies from a saved file.
jpayne@69 1804
jpayne@69 1805 Raises LoadError (or OSError) if reversion is not successful; the
jpayne@69 1806 object's state will not be altered if this happens.
jpayne@69 1807
jpayne@69 1808 """
jpayne@69 1809 if filename is None:
jpayne@69 1810 if self.filename is not None: filename = self.filename
jpayne@69 1811 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@69 1812
jpayne@69 1813 self._cookies_lock.acquire()
jpayne@69 1814 try:
jpayne@69 1815
jpayne@69 1816 old_state = copy.deepcopy(self._cookies)
jpayne@69 1817 self._cookies = {}
jpayne@69 1818 try:
jpayne@69 1819 self.load(filename, ignore_discard, ignore_expires)
jpayne@69 1820 except OSError:
jpayne@69 1821 self._cookies = old_state
jpayne@69 1822 raise
jpayne@69 1823
jpayne@69 1824 finally:
jpayne@69 1825 self._cookies_lock.release()
jpayne@69 1826
jpayne@69 1827
jpayne@69 1828 def lwp_cookie_str(cookie):
jpayne@69 1829 """Return string representation of Cookie in the LWP cookie file format.
jpayne@69 1830
jpayne@69 1831 Actually, the format is extended a bit -- see module docstring.
jpayne@69 1832
jpayne@69 1833 """
jpayne@69 1834 h = [(cookie.name, cookie.value),
jpayne@69 1835 ("path", cookie.path),
jpayne@69 1836 ("domain", cookie.domain)]
jpayne@69 1837 if cookie.port is not None: h.append(("port", cookie.port))
jpayne@69 1838 if cookie.path_specified: h.append(("path_spec", None))
jpayne@69 1839 if cookie.port_specified: h.append(("port_spec", None))
jpayne@69 1840 if cookie.domain_initial_dot: h.append(("domain_dot", None))
jpayne@69 1841 if cookie.secure: h.append(("secure", None))
jpayne@69 1842 if cookie.expires: h.append(("expires",
jpayne@69 1843 time2isoz(float(cookie.expires))))
jpayne@69 1844 if cookie.discard: h.append(("discard", None))
jpayne@69 1845 if cookie.comment: h.append(("comment", cookie.comment))
jpayne@69 1846 if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
jpayne@69 1847
jpayne@69 1848 keys = sorted(cookie._rest.keys())
jpayne@69 1849 for k in keys:
jpayne@69 1850 h.append((k, str(cookie._rest[k])))
jpayne@69 1851
jpayne@69 1852 h.append(("version", str(cookie.version)))
jpayne@69 1853
jpayne@69 1854 return join_header_words([h])
jpayne@69 1855
jpayne@69 1856 class LWPCookieJar(FileCookieJar):
jpayne@69 1857 """
jpayne@69 1858 The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
jpayne@69 1859 "Set-Cookie3" is the format used by the libwww-perl library, not known
jpayne@69 1860 to be compatible with any browser, but which is easy to read and
jpayne@69 1861 doesn't lose information about RFC 2965 cookies.
jpayne@69 1862
jpayne@69 1863 Additional methods
jpayne@69 1864
jpayne@69 1865 as_lwp_str(ignore_discard=True, ignore_expired=True)
jpayne@69 1866
jpayne@69 1867 """
jpayne@69 1868
jpayne@69 1869 def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
jpayne@69 1870 """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers.
jpayne@69 1871
jpayne@69 1872 ignore_discard and ignore_expires: see docstring for FileCookieJar.save
jpayne@69 1873
jpayne@69 1874 """
jpayne@69 1875 now = time.time()
jpayne@69 1876 r = []
jpayne@69 1877 for cookie in self:
jpayne@69 1878 if not ignore_discard and cookie.discard:
jpayne@69 1879 continue
jpayne@69 1880 if not ignore_expires and cookie.is_expired(now):
jpayne@69 1881 continue
jpayne@69 1882 r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
jpayne@69 1883 return "\n".join(r+[""])
jpayne@69 1884
jpayne@69 1885 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@69 1886 if filename is None:
jpayne@69 1887 if self.filename is not None: filename = self.filename
jpayne@69 1888 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@69 1889
jpayne@69 1890 with open(filename, "w") as f:
jpayne@69 1891 # There really isn't an LWP Cookies 2.0 format, but this indicates
jpayne@69 1892 # that there is extra information in here (domain_dot and
jpayne@69 1893 # port_spec) while still being compatible with libwww-perl, I hope.
jpayne@69 1894 f.write("#LWP-Cookies-2.0\n")
jpayne@69 1895 f.write(self.as_lwp_str(ignore_discard, ignore_expires))
jpayne@69 1896
jpayne@69 1897 def _really_load(self, f, filename, ignore_discard, ignore_expires):
jpayne@69 1898 magic = f.readline()
jpayne@69 1899 if not self.magic_re.search(magic):
jpayne@69 1900 msg = ("%r does not look like a Set-Cookie3 (LWP) format "
jpayne@69 1901 "file" % filename)
jpayne@69 1902 raise LoadError(msg)
jpayne@69 1903
jpayne@69 1904 now = time.time()
jpayne@69 1905
jpayne@69 1906 header = "Set-Cookie3:"
jpayne@69 1907 boolean_attrs = ("port_spec", "path_spec", "domain_dot",
jpayne@69 1908 "secure", "discard")
jpayne@69 1909 value_attrs = ("version",
jpayne@69 1910 "port", "path", "domain",
jpayne@69 1911 "expires",
jpayne@69 1912 "comment", "commenturl")
jpayne@69 1913
jpayne@69 1914 try:
jpayne@69 1915 while 1:
jpayne@69 1916 line = f.readline()
jpayne@69 1917 if line == "": break
jpayne@69 1918 if not line.startswith(header):
jpayne@69 1919 continue
jpayne@69 1920 line = line[len(header):].strip()
jpayne@69 1921
jpayne@69 1922 for data in split_header_words([line]):
jpayne@69 1923 name, value = data[0]
jpayne@69 1924 standard = {}
jpayne@69 1925 rest = {}
jpayne@69 1926 for k in boolean_attrs:
jpayne@69 1927 standard[k] = False
jpayne@69 1928 for k, v in data[1:]:
jpayne@69 1929 if k is not None:
jpayne@69 1930 lc = k.lower()
jpayne@69 1931 else:
jpayne@69 1932 lc = None
jpayne@69 1933 # don't lose case distinction for unknown fields
jpayne@69 1934 if (lc in value_attrs) or (lc in boolean_attrs):
jpayne@69 1935 k = lc
jpayne@69 1936 if k in boolean_attrs:
jpayne@69 1937 if v is None: v = True
jpayne@69 1938 standard[k] = v
jpayne@69 1939 elif k in value_attrs:
jpayne@69 1940 standard[k] = v
jpayne@69 1941 else:
jpayne@69 1942 rest[k] = v
jpayne@69 1943
jpayne@69 1944 h = standard.get
jpayne@69 1945 expires = h("expires")
jpayne@69 1946 discard = h("discard")
jpayne@69 1947 if expires is not None:
jpayne@69 1948 expires = iso2time(expires)
jpayne@69 1949 if expires is None:
jpayne@69 1950 discard = True
jpayne@69 1951 domain = h("domain")
jpayne@69 1952 domain_specified = domain.startswith(".")
jpayne@69 1953 c = Cookie(h("version"), name, value,
jpayne@69 1954 h("port"), h("port_spec"),
jpayne@69 1955 domain, domain_specified, h("domain_dot"),
jpayne@69 1956 h("path"), h("path_spec"),
jpayne@69 1957 h("secure"),
jpayne@69 1958 expires,
jpayne@69 1959 discard,
jpayne@69 1960 h("comment"),
jpayne@69 1961 h("commenturl"),
jpayne@69 1962 rest)
jpayne@69 1963 if not ignore_discard and c.discard:
jpayne@69 1964 continue
jpayne@69 1965 if not ignore_expires and c.is_expired(now):
jpayne@69 1966 continue
jpayne@69 1967 self.set_cookie(c)
jpayne@69 1968 except OSError:
jpayne@69 1969 raise
jpayne@69 1970 except Exception:
jpayne@69 1971 _warn_unhandled_exception()
jpayne@69 1972 raise LoadError("invalid Set-Cookie3 format file %r: %r" %
jpayne@69 1973 (filename, line))
jpayne@69 1974
jpayne@69 1975
jpayne@69 1976 class MozillaCookieJar(FileCookieJar):
jpayne@69 1977 """
jpayne@69 1978
jpayne@69 1979 WARNING: you may want to backup your browser's cookies file if you use
jpayne@69 1980 this class to save cookies. I *think* it works, but there have been
jpayne@69 1981 bugs in the past!
jpayne@69 1982
jpayne@69 1983 This class differs from CookieJar only in the format it uses to save and
jpayne@69 1984 load cookies to and from a file. This class uses the Mozilla/Netscape
jpayne@69 1985 `cookies.txt' format. lynx uses this file format, too.
jpayne@69 1986
jpayne@69 1987 Don't expect cookies saved while the browser is running to be noticed by
jpayne@69 1988 the browser (in fact, Mozilla on unix will overwrite your saved cookies if
jpayne@69 1989 you change them on disk while it's running; on Windows, you probably can't
jpayne@69 1990 save at all while the browser is running).
jpayne@69 1991
jpayne@69 1992 Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
jpayne@69 1993 Netscape cookies on saving.
jpayne@69 1994
jpayne@69 1995 In particular, the cookie version and port number information is lost,
jpayne@69 1996 together with information about whether or not Path, Port and Discard were
jpayne@69 1997 specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
jpayne@69 1998 domain as set in the HTTP header started with a dot (yes, I'm aware some
jpayne@69 1999 domains in Netscape files start with a dot and some don't -- trust me, you
jpayne@69 2000 really don't want to know any more about this).
jpayne@69 2001
jpayne@69 2002 Note that though Mozilla and Netscape use the same format, they use
jpayne@69 2003 slightly different headers. The class saves cookies using the Netscape
jpayne@69 2004 header by default (Mozilla can cope with that).
jpayne@69 2005
jpayne@69 2006 """
jpayne@69 2007 magic_re = re.compile("#( Netscape)? HTTP Cookie File")
jpayne@69 2008 header = """\
jpayne@69 2009 # Netscape HTTP Cookie File
jpayne@69 2010 # http://curl.haxx.se/rfc/cookie_spec.html
jpayne@69 2011 # This is a generated file! Do not edit.
jpayne@69 2012
jpayne@69 2013 """
jpayne@69 2014
jpayne@69 2015 def _really_load(self, f, filename, ignore_discard, ignore_expires):
jpayne@69 2016 now = time.time()
jpayne@69 2017
jpayne@69 2018 magic = f.readline()
jpayne@69 2019 if not self.magic_re.search(magic):
jpayne@69 2020 raise LoadError(
jpayne@69 2021 "%r does not look like a Netscape format cookies file" %
jpayne@69 2022 filename)
jpayne@69 2023
jpayne@69 2024 try:
jpayne@69 2025 while 1:
jpayne@69 2026 line = f.readline()
jpayne@69 2027 if line == "": break
jpayne@69 2028
jpayne@69 2029 # last field may be absent, so keep any trailing tab
jpayne@69 2030 if line.endswith("\n"): line = line[:-1]
jpayne@69 2031
jpayne@69 2032 # skip comments and blank lines XXX what is $ for?
jpayne@69 2033 if (line.strip().startswith(("#", "$")) or
jpayne@69 2034 line.strip() == ""):
jpayne@69 2035 continue
jpayne@69 2036
jpayne@69 2037 domain, domain_specified, path, secure, expires, name, value = \
jpayne@69 2038 line.split("\t")
jpayne@69 2039 secure = (secure == "TRUE")
jpayne@69 2040 domain_specified = (domain_specified == "TRUE")
jpayne@69 2041 if name == "":
jpayne@69 2042 # cookies.txt regards 'Set-Cookie: foo' as a cookie
jpayne@69 2043 # with no name, whereas http.cookiejar regards it as a
jpayne@69 2044 # cookie with no value.
jpayne@69 2045 name = value
jpayne@69 2046 value = None
jpayne@69 2047
jpayne@69 2048 initial_dot = domain.startswith(".")
jpayne@69 2049 assert domain_specified == initial_dot
jpayne@69 2050
jpayne@69 2051 discard = False
jpayne@69 2052 if expires == "":
jpayne@69 2053 expires = None
jpayne@69 2054 discard = True
jpayne@69 2055
jpayne@69 2056 # assume path_specified is false
jpayne@69 2057 c = Cookie(0, name, value,
jpayne@69 2058 None, False,
jpayne@69 2059 domain, domain_specified, initial_dot,
jpayne@69 2060 path, False,
jpayne@69 2061 secure,
jpayne@69 2062 expires,
jpayne@69 2063 discard,
jpayne@69 2064 None,
jpayne@69 2065 None,
jpayne@69 2066 {})
jpayne@69 2067 if not ignore_discard and c.discard:
jpayne@69 2068 continue
jpayne@69 2069 if not ignore_expires and c.is_expired(now):
jpayne@69 2070 continue
jpayne@69 2071 self.set_cookie(c)
jpayne@69 2072
jpayne@69 2073 except OSError:
jpayne@69 2074 raise
jpayne@69 2075 except Exception:
jpayne@69 2076 _warn_unhandled_exception()
jpayne@69 2077 raise LoadError("invalid Netscape format cookies file %r: %r" %
jpayne@69 2078 (filename, line))
jpayne@69 2079
jpayne@69 2080 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@69 2081 if filename is None:
jpayne@69 2082 if self.filename is not None: filename = self.filename
jpayne@69 2083 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@69 2084
jpayne@69 2085 with open(filename, "w") as f:
jpayne@69 2086 f.write(self.header)
jpayne@69 2087 now = time.time()
jpayne@69 2088 for cookie in self:
jpayne@69 2089 if not ignore_discard and cookie.discard:
jpayne@69 2090 continue
jpayne@69 2091 if not ignore_expires and cookie.is_expired(now):
jpayne@69 2092 continue
jpayne@69 2093 if cookie.secure: secure = "TRUE"
jpayne@69 2094 else: secure = "FALSE"
jpayne@69 2095 if cookie.domain.startswith("."): initial_dot = "TRUE"
jpayne@69 2096 else: initial_dot = "FALSE"
jpayne@69 2097 if cookie.expires is not None:
jpayne@69 2098 expires = str(cookie.expires)
jpayne@69 2099 else:
jpayne@69 2100 expires = ""
jpayne@69 2101 if cookie.value is None:
jpayne@69 2102 # cookies.txt regards 'Set-Cookie: foo' as a cookie
jpayne@69 2103 # with no name, whereas http.cookiejar regards it as a
jpayne@69 2104 # cookie with no value.
jpayne@69 2105 name = ""
jpayne@69 2106 value = cookie.name
jpayne@69 2107 else:
jpayne@69 2108 name = cookie.name
jpayne@69 2109 value = cookie.value
jpayne@69 2110 f.write(
jpayne@69 2111 "\t".join([cookie.domain, initial_dot, cookie.path,
jpayne@69 2112 secure, expires, name, value])+
jpayne@69 2113 "\n")