Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/http/cookiejar.py @ 68:5028fdace37b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 16:23:26 -0400
parents
children
rev   line source
jpayne@68 1 r"""HTTP cookie handling for web clients.
jpayne@68 2
jpayne@68 3 This module has (now fairly distant) origins in Gisle Aas' Perl module
jpayne@68 4 HTTP::Cookies, from the libwww-perl library.
jpayne@68 5
jpayne@68 6 Docstrings, comments and debug strings in this code refer to the
jpayne@68 7 attributes of the HTTP cookie system as cookie-attributes, to distinguish
jpayne@68 8 them clearly from Python attributes.
jpayne@68 9
jpayne@68 10 Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
jpayne@68 11 distributed with the Python standard library, but are available from
jpayne@68 12 http://wwwsearch.sf.net/):
jpayne@68 13
jpayne@68 14 CookieJar____
jpayne@68 15 / \ \
jpayne@68 16 FileCookieJar \ \
jpayne@68 17 / | \ \ \
jpayne@68 18 MozillaCookieJar | LWPCookieJar \ \
jpayne@68 19 | | \
jpayne@68 20 | ---MSIEBase | \
jpayne@68 21 | / | | \
jpayne@68 22 | / MSIEDBCookieJar BSDDBCookieJar
jpayne@68 23 |/
jpayne@68 24 MSIECookieJar
jpayne@68 25
jpayne@68 26 """
jpayne@68 27
jpayne@68 28 __all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
jpayne@68 29 'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
jpayne@68 30
jpayne@68 31 import os
jpayne@68 32 import copy
jpayne@68 33 import datetime
jpayne@68 34 import re
jpayne@68 35 import time
jpayne@68 36 import urllib.parse, urllib.request
jpayne@68 37 import threading as _threading
jpayne@68 38 import http.client # only for the default HTTP port
jpayne@68 39 from calendar import timegm
jpayne@68 40
jpayne@68 41 debug = False # set to True to enable debugging via the logging module
jpayne@68 42 logger = None
jpayne@68 43
jpayne@68 44 def _debug(*args):
jpayne@68 45 if not debug:
jpayne@68 46 return
jpayne@68 47 global logger
jpayne@68 48 if not logger:
jpayne@68 49 import logging
jpayne@68 50 logger = logging.getLogger("http.cookiejar")
jpayne@68 51 return logger.debug(*args)
jpayne@68 52
jpayne@68 53
jpayne@68 54 DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
jpayne@68 55 MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
jpayne@68 56 "instance initialised with one)")
jpayne@68 57
jpayne@68 58 def _warn_unhandled_exception():
jpayne@68 59 # There are a few catch-all except: statements in this module, for
jpayne@68 60 # catching input that's bad in unexpected ways. Warn if any
jpayne@68 61 # exceptions are caught there.
jpayne@68 62 import io, warnings, traceback
jpayne@68 63 f = io.StringIO()
jpayne@68 64 traceback.print_exc(None, f)
jpayne@68 65 msg = f.getvalue()
jpayne@68 66 warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2)
jpayne@68 67
jpayne@68 68
jpayne@68 69 # Date/time conversion
jpayne@68 70 # -----------------------------------------------------------------------------
jpayne@68 71
jpayne@68 72 EPOCH_YEAR = 1970
jpayne@68 73 def _timegm(tt):
jpayne@68 74 year, month, mday, hour, min, sec = tt[:6]
jpayne@68 75 if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and
jpayne@68 76 (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
jpayne@68 77 return timegm(tt)
jpayne@68 78 else:
jpayne@68 79 return None
jpayne@68 80
jpayne@68 81 DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
jpayne@68 82 MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
jpayne@68 83 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
jpayne@68 84 MONTHS_LOWER = []
jpayne@68 85 for month in MONTHS: MONTHS_LOWER.append(month.lower())
jpayne@68 86
jpayne@68 87 def time2isoz(t=None):
jpayne@68 88 """Return a string representing time in seconds since epoch, t.
jpayne@68 89
jpayne@68 90 If the function is called without an argument, it will use the current
jpayne@68 91 time.
jpayne@68 92
jpayne@68 93 The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
jpayne@68 94 representing Universal Time (UTC, aka GMT). An example of this format is:
jpayne@68 95
jpayne@68 96 1994-11-24 08:49:37Z
jpayne@68 97
jpayne@68 98 """
jpayne@68 99 if t is None:
jpayne@68 100 dt = datetime.datetime.utcnow()
jpayne@68 101 else:
jpayne@68 102 dt = datetime.datetime.utcfromtimestamp(t)
jpayne@68 103 return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
jpayne@68 104 dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
jpayne@68 105
jpayne@68 106 def time2netscape(t=None):
jpayne@68 107 """Return a string representing time in seconds since epoch, t.
jpayne@68 108
jpayne@68 109 If the function is called without an argument, it will use the current
jpayne@68 110 time.
jpayne@68 111
jpayne@68 112 The format of the returned string is like this:
jpayne@68 113
jpayne@68 114 Wed, DD-Mon-YYYY HH:MM:SS GMT
jpayne@68 115
jpayne@68 116 """
jpayne@68 117 if t is None:
jpayne@68 118 dt = datetime.datetime.utcnow()
jpayne@68 119 else:
jpayne@68 120 dt = datetime.datetime.utcfromtimestamp(t)
jpayne@68 121 return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
jpayne@68 122 DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1],
jpayne@68 123 dt.year, dt.hour, dt.minute, dt.second)
jpayne@68 124
jpayne@68 125
jpayne@68 126 UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
jpayne@68 127
jpayne@68 128 TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII)
jpayne@68 129 def offset_from_tz_string(tz):
jpayne@68 130 offset = None
jpayne@68 131 if tz in UTC_ZONES:
jpayne@68 132 offset = 0
jpayne@68 133 else:
jpayne@68 134 m = TIMEZONE_RE.search(tz)
jpayne@68 135 if m:
jpayne@68 136 offset = 3600 * int(m.group(2))
jpayne@68 137 if m.group(3):
jpayne@68 138 offset = offset + 60 * int(m.group(3))
jpayne@68 139 if m.group(1) == '-':
jpayne@68 140 offset = -offset
jpayne@68 141 return offset
jpayne@68 142
jpayne@68 143 def _str2time(day, mon, yr, hr, min, sec, tz):
jpayne@68 144 yr = int(yr)
jpayne@68 145 if yr > datetime.MAXYEAR:
jpayne@68 146 return None
jpayne@68 147
jpayne@68 148 # translate month name to number
jpayne@68 149 # month numbers start with 1 (January)
jpayne@68 150 try:
jpayne@68 151 mon = MONTHS_LOWER.index(mon.lower())+1
jpayne@68 152 except ValueError:
jpayne@68 153 # maybe it's already a number
jpayne@68 154 try:
jpayne@68 155 imon = int(mon)
jpayne@68 156 except ValueError:
jpayne@68 157 return None
jpayne@68 158 if 1 <= imon <= 12:
jpayne@68 159 mon = imon
jpayne@68 160 else:
jpayne@68 161 return None
jpayne@68 162
jpayne@68 163 # make sure clock elements are defined
jpayne@68 164 if hr is None: hr = 0
jpayne@68 165 if min is None: min = 0
jpayne@68 166 if sec is None: sec = 0
jpayne@68 167
jpayne@68 168 day = int(day)
jpayne@68 169 hr = int(hr)
jpayne@68 170 min = int(min)
jpayne@68 171 sec = int(sec)
jpayne@68 172
jpayne@68 173 if yr < 1000:
jpayne@68 174 # find "obvious" year
jpayne@68 175 cur_yr = time.localtime(time.time())[0]
jpayne@68 176 m = cur_yr % 100
jpayne@68 177 tmp = yr
jpayne@68 178 yr = yr + cur_yr - m
jpayne@68 179 m = m - tmp
jpayne@68 180 if abs(m) > 50:
jpayne@68 181 if m > 0: yr = yr + 100
jpayne@68 182 else: yr = yr - 100
jpayne@68 183
jpayne@68 184 # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
jpayne@68 185 t = _timegm((yr, mon, day, hr, min, sec, tz))
jpayne@68 186
jpayne@68 187 if t is not None:
jpayne@68 188 # adjust time using timezone string, to get absolute time since epoch
jpayne@68 189 if tz is None:
jpayne@68 190 tz = "UTC"
jpayne@68 191 tz = tz.upper()
jpayne@68 192 offset = offset_from_tz_string(tz)
jpayne@68 193 if offset is None:
jpayne@68 194 return None
jpayne@68 195 t = t - offset
jpayne@68 196
jpayne@68 197 return t
jpayne@68 198
jpayne@68 199 STRICT_DATE_RE = re.compile(
jpayne@68 200 r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
jpayne@68 201 r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII)
jpayne@68 202 WEEKDAY_RE = re.compile(
jpayne@68 203 r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII)
jpayne@68 204 LOOSE_HTTP_DATE_RE = re.compile(
jpayne@68 205 r"""^
jpayne@68 206 (\d\d?) # day
jpayne@68 207 (?:\s+|[-\/])
jpayne@68 208 (\w+) # month
jpayne@68 209 (?:\s+|[-\/])
jpayne@68 210 (\d+) # year
jpayne@68 211 (?:
jpayne@68 212 (?:\s+|:) # separator before clock
jpayne@68 213 (\d\d?):(\d\d) # hour:min
jpayne@68 214 (?::(\d\d))? # optional seconds
jpayne@68 215 )? # optional clock
jpayne@68 216 \s*
jpayne@68 217 (?:
jpayne@68 218 ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone
jpayne@68 219 \s*
jpayne@68 220 )?
jpayne@68 221 (?:
jpayne@68 222 \(\w+\) # ASCII representation of timezone in parens.
jpayne@68 223 \s*
jpayne@68 224 )?$""", re.X | re.ASCII)
jpayne@68 225 def http2time(text):
jpayne@68 226 """Returns time in seconds since epoch of time represented by a string.
jpayne@68 227
jpayne@68 228 Return value is an integer.
jpayne@68 229
jpayne@68 230 None is returned if the format of str is unrecognized, the time is outside
jpayne@68 231 the representable range, or the timezone string is not recognized. If the
jpayne@68 232 string contains no timezone, UTC is assumed.
jpayne@68 233
jpayne@68 234 The timezone in the string may be numerical (like "-0800" or "+0100") or a
jpayne@68 235 string timezone (like "UTC", "GMT", "BST" or "EST"). Currently, only the
jpayne@68 236 timezone strings equivalent to UTC (zero offset) are known to the function.
jpayne@68 237
jpayne@68 238 The function loosely parses the following formats:
jpayne@68 239
jpayne@68 240 Wed, 09 Feb 1994 22:23:32 GMT -- HTTP format
jpayne@68 241 Tuesday, 08-Feb-94 14:15:29 GMT -- old rfc850 HTTP format
jpayne@68 242 Tuesday, 08-Feb-1994 14:15:29 GMT -- broken rfc850 HTTP format
jpayne@68 243 09 Feb 1994 22:23:32 GMT -- HTTP format (no weekday)
jpayne@68 244 08-Feb-94 14:15:29 GMT -- rfc850 format (no weekday)
jpayne@68 245 08-Feb-1994 14:15:29 GMT -- broken rfc850 format (no weekday)
jpayne@68 246
jpayne@68 247 The parser ignores leading and trailing whitespace. The time may be
jpayne@68 248 absent.
jpayne@68 249
jpayne@68 250 If the year is given with only 2 digits, the function will select the
jpayne@68 251 century that makes the year closest to the current date.
jpayne@68 252
jpayne@68 253 """
jpayne@68 254 # fast exit for strictly conforming string
jpayne@68 255 m = STRICT_DATE_RE.search(text)
jpayne@68 256 if m:
jpayne@68 257 g = m.groups()
jpayne@68 258 mon = MONTHS_LOWER.index(g[1].lower()) + 1
jpayne@68 259 tt = (int(g[2]), mon, int(g[0]),
jpayne@68 260 int(g[3]), int(g[4]), float(g[5]))
jpayne@68 261 return _timegm(tt)
jpayne@68 262
jpayne@68 263 # No, we need some messy parsing...
jpayne@68 264
jpayne@68 265 # clean up
jpayne@68 266 text = text.lstrip()
jpayne@68 267 text = WEEKDAY_RE.sub("", text, 1) # Useless weekday
jpayne@68 268
jpayne@68 269 # tz is time zone specifier string
jpayne@68 270 day, mon, yr, hr, min, sec, tz = [None]*7
jpayne@68 271
jpayne@68 272 # loose regexp parse
jpayne@68 273 m = LOOSE_HTTP_DATE_RE.search(text)
jpayne@68 274 if m is not None:
jpayne@68 275 day, mon, yr, hr, min, sec, tz = m.groups()
jpayne@68 276 else:
jpayne@68 277 return None # bad format
jpayne@68 278
jpayne@68 279 return _str2time(day, mon, yr, hr, min, sec, tz)
jpayne@68 280
jpayne@68 281 ISO_DATE_RE = re.compile(
jpayne@68 282 r"""^
jpayne@68 283 (\d{4}) # year
jpayne@68 284 [-\/]?
jpayne@68 285 (\d\d?) # numerical month
jpayne@68 286 [-\/]?
jpayne@68 287 (\d\d?) # day
jpayne@68 288 (?:
jpayne@68 289 (?:\s+|[-:Tt]) # separator before clock
jpayne@68 290 (\d\d?):?(\d\d) # hour:min
jpayne@68 291 (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional)
jpayne@68 292 )? # optional clock
jpayne@68 293 \s*
jpayne@68 294 (?:
jpayne@68 295 ([-+]?\d\d?:?(:?\d\d)?
jpayne@68 296 |Z|z) # timezone (Z is "zero meridian", i.e. GMT)
jpayne@68 297 \s*
jpayne@68 298 )?$""", re.X | re. ASCII)
jpayne@68 299 def iso2time(text):
jpayne@68 300 """
jpayne@68 301 As for http2time, but parses the ISO 8601 formats:
jpayne@68 302
jpayne@68 303 1994-02-03 14:15:29 -0100 -- ISO 8601 format
jpayne@68 304 1994-02-03 14:15:29 -- zone is optional
jpayne@68 305 1994-02-03 -- only date
jpayne@68 306 1994-02-03T14:15:29 -- Use T as separator
jpayne@68 307 19940203T141529Z -- ISO 8601 compact format
jpayne@68 308 19940203 -- only date
jpayne@68 309
jpayne@68 310 """
jpayne@68 311 # clean up
jpayne@68 312 text = text.lstrip()
jpayne@68 313
jpayne@68 314 # tz is time zone specifier string
jpayne@68 315 day, mon, yr, hr, min, sec, tz = [None]*7
jpayne@68 316
jpayne@68 317 # loose regexp parse
jpayne@68 318 m = ISO_DATE_RE.search(text)
jpayne@68 319 if m is not None:
jpayne@68 320 # XXX there's an extra bit of the timezone I'm ignoring here: is
jpayne@68 321 # this the right thing to do?
jpayne@68 322 yr, mon, day, hr, min, sec, tz, _ = m.groups()
jpayne@68 323 else:
jpayne@68 324 return None # bad format
jpayne@68 325
jpayne@68 326 return _str2time(day, mon, yr, hr, min, sec, tz)
jpayne@68 327
jpayne@68 328
jpayne@68 329 # Header parsing
jpayne@68 330 # -----------------------------------------------------------------------------
jpayne@68 331
jpayne@68 332 def unmatched(match):
jpayne@68 333 """Return unmatched part of re.Match object."""
jpayne@68 334 start, end = match.span(0)
jpayne@68 335 return match.string[:start]+match.string[end:]
jpayne@68 336
jpayne@68 337 HEADER_TOKEN_RE = re.compile(r"^\s*([^=\s;,]+)")
jpayne@68 338 HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")
jpayne@68 339 HEADER_VALUE_RE = re.compile(r"^\s*=\s*([^\s;,]*)")
jpayne@68 340 HEADER_ESCAPE_RE = re.compile(r"\\(.)")
jpayne@68 341 def split_header_words(header_values):
jpayne@68 342 r"""Parse header values into a list of lists containing key,value pairs.
jpayne@68 343
jpayne@68 344 The function knows how to deal with ",", ";" and "=" as well as quoted
jpayne@68 345 values after "=". A list of space separated tokens are parsed as if they
jpayne@68 346 were separated by ";".
jpayne@68 347
jpayne@68 348 If the header_values passed as argument contains multiple values, then they
jpayne@68 349 are treated as if they were a single value separated by comma ",".
jpayne@68 350
jpayne@68 351 This means that this function is useful for parsing header fields that
jpayne@68 352 follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
jpayne@68 353 the requirement for tokens).
jpayne@68 354
jpayne@68 355 headers = #header
jpayne@68 356 header = (token | parameter) *( [";"] (token | parameter))
jpayne@68 357
jpayne@68 358 token = 1*<any CHAR except CTLs or separators>
jpayne@68 359 separators = "(" | ")" | "<" | ">" | "@"
jpayne@68 360 | "," | ";" | ":" | "\" | <">
jpayne@68 361 | "/" | "[" | "]" | "?" | "="
jpayne@68 362 | "{" | "}" | SP | HT
jpayne@68 363
jpayne@68 364 quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
jpayne@68 365 qdtext = <any TEXT except <">>
jpayne@68 366 quoted-pair = "\" CHAR
jpayne@68 367
jpayne@68 368 parameter = attribute "=" value
jpayne@68 369 attribute = token
jpayne@68 370 value = token | quoted-string
jpayne@68 371
jpayne@68 372 Each header is represented by a list of key/value pairs. The value for a
jpayne@68 373 simple token (not part of a parameter) is None. Syntactically incorrect
jpayne@68 374 headers will not necessarily be parsed as you would want.
jpayne@68 375
jpayne@68 376 This is easier to describe with some examples:
jpayne@68 377
jpayne@68 378 >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
jpayne@68 379 [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
jpayne@68 380 >>> split_header_words(['text/html; charset="iso-8859-1"'])
jpayne@68 381 [[('text/html', None), ('charset', 'iso-8859-1')]]
jpayne@68 382 >>> split_header_words([r'Basic realm="\"foo\bar\""'])
jpayne@68 383 [[('Basic', None), ('realm', '"foobar"')]]
jpayne@68 384
jpayne@68 385 """
jpayne@68 386 assert not isinstance(header_values, str)
jpayne@68 387 result = []
jpayne@68 388 for text in header_values:
jpayne@68 389 orig_text = text
jpayne@68 390 pairs = []
jpayne@68 391 while text:
jpayne@68 392 m = HEADER_TOKEN_RE.search(text)
jpayne@68 393 if m:
jpayne@68 394 text = unmatched(m)
jpayne@68 395 name = m.group(1)
jpayne@68 396 m = HEADER_QUOTED_VALUE_RE.search(text)
jpayne@68 397 if m: # quoted value
jpayne@68 398 text = unmatched(m)
jpayne@68 399 value = m.group(1)
jpayne@68 400 value = HEADER_ESCAPE_RE.sub(r"\1", value)
jpayne@68 401 else:
jpayne@68 402 m = HEADER_VALUE_RE.search(text)
jpayne@68 403 if m: # unquoted value
jpayne@68 404 text = unmatched(m)
jpayne@68 405 value = m.group(1)
jpayne@68 406 value = value.rstrip()
jpayne@68 407 else:
jpayne@68 408 # no value, a lone token
jpayne@68 409 value = None
jpayne@68 410 pairs.append((name, value))
jpayne@68 411 elif text.lstrip().startswith(","):
jpayne@68 412 # concatenated headers, as per RFC 2616 section 4.2
jpayne@68 413 text = text.lstrip()[1:]
jpayne@68 414 if pairs: result.append(pairs)
jpayne@68 415 pairs = []
jpayne@68 416 else:
jpayne@68 417 # skip junk
jpayne@68 418 non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text)
jpayne@68 419 assert nr_junk_chars > 0, (
jpayne@68 420 "split_header_words bug: '%s', '%s', %s" %
jpayne@68 421 (orig_text, text, pairs))
jpayne@68 422 text = non_junk
jpayne@68 423 if pairs: result.append(pairs)
jpayne@68 424 return result
jpayne@68 425
jpayne@68 426 HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")
jpayne@68 427 def join_header_words(lists):
jpayne@68 428 """Do the inverse (almost) of the conversion done by split_header_words.
jpayne@68 429
jpayne@68 430 Takes a list of lists of (key, value) pairs and produces a single header
jpayne@68 431 value. Attribute values are quoted if needed.
jpayne@68 432
jpayne@68 433 >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
jpayne@68 434 'text/plain; charset="iso-8859-1"'
jpayne@68 435 >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
jpayne@68 436 'text/plain, charset="iso-8859-1"'
jpayne@68 437
jpayne@68 438 """
jpayne@68 439 headers = []
jpayne@68 440 for pairs in lists:
jpayne@68 441 attr = []
jpayne@68 442 for k, v in pairs:
jpayne@68 443 if v is not None:
jpayne@68 444 if not re.search(r"^\w+$", v):
jpayne@68 445 v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v) # escape " and \
jpayne@68 446 v = '"%s"' % v
jpayne@68 447 k = "%s=%s" % (k, v)
jpayne@68 448 attr.append(k)
jpayne@68 449 if attr: headers.append("; ".join(attr))
jpayne@68 450 return ", ".join(headers)
jpayne@68 451
jpayne@68 452 def strip_quotes(text):
jpayne@68 453 if text.startswith('"'):
jpayne@68 454 text = text[1:]
jpayne@68 455 if text.endswith('"'):
jpayne@68 456 text = text[:-1]
jpayne@68 457 return text
jpayne@68 458
jpayne@68 459 def parse_ns_headers(ns_headers):
jpayne@68 460 """Ad-hoc parser for Netscape protocol cookie-attributes.
jpayne@68 461
jpayne@68 462 The old Netscape cookie format for Set-Cookie can for instance contain
jpayne@68 463 an unquoted "," in the expires field, so we have to use this ad-hoc
jpayne@68 464 parser instead of split_header_words.
jpayne@68 465
jpayne@68 466 XXX This may not make the best possible effort to parse all the crap
jpayne@68 467 that Netscape Cookie headers contain. Ronald Tschalar's HTTPClient
jpayne@68 468 parser is probably better, so could do worse than following that if
jpayne@68 469 this ever gives any trouble.
jpayne@68 470
jpayne@68 471 Currently, this is also used for parsing RFC 2109 cookies.
jpayne@68 472
jpayne@68 473 """
jpayne@68 474 known_attrs = ("expires", "domain", "path", "secure",
jpayne@68 475 # RFC 2109 attrs (may turn up in Netscape cookies, too)
jpayne@68 476 "version", "port", "max-age")
jpayne@68 477
jpayne@68 478 result = []
jpayne@68 479 for ns_header in ns_headers:
jpayne@68 480 pairs = []
jpayne@68 481 version_set = False
jpayne@68 482
jpayne@68 483 # XXX: The following does not strictly adhere to RFCs in that empty
jpayne@68 484 # names and values are legal (the former will only appear once and will
jpayne@68 485 # be overwritten if multiple occurrences are present). This is
jpayne@68 486 # mostly to deal with backwards compatibility.
jpayne@68 487 for ii, param in enumerate(ns_header.split(';')):
jpayne@68 488 param = param.strip()
jpayne@68 489
jpayne@68 490 key, sep, val = param.partition('=')
jpayne@68 491 key = key.strip()
jpayne@68 492
jpayne@68 493 if not key:
jpayne@68 494 if ii == 0:
jpayne@68 495 break
jpayne@68 496 else:
jpayne@68 497 continue
jpayne@68 498
jpayne@68 499 # allow for a distinction between present and empty and missing
jpayne@68 500 # altogether
jpayne@68 501 val = val.strip() if sep else None
jpayne@68 502
jpayne@68 503 if ii != 0:
jpayne@68 504 lc = key.lower()
jpayne@68 505 if lc in known_attrs:
jpayne@68 506 key = lc
jpayne@68 507
jpayne@68 508 if key == "version":
jpayne@68 509 # This is an RFC 2109 cookie.
jpayne@68 510 if val is not None:
jpayne@68 511 val = strip_quotes(val)
jpayne@68 512 version_set = True
jpayne@68 513 elif key == "expires":
jpayne@68 514 # convert expires date to seconds since epoch
jpayne@68 515 if val is not None:
jpayne@68 516 val = http2time(strip_quotes(val)) # None if invalid
jpayne@68 517 pairs.append((key, val))
jpayne@68 518
jpayne@68 519 if pairs:
jpayne@68 520 if not version_set:
jpayne@68 521 pairs.append(("version", "0"))
jpayne@68 522 result.append(pairs)
jpayne@68 523
jpayne@68 524 return result
jpayne@68 525
jpayne@68 526
jpayne@68 527 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
jpayne@68 528 def is_HDN(text):
jpayne@68 529 """Return True if text is a host domain name."""
jpayne@68 530 # XXX
jpayne@68 531 # This may well be wrong. Which RFC is HDN defined in, if any (for
jpayne@68 532 # the purposes of RFC 2965)?
jpayne@68 533 # For the current implementation, what about IPv6? Remember to look
jpayne@68 534 # at other uses of IPV4_RE also, if change this.
jpayne@68 535 if IPV4_RE.search(text):
jpayne@68 536 return False
jpayne@68 537 if text == "":
jpayne@68 538 return False
jpayne@68 539 if text[0] == "." or text[-1] == ".":
jpayne@68 540 return False
jpayne@68 541 return True
jpayne@68 542
jpayne@68 543 def domain_match(A, B):
jpayne@68 544 """Return True if domain A domain-matches domain B, according to RFC 2965.
jpayne@68 545
jpayne@68 546 A and B may be host domain names or IP addresses.
jpayne@68 547
jpayne@68 548 RFC 2965, section 1:
jpayne@68 549
jpayne@68 550 Host names can be specified either as an IP address or a HDN string.
jpayne@68 551 Sometimes we compare one host name with another. (Such comparisons SHALL
jpayne@68 552 be case-insensitive.) Host A's name domain-matches host B's if
jpayne@68 553
jpayne@68 554 * their host name strings string-compare equal; or
jpayne@68 555
jpayne@68 556 * A is a HDN string and has the form NB, where N is a non-empty
jpayne@68 557 name string, B has the form .B', and B' is a HDN string. (So,
jpayne@68 558 x.y.com domain-matches .Y.com but not Y.com.)
jpayne@68 559
jpayne@68 560 Note that domain-match is not a commutative operation: a.b.c.com
jpayne@68 561 domain-matches .c.com, but not the reverse.
jpayne@68 562
jpayne@68 563 """
jpayne@68 564 # Note that, if A or B are IP addresses, the only relevant part of the
jpayne@68 565 # definition of the domain-match algorithm is the direct string-compare.
jpayne@68 566 A = A.lower()
jpayne@68 567 B = B.lower()
jpayne@68 568 if A == B:
jpayne@68 569 return True
jpayne@68 570 if not is_HDN(A):
jpayne@68 571 return False
jpayne@68 572 i = A.rfind(B)
jpayne@68 573 if i == -1 or i == 0:
jpayne@68 574 # A does not have form NB, or N is the empty string
jpayne@68 575 return False
jpayne@68 576 if not B.startswith("."):
jpayne@68 577 return False
jpayne@68 578 if not is_HDN(B[1:]):
jpayne@68 579 return False
jpayne@68 580 return True
jpayne@68 581
jpayne@68 582 def liberal_is_HDN(text):
jpayne@68 583 """Return True if text is a sort-of-like a host domain name.
jpayne@68 584
jpayne@68 585 For accepting/blocking domains.
jpayne@68 586
jpayne@68 587 """
jpayne@68 588 if IPV4_RE.search(text):
jpayne@68 589 return False
jpayne@68 590 return True
jpayne@68 591
jpayne@68 592 def user_domain_match(A, B):
jpayne@68 593 """For blocking/accepting domains.
jpayne@68 594
jpayne@68 595 A and B may be host domain names or IP addresses.
jpayne@68 596
jpayne@68 597 """
jpayne@68 598 A = A.lower()
jpayne@68 599 B = B.lower()
jpayne@68 600 if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
jpayne@68 601 if A == B:
jpayne@68 602 # equal IP addresses
jpayne@68 603 return True
jpayne@68 604 return False
jpayne@68 605 initial_dot = B.startswith(".")
jpayne@68 606 if initial_dot and A.endswith(B):
jpayne@68 607 return True
jpayne@68 608 if not initial_dot and A == B:
jpayne@68 609 return True
jpayne@68 610 return False
jpayne@68 611
jpayne@68 612 cut_port_re = re.compile(r":\d+$", re.ASCII)
jpayne@68 613 def request_host(request):
jpayne@68 614 """Return request-host, as defined by RFC 2965.
jpayne@68 615
jpayne@68 616 Variation from RFC: returned value is lowercased, for convenient
jpayne@68 617 comparison.
jpayne@68 618
jpayne@68 619 """
jpayne@68 620 url = request.get_full_url()
jpayne@68 621 host = urllib.parse.urlparse(url)[1]
jpayne@68 622 if host == "":
jpayne@68 623 host = request.get_header("Host", "")
jpayne@68 624
jpayne@68 625 # remove port, if present
jpayne@68 626 host = cut_port_re.sub("", host, 1)
jpayne@68 627 return host.lower()
jpayne@68 628
jpayne@68 629 def eff_request_host(request):
jpayne@68 630 """Return a tuple (request-host, effective request-host name).
jpayne@68 631
jpayne@68 632 As defined by RFC 2965, except both are lowercased.
jpayne@68 633
jpayne@68 634 """
jpayne@68 635 erhn = req_host = request_host(request)
jpayne@68 636 if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
jpayne@68 637 erhn = req_host + ".local"
jpayne@68 638 return req_host, erhn
jpayne@68 639
jpayne@68 640 def request_path(request):
jpayne@68 641 """Path component of request-URI, as defined by RFC 2965."""
jpayne@68 642 url = request.get_full_url()
jpayne@68 643 parts = urllib.parse.urlsplit(url)
jpayne@68 644 path = escape_path(parts.path)
jpayne@68 645 if not path.startswith("/"):
jpayne@68 646 # fix bad RFC 2396 absoluteURI
jpayne@68 647 path = "/" + path
jpayne@68 648 return path
jpayne@68 649
jpayne@68 650 def request_port(request):
jpayne@68 651 host = request.host
jpayne@68 652 i = host.find(':')
jpayne@68 653 if i >= 0:
jpayne@68 654 port = host[i+1:]
jpayne@68 655 try:
jpayne@68 656 int(port)
jpayne@68 657 except ValueError:
jpayne@68 658 _debug("nonnumeric port: '%s'", port)
jpayne@68 659 return None
jpayne@68 660 else:
jpayne@68 661 port = DEFAULT_HTTP_PORT
jpayne@68 662 return port
jpayne@68 663
jpayne@68 664 # Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
jpayne@68 665 # need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
jpayne@68 666 HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
jpayne@68 667 ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
jpayne@68 668 def uppercase_escaped_char(match):
jpayne@68 669 return "%%%s" % match.group(1).upper()
jpayne@68 670 def escape_path(path):
jpayne@68 671 """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
jpayne@68 672 # There's no knowing what character encoding was used to create URLs
jpayne@68 673 # containing %-escapes, but since we have to pick one to escape invalid
jpayne@68 674 # path characters, we pick UTF-8, as recommended in the HTML 4.0
jpayne@68 675 # specification:
jpayne@68 676 # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
jpayne@68 677 # And here, kind of: draft-fielding-uri-rfc2396bis-03
jpayne@68 678 # (And in draft IRI specification: draft-duerst-iri-05)
jpayne@68 679 # (And here, for new URI schemes: RFC 2718)
jpayne@68 680 path = urllib.parse.quote(path, HTTP_PATH_SAFE)
jpayne@68 681 path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
jpayne@68 682 return path
jpayne@68 683
jpayne@68 684 def reach(h):
jpayne@68 685 """Return reach of host h, as defined by RFC 2965, section 1.
jpayne@68 686
jpayne@68 687 The reach R of a host name H is defined as follows:
jpayne@68 688
jpayne@68 689 * If
jpayne@68 690
jpayne@68 691 - H is the host domain name of a host; and,
jpayne@68 692
jpayne@68 693 - H has the form A.B; and
jpayne@68 694
jpayne@68 695 - A has no embedded (that is, interior) dots; and
jpayne@68 696
jpayne@68 697 - B has at least one embedded dot, or B is the string "local".
jpayne@68 698 then the reach of H is .B.
jpayne@68 699
jpayne@68 700 * Otherwise, the reach of H is H.
jpayne@68 701
jpayne@68 702 >>> reach("www.acme.com")
jpayne@68 703 '.acme.com'
jpayne@68 704 >>> reach("acme.com")
jpayne@68 705 'acme.com'
jpayne@68 706 >>> reach("acme.local")
jpayne@68 707 '.local'
jpayne@68 708
jpayne@68 709 """
jpayne@68 710 i = h.find(".")
jpayne@68 711 if i >= 0:
jpayne@68 712 #a = h[:i] # this line is only here to show what a is
jpayne@68 713 b = h[i+1:]
jpayne@68 714 i = b.find(".")
jpayne@68 715 if is_HDN(h) and (i >= 0 or b == "local"):
jpayne@68 716 return "."+b
jpayne@68 717 return h
jpayne@68 718
jpayne@68 719 def is_third_party(request):
jpayne@68 720 """
jpayne@68 721
jpayne@68 722 RFC 2965, section 3.3.6:
jpayne@68 723
jpayne@68 724 An unverifiable transaction is to a third-party host if its request-
jpayne@68 725 host U does not domain-match the reach R of the request-host O in the
jpayne@68 726 origin transaction.
jpayne@68 727
jpayne@68 728 """
jpayne@68 729 req_host = request_host(request)
jpayne@68 730 if not domain_match(req_host, reach(request.origin_req_host)):
jpayne@68 731 return True
jpayne@68 732 else:
jpayne@68 733 return False
jpayne@68 734
jpayne@68 735
jpayne@68 736 class Cookie:
jpayne@68 737 """HTTP Cookie.
jpayne@68 738
jpayne@68 739 This class represents both Netscape and RFC 2965 cookies.
jpayne@68 740
jpayne@68 741 This is deliberately a very simple class. It just holds attributes. It's
jpayne@68 742 possible to construct Cookie instances that don't comply with the cookie
jpayne@68 743 standards. CookieJar.make_cookies is the factory function for Cookie
jpayne@68 744 objects -- it deals with cookie parsing, supplying defaults, and
jpayne@68 745 normalising to the representation used in this class. CookiePolicy is
jpayne@68 746 responsible for checking them to see whether they should be accepted from
jpayne@68 747 and returned to the server.
jpayne@68 748
jpayne@68 749 Note that the port may be present in the headers, but unspecified ("Port"
jpayne@68 750 rather than"Port=80", for example); if this is the case, port is None.
jpayne@68 751
jpayne@68 752 """
jpayne@68 753
jpayne@68 754 def __init__(self, version, name, value,
jpayne@68 755 port, port_specified,
jpayne@68 756 domain, domain_specified, domain_initial_dot,
jpayne@68 757 path, path_specified,
jpayne@68 758 secure,
jpayne@68 759 expires,
jpayne@68 760 discard,
jpayne@68 761 comment,
jpayne@68 762 comment_url,
jpayne@68 763 rest,
jpayne@68 764 rfc2109=False,
jpayne@68 765 ):
jpayne@68 766
jpayne@68 767 if version is not None: version = int(version)
jpayne@68 768 if expires is not None: expires = int(float(expires))
jpayne@68 769 if port is None and port_specified is True:
jpayne@68 770 raise ValueError("if port is None, port_specified must be false")
jpayne@68 771
jpayne@68 772 self.version = version
jpayne@68 773 self.name = name
jpayne@68 774 self.value = value
jpayne@68 775 self.port = port
jpayne@68 776 self.port_specified = port_specified
jpayne@68 777 # normalise case, as per RFC 2965 section 3.3.3
jpayne@68 778 self.domain = domain.lower()
jpayne@68 779 self.domain_specified = domain_specified
jpayne@68 780 # Sigh. We need to know whether the domain given in the
jpayne@68 781 # cookie-attribute had an initial dot, in order to follow RFC 2965
jpayne@68 782 # (as clarified in draft errata). Needed for the returned $Domain
jpayne@68 783 # value.
jpayne@68 784 self.domain_initial_dot = domain_initial_dot
jpayne@68 785 self.path = path
jpayne@68 786 self.path_specified = path_specified
jpayne@68 787 self.secure = secure
jpayne@68 788 self.expires = expires
jpayne@68 789 self.discard = discard
jpayne@68 790 self.comment = comment
jpayne@68 791 self.comment_url = comment_url
jpayne@68 792 self.rfc2109 = rfc2109
jpayne@68 793
jpayne@68 794 self._rest = copy.copy(rest)
jpayne@68 795
jpayne@68 796 def has_nonstandard_attr(self, name):
jpayne@68 797 return name in self._rest
jpayne@68 798 def get_nonstandard_attr(self, name, default=None):
jpayne@68 799 return self._rest.get(name, default)
jpayne@68 800 def set_nonstandard_attr(self, name, value):
jpayne@68 801 self._rest[name] = value
jpayne@68 802
jpayne@68 803 def is_expired(self, now=None):
jpayne@68 804 if now is None: now = time.time()
jpayne@68 805 if (self.expires is not None) and (self.expires <= now):
jpayne@68 806 return True
jpayne@68 807 return False
jpayne@68 808
jpayne@68 809 def __str__(self):
jpayne@68 810 if self.port is None: p = ""
jpayne@68 811 else: p = ":"+self.port
jpayne@68 812 limit = self.domain + p + self.path
jpayne@68 813 if self.value is not None:
jpayne@68 814 namevalue = "%s=%s" % (self.name, self.value)
jpayne@68 815 else:
jpayne@68 816 namevalue = self.name
jpayne@68 817 return "<Cookie %s for %s>" % (namevalue, limit)
jpayne@68 818
jpayne@68 819 def __repr__(self):
jpayne@68 820 args = []
jpayne@68 821 for name in ("version", "name", "value",
jpayne@68 822 "port", "port_specified",
jpayne@68 823 "domain", "domain_specified", "domain_initial_dot",
jpayne@68 824 "path", "path_specified",
jpayne@68 825 "secure", "expires", "discard", "comment", "comment_url",
jpayne@68 826 ):
jpayne@68 827 attr = getattr(self, name)
jpayne@68 828 args.append("%s=%s" % (name, repr(attr)))
jpayne@68 829 args.append("rest=%s" % repr(self._rest))
jpayne@68 830 args.append("rfc2109=%s" % repr(self.rfc2109))
jpayne@68 831 return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
jpayne@68 832
jpayne@68 833
jpayne@68 834 class CookiePolicy:
jpayne@68 835 """Defines which cookies get accepted from and returned to server.
jpayne@68 836
jpayne@68 837 May also modify cookies, though this is probably a bad idea.
jpayne@68 838
jpayne@68 839 The subclass DefaultCookiePolicy defines the standard rules for Netscape
jpayne@68 840 and RFC 2965 cookies -- override that if you want a customized policy.
jpayne@68 841
jpayne@68 842 """
jpayne@68 843 def set_ok(self, cookie, request):
jpayne@68 844 """Return true if (and only if) cookie should be accepted from server.
jpayne@68 845
jpayne@68 846 Currently, pre-expired cookies never get this far -- the CookieJar
jpayne@68 847 class deletes such cookies itself.
jpayne@68 848
jpayne@68 849 """
jpayne@68 850 raise NotImplementedError()
jpayne@68 851
jpayne@68 852 def return_ok(self, cookie, request):
jpayne@68 853 """Return true if (and only if) cookie should be returned to server."""
jpayne@68 854 raise NotImplementedError()
jpayne@68 855
jpayne@68 856 def domain_return_ok(self, domain, request):
jpayne@68 857 """Return false if cookies should not be returned, given cookie domain.
jpayne@68 858 """
jpayne@68 859 return True
jpayne@68 860
jpayne@68 861 def path_return_ok(self, path, request):
jpayne@68 862 """Return false if cookies should not be returned, given cookie path.
jpayne@68 863 """
jpayne@68 864 return True
jpayne@68 865
jpayne@68 866
jpayne@68 867 class DefaultCookiePolicy(CookiePolicy):
jpayne@68 868 """Implements the standard rules for accepting and returning cookies."""
jpayne@68 869
jpayne@68 870 DomainStrictNoDots = 1
jpayne@68 871 DomainStrictNonDomain = 2
jpayne@68 872 DomainRFC2965Match = 4
jpayne@68 873
jpayne@68 874 DomainLiberal = 0
jpayne@68 875 DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
jpayne@68 876
jpayne@68 877 def __init__(self,
jpayne@68 878 blocked_domains=None, allowed_domains=None,
jpayne@68 879 netscape=True, rfc2965=False,
jpayne@68 880 rfc2109_as_netscape=None,
jpayne@68 881 hide_cookie2=False,
jpayne@68 882 strict_domain=False,
jpayne@68 883 strict_rfc2965_unverifiable=True,
jpayne@68 884 strict_ns_unverifiable=False,
jpayne@68 885 strict_ns_domain=DomainLiberal,
jpayne@68 886 strict_ns_set_initial_dollar=False,
jpayne@68 887 strict_ns_set_path=False,
jpayne@68 888 secure_protocols=("https", "wss")
jpayne@68 889 ):
jpayne@68 890 """Constructor arguments should be passed as keyword arguments only."""
jpayne@68 891 self.netscape = netscape
jpayne@68 892 self.rfc2965 = rfc2965
jpayne@68 893 self.rfc2109_as_netscape = rfc2109_as_netscape
jpayne@68 894 self.hide_cookie2 = hide_cookie2
jpayne@68 895 self.strict_domain = strict_domain
jpayne@68 896 self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
jpayne@68 897 self.strict_ns_unverifiable = strict_ns_unverifiable
jpayne@68 898 self.strict_ns_domain = strict_ns_domain
jpayne@68 899 self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
jpayne@68 900 self.strict_ns_set_path = strict_ns_set_path
jpayne@68 901 self.secure_protocols = secure_protocols
jpayne@68 902
jpayne@68 903 if blocked_domains is not None:
jpayne@68 904 self._blocked_domains = tuple(blocked_domains)
jpayne@68 905 else:
jpayne@68 906 self._blocked_domains = ()
jpayne@68 907
jpayne@68 908 if allowed_domains is not None:
jpayne@68 909 allowed_domains = tuple(allowed_domains)
jpayne@68 910 self._allowed_domains = allowed_domains
jpayne@68 911
jpayne@68 912 def blocked_domains(self):
jpayne@68 913 """Return the sequence of blocked domains (as a tuple)."""
jpayne@68 914 return self._blocked_domains
jpayne@68 915 def set_blocked_domains(self, blocked_domains):
jpayne@68 916 """Set the sequence of blocked domains."""
jpayne@68 917 self._blocked_domains = tuple(blocked_domains)
jpayne@68 918
jpayne@68 919 def is_blocked(self, domain):
jpayne@68 920 for blocked_domain in self._blocked_domains:
jpayne@68 921 if user_domain_match(domain, blocked_domain):
jpayne@68 922 return True
jpayne@68 923 return False
jpayne@68 924
jpayne@68 925 def allowed_domains(self):
jpayne@68 926 """Return None, or the sequence of allowed domains (as a tuple)."""
jpayne@68 927 return self._allowed_domains
jpayne@68 928 def set_allowed_domains(self, allowed_domains):
jpayne@68 929 """Set the sequence of allowed domains, or None."""
jpayne@68 930 if allowed_domains is not None:
jpayne@68 931 allowed_domains = tuple(allowed_domains)
jpayne@68 932 self._allowed_domains = allowed_domains
jpayne@68 933
jpayne@68 934 def is_not_allowed(self, domain):
jpayne@68 935 if self._allowed_domains is None:
jpayne@68 936 return False
jpayne@68 937 for allowed_domain in self._allowed_domains:
jpayne@68 938 if user_domain_match(domain, allowed_domain):
jpayne@68 939 return False
jpayne@68 940 return True
jpayne@68 941
jpayne@68 942 def set_ok(self, cookie, request):
jpayne@68 943 """
jpayne@68 944 If you override .set_ok(), be sure to call this method. If it returns
jpayne@68 945 false, so should your subclass (assuming your subclass wants to be more
jpayne@68 946 strict about which cookies to accept).
jpayne@68 947
jpayne@68 948 """
jpayne@68 949 _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
jpayne@68 950
jpayne@68 951 assert cookie.name is not None
jpayne@68 952
jpayne@68 953 for n in "version", "verifiability", "name", "path", "domain", "port":
jpayne@68 954 fn_name = "set_ok_"+n
jpayne@68 955 fn = getattr(self, fn_name)
jpayne@68 956 if not fn(cookie, request):
jpayne@68 957 return False
jpayne@68 958
jpayne@68 959 return True
jpayne@68 960
jpayne@68 961 def set_ok_version(self, cookie, request):
jpayne@68 962 if cookie.version is None:
jpayne@68 963 # Version is always set to 0 by parse_ns_headers if it's a Netscape
jpayne@68 964 # cookie, so this must be an invalid RFC 2965 cookie.
jpayne@68 965 _debug(" Set-Cookie2 without version attribute (%s=%s)",
jpayne@68 966 cookie.name, cookie.value)
jpayne@68 967 return False
jpayne@68 968 if cookie.version > 0 and not self.rfc2965:
jpayne@68 969 _debug(" RFC 2965 cookies are switched off")
jpayne@68 970 return False
jpayne@68 971 elif cookie.version == 0 and not self.netscape:
jpayne@68 972 _debug(" Netscape cookies are switched off")
jpayne@68 973 return False
jpayne@68 974 return True
jpayne@68 975
jpayne@68 976 def set_ok_verifiability(self, cookie, request):
jpayne@68 977 if request.unverifiable and is_third_party(request):
jpayne@68 978 if cookie.version > 0 and self.strict_rfc2965_unverifiable:
jpayne@68 979 _debug(" third-party RFC 2965 cookie during "
jpayne@68 980 "unverifiable transaction")
jpayne@68 981 return False
jpayne@68 982 elif cookie.version == 0 and self.strict_ns_unverifiable:
jpayne@68 983 _debug(" third-party Netscape cookie during "
jpayne@68 984 "unverifiable transaction")
jpayne@68 985 return False
jpayne@68 986 return True
jpayne@68 987
jpayne@68 988 def set_ok_name(self, cookie, request):
jpayne@68 989 # Try and stop servers setting V0 cookies designed to hack other
jpayne@68 990 # servers that know both V0 and V1 protocols.
jpayne@68 991 if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
jpayne@68 992 cookie.name.startswith("$")):
jpayne@68 993 _debug(" illegal name (starts with '$'): '%s'", cookie.name)
jpayne@68 994 return False
jpayne@68 995 return True
jpayne@68 996
jpayne@68 997 def set_ok_path(self, cookie, request):
jpayne@68 998 if cookie.path_specified:
jpayne@68 999 req_path = request_path(request)
jpayne@68 1000 if ((cookie.version > 0 or
jpayne@68 1001 (cookie.version == 0 and self.strict_ns_set_path)) and
jpayne@68 1002 not self.path_return_ok(cookie.path, request)):
jpayne@68 1003 _debug(" path attribute %s is not a prefix of request "
jpayne@68 1004 "path %s", cookie.path, req_path)
jpayne@68 1005 return False
jpayne@68 1006 return True
jpayne@68 1007
jpayne@68 1008 def set_ok_domain(self, cookie, request):
jpayne@68 1009 if self.is_blocked(cookie.domain):
jpayne@68 1010 _debug(" domain %s is in user block-list", cookie.domain)
jpayne@68 1011 return False
jpayne@68 1012 if self.is_not_allowed(cookie.domain):
jpayne@68 1013 _debug(" domain %s is not in user allow-list", cookie.domain)
jpayne@68 1014 return False
jpayne@68 1015 if cookie.domain_specified:
jpayne@68 1016 req_host, erhn = eff_request_host(request)
jpayne@68 1017 domain = cookie.domain
jpayne@68 1018 if self.strict_domain and (domain.count(".") >= 2):
jpayne@68 1019 # XXX This should probably be compared with the Konqueror
jpayne@68 1020 # (kcookiejar.cpp) and Mozilla implementations, but it's a
jpayne@68 1021 # losing battle.
jpayne@68 1022 i = domain.rfind(".")
jpayne@68 1023 j = domain.rfind(".", 0, i)
jpayne@68 1024 if j == 0: # domain like .foo.bar
jpayne@68 1025 tld = domain[i+1:]
jpayne@68 1026 sld = domain[j+1:i]
jpayne@68 1027 if sld.lower() in ("co", "ac", "com", "edu", "org", "net",
jpayne@68 1028 "gov", "mil", "int", "aero", "biz", "cat", "coop",
jpayne@68 1029 "info", "jobs", "mobi", "museum", "name", "pro",
jpayne@68 1030 "travel", "eu") and len(tld) == 2:
jpayne@68 1031 # domain like .co.uk
jpayne@68 1032 _debug(" country-code second level domain %s", domain)
jpayne@68 1033 return False
jpayne@68 1034 if domain.startswith("."):
jpayne@68 1035 undotted_domain = domain[1:]
jpayne@68 1036 else:
jpayne@68 1037 undotted_domain = domain
jpayne@68 1038 embedded_dots = (undotted_domain.find(".") >= 0)
jpayne@68 1039 if not embedded_dots and domain != ".local":
jpayne@68 1040 _debug(" non-local domain %s contains no embedded dot",
jpayne@68 1041 domain)
jpayne@68 1042 return False
jpayne@68 1043 if cookie.version == 0:
jpayne@68 1044 if (not erhn.endswith(domain) and
jpayne@68 1045 (not erhn.startswith(".") and
jpayne@68 1046 not ("."+erhn).endswith(domain))):
jpayne@68 1047 _debug(" effective request-host %s (even with added "
jpayne@68 1048 "initial dot) does not end with %s",
jpayne@68 1049 erhn, domain)
jpayne@68 1050 return False
jpayne@68 1051 if (cookie.version > 0 or
jpayne@68 1052 (self.strict_ns_domain & self.DomainRFC2965Match)):
jpayne@68 1053 if not domain_match(erhn, domain):
jpayne@68 1054 _debug(" effective request-host %s does not domain-match "
jpayne@68 1055 "%s", erhn, domain)
jpayne@68 1056 return False
jpayne@68 1057 if (cookie.version > 0 or
jpayne@68 1058 (self.strict_ns_domain & self.DomainStrictNoDots)):
jpayne@68 1059 host_prefix = req_host[:-len(domain)]
jpayne@68 1060 if (host_prefix.find(".") >= 0 and
jpayne@68 1061 not IPV4_RE.search(req_host)):
jpayne@68 1062 _debug(" host prefix %s for domain %s contains a dot",
jpayne@68 1063 host_prefix, domain)
jpayne@68 1064 return False
jpayne@68 1065 return True
jpayne@68 1066
jpayne@68 1067 def set_ok_port(self, cookie, request):
jpayne@68 1068 if cookie.port_specified:
jpayne@68 1069 req_port = request_port(request)
jpayne@68 1070 if req_port is None:
jpayne@68 1071 req_port = "80"
jpayne@68 1072 else:
jpayne@68 1073 req_port = str(req_port)
jpayne@68 1074 for p in cookie.port.split(","):
jpayne@68 1075 try:
jpayne@68 1076 int(p)
jpayne@68 1077 except ValueError:
jpayne@68 1078 _debug(" bad port %s (not numeric)", p)
jpayne@68 1079 return False
jpayne@68 1080 if p == req_port:
jpayne@68 1081 break
jpayne@68 1082 else:
jpayne@68 1083 _debug(" request port (%s) not found in %s",
jpayne@68 1084 req_port, cookie.port)
jpayne@68 1085 return False
jpayne@68 1086 return True
jpayne@68 1087
jpayne@68 1088 def return_ok(self, cookie, request):
jpayne@68 1089 """
jpayne@68 1090 If you override .return_ok(), be sure to call this method. If it
jpayne@68 1091 returns false, so should your subclass (assuming your subclass wants to
jpayne@68 1092 be more strict about which cookies to return).
jpayne@68 1093
jpayne@68 1094 """
jpayne@68 1095 # Path has already been checked by .path_return_ok(), and domain
jpayne@68 1096 # blocking done by .domain_return_ok().
jpayne@68 1097 _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
jpayne@68 1098
jpayne@68 1099 for n in "version", "verifiability", "secure", "expires", "port", "domain":
jpayne@68 1100 fn_name = "return_ok_"+n
jpayne@68 1101 fn = getattr(self, fn_name)
jpayne@68 1102 if not fn(cookie, request):
jpayne@68 1103 return False
jpayne@68 1104 return True
jpayne@68 1105
jpayne@68 1106 def return_ok_version(self, cookie, request):
jpayne@68 1107 if cookie.version > 0 and not self.rfc2965:
jpayne@68 1108 _debug(" RFC 2965 cookies are switched off")
jpayne@68 1109 return False
jpayne@68 1110 elif cookie.version == 0 and not self.netscape:
jpayne@68 1111 _debug(" Netscape cookies are switched off")
jpayne@68 1112 return False
jpayne@68 1113 return True
jpayne@68 1114
jpayne@68 1115 def return_ok_verifiability(self, cookie, request):
jpayne@68 1116 if request.unverifiable and is_third_party(request):
jpayne@68 1117 if cookie.version > 0 and self.strict_rfc2965_unverifiable:
jpayne@68 1118 _debug(" third-party RFC 2965 cookie during unverifiable "
jpayne@68 1119 "transaction")
jpayne@68 1120 return False
jpayne@68 1121 elif cookie.version == 0 and self.strict_ns_unverifiable:
jpayne@68 1122 _debug(" third-party Netscape cookie during unverifiable "
jpayne@68 1123 "transaction")
jpayne@68 1124 return False
jpayne@68 1125 return True
jpayne@68 1126
jpayne@68 1127 def return_ok_secure(self, cookie, request):
jpayne@68 1128 if cookie.secure and request.type not in self.secure_protocols:
jpayne@68 1129 _debug(" secure cookie with non-secure request")
jpayne@68 1130 return False
jpayne@68 1131 return True
jpayne@68 1132
jpayne@68 1133 def return_ok_expires(self, cookie, request):
jpayne@68 1134 if cookie.is_expired(self._now):
jpayne@68 1135 _debug(" cookie expired")
jpayne@68 1136 return False
jpayne@68 1137 return True
jpayne@68 1138
jpayne@68 1139 def return_ok_port(self, cookie, request):
jpayne@68 1140 if cookie.port:
jpayne@68 1141 req_port = request_port(request)
jpayne@68 1142 if req_port is None:
jpayne@68 1143 req_port = "80"
jpayne@68 1144 for p in cookie.port.split(","):
jpayne@68 1145 if p == req_port:
jpayne@68 1146 break
jpayne@68 1147 else:
jpayne@68 1148 _debug(" request port %s does not match cookie port %s",
jpayne@68 1149 req_port, cookie.port)
jpayne@68 1150 return False
jpayne@68 1151 return True
jpayne@68 1152
jpayne@68 1153 def return_ok_domain(self, cookie, request):
jpayne@68 1154 req_host, erhn = eff_request_host(request)
jpayne@68 1155 domain = cookie.domain
jpayne@68 1156
jpayne@68 1157 if domain and not domain.startswith("."):
jpayne@68 1158 dotdomain = "." + domain
jpayne@68 1159 else:
jpayne@68 1160 dotdomain = domain
jpayne@68 1161
jpayne@68 1162 # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
jpayne@68 1163 if (cookie.version == 0 and
jpayne@68 1164 (self.strict_ns_domain & self.DomainStrictNonDomain) and
jpayne@68 1165 not cookie.domain_specified and domain != erhn):
jpayne@68 1166 _debug(" cookie with unspecified domain does not string-compare "
jpayne@68 1167 "equal to request domain")
jpayne@68 1168 return False
jpayne@68 1169
jpayne@68 1170 if cookie.version > 0 and not domain_match(erhn, domain):
jpayne@68 1171 _debug(" effective request-host name %s does not domain-match "
jpayne@68 1172 "RFC 2965 cookie domain %s", erhn, domain)
jpayne@68 1173 return False
jpayne@68 1174 if cookie.version == 0 and not ("."+erhn).endswith(dotdomain):
jpayne@68 1175 _debug(" request-host %s does not match Netscape cookie domain "
jpayne@68 1176 "%s", req_host, domain)
jpayne@68 1177 return False
jpayne@68 1178 return True
jpayne@68 1179
jpayne@68 1180 def domain_return_ok(self, domain, request):
jpayne@68 1181 # Liberal check of. This is here as an optimization to avoid
jpayne@68 1182 # having to load lots of MSIE cookie files unless necessary.
jpayne@68 1183 req_host, erhn = eff_request_host(request)
jpayne@68 1184 if not req_host.startswith("."):
jpayne@68 1185 req_host = "."+req_host
jpayne@68 1186 if not erhn.startswith("."):
jpayne@68 1187 erhn = "."+erhn
jpayne@68 1188 if domain and not domain.startswith("."):
jpayne@68 1189 dotdomain = "." + domain
jpayne@68 1190 else:
jpayne@68 1191 dotdomain = domain
jpayne@68 1192 if not (req_host.endswith(dotdomain) or erhn.endswith(dotdomain)):
jpayne@68 1193 #_debug(" request domain %s does not match cookie domain %s",
jpayne@68 1194 # req_host, domain)
jpayne@68 1195 return False
jpayne@68 1196
jpayne@68 1197 if self.is_blocked(domain):
jpayne@68 1198 _debug(" domain %s is in user block-list", domain)
jpayne@68 1199 return False
jpayne@68 1200 if self.is_not_allowed(domain):
jpayne@68 1201 _debug(" domain %s is not in user allow-list", domain)
jpayne@68 1202 return False
jpayne@68 1203
jpayne@68 1204 return True
jpayne@68 1205
jpayne@68 1206 def path_return_ok(self, path, request):
jpayne@68 1207 _debug("- checking cookie path=%s", path)
jpayne@68 1208 req_path = request_path(request)
jpayne@68 1209 pathlen = len(path)
jpayne@68 1210 if req_path == path:
jpayne@68 1211 return True
jpayne@68 1212 elif (req_path.startswith(path) and
jpayne@68 1213 (path.endswith("/") or req_path[pathlen:pathlen+1] == "/")):
jpayne@68 1214 return True
jpayne@68 1215
jpayne@68 1216 _debug(" %s does not path-match %s", req_path, path)
jpayne@68 1217 return False
jpayne@68 1218
jpayne@68 1219 def vals_sorted_by_key(adict):
jpayne@68 1220 keys = sorted(adict.keys())
jpayne@68 1221 return map(adict.get, keys)
jpayne@68 1222
jpayne@68 1223 def deepvalues(mapping):
jpayne@68 1224 """Iterates over nested mapping, depth-first, in sorted order by key."""
jpayne@68 1225 values = vals_sorted_by_key(mapping)
jpayne@68 1226 for obj in values:
jpayne@68 1227 mapping = False
jpayne@68 1228 try:
jpayne@68 1229 obj.items
jpayne@68 1230 except AttributeError:
jpayne@68 1231 pass
jpayne@68 1232 else:
jpayne@68 1233 mapping = True
jpayne@68 1234 yield from deepvalues(obj)
jpayne@68 1235 if not mapping:
jpayne@68 1236 yield obj
jpayne@68 1237
jpayne@68 1238
jpayne@68 1239 # Used as second parameter to dict.get() method, to distinguish absent
jpayne@68 1240 # dict key from one with a None value.
jpayne@68 1241 class Absent: pass
jpayne@68 1242
jpayne@68 1243 class CookieJar:
jpayne@68 1244 """Collection of HTTP cookies.
jpayne@68 1245
jpayne@68 1246 You may not need to know about this class: try
jpayne@68 1247 urllib.request.build_opener(HTTPCookieProcessor).open(url).
jpayne@68 1248 """
jpayne@68 1249
jpayne@68 1250 non_word_re = re.compile(r"\W")
jpayne@68 1251 quote_re = re.compile(r"([\"\\])")
jpayne@68 1252 strict_domain_re = re.compile(r"\.?[^.]*")
jpayne@68 1253 domain_re = re.compile(r"[^.]*")
jpayne@68 1254 dots_re = re.compile(r"^\.+")
jpayne@68 1255
jpayne@68 1256 magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII)
jpayne@68 1257
jpayne@68 1258 def __init__(self, policy=None):
jpayne@68 1259 if policy is None:
jpayne@68 1260 policy = DefaultCookiePolicy()
jpayne@68 1261 self._policy = policy
jpayne@68 1262
jpayne@68 1263 self._cookies_lock = _threading.RLock()
jpayne@68 1264 self._cookies = {}
jpayne@68 1265
jpayne@68 1266 def set_policy(self, policy):
jpayne@68 1267 self._policy = policy
jpayne@68 1268
jpayne@68 1269 def _cookies_for_domain(self, domain, request):
jpayne@68 1270 cookies = []
jpayne@68 1271 if not self._policy.domain_return_ok(domain, request):
jpayne@68 1272 return []
jpayne@68 1273 _debug("Checking %s for cookies to return", domain)
jpayne@68 1274 cookies_by_path = self._cookies[domain]
jpayne@68 1275 for path in cookies_by_path.keys():
jpayne@68 1276 if not self._policy.path_return_ok(path, request):
jpayne@68 1277 continue
jpayne@68 1278 cookies_by_name = cookies_by_path[path]
jpayne@68 1279 for cookie in cookies_by_name.values():
jpayne@68 1280 if not self._policy.return_ok(cookie, request):
jpayne@68 1281 _debug(" not returning cookie")
jpayne@68 1282 continue
jpayne@68 1283 _debug(" it's a match")
jpayne@68 1284 cookies.append(cookie)
jpayne@68 1285 return cookies
jpayne@68 1286
jpayne@68 1287 def _cookies_for_request(self, request):
jpayne@68 1288 """Return a list of cookies to be returned to server."""
jpayne@68 1289 cookies = []
jpayne@68 1290 for domain in self._cookies.keys():
jpayne@68 1291 cookies.extend(self._cookies_for_domain(domain, request))
jpayne@68 1292 return cookies
jpayne@68 1293
jpayne@68 1294 def _cookie_attrs(self, cookies):
jpayne@68 1295 """Return a list of cookie-attributes to be returned to server.
jpayne@68 1296
jpayne@68 1297 like ['foo="bar"; $Path="/"', ...]
jpayne@68 1298
jpayne@68 1299 The $Version attribute is also added when appropriate (currently only
jpayne@68 1300 once per request).
jpayne@68 1301
jpayne@68 1302 """
jpayne@68 1303 # add cookies in order of most specific (ie. longest) path first
jpayne@68 1304 cookies.sort(key=lambda a: len(a.path), reverse=True)
jpayne@68 1305
jpayne@68 1306 version_set = False
jpayne@68 1307
jpayne@68 1308 attrs = []
jpayne@68 1309 for cookie in cookies:
jpayne@68 1310 # set version of Cookie header
jpayne@68 1311 # XXX
jpayne@68 1312 # What should it be if multiple matching Set-Cookie headers have
jpayne@68 1313 # different versions themselves?
jpayne@68 1314 # Answer: there is no answer; was supposed to be settled by
jpayne@68 1315 # RFC 2965 errata, but that may never appear...
jpayne@68 1316 version = cookie.version
jpayne@68 1317 if not version_set:
jpayne@68 1318 version_set = True
jpayne@68 1319 if version > 0:
jpayne@68 1320 attrs.append("$Version=%s" % version)
jpayne@68 1321
jpayne@68 1322 # quote cookie value if necessary
jpayne@68 1323 # (not for Netscape protocol, which already has any quotes
jpayne@68 1324 # intact, due to the poorly-specified Netscape Cookie: syntax)
jpayne@68 1325 if ((cookie.value is not None) and
jpayne@68 1326 self.non_word_re.search(cookie.value) and version > 0):
jpayne@68 1327 value = self.quote_re.sub(r"\\\1", cookie.value)
jpayne@68 1328 else:
jpayne@68 1329 value = cookie.value
jpayne@68 1330
jpayne@68 1331 # add cookie-attributes to be returned in Cookie header
jpayne@68 1332 if cookie.value is None:
jpayne@68 1333 attrs.append(cookie.name)
jpayne@68 1334 else:
jpayne@68 1335 attrs.append("%s=%s" % (cookie.name, value))
jpayne@68 1336 if version > 0:
jpayne@68 1337 if cookie.path_specified:
jpayne@68 1338 attrs.append('$Path="%s"' % cookie.path)
jpayne@68 1339 if cookie.domain.startswith("."):
jpayne@68 1340 domain = cookie.domain
jpayne@68 1341 if (not cookie.domain_initial_dot and
jpayne@68 1342 domain.startswith(".")):
jpayne@68 1343 domain = domain[1:]
jpayne@68 1344 attrs.append('$Domain="%s"' % domain)
jpayne@68 1345 if cookie.port is not None:
jpayne@68 1346 p = "$Port"
jpayne@68 1347 if cookie.port_specified:
jpayne@68 1348 p = p + ('="%s"' % cookie.port)
jpayne@68 1349 attrs.append(p)
jpayne@68 1350
jpayne@68 1351 return attrs
jpayne@68 1352
jpayne@68 1353 def add_cookie_header(self, request):
jpayne@68 1354 """Add correct Cookie: header to request (urllib.request.Request object).
jpayne@68 1355
jpayne@68 1356 The Cookie2 header is also added unless policy.hide_cookie2 is true.
jpayne@68 1357
jpayne@68 1358 """
jpayne@68 1359 _debug("add_cookie_header")
jpayne@68 1360 self._cookies_lock.acquire()
jpayne@68 1361 try:
jpayne@68 1362
jpayne@68 1363 self._policy._now = self._now = int(time.time())
jpayne@68 1364
jpayne@68 1365 cookies = self._cookies_for_request(request)
jpayne@68 1366
jpayne@68 1367 attrs = self._cookie_attrs(cookies)
jpayne@68 1368 if attrs:
jpayne@68 1369 if not request.has_header("Cookie"):
jpayne@68 1370 request.add_unredirected_header(
jpayne@68 1371 "Cookie", "; ".join(attrs))
jpayne@68 1372
jpayne@68 1373 # if necessary, advertise that we know RFC 2965
jpayne@68 1374 if (self._policy.rfc2965 and not self._policy.hide_cookie2 and
jpayne@68 1375 not request.has_header("Cookie2")):
jpayne@68 1376 for cookie in cookies:
jpayne@68 1377 if cookie.version != 1:
jpayne@68 1378 request.add_unredirected_header("Cookie2", '$Version="1"')
jpayne@68 1379 break
jpayne@68 1380
jpayne@68 1381 finally:
jpayne@68 1382 self._cookies_lock.release()
jpayne@68 1383
jpayne@68 1384 self.clear_expired_cookies()
jpayne@68 1385
jpayne@68 1386 def _normalized_cookie_tuples(self, attrs_set):
jpayne@68 1387 """Return list of tuples containing normalised cookie information.
jpayne@68 1388
jpayne@68 1389 attrs_set is the list of lists of key,value pairs extracted from
jpayne@68 1390 the Set-Cookie or Set-Cookie2 headers.
jpayne@68 1391
jpayne@68 1392 Tuples are name, value, standard, rest, where name and value are the
jpayne@68 1393 cookie name and value, standard is a dictionary containing the standard
jpayne@68 1394 cookie-attributes (discard, secure, version, expires or max-age,
jpayne@68 1395 domain, path and port) and rest is a dictionary containing the rest of
jpayne@68 1396 the cookie-attributes.
jpayne@68 1397
jpayne@68 1398 """
jpayne@68 1399 cookie_tuples = []
jpayne@68 1400
jpayne@68 1401 boolean_attrs = "discard", "secure"
jpayne@68 1402 value_attrs = ("version",
jpayne@68 1403 "expires", "max-age",
jpayne@68 1404 "domain", "path", "port",
jpayne@68 1405 "comment", "commenturl")
jpayne@68 1406
jpayne@68 1407 for cookie_attrs in attrs_set:
jpayne@68 1408 name, value = cookie_attrs[0]
jpayne@68 1409
jpayne@68 1410 # Build dictionary of standard cookie-attributes (standard) and
jpayne@68 1411 # dictionary of other cookie-attributes (rest).
jpayne@68 1412
jpayne@68 1413 # Note: expiry time is normalised to seconds since epoch. V0
jpayne@68 1414 # cookies should have the Expires cookie-attribute, and V1 cookies
jpayne@68 1415 # should have Max-Age, but since V1 includes RFC 2109 cookies (and
jpayne@68 1416 # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
jpayne@68 1417 # accept either (but prefer Max-Age).
jpayne@68 1418 max_age_set = False
jpayne@68 1419
jpayne@68 1420 bad_cookie = False
jpayne@68 1421
jpayne@68 1422 standard = {}
jpayne@68 1423 rest = {}
jpayne@68 1424 for k, v in cookie_attrs[1:]:
jpayne@68 1425 lc = k.lower()
jpayne@68 1426 # don't lose case distinction for unknown fields
jpayne@68 1427 if lc in value_attrs or lc in boolean_attrs:
jpayne@68 1428 k = lc
jpayne@68 1429 if k in boolean_attrs and v is None:
jpayne@68 1430 # boolean cookie-attribute is present, but has no value
jpayne@68 1431 # (like "discard", rather than "port=80")
jpayne@68 1432 v = True
jpayne@68 1433 if k in standard:
jpayne@68 1434 # only first value is significant
jpayne@68 1435 continue
jpayne@68 1436 if k == "domain":
jpayne@68 1437 if v is None:
jpayne@68 1438 _debug(" missing value for domain attribute")
jpayne@68 1439 bad_cookie = True
jpayne@68 1440 break
jpayne@68 1441 # RFC 2965 section 3.3.3
jpayne@68 1442 v = v.lower()
jpayne@68 1443 if k == "expires":
jpayne@68 1444 if max_age_set:
jpayne@68 1445 # Prefer max-age to expires (like Mozilla)
jpayne@68 1446 continue
jpayne@68 1447 if v is None:
jpayne@68 1448 _debug(" missing or invalid value for expires "
jpayne@68 1449 "attribute: treating as session cookie")
jpayne@68 1450 continue
jpayne@68 1451 if k == "max-age":
jpayne@68 1452 max_age_set = True
jpayne@68 1453 try:
jpayne@68 1454 v = int(v)
jpayne@68 1455 except ValueError:
jpayne@68 1456 _debug(" missing or invalid (non-numeric) value for "
jpayne@68 1457 "max-age attribute")
jpayne@68 1458 bad_cookie = True
jpayne@68 1459 break
jpayne@68 1460 # convert RFC 2965 Max-Age to seconds since epoch
jpayne@68 1461 # XXX Strictly you're supposed to follow RFC 2616
jpayne@68 1462 # age-calculation rules. Remember that zero Max-Age
jpayne@68 1463 # is a request to discard (old and new) cookie, though.
jpayne@68 1464 k = "expires"
jpayne@68 1465 v = self._now + v
jpayne@68 1466 if (k in value_attrs) or (k in boolean_attrs):
jpayne@68 1467 if (v is None and
jpayne@68 1468 k not in ("port", "comment", "commenturl")):
jpayne@68 1469 _debug(" missing value for %s attribute" % k)
jpayne@68 1470 bad_cookie = True
jpayne@68 1471 break
jpayne@68 1472 standard[k] = v
jpayne@68 1473 else:
jpayne@68 1474 rest[k] = v
jpayne@68 1475
jpayne@68 1476 if bad_cookie:
jpayne@68 1477 continue
jpayne@68 1478
jpayne@68 1479 cookie_tuples.append((name, value, standard, rest))
jpayne@68 1480
jpayne@68 1481 return cookie_tuples
jpayne@68 1482
jpayne@68 1483 def _cookie_from_cookie_tuple(self, tup, request):
jpayne@68 1484 # standard is dict of standard cookie-attributes, rest is dict of the
jpayne@68 1485 # rest of them
jpayne@68 1486 name, value, standard, rest = tup
jpayne@68 1487
jpayne@68 1488 domain = standard.get("domain", Absent)
jpayne@68 1489 path = standard.get("path", Absent)
jpayne@68 1490 port = standard.get("port", Absent)
jpayne@68 1491 expires = standard.get("expires", Absent)
jpayne@68 1492
jpayne@68 1493 # set the easy defaults
jpayne@68 1494 version = standard.get("version", None)
jpayne@68 1495 if version is not None:
jpayne@68 1496 try:
jpayne@68 1497 version = int(version)
jpayne@68 1498 except ValueError:
jpayne@68 1499 return None # invalid version, ignore cookie
jpayne@68 1500 secure = standard.get("secure", False)
jpayne@68 1501 # (discard is also set if expires is Absent)
jpayne@68 1502 discard = standard.get("discard", False)
jpayne@68 1503 comment = standard.get("comment", None)
jpayne@68 1504 comment_url = standard.get("commenturl", None)
jpayne@68 1505
jpayne@68 1506 # set default path
jpayne@68 1507 if path is not Absent and path != "":
jpayne@68 1508 path_specified = True
jpayne@68 1509 path = escape_path(path)
jpayne@68 1510 else:
jpayne@68 1511 path_specified = False
jpayne@68 1512 path = request_path(request)
jpayne@68 1513 i = path.rfind("/")
jpayne@68 1514 if i != -1:
jpayne@68 1515 if version == 0:
jpayne@68 1516 # Netscape spec parts company from reality here
jpayne@68 1517 path = path[:i]
jpayne@68 1518 else:
jpayne@68 1519 path = path[:i+1]
jpayne@68 1520 if len(path) == 0: path = "/"
jpayne@68 1521
jpayne@68 1522 # set default domain
jpayne@68 1523 domain_specified = domain is not Absent
jpayne@68 1524 # but first we have to remember whether it starts with a dot
jpayne@68 1525 domain_initial_dot = False
jpayne@68 1526 if domain_specified:
jpayne@68 1527 domain_initial_dot = bool(domain.startswith("."))
jpayne@68 1528 if domain is Absent:
jpayne@68 1529 req_host, erhn = eff_request_host(request)
jpayne@68 1530 domain = erhn
jpayne@68 1531 elif not domain.startswith("."):
jpayne@68 1532 domain = "."+domain
jpayne@68 1533
jpayne@68 1534 # set default port
jpayne@68 1535 port_specified = False
jpayne@68 1536 if port is not Absent:
jpayne@68 1537 if port is None:
jpayne@68 1538 # Port attr present, but has no value: default to request port.
jpayne@68 1539 # Cookie should then only be sent back on that port.
jpayne@68 1540 port = request_port(request)
jpayne@68 1541 else:
jpayne@68 1542 port_specified = True
jpayne@68 1543 port = re.sub(r"\s+", "", port)
jpayne@68 1544 else:
jpayne@68 1545 # No port attr present. Cookie can be sent back on any port.
jpayne@68 1546 port = None
jpayne@68 1547
jpayne@68 1548 # set default expires and discard
jpayne@68 1549 if expires is Absent:
jpayne@68 1550 expires = None
jpayne@68 1551 discard = True
jpayne@68 1552 elif expires <= self._now:
jpayne@68 1553 # Expiry date in past is request to delete cookie. This can't be
jpayne@68 1554 # in DefaultCookiePolicy, because can't delete cookies there.
jpayne@68 1555 try:
jpayne@68 1556 self.clear(domain, path, name)
jpayne@68 1557 except KeyError:
jpayne@68 1558 pass
jpayne@68 1559 _debug("Expiring cookie, domain='%s', path='%s', name='%s'",
jpayne@68 1560 domain, path, name)
jpayne@68 1561 return None
jpayne@68 1562
jpayne@68 1563 return Cookie(version,
jpayne@68 1564 name, value,
jpayne@68 1565 port, port_specified,
jpayne@68 1566 domain, domain_specified, domain_initial_dot,
jpayne@68 1567 path, path_specified,
jpayne@68 1568 secure,
jpayne@68 1569 expires,
jpayne@68 1570 discard,
jpayne@68 1571 comment,
jpayne@68 1572 comment_url,
jpayne@68 1573 rest)
jpayne@68 1574
jpayne@68 1575 def _cookies_from_attrs_set(self, attrs_set, request):
jpayne@68 1576 cookie_tuples = self._normalized_cookie_tuples(attrs_set)
jpayne@68 1577
jpayne@68 1578 cookies = []
jpayne@68 1579 for tup in cookie_tuples:
jpayne@68 1580 cookie = self._cookie_from_cookie_tuple(tup, request)
jpayne@68 1581 if cookie: cookies.append(cookie)
jpayne@68 1582 return cookies
jpayne@68 1583
jpayne@68 1584 def _process_rfc2109_cookies(self, cookies):
jpayne@68 1585 rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
jpayne@68 1586 if rfc2109_as_ns is None:
jpayne@68 1587 rfc2109_as_ns = not self._policy.rfc2965
jpayne@68 1588 for cookie in cookies:
jpayne@68 1589 if cookie.version == 1:
jpayne@68 1590 cookie.rfc2109 = True
jpayne@68 1591 if rfc2109_as_ns:
jpayne@68 1592 # treat 2109 cookies as Netscape cookies rather than
jpayne@68 1593 # as RFC2965 cookies
jpayne@68 1594 cookie.version = 0
jpayne@68 1595
jpayne@68 1596 def make_cookies(self, response, request):
jpayne@68 1597 """Return sequence of Cookie objects extracted from response object."""
jpayne@68 1598 # get cookie-attributes for RFC 2965 and Netscape protocols
jpayne@68 1599 headers = response.info()
jpayne@68 1600 rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
jpayne@68 1601 ns_hdrs = headers.get_all("Set-Cookie", [])
jpayne@68 1602 self._policy._now = self._now = int(time.time())
jpayne@68 1603
jpayne@68 1604 rfc2965 = self._policy.rfc2965
jpayne@68 1605 netscape = self._policy.netscape
jpayne@68 1606
jpayne@68 1607 if ((not rfc2965_hdrs and not ns_hdrs) or
jpayne@68 1608 (not ns_hdrs and not rfc2965) or
jpayne@68 1609 (not rfc2965_hdrs and not netscape) or
jpayne@68 1610 (not netscape and not rfc2965)):
jpayne@68 1611 return [] # no relevant cookie headers: quick exit
jpayne@68 1612
jpayne@68 1613 try:
jpayne@68 1614 cookies = self._cookies_from_attrs_set(
jpayne@68 1615 split_header_words(rfc2965_hdrs), request)
jpayne@68 1616 except Exception:
jpayne@68 1617 _warn_unhandled_exception()
jpayne@68 1618 cookies = []
jpayne@68 1619
jpayne@68 1620 if ns_hdrs and netscape:
jpayne@68 1621 try:
jpayne@68 1622 # RFC 2109 and Netscape cookies
jpayne@68 1623 ns_cookies = self._cookies_from_attrs_set(
jpayne@68 1624 parse_ns_headers(ns_hdrs), request)
jpayne@68 1625 except Exception:
jpayne@68 1626 _warn_unhandled_exception()
jpayne@68 1627 ns_cookies = []
jpayne@68 1628 self._process_rfc2109_cookies(ns_cookies)
jpayne@68 1629
jpayne@68 1630 # Look for Netscape cookies (from Set-Cookie headers) that match
jpayne@68 1631 # corresponding RFC 2965 cookies (from Set-Cookie2 headers).
jpayne@68 1632 # For each match, keep the RFC 2965 cookie and ignore the Netscape
jpayne@68 1633 # cookie (RFC 2965 section 9.1). Actually, RFC 2109 cookies are
jpayne@68 1634 # bundled in with the Netscape cookies for this purpose, which is
jpayne@68 1635 # reasonable behaviour.
jpayne@68 1636 if rfc2965:
jpayne@68 1637 lookup = {}
jpayne@68 1638 for cookie in cookies:
jpayne@68 1639 lookup[(cookie.domain, cookie.path, cookie.name)] = None
jpayne@68 1640
jpayne@68 1641 def no_matching_rfc2965(ns_cookie, lookup=lookup):
jpayne@68 1642 key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
jpayne@68 1643 return key not in lookup
jpayne@68 1644 ns_cookies = filter(no_matching_rfc2965, ns_cookies)
jpayne@68 1645
jpayne@68 1646 if ns_cookies:
jpayne@68 1647 cookies.extend(ns_cookies)
jpayne@68 1648
jpayne@68 1649 return cookies
jpayne@68 1650
jpayne@68 1651 def set_cookie_if_ok(self, cookie, request):
jpayne@68 1652 """Set a cookie if policy says it's OK to do so."""
jpayne@68 1653 self._cookies_lock.acquire()
jpayne@68 1654 try:
jpayne@68 1655 self._policy._now = self._now = int(time.time())
jpayne@68 1656
jpayne@68 1657 if self._policy.set_ok(cookie, request):
jpayne@68 1658 self.set_cookie(cookie)
jpayne@68 1659
jpayne@68 1660
jpayne@68 1661 finally:
jpayne@68 1662 self._cookies_lock.release()
jpayne@68 1663
jpayne@68 1664 def set_cookie(self, cookie):
jpayne@68 1665 """Set a cookie, without checking whether or not it should be set."""
jpayne@68 1666 c = self._cookies
jpayne@68 1667 self._cookies_lock.acquire()
jpayne@68 1668 try:
jpayne@68 1669 if cookie.domain not in c: c[cookie.domain] = {}
jpayne@68 1670 c2 = c[cookie.domain]
jpayne@68 1671 if cookie.path not in c2: c2[cookie.path] = {}
jpayne@68 1672 c3 = c2[cookie.path]
jpayne@68 1673 c3[cookie.name] = cookie
jpayne@68 1674 finally:
jpayne@68 1675 self._cookies_lock.release()
jpayne@68 1676
jpayne@68 1677 def extract_cookies(self, response, request):
jpayne@68 1678 """Extract cookies from response, where allowable given the request."""
jpayne@68 1679 _debug("extract_cookies: %s", response.info())
jpayne@68 1680 self._cookies_lock.acquire()
jpayne@68 1681 try:
jpayne@68 1682 for cookie in self.make_cookies(response, request):
jpayne@68 1683 if self._policy.set_ok(cookie, request):
jpayne@68 1684 _debug(" setting cookie: %s", cookie)
jpayne@68 1685 self.set_cookie(cookie)
jpayne@68 1686 finally:
jpayne@68 1687 self._cookies_lock.release()
jpayne@68 1688
jpayne@68 1689 def clear(self, domain=None, path=None, name=None):
jpayne@68 1690 """Clear some cookies.
jpayne@68 1691
jpayne@68 1692 Invoking this method without arguments will clear all cookies. If
jpayne@68 1693 given a single argument, only cookies belonging to that domain will be
jpayne@68 1694 removed. If given two arguments, cookies belonging to the specified
jpayne@68 1695 path within that domain are removed. If given three arguments, then
jpayne@68 1696 the cookie with the specified name, path and domain is removed.
jpayne@68 1697
jpayne@68 1698 Raises KeyError if no matching cookie exists.
jpayne@68 1699
jpayne@68 1700 """
jpayne@68 1701 if name is not None:
jpayne@68 1702 if (domain is None) or (path is None):
jpayne@68 1703 raise ValueError(
jpayne@68 1704 "domain and path must be given to remove a cookie by name")
jpayne@68 1705 del self._cookies[domain][path][name]
jpayne@68 1706 elif path is not None:
jpayne@68 1707 if domain is None:
jpayne@68 1708 raise ValueError(
jpayne@68 1709 "domain must be given to remove cookies by path")
jpayne@68 1710 del self._cookies[domain][path]
jpayne@68 1711 elif domain is not None:
jpayne@68 1712 del self._cookies[domain]
jpayne@68 1713 else:
jpayne@68 1714 self._cookies = {}
jpayne@68 1715
jpayne@68 1716 def clear_session_cookies(self):
jpayne@68 1717 """Discard all session cookies.
jpayne@68 1718
jpayne@68 1719 Note that the .save() method won't save session cookies anyway, unless
jpayne@68 1720 you ask otherwise by passing a true ignore_discard argument.
jpayne@68 1721
jpayne@68 1722 """
jpayne@68 1723 self._cookies_lock.acquire()
jpayne@68 1724 try:
jpayne@68 1725 for cookie in self:
jpayne@68 1726 if cookie.discard:
jpayne@68 1727 self.clear(cookie.domain, cookie.path, cookie.name)
jpayne@68 1728 finally:
jpayne@68 1729 self._cookies_lock.release()
jpayne@68 1730
jpayne@68 1731 def clear_expired_cookies(self):
jpayne@68 1732 """Discard all expired cookies.
jpayne@68 1733
jpayne@68 1734 You probably don't need to call this method: expired cookies are never
jpayne@68 1735 sent back to the server (provided you're using DefaultCookiePolicy),
jpayne@68 1736 this method is called by CookieJar itself every so often, and the
jpayne@68 1737 .save() method won't save expired cookies anyway (unless you ask
jpayne@68 1738 otherwise by passing a true ignore_expires argument).
jpayne@68 1739
jpayne@68 1740 """
jpayne@68 1741 self._cookies_lock.acquire()
jpayne@68 1742 try:
jpayne@68 1743 now = time.time()
jpayne@68 1744 for cookie in self:
jpayne@68 1745 if cookie.is_expired(now):
jpayne@68 1746 self.clear(cookie.domain, cookie.path, cookie.name)
jpayne@68 1747 finally:
jpayne@68 1748 self._cookies_lock.release()
jpayne@68 1749
jpayne@68 1750 def __iter__(self):
jpayne@68 1751 return deepvalues(self._cookies)
jpayne@68 1752
jpayne@68 1753 def __len__(self):
jpayne@68 1754 """Return number of contained cookies."""
jpayne@68 1755 i = 0
jpayne@68 1756 for cookie in self: i = i + 1
jpayne@68 1757 return i
jpayne@68 1758
jpayne@68 1759 def __repr__(self):
jpayne@68 1760 r = []
jpayne@68 1761 for cookie in self: r.append(repr(cookie))
jpayne@68 1762 return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
jpayne@68 1763
jpayne@68 1764 def __str__(self):
jpayne@68 1765 r = []
jpayne@68 1766 for cookie in self: r.append(str(cookie))
jpayne@68 1767 return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
jpayne@68 1768
jpayne@68 1769
jpayne@68 1770 # derives from OSError for backwards-compatibility with Python 2.4.0
jpayne@68 1771 class LoadError(OSError): pass
jpayne@68 1772
jpayne@68 1773 class FileCookieJar(CookieJar):
jpayne@68 1774 """CookieJar that can be loaded from and saved to a file."""
jpayne@68 1775
jpayne@68 1776 def __init__(self, filename=None, delayload=False, policy=None):
jpayne@68 1777 """
jpayne@68 1778 Cookies are NOT loaded from the named file until either the .load() or
jpayne@68 1779 .revert() method is called.
jpayne@68 1780
jpayne@68 1781 """
jpayne@68 1782 CookieJar.__init__(self, policy)
jpayne@68 1783 if filename is not None:
jpayne@68 1784 filename = os.fspath(filename)
jpayne@68 1785 self.filename = filename
jpayne@68 1786 self.delayload = bool(delayload)
jpayne@68 1787
jpayne@68 1788 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@68 1789 """Save cookies to a file."""
jpayne@68 1790 raise NotImplementedError()
jpayne@68 1791
jpayne@68 1792 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@68 1793 """Load cookies from a file."""
jpayne@68 1794 if filename is None:
jpayne@68 1795 if self.filename is not None: filename = self.filename
jpayne@68 1796 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@68 1797
jpayne@68 1798 with open(filename) as f:
jpayne@68 1799 self._really_load(f, filename, ignore_discard, ignore_expires)
jpayne@68 1800
jpayne@68 1801 def revert(self, filename=None,
jpayne@68 1802 ignore_discard=False, ignore_expires=False):
jpayne@68 1803 """Clear all cookies and reload cookies from a saved file.
jpayne@68 1804
jpayne@68 1805 Raises LoadError (or OSError) if reversion is not successful; the
jpayne@68 1806 object's state will not be altered if this happens.
jpayne@68 1807
jpayne@68 1808 """
jpayne@68 1809 if filename is None:
jpayne@68 1810 if self.filename is not None: filename = self.filename
jpayne@68 1811 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@68 1812
jpayne@68 1813 self._cookies_lock.acquire()
jpayne@68 1814 try:
jpayne@68 1815
jpayne@68 1816 old_state = copy.deepcopy(self._cookies)
jpayne@68 1817 self._cookies = {}
jpayne@68 1818 try:
jpayne@68 1819 self.load(filename, ignore_discard, ignore_expires)
jpayne@68 1820 except OSError:
jpayne@68 1821 self._cookies = old_state
jpayne@68 1822 raise
jpayne@68 1823
jpayne@68 1824 finally:
jpayne@68 1825 self._cookies_lock.release()
jpayne@68 1826
jpayne@68 1827
jpayne@68 1828 def lwp_cookie_str(cookie):
jpayne@68 1829 """Return string representation of Cookie in the LWP cookie file format.
jpayne@68 1830
jpayne@68 1831 Actually, the format is extended a bit -- see module docstring.
jpayne@68 1832
jpayne@68 1833 """
jpayne@68 1834 h = [(cookie.name, cookie.value),
jpayne@68 1835 ("path", cookie.path),
jpayne@68 1836 ("domain", cookie.domain)]
jpayne@68 1837 if cookie.port is not None: h.append(("port", cookie.port))
jpayne@68 1838 if cookie.path_specified: h.append(("path_spec", None))
jpayne@68 1839 if cookie.port_specified: h.append(("port_spec", None))
jpayne@68 1840 if cookie.domain_initial_dot: h.append(("domain_dot", None))
jpayne@68 1841 if cookie.secure: h.append(("secure", None))
jpayne@68 1842 if cookie.expires: h.append(("expires",
jpayne@68 1843 time2isoz(float(cookie.expires))))
jpayne@68 1844 if cookie.discard: h.append(("discard", None))
jpayne@68 1845 if cookie.comment: h.append(("comment", cookie.comment))
jpayne@68 1846 if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
jpayne@68 1847
jpayne@68 1848 keys = sorted(cookie._rest.keys())
jpayne@68 1849 for k in keys:
jpayne@68 1850 h.append((k, str(cookie._rest[k])))
jpayne@68 1851
jpayne@68 1852 h.append(("version", str(cookie.version)))
jpayne@68 1853
jpayne@68 1854 return join_header_words([h])
jpayne@68 1855
jpayne@68 1856 class LWPCookieJar(FileCookieJar):
jpayne@68 1857 """
jpayne@68 1858 The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
jpayne@68 1859 "Set-Cookie3" is the format used by the libwww-perl library, not known
jpayne@68 1860 to be compatible with any browser, but which is easy to read and
jpayne@68 1861 doesn't lose information about RFC 2965 cookies.
jpayne@68 1862
jpayne@68 1863 Additional methods
jpayne@68 1864
jpayne@68 1865 as_lwp_str(ignore_discard=True, ignore_expired=True)
jpayne@68 1866
jpayne@68 1867 """
jpayne@68 1868
jpayne@68 1869 def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
jpayne@68 1870 """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers.
jpayne@68 1871
jpayne@68 1872 ignore_discard and ignore_expires: see docstring for FileCookieJar.save
jpayne@68 1873
jpayne@68 1874 """
jpayne@68 1875 now = time.time()
jpayne@68 1876 r = []
jpayne@68 1877 for cookie in self:
jpayne@68 1878 if not ignore_discard and cookie.discard:
jpayne@68 1879 continue
jpayne@68 1880 if not ignore_expires and cookie.is_expired(now):
jpayne@68 1881 continue
jpayne@68 1882 r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
jpayne@68 1883 return "\n".join(r+[""])
jpayne@68 1884
jpayne@68 1885 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@68 1886 if filename is None:
jpayne@68 1887 if self.filename is not None: filename = self.filename
jpayne@68 1888 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@68 1889
jpayne@68 1890 with open(filename, "w") as f:
jpayne@68 1891 # There really isn't an LWP Cookies 2.0 format, but this indicates
jpayne@68 1892 # that there is extra information in here (domain_dot and
jpayne@68 1893 # port_spec) while still being compatible with libwww-perl, I hope.
jpayne@68 1894 f.write("#LWP-Cookies-2.0\n")
jpayne@68 1895 f.write(self.as_lwp_str(ignore_discard, ignore_expires))
jpayne@68 1896
jpayne@68 1897 def _really_load(self, f, filename, ignore_discard, ignore_expires):
jpayne@68 1898 magic = f.readline()
jpayne@68 1899 if not self.magic_re.search(magic):
jpayne@68 1900 msg = ("%r does not look like a Set-Cookie3 (LWP) format "
jpayne@68 1901 "file" % filename)
jpayne@68 1902 raise LoadError(msg)
jpayne@68 1903
jpayne@68 1904 now = time.time()
jpayne@68 1905
jpayne@68 1906 header = "Set-Cookie3:"
jpayne@68 1907 boolean_attrs = ("port_spec", "path_spec", "domain_dot",
jpayne@68 1908 "secure", "discard")
jpayne@68 1909 value_attrs = ("version",
jpayne@68 1910 "port", "path", "domain",
jpayne@68 1911 "expires",
jpayne@68 1912 "comment", "commenturl")
jpayne@68 1913
jpayne@68 1914 try:
jpayne@68 1915 while 1:
jpayne@68 1916 line = f.readline()
jpayne@68 1917 if line == "": break
jpayne@68 1918 if not line.startswith(header):
jpayne@68 1919 continue
jpayne@68 1920 line = line[len(header):].strip()
jpayne@68 1921
jpayne@68 1922 for data in split_header_words([line]):
jpayne@68 1923 name, value = data[0]
jpayne@68 1924 standard = {}
jpayne@68 1925 rest = {}
jpayne@68 1926 for k in boolean_attrs:
jpayne@68 1927 standard[k] = False
jpayne@68 1928 for k, v in data[1:]:
jpayne@68 1929 if k is not None:
jpayne@68 1930 lc = k.lower()
jpayne@68 1931 else:
jpayne@68 1932 lc = None
jpayne@68 1933 # don't lose case distinction for unknown fields
jpayne@68 1934 if (lc in value_attrs) or (lc in boolean_attrs):
jpayne@68 1935 k = lc
jpayne@68 1936 if k in boolean_attrs:
jpayne@68 1937 if v is None: v = True
jpayne@68 1938 standard[k] = v
jpayne@68 1939 elif k in value_attrs:
jpayne@68 1940 standard[k] = v
jpayne@68 1941 else:
jpayne@68 1942 rest[k] = v
jpayne@68 1943
jpayne@68 1944 h = standard.get
jpayne@68 1945 expires = h("expires")
jpayne@68 1946 discard = h("discard")
jpayne@68 1947 if expires is not None:
jpayne@68 1948 expires = iso2time(expires)
jpayne@68 1949 if expires is None:
jpayne@68 1950 discard = True
jpayne@68 1951 domain = h("domain")
jpayne@68 1952 domain_specified = domain.startswith(".")
jpayne@68 1953 c = Cookie(h("version"), name, value,
jpayne@68 1954 h("port"), h("port_spec"),
jpayne@68 1955 domain, domain_specified, h("domain_dot"),
jpayne@68 1956 h("path"), h("path_spec"),
jpayne@68 1957 h("secure"),
jpayne@68 1958 expires,
jpayne@68 1959 discard,
jpayne@68 1960 h("comment"),
jpayne@68 1961 h("commenturl"),
jpayne@68 1962 rest)
jpayne@68 1963 if not ignore_discard and c.discard:
jpayne@68 1964 continue
jpayne@68 1965 if not ignore_expires and c.is_expired(now):
jpayne@68 1966 continue
jpayne@68 1967 self.set_cookie(c)
jpayne@68 1968 except OSError:
jpayne@68 1969 raise
jpayne@68 1970 except Exception:
jpayne@68 1971 _warn_unhandled_exception()
jpayne@68 1972 raise LoadError("invalid Set-Cookie3 format file %r: %r" %
jpayne@68 1973 (filename, line))
jpayne@68 1974
jpayne@68 1975
jpayne@68 1976 class MozillaCookieJar(FileCookieJar):
jpayne@68 1977 """
jpayne@68 1978
jpayne@68 1979 WARNING: you may want to backup your browser's cookies file if you use
jpayne@68 1980 this class to save cookies. I *think* it works, but there have been
jpayne@68 1981 bugs in the past!
jpayne@68 1982
jpayne@68 1983 This class differs from CookieJar only in the format it uses to save and
jpayne@68 1984 load cookies to and from a file. This class uses the Mozilla/Netscape
jpayne@68 1985 `cookies.txt' format. lynx uses this file format, too.
jpayne@68 1986
jpayne@68 1987 Don't expect cookies saved while the browser is running to be noticed by
jpayne@68 1988 the browser (in fact, Mozilla on unix will overwrite your saved cookies if
jpayne@68 1989 you change them on disk while it's running; on Windows, you probably can't
jpayne@68 1990 save at all while the browser is running).
jpayne@68 1991
jpayne@68 1992 Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
jpayne@68 1993 Netscape cookies on saving.
jpayne@68 1994
jpayne@68 1995 In particular, the cookie version and port number information is lost,
jpayne@68 1996 together with information about whether or not Path, Port and Discard were
jpayne@68 1997 specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
jpayne@68 1998 domain as set in the HTTP header started with a dot (yes, I'm aware some
jpayne@68 1999 domains in Netscape files start with a dot and some don't -- trust me, you
jpayne@68 2000 really don't want to know any more about this).
jpayne@68 2001
jpayne@68 2002 Note that though Mozilla and Netscape use the same format, they use
jpayne@68 2003 slightly different headers. The class saves cookies using the Netscape
jpayne@68 2004 header by default (Mozilla can cope with that).
jpayne@68 2005
jpayne@68 2006 """
jpayne@68 2007 magic_re = re.compile("#( Netscape)? HTTP Cookie File")
jpayne@68 2008 header = """\
jpayne@68 2009 # Netscape HTTP Cookie File
jpayne@68 2010 # http://curl.haxx.se/rfc/cookie_spec.html
jpayne@68 2011 # This is a generated file! Do not edit.
jpayne@68 2012
jpayne@68 2013 """
jpayne@68 2014
jpayne@68 2015 def _really_load(self, f, filename, ignore_discard, ignore_expires):
jpayne@68 2016 now = time.time()
jpayne@68 2017
jpayne@68 2018 magic = f.readline()
jpayne@68 2019 if not self.magic_re.search(magic):
jpayne@68 2020 raise LoadError(
jpayne@68 2021 "%r does not look like a Netscape format cookies file" %
jpayne@68 2022 filename)
jpayne@68 2023
jpayne@68 2024 try:
jpayne@68 2025 while 1:
jpayne@68 2026 line = f.readline()
jpayne@68 2027 if line == "": break
jpayne@68 2028
jpayne@68 2029 # last field may be absent, so keep any trailing tab
jpayne@68 2030 if line.endswith("\n"): line = line[:-1]
jpayne@68 2031
jpayne@68 2032 # skip comments and blank lines XXX what is $ for?
jpayne@68 2033 if (line.strip().startswith(("#", "$")) or
jpayne@68 2034 line.strip() == ""):
jpayne@68 2035 continue
jpayne@68 2036
jpayne@68 2037 domain, domain_specified, path, secure, expires, name, value = \
jpayne@68 2038 line.split("\t")
jpayne@68 2039 secure = (secure == "TRUE")
jpayne@68 2040 domain_specified = (domain_specified == "TRUE")
jpayne@68 2041 if name == "":
jpayne@68 2042 # cookies.txt regards 'Set-Cookie: foo' as a cookie
jpayne@68 2043 # with no name, whereas http.cookiejar regards it as a
jpayne@68 2044 # cookie with no value.
jpayne@68 2045 name = value
jpayne@68 2046 value = None
jpayne@68 2047
jpayne@68 2048 initial_dot = domain.startswith(".")
jpayne@68 2049 assert domain_specified == initial_dot
jpayne@68 2050
jpayne@68 2051 discard = False
jpayne@68 2052 if expires == "":
jpayne@68 2053 expires = None
jpayne@68 2054 discard = True
jpayne@68 2055
jpayne@68 2056 # assume path_specified is false
jpayne@68 2057 c = Cookie(0, name, value,
jpayne@68 2058 None, False,
jpayne@68 2059 domain, domain_specified, initial_dot,
jpayne@68 2060 path, False,
jpayne@68 2061 secure,
jpayne@68 2062 expires,
jpayne@68 2063 discard,
jpayne@68 2064 None,
jpayne@68 2065 None,
jpayne@68 2066 {})
jpayne@68 2067 if not ignore_discard and c.discard:
jpayne@68 2068 continue
jpayne@68 2069 if not ignore_expires and c.is_expired(now):
jpayne@68 2070 continue
jpayne@68 2071 self.set_cookie(c)
jpayne@68 2072
jpayne@68 2073 except OSError:
jpayne@68 2074 raise
jpayne@68 2075 except Exception:
jpayne@68 2076 _warn_unhandled_exception()
jpayne@68 2077 raise LoadError("invalid Netscape format cookies file %r: %r" %
jpayne@68 2078 (filename, line))
jpayne@68 2079
jpayne@68 2080 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
jpayne@68 2081 if filename is None:
jpayne@68 2082 if self.filename is not None: filename = self.filename
jpayne@68 2083 else: raise ValueError(MISSING_FILENAME_TEXT)
jpayne@68 2084
jpayne@68 2085 with open(filename, "w") as f:
jpayne@68 2086 f.write(self.header)
jpayne@68 2087 now = time.time()
jpayne@68 2088 for cookie in self:
jpayne@68 2089 if not ignore_discard and cookie.discard:
jpayne@68 2090 continue
jpayne@68 2091 if not ignore_expires and cookie.is_expired(now):
jpayne@68 2092 continue
jpayne@68 2093 if cookie.secure: secure = "TRUE"
jpayne@68 2094 else: secure = "FALSE"
jpayne@68 2095 if cookie.domain.startswith("."): initial_dot = "TRUE"
jpayne@68 2096 else: initial_dot = "FALSE"
jpayne@68 2097 if cookie.expires is not None:
jpayne@68 2098 expires = str(cookie.expires)
jpayne@68 2099 else:
jpayne@68 2100 expires = ""
jpayne@68 2101 if cookie.value is None:
jpayne@68 2102 # cookies.txt regards 'Set-Cookie: foo' as a cookie
jpayne@68 2103 # with no name, whereas http.cookiejar regards it as a
jpayne@68 2104 # cookie with no value.
jpayne@68 2105 name = ""
jpayne@68 2106 value = cookie.name
jpayne@68 2107 else:
jpayne@68 2108 name = cookie.name
jpayne@68 2109 value = cookie.value
jpayne@68 2110 f.write(
jpayne@68 2111 "\t".join([cookie.domain, initial_dot, cookie.path,
jpayne@68 2112 secure, expires, name, value])+
jpayne@68 2113 "\n")