Mercurial > repos > rliterman > csp2

annotate CSP2/CSP2_env/env-d9b9114564458d9d-741b3de822f2aaca6c6caa4325c4afce/lib/python3.8/email/charset.py @ 69:33d812a61356

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit 2e9511a184a1ca667c7be0c6321a36dc4e3d116d
author jpayne
date Tue, 18 Mar 2025 17:55:14 -0400
parents
children
rev   line source
jpayne@69 1 # Copyright (C) 2001-2007 Python Software Foundation
jpayne@69 2 # Author: Ben Gertzfield, Barry Warsaw
jpayne@69 3 # Contact: email-sig@python.org
jpayne@69 4
jpayne@69 5 __all__ = [
jpayne@69 6 'Charset',
jpayne@69 7 'add_alias',
jpayne@69 8 'add_charset',
jpayne@69 9 'add_codec',
jpayne@69 10 ]
jpayne@69 11
jpayne@69 12 from functools import partial
jpayne@69 13
jpayne@69 14 import email.base64mime
jpayne@69 15 import email.quoprimime
jpayne@69 16
jpayne@69 17 from email import errors
jpayne@69 18 from email.encoders import encode_7or8bit
jpayne@69 19
jpayne@69 20
jpayne@69 21
jpayne@69 22 # Flags for types of header encodings
jpayne@69 23 QP = 1 # Quoted-Printable
jpayne@69 24 BASE64 = 2 # Base64
jpayne@69 25 SHORTEST = 3 # the shorter of QP and base64, but only for headers
jpayne@69 26
jpayne@69 27 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
jpayne@69 28 RFC2047_CHROME_LEN = 7
jpayne@69 29
jpayne@69 30 DEFAULT_CHARSET = 'us-ascii'
jpayne@69 31 UNKNOWN8BIT = 'unknown-8bit'
jpayne@69 32 EMPTYSTRING = ''
jpayne@69 33
jpayne@69 34
jpayne@69 35
jpayne@69 36 # Defaults
jpayne@69 37 CHARSETS = {
jpayne@69 38 # input header enc body enc output conv
jpayne@69 39 'iso-8859-1': (QP, QP, None),
jpayne@69 40 'iso-8859-2': (QP, QP, None),
jpayne@69 41 'iso-8859-3': (QP, QP, None),
jpayne@69 42 'iso-8859-4': (QP, QP, None),
jpayne@69 43 # iso-8859-5 is Cyrillic, and not especially used
jpayne@69 44 # iso-8859-6 is Arabic, also not particularly used
jpayne@69 45 # iso-8859-7 is Greek, QP will not make it readable
jpayne@69 46 # iso-8859-8 is Hebrew, QP will not make it readable
jpayne@69 47 'iso-8859-9': (QP, QP, None),
jpayne@69 48 'iso-8859-10': (QP, QP, None),
jpayne@69 49 # iso-8859-11 is Thai, QP will not make it readable
jpayne@69 50 'iso-8859-13': (QP, QP, None),
jpayne@69 51 'iso-8859-14': (QP, QP, None),
jpayne@69 52 'iso-8859-15': (QP, QP, None),
jpayne@69 53 'iso-8859-16': (QP, QP, None),
jpayne@69 54 'windows-1252':(QP, QP, None),
jpayne@69 55 'viscii': (QP, QP, None),
jpayne@69 56 'us-ascii': (None, None, None),
jpayne@69 57 'big5': (BASE64, BASE64, None),
jpayne@69 58 'gb2312': (BASE64, BASE64, None),
jpayne@69 59 'euc-jp': (BASE64, None, 'iso-2022-jp'),
jpayne@69 60 'shift_jis': (BASE64, None, 'iso-2022-jp'),
jpayne@69 61 'iso-2022-jp': (BASE64, None, None),
jpayne@69 62 'koi8-r': (BASE64, BASE64, None),
jpayne@69 63 'utf-8': (SHORTEST, BASE64, 'utf-8'),
jpayne@69 64 }
jpayne@69 65
jpayne@69 66 # Aliases for other commonly-used names for character sets. Map
jpayne@69 67 # them to the real ones used in email.
jpayne@69 68 ALIASES = {
jpayne@69 69 'latin_1': 'iso-8859-1',
jpayne@69 70 'latin-1': 'iso-8859-1',
jpayne@69 71 'latin_2': 'iso-8859-2',
jpayne@69 72 'latin-2': 'iso-8859-2',
jpayne@69 73 'latin_3': 'iso-8859-3',
jpayne@69 74 'latin-3': 'iso-8859-3',
jpayne@69 75 'latin_4': 'iso-8859-4',
jpayne@69 76 'latin-4': 'iso-8859-4',
jpayne@69 77 'latin_5': 'iso-8859-9',
jpayne@69 78 'latin-5': 'iso-8859-9',
jpayne@69 79 'latin_6': 'iso-8859-10',
jpayne@69 80 'latin-6': 'iso-8859-10',
jpayne@69 81 'latin_7': 'iso-8859-13',
jpayne@69 82 'latin-7': 'iso-8859-13',
jpayne@69 83 'latin_8': 'iso-8859-14',
jpayne@69 84 'latin-8': 'iso-8859-14',
jpayne@69 85 'latin_9': 'iso-8859-15',
jpayne@69 86 'latin-9': 'iso-8859-15',
jpayne@69 87 'latin_10':'iso-8859-16',
jpayne@69 88 'latin-10':'iso-8859-16',
jpayne@69 89 'cp949': 'ks_c_5601-1987',
jpayne@69 90 'euc_jp': 'euc-jp',
jpayne@69 91 'euc_kr': 'euc-kr',
jpayne@69 92 'ascii': 'us-ascii',
jpayne@69 93 }
jpayne@69 94
jpayne@69 95
jpayne@69 96 # Map charsets to their Unicode codec strings.
jpayne@69 97 CODEC_MAP = {
jpayne@69 98 'gb2312': 'eucgb2312_cn',
jpayne@69 99 'big5': 'big5_tw',
jpayne@69 100 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
jpayne@69 101 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
jpayne@69 102 # Let that stuff pass through without conversion to/from Unicode.
jpayne@69 103 'us-ascii': None,
jpayne@69 104 }
jpayne@69 105
jpayne@69 106
jpayne@69 107
jpayne@69 108 # Convenience functions for extending the above mappings
jpayne@69 109 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
jpayne@69 110 """Add character set properties to the global registry.
jpayne@69 111
jpayne@69 112 charset is the input character set, and must be the canonical name of a
jpayne@69 113 character set.
jpayne@69 114
jpayne@69 115 Optional header_enc and body_enc is either Charset.QP for
jpayne@69 116 quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
jpayne@69 117 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
jpayne@69 118 is only valid for header_enc. It describes how message headers and
jpayne@69 119 message bodies in the input charset are to be encoded. Default is no
jpayne@69 120 encoding.
jpayne@69 121
jpayne@69 122 Optional output_charset is the character set that the output should be
jpayne@69 123 in. Conversions will proceed from input charset, to Unicode, to the
jpayne@69 124 output charset when the method Charset.convert() is called. The default
jpayne@69 125 is to output in the same character set as the input.
jpayne@69 126
jpayne@69 127 Both input_charset and output_charset must have Unicode codec entries in
jpayne@69 128 the module's charset-to-codec mapping; use add_codec(charset, codecname)
jpayne@69 129 to add codecs the module does not know about. See the codecs module's
jpayne@69 130 documentation for more information.
jpayne@69 131 """
jpayne@69 132 if body_enc == SHORTEST:
jpayne@69 133 raise ValueError('SHORTEST not allowed for body_enc')
jpayne@69 134 CHARSETS[charset] = (header_enc, body_enc, output_charset)
jpayne@69 135
jpayne@69 136
jpayne@69 137 def add_alias(alias, canonical):
jpayne@69 138 """Add a character set alias.
jpayne@69 139
jpayne@69 140 alias is the alias name, e.g. latin-1
jpayne@69 141 canonical is the character set's canonical name, e.g. iso-8859-1
jpayne@69 142 """
jpayne@69 143 ALIASES[alias] = canonical
jpayne@69 144
jpayne@69 145
jpayne@69 146 def add_codec(charset, codecname):
jpayne@69 147 """Add a codec that map characters in the given charset to/from Unicode.
jpayne@69 148
jpayne@69 149 charset is the canonical name of a character set. codecname is the name
jpayne@69 150 of a Python codec, as appropriate for the second argument to the unicode()
jpayne@69 151 built-in, or to the encode() method of a Unicode string.
jpayne@69 152 """
jpayne@69 153 CODEC_MAP[charset] = codecname
jpayne@69 154
jpayne@69 155
jpayne@69 156
jpayne@69 157 # Convenience function for encoding strings, taking into account
jpayne@69 158 # that they might be unknown-8bit (ie: have surrogate-escaped bytes)
jpayne@69 159 def _encode(string, codec):
jpayne@69 160 if codec == UNKNOWN8BIT:
jpayne@69 161 return string.encode('ascii', 'surrogateescape')
jpayne@69 162 else:
jpayne@69 163 return string.encode(codec)
jpayne@69 164
jpayne@69 165
jpayne@69 166
jpayne@69 167 class Charset:
jpayne@69 168 """Map character sets to their email properties.
jpayne@69 169
jpayne@69 170 This class provides information about the requirements imposed on email
jpayne@69 171 for a specific character set. It also provides convenience routines for
jpayne@69 172 converting between character sets, given the availability of the
jpayne@69 173 applicable codecs. Given a character set, it will do its best to provide
jpayne@69 174 information on how to use that character set in an email in an
jpayne@69 175 RFC-compliant way.
jpayne@69 176
jpayne@69 177 Certain character sets must be encoded with quoted-printable or base64
jpayne@69 178 when used in email headers or bodies. Certain character sets must be
jpayne@69 179 converted outright, and are not allowed in email. Instances of this
jpayne@69 180 module expose the following information about a character set:
jpayne@69 181
jpayne@69 182 input_charset: The initial character set specified. Common aliases
jpayne@69 183 are converted to their `official' email names (e.g. latin_1
jpayne@69 184 is converted to iso-8859-1). Defaults to 7-bit us-ascii.
jpayne@69 185
jpayne@69 186 header_encoding: If the character set must be encoded before it can be
jpayne@69 187 used in an email header, this attribute will be set to
jpayne@69 188 Charset.QP (for quoted-printable), Charset.BASE64 (for
jpayne@69 189 base64 encoding), or Charset.SHORTEST for the shortest of
jpayne@69 190 QP or BASE64 encoding. Otherwise, it will be None.
jpayne@69 191
jpayne@69 192 body_encoding: Same as header_encoding, but describes the encoding for the
jpayne@69 193 mail message's body, which indeed may be different than the
jpayne@69 194 header encoding. Charset.SHORTEST is not allowed for
jpayne@69 195 body_encoding.
jpayne@69 196
jpayne@69 197 output_charset: Some character sets must be converted before they can be
jpayne@69 198 used in email headers or bodies. If the input_charset is
jpayne@69 199 one of them, this attribute will contain the name of the
jpayne@69 200 charset output will be converted to. Otherwise, it will
jpayne@69 201 be None.
jpayne@69 202
jpayne@69 203 input_codec: The name of the Python codec used to convert the
jpayne@69 204 input_charset to Unicode. If no conversion codec is
jpayne@69 205 necessary, this attribute will be None.
jpayne@69 206
jpayne@69 207 output_codec: The name of the Python codec used to convert Unicode
jpayne@69 208 to the output_charset. If no conversion codec is necessary,
jpayne@69 209 this attribute will have the same value as the input_codec.
jpayne@69 210 """
jpayne@69 211 def __init__(self, input_charset=DEFAULT_CHARSET):
jpayne@69 212 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
jpayne@69 213 # unicode because its .lower() is locale insensitive. If the argument
jpayne@69 214 # is already a unicode, we leave it at that, but ensure that the
jpayne@69 215 # charset is ASCII, as the standard (RFC XXX) requires.
jpayne@69 216 try:
jpayne@69 217 if isinstance(input_charset, str):
jpayne@69 218 input_charset.encode('ascii')
jpayne@69 219 else:
jpayne@69 220 input_charset = str(input_charset, 'ascii')
jpayne@69 221 except UnicodeError:
jpayne@69 222 raise errors.CharsetError(input_charset)
jpayne@69 223 input_charset = input_charset.lower()
jpayne@69 224 # Set the input charset after filtering through the aliases
jpayne@69 225 self.input_charset = ALIASES.get(input_charset, input_charset)
jpayne@69 226 # We can try to guess which encoding and conversion to use by the
jpayne@69 227 # charset_map dictionary. Try that first, but let the user override
jpayne@69 228 # it.
jpayne@69 229 henc, benc, conv = CHARSETS.get(self.input_charset,
jpayne@69 230 (SHORTEST, BASE64, None))
jpayne@69 231 if not conv:
jpayne@69 232 conv = self.input_charset
jpayne@69 233 # Set the attributes, allowing the arguments to override the default.
jpayne@69 234 self.header_encoding = henc
jpayne@69 235 self.body_encoding = benc
jpayne@69 236 self.output_charset = ALIASES.get(conv, conv)
jpayne@69 237 # Now set the codecs. If one isn't defined for input_charset,
jpayne@69 238 # guess and try a Unicode codec with the same name as input_codec.
jpayne@69 239 self.input_codec = CODEC_MAP.get(self.input_charset,
jpayne@69 240 self.input_charset)
jpayne@69 241 self.output_codec = CODEC_MAP.get(self.output_charset,
jpayne@69 242 self.output_charset)
jpayne@69 243
jpayne@69 244 def __repr__(self):
jpayne@69 245 return self.input_charset.lower()
jpayne@69 246
jpayne@69 247 def __eq__(self, other):
jpayne@69 248 return str(self) == str(other).lower()
jpayne@69 249
jpayne@69 250 def get_body_encoding(self):
jpayne@69 251 """Return the content-transfer-encoding used for body encoding.
jpayne@69 252
jpayne@69 253 This is either the string `quoted-printable' or `base64' depending on
jpayne@69 254 the encoding used, or it is a function in which case you should call
jpayne@69 255 the function with a single argument, the Message object being
jpayne@69 256 encoded. The function should then set the Content-Transfer-Encoding
jpayne@69 257 header itself to whatever is appropriate.
jpayne@69 258
jpayne@69 259 Returns "quoted-printable" if self.body_encoding is QP.
jpayne@69 260 Returns "base64" if self.body_encoding is BASE64.
jpayne@69 261 Returns conversion function otherwise.
jpayne@69 262 """
jpayne@69 263 assert self.body_encoding != SHORTEST
jpayne@69 264 if self.body_encoding == QP:
jpayne@69 265 return 'quoted-printable'
jpayne@69 266 elif self.body_encoding == BASE64:
jpayne@69 267 return 'base64'
jpayne@69 268 else:
jpayne@69 269 return encode_7or8bit
jpayne@69 270
jpayne@69 271 def get_output_charset(self):
jpayne@69 272 """Return the output character set.
jpayne@69 273
jpayne@69 274 This is self.output_charset if that is not None, otherwise it is
jpayne@69 275 self.input_charset.
jpayne@69 276 """
jpayne@69 277 return self.output_charset or self.input_charset
jpayne@69 278
jpayne@69 279 def header_encode(self, string):
jpayne@69 280 """Header-encode a string by converting it first to bytes.
jpayne@69 281
jpayne@69 282 The type of encoding (base64 or quoted-printable) will be based on
jpayne@69 283 this charset's `header_encoding`.
jpayne@69 284
jpayne@69 285 :param string: A unicode string for the header. It must be possible
jpayne@69 286 to encode this string to bytes using the character set's
jpayne@69 287 output codec.
jpayne@69 288 :return: The encoded string, with RFC 2047 chrome.
jpayne@69 289 """
jpayne@69 290 codec = self.output_codec or 'us-ascii'
jpayne@69 291 header_bytes = _encode(string, codec)
jpayne@69 292 # 7bit/8bit encodings return the string unchanged (modulo conversions)
jpayne@69 293 encoder_module = self._get_encoder(header_bytes)
jpayne@69 294 if encoder_module is None:
jpayne@69 295 return string
jpayne@69 296 return encoder_module.header_encode(header_bytes, codec)
jpayne@69 297
jpayne@69 298 def header_encode_lines(self, string, maxlengths):
jpayne@69 299 """Header-encode a string by converting it first to bytes.
jpayne@69 300
jpayne@69 301 This is similar to `header_encode()` except that the string is fit
jpayne@69 302 into maximum line lengths as given by the argument.
jpayne@69 303
jpayne@69 304 :param string: A unicode string for the header. It must be possible
jpayne@69 305 to encode this string to bytes using the character set's
jpayne@69 306 output codec.
jpayne@69 307 :param maxlengths: Maximum line length iterator. Each element
jpayne@69 308 returned from this iterator will provide the next maximum line
jpayne@69 309 length. This parameter is used as an argument to built-in next()
jpayne@69 310 and should never be exhausted. The maximum line lengths should
jpayne@69 311 not count the RFC 2047 chrome. These line lengths are only a
jpayne@69 312 hint; the splitter does the best it can.
jpayne@69 313 :return: Lines of encoded strings, each with RFC 2047 chrome.
jpayne@69 314 """
jpayne@69 315 # See which encoding we should use.
jpayne@69 316 codec = self.output_codec or 'us-ascii'
jpayne@69 317 header_bytes = _encode(string, codec)
jpayne@69 318 encoder_module = self._get_encoder(header_bytes)
jpayne@69 319 encoder = partial(encoder_module.header_encode, charset=codec)
jpayne@69 320 # Calculate the number of characters that the RFC 2047 chrome will
jpayne@69 321 # contribute to each line.
jpayne@69 322 charset = self.get_output_charset()
jpayne@69 323 extra = len(charset) + RFC2047_CHROME_LEN
jpayne@69 324 # Now comes the hard part. We must encode bytes but we can't split on
jpayne@69 325 # bytes because some character sets are variable length and each
jpayne@69 326 # encoded word must stand on its own. So the problem is you have to
jpayne@69 327 # encode to bytes to figure out this word's length, but you must split
jpayne@69 328 # on characters. This causes two problems: first, we don't know how
jpayne@69 329 # many octets a specific substring of unicode characters will get
jpayne@69 330 # encoded to, and second, we don't know how many ASCII characters
jpayne@69 331 # those octets will get encoded to. Unless we try it. Which seems
jpayne@69 332 # inefficient. In the interest of being correct rather than fast (and
jpayne@69 333 # in the hope that there will be few encoded headers in any such
jpayne@69 334 # message), brute force it. :(
jpayne@69 335 lines = []
jpayne@69 336 current_line = []
jpayne@69 337 maxlen = next(maxlengths) - extra
jpayne@69 338 for character in string:
jpayne@69 339 current_line.append(character)
jpayne@69 340 this_line = EMPTYSTRING.join(current_line)
jpayne@69 341 length = encoder_module.header_length(_encode(this_line, charset))
jpayne@69 342 if length > maxlen:
jpayne@69 343 # This last character doesn't fit so pop it off.
jpayne@69 344 current_line.pop()
jpayne@69 345 # Does nothing fit on the first line?
jpayne@69 346 if not lines and not current_line:
jpayne@69 347 lines.append(None)
jpayne@69 348 else:
jpayne@69 349 separator = (' ' if lines else '')
jpayne@69 350 joined_line = EMPTYSTRING.join(current_line)
jpayne@69 351 header_bytes = _encode(joined_line, codec)
jpayne@69 352 lines.append(encoder(header_bytes))
jpayne@69 353 current_line = [character]
jpayne@69 354 maxlen = next(maxlengths) - extra
jpayne@69 355 joined_line = EMPTYSTRING.join(current_line)
jpayne@69 356 header_bytes = _encode(joined_line, codec)
jpayne@69 357 lines.append(encoder(header_bytes))
jpayne@69 358 return lines
jpayne@69 359
jpayne@69 360 def _get_encoder(self, header_bytes):
jpayne@69 361 if self.header_encoding == BASE64:
jpayne@69 362 return email.base64mime
jpayne@69 363 elif self.header_encoding == QP:
jpayne@69 364 return email.quoprimime
jpayne@69 365 elif self.header_encoding == SHORTEST:
jpayne@69 366 len64 = email.base64mime.header_length(header_bytes)
jpayne@69 367 lenqp = email.quoprimime.header_length(header_bytes)
jpayne@69 368 if len64 < lenqp:
jpayne@69 369 return email.base64mime
jpayne@69 370 else:
jpayne@69 371 return email.quoprimime
jpayne@69 372 else:
jpayne@69 373 return None
jpayne@69 374
jpayne@69 375 def body_encode(self, string):
jpayne@69 376 """Body-encode a string by converting it first to bytes.
jpayne@69 377
jpayne@69 378 The type of encoding (base64 or quoted-printable) will be based on
jpayne@69 379 self.body_encoding. If body_encoding is None, we assume the
jpayne@69 380 output charset is a 7bit encoding, so re-encoding the decoded
jpayne@69 381 string using the ascii codec produces the correct string version
jpayne@69 382 of the content.
jpayne@69 383 """
jpayne@69 384 if not string:
jpayne@69 385 return string
jpayne@69 386 if self.body_encoding is BASE64:
jpayne@69 387 if isinstance(string, str):
jpayne@69 388 string = string.encode(self.output_charset)
jpayne@69 389 return email.base64mime.body_encode(string)
jpayne@69 390 elif self.body_encoding is QP:
jpayne@69 391 # quopromime.body_encode takes a string, but operates on it as if
jpayne@69 392 # it were a list of byte codes. For a (minimal) history on why
jpayne@69 393 # this is so, see changeset 0cf700464177. To correctly encode a
jpayne@69 394 # character set, then, we must turn it into pseudo bytes via the
jpayne@69 395 # latin1 charset, which will encode any byte as a single code point
jpayne@69 396 # between 0 and 255, which is what body_encode is expecting.
jpayne@69 397 if isinstance(string, str):
jpayne@69 398 string = string.encode(self.output_charset)
jpayne@69 399 string = string.decode('latin1')
jpayne@69 400 return email.quoprimime.body_encode(string)
jpayne@69 401 else:
jpayne@69 402 if isinstance(string, str):
jpayne@69 403 string = string.encode(self.output_charset).decode('ascii')
jpayne@69 404 return string