jpayne@68: # Copyright (C) 2001-2007 Python Software Foundation jpayne@68: # Author: Ben Gertzfield, Barry Warsaw jpayne@68: # Contact: email-sig@python.org jpayne@68: jpayne@68: __all__ = [ jpayne@68: 'Charset', jpayne@68: 'add_alias', jpayne@68: 'add_charset', jpayne@68: 'add_codec', jpayne@68: ] jpayne@68: jpayne@68: from functools import partial jpayne@68: jpayne@68: import email.base64mime jpayne@68: import email.quoprimime jpayne@68: jpayne@68: from email import errors jpayne@68: from email.encoders import encode_7or8bit jpayne@68: jpayne@68: jpayne@68: jpayne@68: # Flags for types of header encodings jpayne@68: QP = 1 # Quoted-Printable jpayne@68: BASE64 = 2 # Base64 jpayne@68: SHORTEST = 3 # the shorter of QP and base64, but only for headers jpayne@68: jpayne@68: # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 jpayne@68: RFC2047_CHROME_LEN = 7 jpayne@68: jpayne@68: DEFAULT_CHARSET = 'us-ascii' jpayne@68: UNKNOWN8BIT = 'unknown-8bit' jpayne@68: EMPTYSTRING = '' jpayne@68: jpayne@68: jpayne@68: jpayne@68: # Defaults jpayne@68: CHARSETS = { jpayne@68: # input header enc body enc output conv jpayne@68: 'iso-8859-1': (QP, QP, None), jpayne@68: 'iso-8859-2': (QP, QP, None), jpayne@68: 'iso-8859-3': (QP, QP, None), jpayne@68: 'iso-8859-4': (QP, QP, None), jpayne@68: # iso-8859-5 is Cyrillic, and not especially used jpayne@68: # iso-8859-6 is Arabic, also not particularly used jpayne@68: # iso-8859-7 is Greek, QP will not make it readable jpayne@68: # iso-8859-8 is Hebrew, QP will not make it readable jpayne@68: 'iso-8859-9': (QP, QP, None), jpayne@68: 'iso-8859-10': (QP, QP, None), jpayne@68: # iso-8859-11 is Thai, QP will not make it readable jpayne@68: 'iso-8859-13': (QP, QP, None), jpayne@68: 'iso-8859-14': (QP, QP, None), jpayne@68: 'iso-8859-15': (QP, QP, None), jpayne@68: 'iso-8859-16': (QP, QP, None), jpayne@68: 'windows-1252':(QP, QP, None), jpayne@68: 'viscii': (QP, QP, None), jpayne@68: 'us-ascii': (None, None, None), jpayne@68: 'big5': (BASE64, BASE64, None), jpayne@68: 'gb2312': (BASE64, BASE64, None), jpayne@68: 'euc-jp': (BASE64, None, 'iso-2022-jp'), jpayne@68: 'shift_jis': (BASE64, None, 'iso-2022-jp'), jpayne@68: 'iso-2022-jp': (BASE64, None, None), jpayne@68: 'koi8-r': (BASE64, BASE64, None), jpayne@68: 'utf-8': (SHORTEST, BASE64, 'utf-8'), jpayne@68: } jpayne@68: jpayne@68: # Aliases for other commonly-used names for character sets. Map jpayne@68: # them to the real ones used in email. jpayne@68: ALIASES = { jpayne@68: 'latin_1': 'iso-8859-1', jpayne@68: 'latin-1': 'iso-8859-1', jpayne@68: 'latin_2': 'iso-8859-2', jpayne@68: 'latin-2': 'iso-8859-2', jpayne@68: 'latin_3': 'iso-8859-3', jpayne@68: 'latin-3': 'iso-8859-3', jpayne@68: 'latin_4': 'iso-8859-4', jpayne@68: 'latin-4': 'iso-8859-4', jpayne@68: 'latin_5': 'iso-8859-9', jpayne@68: 'latin-5': 'iso-8859-9', jpayne@68: 'latin_6': 'iso-8859-10', jpayne@68: 'latin-6': 'iso-8859-10', jpayne@68: 'latin_7': 'iso-8859-13', jpayne@68: 'latin-7': 'iso-8859-13', jpayne@68: 'latin_8': 'iso-8859-14', jpayne@68: 'latin-8': 'iso-8859-14', jpayne@68: 'latin_9': 'iso-8859-15', jpayne@68: 'latin-9': 'iso-8859-15', jpayne@68: 'latin_10':'iso-8859-16', jpayne@68: 'latin-10':'iso-8859-16', jpayne@68: 'cp949': 'ks_c_5601-1987', jpayne@68: 'euc_jp': 'euc-jp', jpayne@68: 'euc_kr': 'euc-kr', jpayne@68: 'ascii': 'us-ascii', jpayne@68: } jpayne@68: jpayne@68: jpayne@68: # Map charsets to their Unicode codec strings. jpayne@68: CODEC_MAP = { jpayne@68: 'gb2312': 'eucgb2312_cn', jpayne@68: 'big5': 'big5_tw', jpayne@68: # Hack: We don't want *any* conversion for stuff marked us-ascii, as all jpayne@68: # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. jpayne@68: # Let that stuff pass through without conversion to/from Unicode. jpayne@68: 'us-ascii': None, jpayne@68: } jpayne@68: jpayne@68: jpayne@68: jpayne@68: # Convenience functions for extending the above mappings jpayne@68: def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): jpayne@68: """Add character set properties to the global registry. jpayne@68: jpayne@68: charset is the input character set, and must be the canonical name of a jpayne@68: character set. jpayne@68: jpayne@68: Optional header_enc and body_enc is either Charset.QP for jpayne@68: quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for jpayne@68: the shortest of qp or base64 encoding, or None for no encoding. SHORTEST jpayne@68: is only valid for header_enc. It describes how message headers and jpayne@68: message bodies in the input charset are to be encoded. Default is no jpayne@68: encoding. jpayne@68: jpayne@68: Optional output_charset is the character set that the output should be jpayne@68: in. Conversions will proceed from input charset, to Unicode, to the jpayne@68: output charset when the method Charset.convert() is called. The default jpayne@68: is to output in the same character set as the input. jpayne@68: jpayne@68: Both input_charset and output_charset must have Unicode codec entries in jpayne@68: the module's charset-to-codec mapping; use add_codec(charset, codecname) jpayne@68: to add codecs the module does not know about. See the codecs module's jpayne@68: documentation for more information. jpayne@68: """ jpayne@68: if body_enc == SHORTEST: jpayne@68: raise ValueError('SHORTEST not allowed for body_enc') jpayne@68: CHARSETS[charset] = (header_enc, body_enc, output_charset) jpayne@68: jpayne@68: jpayne@68: def add_alias(alias, canonical): jpayne@68: """Add a character set alias. jpayne@68: jpayne@68: alias is the alias name, e.g. latin-1 jpayne@68: canonical is the character set's canonical name, e.g. iso-8859-1 jpayne@68: """ jpayne@68: ALIASES[alias] = canonical jpayne@68: jpayne@68: jpayne@68: def add_codec(charset, codecname): jpayne@68: """Add a codec that map characters in the given charset to/from Unicode. jpayne@68: jpayne@68: charset is the canonical name of a character set. codecname is the name jpayne@68: of a Python codec, as appropriate for the second argument to the unicode() jpayne@68: built-in, or to the encode() method of a Unicode string. jpayne@68: """ jpayne@68: CODEC_MAP[charset] = codecname jpayne@68: jpayne@68: jpayne@68: jpayne@68: # Convenience function for encoding strings, taking into account jpayne@68: # that they might be unknown-8bit (ie: have surrogate-escaped bytes) jpayne@68: def _encode(string, codec): jpayne@68: if codec == UNKNOWN8BIT: jpayne@68: return string.encode('ascii', 'surrogateescape') jpayne@68: else: jpayne@68: return string.encode(codec) jpayne@68: jpayne@68: jpayne@68: jpayne@68: class Charset: jpayne@68: """Map character sets to their email properties. jpayne@68: jpayne@68: This class provides information about the requirements imposed on email jpayne@68: for a specific character set. It also provides convenience routines for jpayne@68: converting between character sets, given the availability of the jpayne@68: applicable codecs. Given a character set, it will do its best to provide jpayne@68: information on how to use that character set in an email in an jpayne@68: RFC-compliant way. jpayne@68: jpayne@68: Certain character sets must be encoded with quoted-printable or base64 jpayne@68: when used in email headers or bodies. Certain character sets must be jpayne@68: converted outright, and are not allowed in email. Instances of this jpayne@68: module expose the following information about a character set: jpayne@68: jpayne@68: input_charset: The initial character set specified. Common aliases jpayne@68: are converted to their `official' email names (e.g. latin_1 jpayne@68: is converted to iso-8859-1). Defaults to 7-bit us-ascii. jpayne@68: jpayne@68: header_encoding: If the character set must be encoded before it can be jpayne@68: used in an email header, this attribute will be set to jpayne@68: Charset.QP (for quoted-printable), Charset.BASE64 (for jpayne@68: base64 encoding), or Charset.SHORTEST for the shortest of jpayne@68: QP or BASE64 encoding. Otherwise, it will be None. jpayne@68: jpayne@68: body_encoding: Same as header_encoding, but describes the encoding for the jpayne@68: mail message's body, which indeed may be different than the jpayne@68: header encoding. Charset.SHORTEST is not allowed for jpayne@68: body_encoding. jpayne@68: jpayne@68: output_charset: Some character sets must be converted before they can be jpayne@68: used in email headers or bodies. If the input_charset is jpayne@68: one of them, this attribute will contain the name of the jpayne@68: charset output will be converted to. Otherwise, it will jpayne@68: be None. jpayne@68: jpayne@68: input_codec: The name of the Python codec used to convert the jpayne@68: input_charset to Unicode. If no conversion codec is jpayne@68: necessary, this attribute will be None. jpayne@68: jpayne@68: output_codec: The name of the Python codec used to convert Unicode jpayne@68: to the output_charset. If no conversion codec is necessary, jpayne@68: this attribute will have the same value as the input_codec. jpayne@68: """ jpayne@68: def __init__(self, input_charset=DEFAULT_CHARSET): jpayne@68: # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to jpayne@68: # unicode because its .lower() is locale insensitive. If the argument jpayne@68: # is already a unicode, we leave it at that, but ensure that the jpayne@68: # charset is ASCII, as the standard (RFC XXX) requires. jpayne@68: try: jpayne@68: if isinstance(input_charset, str): jpayne@68: input_charset.encode('ascii') jpayne@68: else: jpayne@68: input_charset = str(input_charset, 'ascii') jpayne@68: except UnicodeError: jpayne@68: raise errors.CharsetError(input_charset) jpayne@68: input_charset = input_charset.lower() jpayne@68: # Set the input charset after filtering through the aliases jpayne@68: self.input_charset = ALIASES.get(input_charset, input_charset) jpayne@68: # We can try to guess which encoding and conversion to use by the jpayne@68: # charset_map dictionary. Try that first, but let the user override jpayne@68: # it. jpayne@68: henc, benc, conv = CHARSETS.get(self.input_charset, jpayne@68: (SHORTEST, BASE64, None)) jpayne@68: if not conv: jpayne@68: conv = self.input_charset jpayne@68: # Set the attributes, allowing the arguments to override the default. jpayne@68: self.header_encoding = henc jpayne@68: self.body_encoding = benc jpayne@68: self.output_charset = ALIASES.get(conv, conv) jpayne@68: # Now set the codecs. If one isn't defined for input_charset, jpayne@68: # guess and try a Unicode codec with the same name as input_codec. jpayne@68: self.input_codec = CODEC_MAP.get(self.input_charset, jpayne@68: self.input_charset) jpayne@68: self.output_codec = CODEC_MAP.get(self.output_charset, jpayne@68: self.output_charset) jpayne@68: jpayne@68: def __repr__(self): jpayne@68: return self.input_charset.lower() jpayne@68: jpayne@68: def __eq__(self, other): jpayne@68: return str(self) == str(other).lower() jpayne@68: jpayne@68: def get_body_encoding(self): jpayne@68: """Return the content-transfer-encoding used for body encoding. jpayne@68: jpayne@68: This is either the string `quoted-printable' or `base64' depending on jpayne@68: the encoding used, or it is a function in which case you should call jpayne@68: the function with a single argument, the Message object being jpayne@68: encoded. The function should then set the Content-Transfer-Encoding jpayne@68: header itself to whatever is appropriate. jpayne@68: jpayne@68: Returns "quoted-printable" if self.body_encoding is QP. jpayne@68: Returns "base64" if self.body_encoding is BASE64. jpayne@68: Returns conversion function otherwise. jpayne@68: """ jpayne@68: assert self.body_encoding != SHORTEST jpayne@68: if self.body_encoding == QP: jpayne@68: return 'quoted-printable' jpayne@68: elif self.body_encoding == BASE64: jpayne@68: return 'base64' jpayne@68: else: jpayne@68: return encode_7or8bit jpayne@68: jpayne@68: def get_output_charset(self): jpayne@68: """Return the output character set. jpayne@68: jpayne@68: This is self.output_charset if that is not None, otherwise it is jpayne@68: self.input_charset. jpayne@68: """ jpayne@68: return self.output_charset or self.input_charset jpayne@68: jpayne@68: def header_encode(self, string): jpayne@68: """Header-encode a string by converting it first to bytes. jpayne@68: jpayne@68: The type of encoding (base64 or quoted-printable) will be based on jpayne@68: this charset's `header_encoding`. jpayne@68: jpayne@68: :param string: A unicode string for the header. It must be possible jpayne@68: to encode this string to bytes using the character set's jpayne@68: output codec. jpayne@68: :return: The encoded string, with RFC 2047 chrome. jpayne@68: """ jpayne@68: codec = self.output_codec or 'us-ascii' jpayne@68: header_bytes = _encode(string, codec) jpayne@68: # 7bit/8bit encodings return the string unchanged (modulo conversions) jpayne@68: encoder_module = self._get_encoder(header_bytes) jpayne@68: if encoder_module is None: jpayne@68: return string jpayne@68: return encoder_module.header_encode(header_bytes, codec) jpayne@68: jpayne@68: def header_encode_lines(self, string, maxlengths): jpayne@68: """Header-encode a string by converting it first to bytes. jpayne@68: jpayne@68: This is similar to `header_encode()` except that the string is fit jpayne@68: into maximum line lengths as given by the argument. jpayne@68: jpayne@68: :param string: A unicode string for the header. It must be possible jpayne@68: to encode this string to bytes using the character set's jpayne@68: output codec. jpayne@68: :param maxlengths: Maximum line length iterator. Each element jpayne@68: returned from this iterator will provide the next maximum line jpayne@68: length. This parameter is used as an argument to built-in next() jpayne@68: and should never be exhausted. The maximum line lengths should jpayne@68: not count the RFC 2047 chrome. These line lengths are only a jpayne@68: hint; the splitter does the best it can. jpayne@68: :return: Lines of encoded strings, each with RFC 2047 chrome. jpayne@68: """ jpayne@68: # See which encoding we should use. jpayne@68: codec = self.output_codec or 'us-ascii' jpayne@68: header_bytes = _encode(string, codec) jpayne@68: encoder_module = self._get_encoder(header_bytes) jpayne@68: encoder = partial(encoder_module.header_encode, charset=codec) jpayne@68: # Calculate the number of characters that the RFC 2047 chrome will jpayne@68: # contribute to each line. jpayne@68: charset = self.get_output_charset() jpayne@68: extra = len(charset) + RFC2047_CHROME_LEN jpayne@68: # Now comes the hard part. We must encode bytes but we can't split on jpayne@68: # bytes because some character sets are variable length and each jpayne@68: # encoded word must stand on its own. So the problem is you have to jpayne@68: # encode to bytes to figure out this word's length, but you must split jpayne@68: # on characters. This causes two problems: first, we don't know how jpayne@68: # many octets a specific substring of unicode characters will get jpayne@68: # encoded to, and second, we don't know how many ASCII characters jpayne@68: # those octets will get encoded to. Unless we try it. Which seems jpayne@68: # inefficient. In the interest of being correct rather than fast (and jpayne@68: # in the hope that there will be few encoded headers in any such jpayne@68: # message), brute force it. :( jpayne@68: lines = [] jpayne@68: current_line = [] jpayne@68: maxlen = next(maxlengths) - extra jpayne@68: for character in string: jpayne@68: current_line.append(character) jpayne@68: this_line = EMPTYSTRING.join(current_line) jpayne@68: length = encoder_module.header_length(_encode(this_line, charset)) jpayne@68: if length > maxlen: jpayne@68: # This last character doesn't fit so pop it off. jpayne@68: current_line.pop() jpayne@68: # Does nothing fit on the first line? jpayne@68: if not lines and not current_line: jpayne@68: lines.append(None) jpayne@68: else: jpayne@68: separator = (' ' if lines else '') jpayne@68: joined_line = EMPTYSTRING.join(current_line) jpayne@68: header_bytes = _encode(joined_line, codec) jpayne@68: lines.append(encoder(header_bytes)) jpayne@68: current_line = [character] jpayne@68: maxlen = next(maxlengths) - extra jpayne@68: joined_line = EMPTYSTRING.join(current_line) jpayne@68: header_bytes = _encode(joined_line, codec) jpayne@68: lines.append(encoder(header_bytes)) jpayne@68: return lines jpayne@68: jpayne@68: def _get_encoder(self, header_bytes): jpayne@68: if self.header_encoding == BASE64: jpayne@68: return email.base64mime jpayne@68: elif self.header_encoding == QP: jpayne@68: return email.quoprimime jpayne@68: elif self.header_encoding == SHORTEST: jpayne@68: len64 = email.base64mime.header_length(header_bytes) jpayne@68: lenqp = email.quoprimime.header_length(header_bytes) jpayne@68: if len64 < lenqp: jpayne@68: return email.base64mime jpayne@68: else: jpayne@68: return email.quoprimime jpayne@68: else: jpayne@68: return None jpayne@68: jpayne@68: def body_encode(self, string): jpayne@68: """Body-encode a string by converting it first to bytes. jpayne@68: jpayne@68: The type of encoding (base64 or quoted-printable) will be based on jpayne@68: self.body_encoding. If body_encoding is None, we assume the jpayne@68: output charset is a 7bit encoding, so re-encoding the decoded jpayne@68: string using the ascii codec produces the correct string version jpayne@68: of the content. jpayne@68: """ jpayne@68: if not string: jpayne@68: return string jpayne@68: if self.body_encoding is BASE64: jpayne@68: if isinstance(string, str): jpayne@68: string = string.encode(self.output_charset) jpayne@68: return email.base64mime.body_encode(string) jpayne@68: elif self.body_encoding is QP: jpayne@68: # quopromime.body_encode takes a string, but operates on it as if jpayne@68: # it were a list of byte codes. For a (minimal) history on why jpayne@68: # this is so, see changeset 0cf700464177. To correctly encode a jpayne@68: # character set, then, we must turn it into pseudo bytes via the jpayne@68: # latin1 charset, which will encode any byte as a single code point jpayne@68: # between 0 and 255, which is what body_encode is expecting. jpayne@68: if isinstance(string, str): jpayne@68: string = string.encode(self.output_charset) jpayne@68: string = string.decode('latin1') jpayne@68: return email.quoprimime.body_encode(string) jpayne@68: else: jpayne@68: if isinstance(string, str): jpayne@68: string = string.encode(self.output_charset).decode('ascii') jpayne@68: return string