1 | # Copyright (C) 2001-2006 Python Software Foundation |
---|
2 | # Author: Ben Gertzfield, Barry Warsaw |
---|
3 | # Contact: email-sig@python.org |
---|
4 | |
---|
5 | __all__ = [ |
---|
6 | 'Charset', |
---|
7 | 'add_alias', |
---|
8 | 'add_charset', |
---|
9 | 'add_codec', |
---|
10 | ] |
---|
11 | |
---|
12 | import codecs |
---|
13 | import email.base64mime |
---|
14 | import email.quoprimime |
---|
15 | |
---|
16 | from email import errors |
---|
17 | from email.encoders import encode_7or8bit |
---|
18 | |
---|
19 | |
---|
20 | |
---|
21 | # Flags for types of header encodings |
---|
22 | QP = 1 # Quoted-Printable |
---|
23 | BASE64 = 2 # Base64 |
---|
24 | SHORTEST = 3 # the shorter of QP and base64, but only for headers |
---|
25 | |
---|
26 | # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 |
---|
27 | MISC_LEN = 7 |
---|
28 | |
---|
29 | DEFAULT_CHARSET = 'us-ascii' |
---|
30 | |
---|
31 | |
---|
32 | |
---|
33 | # Defaults |
---|
34 | CHARSETS = { |
---|
35 | # input header enc body enc output conv |
---|
36 | 'iso-8859-1': (QP, QP, None), |
---|
37 | 'iso-8859-2': (QP, QP, None), |
---|
38 | 'iso-8859-3': (QP, QP, None), |
---|
39 | 'iso-8859-4': (QP, QP, None), |
---|
40 | # iso-8859-5 is Cyrillic, and not especially used |
---|
41 | # iso-8859-6 is Arabic, also not particularly used |
---|
42 | # iso-8859-7 is Greek, QP will not make it readable |
---|
43 | # iso-8859-8 is Hebrew, QP will not make it readable |
---|
44 | 'iso-8859-9': (QP, QP, None), |
---|
45 | 'iso-8859-10': (QP, QP, None), |
---|
46 | # iso-8859-11 is Thai, QP will not make it readable |
---|
47 | 'iso-8859-13': (QP, QP, None), |
---|
48 | 'iso-8859-14': (QP, QP, None), |
---|
49 | 'iso-8859-15': (QP, QP, None), |
---|
50 | 'iso-8859-16': (QP, QP, None), |
---|
51 | 'windows-1252':(QP, QP, None), |
---|
52 | 'viscii': (QP, QP, None), |
---|
53 | 'us-ascii': (None, None, None), |
---|
54 | 'big5': (BASE64, BASE64, None), |
---|
55 | 'gb2312': (BASE64, BASE64, None), |
---|
56 | 'euc-jp': (BASE64, None, 'iso-2022-jp'), |
---|
57 | 'shift_jis': (BASE64, None, 'iso-2022-jp'), |
---|
58 | 'iso-2022-jp': (BASE64, None, None), |
---|
59 | 'koi8-r': (BASE64, BASE64, None), |
---|
60 | 'utf-8': (SHORTEST, BASE64, 'utf-8'), |
---|
61 | # We're making this one up to represent raw unencoded 8-bit |
---|
62 | '8bit': (None, BASE64, 'utf-8'), |
---|
63 | } |
---|
64 | |
---|
65 | # Aliases for other commonly-used names for character sets. Map |
---|
66 | # them to the real ones used in email. |
---|
67 | ALIASES = { |
---|
68 | 'latin_1': 'iso-8859-1', |
---|
69 | 'latin-1': 'iso-8859-1', |
---|
70 | 'latin_2': 'iso-8859-2', |
---|
71 | 'latin-2': 'iso-8859-2', |
---|
72 | 'latin_3': 'iso-8859-3', |
---|
73 | 'latin-3': 'iso-8859-3', |
---|
74 | 'latin_4': 'iso-8859-4', |
---|
75 | 'latin-4': 'iso-8859-4', |
---|
76 | 'latin_5': 'iso-8859-9', |
---|
77 | 'latin-5': 'iso-8859-9', |
---|
78 | 'latin_6': 'iso-8859-10', |
---|
79 | 'latin-6': 'iso-8859-10', |
---|
80 | 'latin_7': 'iso-8859-13', |
---|
81 | 'latin-7': 'iso-8859-13', |
---|
82 | 'latin_8': 'iso-8859-14', |
---|
83 | 'latin-8': 'iso-8859-14', |
---|
84 | 'latin_9': 'iso-8859-15', |
---|
85 | 'latin-9': 'iso-8859-15', |
---|
86 | 'latin_10':'iso-8859-16', |
---|
87 | 'latin-10':'iso-8859-16', |
---|
88 | 'cp949': 'ks_c_5601-1987', |
---|
89 | 'euc_jp': 'euc-jp', |
---|
90 | 'euc_kr': 'euc-kr', |
---|
91 | 'ascii': 'us-ascii', |
---|
92 | } |
---|
93 | |
---|
94 | |
---|
95 | # Map charsets to their Unicode codec strings. |
---|
96 | CODEC_MAP = { |
---|
97 | 'gb2312': 'eucgb2312_cn', |
---|
98 | 'big5': 'big5_tw', |
---|
99 | # Hack: We don't want *any* conversion for stuff marked us-ascii, as all |
---|
100 | # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. |
---|
101 | # Let that stuff pass through without conversion to/from Unicode. |
---|
102 | 'us-ascii': None, |
---|
103 | } |
---|
104 | |
---|
105 | |
---|
106 | |
---|
107 | # Convenience functions for extending the above mappings |
---|
108 | def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): |
---|
109 | """Add character set properties to the global registry. |
---|
110 | |
---|
111 | charset is the input character set, and must be the canonical name of a |
---|
112 | character set. |
---|
113 | |
---|
114 | Optional header_enc and body_enc is either Charset.QP for |
---|
115 | quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for |
---|
116 | the shortest of qp or base64 encoding, or None for no encoding. SHORTEST |
---|
117 | is only valid for header_enc. It describes how message headers and |
---|
118 | message bodies in the input charset are to be encoded. Default is no |
---|
119 | encoding. |
---|
120 | |
---|
121 | Optional output_charset is the character set that the output should be |
---|
122 | in. Conversions will proceed from input charset, to Unicode, to the |
---|
123 | output charset when the method Charset.convert() is called. The default |
---|
124 | is to output in the same character set as the input. |
---|
125 | |
---|
126 | Both input_charset and output_charset must have Unicode codec entries in |
---|
127 | the module's charset-to-codec mapping; use add_codec(charset, codecname) |
---|
128 | to add codecs the module does not know about. See the codecs module's |
---|
129 | documentation for more information. |
---|
130 | """ |
---|
131 | if body_enc == SHORTEST: |
---|
132 | raise ValueError('SHORTEST not allowed for body_enc') |
---|
133 | CHARSETS[charset] = (header_enc, body_enc, output_charset) |
---|
134 | |
---|
135 | |
---|
136 | def add_alias(alias, canonical): |
---|
137 | """Add a character set alias. |
---|
138 | |
---|
139 | alias is the alias name, e.g. latin-1 |
---|
140 | canonical is the character set's canonical name, e.g. iso-8859-1 |
---|
141 | """ |
---|
142 | ALIASES[alias] = canonical |
---|
143 | |
---|
144 | |
---|
145 | def add_codec(charset, codecname): |
---|
146 | """Add a codec that map characters in the given charset to/from Unicode. |
---|
147 | |
---|
148 | charset is the canonical name of a character set. codecname is the name |
---|
149 | of a Python codec, as appropriate for the second argument to the unicode() |
---|
150 | built-in, or to the encode() method of a Unicode string. |
---|
151 | """ |
---|
152 | CODEC_MAP[charset] = codecname |
---|
153 | |
---|
154 | |
---|
155 | |
---|
156 | class Charset: |
---|
157 | """Map character sets to their email properties. |
---|
158 | |
---|
159 | This class provides information about the requirements imposed on email |
---|
160 | for a specific character set. It also provides convenience routines for |
---|
161 | converting between character sets, given the availability of the |
---|
162 | applicable codecs. Given a character set, it will do its best to provide |
---|
163 | information on how to use that character set in an email in an |
---|
164 | RFC-compliant way. |
---|
165 | |
---|
166 | Certain character sets must be encoded with quoted-printable or base64 |
---|
167 | when used in email headers or bodies. Certain character sets must be |
---|
168 | converted outright, and are not allowed in email. Instances of this |
---|
169 | module expose the following information about a character set: |
---|
170 | |
---|
171 | input_charset: The initial character set specified. Common aliases |
---|
172 | are converted to their `official' email names (e.g. latin_1 |
---|
173 | is converted to iso-8859-1). Defaults to 7-bit us-ascii. |
---|
174 | |
---|
175 | header_encoding: If the character set must be encoded before it can be |
---|
176 | used in an email header, this attribute will be set to |
---|
177 | Charset.QP (for quoted-printable), Charset.BASE64 (for |
---|
178 | base64 encoding), or Charset.SHORTEST for the shortest of |
---|
179 | QP or BASE64 encoding. Otherwise, it will be None. |
---|
180 | |
---|
181 | body_encoding: Same as header_encoding, but describes the encoding for the |
---|
182 | mail message's body, which indeed may be different than the |
---|
183 | header encoding. Charset.SHORTEST is not allowed for |
---|
184 | body_encoding. |
---|
185 | |
---|
186 | output_charset: Some character sets must be converted before they can be |
---|
187 | used in email headers or bodies. If the input_charset is |
---|
188 | one of them, this attribute will contain the name of the |
---|
189 | charset output will be converted to. Otherwise, it will |
---|
190 | be None. |
---|
191 | |
---|
192 | input_codec: The name of the Python codec used to convert the |
---|
193 | input_charset to Unicode. If no conversion codec is |
---|
194 | necessary, this attribute will be None. |
---|
195 | |
---|
196 | output_codec: The name of the Python codec used to convert Unicode |
---|
197 | to the output_charset. If no conversion codec is necessary, |
---|
198 | this attribute will have the same value as the input_codec. |
---|
199 | """ |
---|
200 | def __init__(self, input_charset=DEFAULT_CHARSET): |
---|
201 | # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to |
---|
202 | # unicode because its .lower() is locale insensitive. If the argument |
---|
203 | # is already a unicode, we leave it at that, but ensure that the |
---|
204 | # charset is ASCII, as the standard (RFC XXX) requires. |
---|
205 | try: |
---|
206 | if isinstance(input_charset, unicode): |
---|
207 | input_charset.encode('ascii') |
---|
208 | else: |
---|
209 | input_charset = unicode(input_charset, 'ascii') |
---|
210 | except UnicodeError: |
---|
211 | raise errors.CharsetError(input_charset) |
---|
212 | input_charset = input_charset.lower().encode('ascii') |
---|
213 | # Set the input charset after filtering through the aliases and/or codecs |
---|
214 | if not (input_charset in ALIASES or input_charset in CHARSETS): |
---|
215 | try: |
---|
216 | input_charset = codecs.lookup(input_charset).name |
---|
217 | except LookupError: |
---|
218 | pass |
---|
219 | self.input_charset = ALIASES.get(input_charset, input_charset) |
---|
220 | # We can try to guess which encoding and conversion to use by the |
---|
221 | # charset_map dictionary. Try that first, but let the user override |
---|
222 | # it. |
---|
223 | henc, benc, conv = CHARSETS.get(self.input_charset, |
---|
224 | (SHORTEST, BASE64, None)) |
---|
225 | if not conv: |
---|
226 | conv = self.input_charset |
---|
227 | # Set the attributes, allowing the arguments to override the default. |
---|
228 | self.header_encoding = henc |
---|
229 | self.body_encoding = benc |
---|
230 | self.output_charset = ALIASES.get(conv, conv) |
---|
231 | # Now set the codecs. If one isn't defined for input_charset, |
---|
232 | # guess and try a Unicode codec with the same name as input_codec. |
---|
233 | self.input_codec = CODEC_MAP.get(self.input_charset, |
---|
234 | self.input_charset) |
---|
235 | self.output_codec = CODEC_MAP.get(self.output_charset, |
---|
236 | self.output_charset) |
---|
237 | |
---|
238 | def __str__(self): |
---|
239 | return self.input_charset.lower() |
---|
240 | |
---|
241 | __repr__ = __str__ |
---|
242 | |
---|
243 | def __eq__(self, other): |
---|
244 | return str(self) == str(other).lower() |
---|
245 | |
---|
246 | def __ne__(self, other): |
---|
247 | return not self.__eq__(other) |
---|
248 | |
---|
249 | def get_body_encoding(self): |
---|
250 | """Return the content-transfer-encoding used for body encoding. |
---|
251 | |
---|
252 | This is either the string `quoted-printable' or `base64' depending on |
---|
253 | the encoding used, or it is a function in which case you should call |
---|
254 | the function with a single argument, the Message object being |
---|
255 | encoded. The function should then set the Content-Transfer-Encoding |
---|
256 | header itself to whatever is appropriate. |
---|
257 | |
---|
258 | Returns "quoted-printable" if self.body_encoding is QP. |
---|
259 | Returns "base64" if self.body_encoding is BASE64. |
---|
260 | Returns "7bit" otherwise. |
---|
261 | """ |
---|
262 | assert self.body_encoding != SHORTEST |
---|
263 | if self.body_encoding == QP: |
---|
264 | return 'quoted-printable' |
---|
265 | elif self.body_encoding == BASE64: |
---|
266 | return 'base64' |
---|
267 | else: |
---|
268 | return encode_7or8bit |
---|
269 | |
---|
270 | def convert(self, s): |
---|
271 | """Convert a string from the input_codec to the output_codec.""" |
---|
272 | if self.input_codec != self.output_codec: |
---|
273 | return unicode(s, self.input_codec).encode(self.output_codec) |
---|
274 | else: |
---|
275 | return s |
---|
276 | |
---|
277 | def to_splittable(self, s): |
---|
278 | """Convert a possibly multibyte string to a safely splittable format. |
---|
279 | |
---|
280 | Uses the input_codec to try and convert the string to Unicode, so it |
---|
281 | can be safely split on character boundaries (even for multibyte |
---|
282 | characters). |
---|
283 | |
---|
284 | Returns the string as-is if it isn't known how to convert it to |
---|
285 | Unicode with the input_charset. |
---|
286 | |
---|
287 | Characters that could not be converted to Unicode will be replaced |
---|
288 | with the Unicode replacement character U+FFFD. |
---|
289 | """ |
---|
290 | if isinstance(s, unicode) or self.input_codec is None: |
---|
291 | return s |
---|
292 | try: |
---|
293 | return unicode(s, self.input_codec, 'replace') |
---|
294 | except LookupError: |
---|
295 | # Input codec not installed on system, so return the original |
---|
296 | # string unchanged. |
---|
297 | return s |
---|
298 | |
---|
299 | def from_splittable(self, ustr, to_output=True): |
---|
300 | """Convert a splittable string back into an encoded string. |
---|
301 | |
---|
302 | Uses the proper codec to try and convert the string from Unicode back |
---|
303 | into an encoded format. Return the string as-is if it is not Unicode, |
---|
304 | or if it could not be converted from Unicode. |
---|
305 | |
---|
306 | Characters that could not be converted from Unicode will be replaced |
---|
307 | with an appropriate character (usually '?'). |
---|
308 | |
---|
309 | If to_output is True (the default), uses output_codec to convert to an |
---|
310 | encoded format. If to_output is False, uses input_codec. |
---|
311 | """ |
---|
312 | if to_output: |
---|
313 | codec = self.output_codec |
---|
314 | else: |
---|
315 | codec = self.input_codec |
---|
316 | if not isinstance(ustr, unicode) or codec is None: |
---|
317 | return ustr |
---|
318 | try: |
---|
319 | return ustr.encode(codec, 'replace') |
---|
320 | except LookupError: |
---|
321 | # Output codec not installed |
---|
322 | return ustr |
---|
323 | |
---|
324 | def get_output_charset(self): |
---|
325 | """Return the output character set. |
---|
326 | |
---|
327 | This is self.output_charset if that is not None, otherwise it is |
---|
328 | self.input_charset. |
---|
329 | """ |
---|
330 | return self.output_charset or self.input_charset |
---|
331 | |
---|
332 | def encoded_header_len(self, s): |
---|
333 | """Return the length of the encoded header string.""" |
---|
334 | cset = self.get_output_charset() |
---|
335 | # The len(s) of a 7bit encoding is len(s) |
---|
336 | if self.header_encoding == BASE64: |
---|
337 | return email.base64mime.base64_len(s) + len(cset) + MISC_LEN |
---|
338 | elif self.header_encoding == QP: |
---|
339 | return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN |
---|
340 | elif self.header_encoding == SHORTEST: |
---|
341 | lenb64 = email.base64mime.base64_len(s) |
---|
342 | lenqp = email.quoprimime.header_quopri_len(s) |
---|
343 | return min(lenb64, lenqp) + len(cset) + MISC_LEN |
---|
344 | else: |
---|
345 | return len(s) |
---|
346 | |
---|
347 | def header_encode(self, s, convert=False): |
---|
348 | """Header-encode a string, optionally converting it to output_charset. |
---|
349 | |
---|
350 | If convert is True, the string will be converted from the input |
---|
351 | charset to the output charset automatically. This is not useful for |
---|
352 | multibyte character sets, which have line length issues (multibyte |
---|
353 | characters must be split on a character, not a byte boundary); use the |
---|
354 | high-level Header class to deal with these issues. convert defaults |
---|
355 | to False. |
---|
356 | |
---|
357 | The type of encoding (base64 or quoted-printable) will be based on |
---|
358 | self.header_encoding. |
---|
359 | """ |
---|
360 | cset = self.get_output_charset() |
---|
361 | if convert: |
---|
362 | s = self.convert(s) |
---|
363 | # 7bit/8bit encodings return the string unchanged (modulo conversions) |
---|
364 | if self.header_encoding == BASE64: |
---|
365 | return email.base64mime.header_encode(s, cset) |
---|
366 | elif self.header_encoding == QP: |
---|
367 | return email.quoprimime.header_encode(s, cset, maxlinelen=None) |
---|
368 | elif self.header_encoding == SHORTEST: |
---|
369 | lenb64 = email.base64mime.base64_len(s) |
---|
370 | lenqp = email.quoprimime.header_quopri_len(s) |
---|
371 | if lenb64 < lenqp: |
---|
372 | return email.base64mime.header_encode(s, cset) |
---|
373 | else: |
---|
374 | return email.quoprimime.header_encode(s, cset, maxlinelen=None) |
---|
375 | else: |
---|
376 | return s |
---|
377 | |
---|
378 | def body_encode(self, s, convert=True): |
---|
379 | """Body-encode a string and convert it to output_charset. |
---|
380 | |
---|
381 | If convert is True (the default), the string will be converted from |
---|
382 | the input charset to output charset automatically. Unlike |
---|
383 | header_encode(), there are no issues with byte boundaries and |
---|
384 | multibyte charsets in email bodies, so this is usually pretty safe. |
---|
385 | |
---|
386 | The type of encoding (base64 or quoted-printable) will be based on |
---|
387 | self.body_encoding. |
---|
388 | """ |
---|
389 | if convert: |
---|
390 | s = self.convert(s) |
---|
391 | # 7bit/8bit encodings return the string unchanged (module conversions) |
---|
392 | if self.body_encoding is BASE64: |
---|
393 | return email.base64mime.body_encode(s) |
---|
394 | elif self.body_encoding is QP: |
---|
395 | return email.quoprimime.body_encode(s) |
---|
396 | else: |
---|
397 | return s |
---|