jpayne@68: # Copyright (C) 2004-2006 Python Software Foundation jpayne@68: # Authors: Baxter, Wouters and Warsaw jpayne@68: # Contact: email-sig@python.org jpayne@68: jpayne@68: """FeedParser - An email feed parser. jpayne@68: jpayne@68: The feed parser implements an interface for incrementally parsing an email jpayne@68: message, line by line. This has advantages for certain applications, such as jpayne@68: those reading email messages off a socket. jpayne@68: jpayne@68: FeedParser.feed() is the primary interface for pushing new data into the jpayne@68: parser. It returns when there's nothing more it can do with the available jpayne@68: data. When you have no more data to push into the parser, call .close(). jpayne@68: This completes the parsing and returns the root message object. jpayne@68: jpayne@68: The other advantage of this parser is that it will never raise a parsing jpayne@68: exception. Instead, when it finds something unexpected, it adds a 'defect' to jpayne@68: the current message. Defects are just instances that live on the message jpayne@68: object's .defects attribute. jpayne@68: """ jpayne@68: jpayne@68: __all__ = ['FeedParser', 'BytesFeedParser'] jpayne@68: jpayne@68: import re jpayne@68: jpayne@68: from email import errors jpayne@68: from email._policybase import compat32 jpayne@68: from collections import deque jpayne@68: from io import StringIO jpayne@68: jpayne@68: NLCRE = re.compile(r'\r\n|\r|\n') jpayne@68: NLCRE_bol = re.compile(r'(\r\n|\r|\n)') jpayne@68: NLCRE_eol = re.compile(r'(\r\n|\r|\n)\Z') jpayne@68: NLCRE_crack = re.compile(r'(\r\n|\r|\n)') jpayne@68: # RFC 2822 $3.6.8 Optional fields. ftext is %d33-57 / %d59-126, Any character jpayne@68: # except controls, SP, and ":". jpayne@68: headerRE = re.compile(r'^(From |[\041-\071\073-\176]*:|[\t ])') jpayne@68: EMPTYSTRING = '' jpayne@68: NL = '\n' jpayne@68: jpayne@68: NeedMoreData = object() jpayne@68: jpayne@68: jpayne@68: jpayne@68: class BufferedSubFile(object): jpayne@68: """A file-ish object that can have new data loaded into it. jpayne@68: jpayne@68: You can also push and pop line-matching predicates onto a stack. When the jpayne@68: current predicate matches the current line, a false EOF response jpayne@68: (i.e. empty string) is returned instead. This lets the parser adhere to a jpayne@68: simple abstraction -- it parses until EOF closes the current message. jpayne@68: """ jpayne@68: def __init__(self): jpayne@68: # Text stream of the last partial line pushed into this object. jpayne@68: # See issue 22233 for why this is a text stream and not a list. jpayne@68: self._partial = StringIO(newline='') jpayne@68: # A deque of full, pushed lines jpayne@68: self._lines = deque() jpayne@68: # The stack of false-EOF checking predicates. jpayne@68: self._eofstack = [] jpayne@68: # A flag indicating whether the file has been closed or not. jpayne@68: self._closed = False jpayne@68: jpayne@68: def push_eof_matcher(self, pred): jpayne@68: self._eofstack.append(pred) jpayne@68: jpayne@68: def pop_eof_matcher(self): jpayne@68: return self._eofstack.pop() jpayne@68: jpayne@68: def close(self): jpayne@68: # Don't forget any trailing partial line. jpayne@68: self._partial.seek(0) jpayne@68: self.pushlines(self._partial.readlines()) jpayne@68: self._partial.seek(0) jpayne@68: self._partial.truncate() jpayne@68: self._closed = True jpayne@68: jpayne@68: def readline(self): jpayne@68: if not self._lines: jpayne@68: if self._closed: jpayne@68: return '' jpayne@68: return NeedMoreData jpayne@68: # Pop the line off the stack and see if it matches the current jpayne@68: # false-EOF predicate. jpayne@68: line = self._lines.popleft() jpayne@68: # RFC 2046, section 5.1.2 requires us to recognize outer level jpayne@68: # boundaries at any level of inner nesting. Do this, but be sure it's jpayne@68: # in the order of most to least nested. jpayne@68: for ateof in reversed(self._eofstack): jpayne@68: if ateof(line): jpayne@68: # We're at the false EOF. But push the last line back first. jpayne@68: self._lines.appendleft(line) jpayne@68: return '' jpayne@68: return line jpayne@68: jpayne@68: def unreadline(self, line): jpayne@68: # Let the consumer push a line back into the buffer. jpayne@68: assert line is not NeedMoreData jpayne@68: self._lines.appendleft(line) jpayne@68: jpayne@68: def push(self, data): jpayne@68: """Push some new data into this object.""" jpayne@68: self._partial.write(data) jpayne@68: if '\n' not in data and '\r' not in data: jpayne@68: # No new complete lines, wait for more. jpayne@68: return jpayne@68: jpayne@68: # Crack into lines, preserving the linesep characters. jpayne@68: self._partial.seek(0) jpayne@68: parts = self._partial.readlines() jpayne@68: self._partial.seek(0) jpayne@68: self._partial.truncate() jpayne@68: jpayne@68: # If the last element of the list does not end in a newline, then treat jpayne@68: # it as a partial line. We only check for '\n' here because a line jpayne@68: # ending with '\r' might be a line that was split in the middle of a jpayne@68: # '\r\n' sequence (see bugs 1555570 and 1721862). jpayne@68: if not parts[-1].endswith('\n'): jpayne@68: self._partial.write(parts.pop()) jpayne@68: self.pushlines(parts) jpayne@68: jpayne@68: def pushlines(self, lines): jpayne@68: self._lines.extend(lines) jpayne@68: jpayne@68: def __iter__(self): jpayne@68: return self jpayne@68: jpayne@68: def __next__(self): jpayne@68: line = self.readline() jpayne@68: if line == '': jpayne@68: raise StopIteration jpayne@68: return line jpayne@68: jpayne@68: jpayne@68: jpayne@68: class FeedParser: jpayne@68: """A feed-style parser of email.""" jpayne@68: jpayne@68: def __init__(self, _factory=None, *, policy=compat32): jpayne@68: """_factory is called with no arguments to create a new message obj jpayne@68: jpayne@68: The policy keyword specifies a policy object that controls a number of jpayne@68: aspects of the parser's operation. The default policy maintains jpayne@68: backward compatibility. jpayne@68: jpayne@68: """ jpayne@68: self.policy = policy jpayne@68: self._old_style_factory = False jpayne@68: if _factory is None: jpayne@68: if policy.message_factory is None: jpayne@68: from email.message import Message jpayne@68: self._factory = Message jpayne@68: else: jpayne@68: self._factory = policy.message_factory jpayne@68: else: jpayne@68: self._factory = _factory jpayne@68: try: jpayne@68: _factory(policy=self.policy) jpayne@68: except TypeError: jpayne@68: # Assume this is an old-style factory jpayne@68: self._old_style_factory = True jpayne@68: self._input = BufferedSubFile() jpayne@68: self._msgstack = [] jpayne@68: self._parse = self._parsegen().__next__ jpayne@68: self._cur = None jpayne@68: self._last = None jpayne@68: self._headersonly = False jpayne@68: jpayne@68: # Non-public interface for supporting Parser's headersonly flag jpayne@68: def _set_headersonly(self): jpayne@68: self._headersonly = True jpayne@68: jpayne@68: def feed(self, data): jpayne@68: """Push more data into the parser.""" jpayne@68: self._input.push(data) jpayne@68: self._call_parse() jpayne@68: jpayne@68: def _call_parse(self): jpayne@68: try: jpayne@68: self._parse() jpayne@68: except StopIteration: jpayne@68: pass jpayne@68: jpayne@68: def close(self): jpayne@68: """Parse all remaining data and return the root message object.""" jpayne@68: self._input.close() jpayne@68: self._call_parse() jpayne@68: root = self._pop_message() jpayne@68: assert not self._msgstack jpayne@68: # Look for final set of defects jpayne@68: if root.get_content_maintype() == 'multipart' \ jpayne@68: and not root.is_multipart(): jpayne@68: defect = errors.MultipartInvariantViolationDefect() jpayne@68: self.policy.handle_defect(root, defect) jpayne@68: return root jpayne@68: jpayne@68: def _new_message(self): jpayne@68: if self._old_style_factory: jpayne@68: msg = self._factory() jpayne@68: else: jpayne@68: msg = self._factory(policy=self.policy) jpayne@68: if self._cur and self._cur.get_content_type() == 'multipart/digest': jpayne@68: msg.set_default_type('message/rfc822') jpayne@68: if self._msgstack: jpayne@68: self._msgstack[-1].attach(msg) jpayne@68: self._msgstack.append(msg) jpayne@68: self._cur = msg jpayne@68: self._last = msg jpayne@68: jpayne@68: def _pop_message(self): jpayne@68: retval = self._msgstack.pop() jpayne@68: if self._msgstack: jpayne@68: self._cur = self._msgstack[-1] jpayne@68: else: jpayne@68: self._cur = None jpayne@68: return retval jpayne@68: jpayne@68: def _parsegen(self): jpayne@68: # Create a new message and start by parsing headers. jpayne@68: self._new_message() jpayne@68: headers = [] jpayne@68: # Collect the headers, searching for a line that doesn't match the RFC jpayne@68: # 2822 header or continuation pattern (including an empty line). jpayne@68: for line in self._input: jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: if not headerRE.match(line): jpayne@68: # If we saw the RFC defined header/body separator jpayne@68: # (i.e. newline), just throw it away. Otherwise the line is jpayne@68: # part of the body so push it back. jpayne@68: if not NLCRE.match(line): jpayne@68: defect = errors.MissingHeaderBodySeparatorDefect() jpayne@68: self.policy.handle_defect(self._cur, defect) jpayne@68: self._input.unreadline(line) jpayne@68: break jpayne@68: headers.append(line) jpayne@68: # Done with the headers, so parse them and figure out what we're jpayne@68: # supposed to see in the body of the message. jpayne@68: self._parse_headers(headers) jpayne@68: # Headers-only parsing is a backwards compatibility hack, which was jpayne@68: # necessary in the older parser, which could raise errors. All jpayne@68: # remaining lines in the input are thrown into the message body. jpayne@68: if self._headersonly: jpayne@68: lines = [] jpayne@68: while True: jpayne@68: line = self._input.readline() jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: if line == '': jpayne@68: break jpayne@68: lines.append(line) jpayne@68: self._cur.set_payload(EMPTYSTRING.join(lines)) jpayne@68: return jpayne@68: if self._cur.get_content_type() == 'message/delivery-status': jpayne@68: # message/delivery-status contains blocks of headers separated by jpayne@68: # a blank line. We'll represent each header block as a separate jpayne@68: # nested message object, but the processing is a bit different jpayne@68: # than standard message/* types because there is no body for the jpayne@68: # nested messages. A blank line separates the subparts. jpayne@68: while True: jpayne@68: self._input.push_eof_matcher(NLCRE.match) jpayne@68: for retval in self._parsegen(): jpayne@68: if retval is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: break jpayne@68: msg = self._pop_message() jpayne@68: # We need to pop the EOF matcher in order to tell if we're at jpayne@68: # the end of the current file, not the end of the last block jpayne@68: # of message headers. jpayne@68: self._input.pop_eof_matcher() jpayne@68: # The input stream must be sitting at the newline or at the jpayne@68: # EOF. We want to see if we're at the end of this subpart, so jpayne@68: # first consume the blank line, then test the next line to see jpayne@68: # if we're at this subpart's EOF. jpayne@68: while True: jpayne@68: line = self._input.readline() jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: break jpayne@68: while True: jpayne@68: line = self._input.readline() jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: break jpayne@68: if line == '': jpayne@68: break jpayne@68: # Not at EOF so this is a line we're going to need. jpayne@68: self._input.unreadline(line) jpayne@68: return jpayne@68: if self._cur.get_content_maintype() == 'message': jpayne@68: # The message claims to be a message/* type, then what follows is jpayne@68: # another RFC 2822 message. jpayne@68: for retval in self._parsegen(): jpayne@68: if retval is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: break jpayne@68: self._pop_message() jpayne@68: return jpayne@68: if self._cur.get_content_maintype() == 'multipart': jpayne@68: boundary = self._cur.get_boundary() jpayne@68: if boundary is None: jpayne@68: # The message /claims/ to be a multipart but it has not jpayne@68: # defined a boundary. That's a problem which we'll handle by jpayne@68: # reading everything until the EOF and marking the message as jpayne@68: # defective. jpayne@68: defect = errors.NoBoundaryInMultipartDefect() jpayne@68: self.policy.handle_defect(self._cur, defect) jpayne@68: lines = [] jpayne@68: for line in self._input: jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: lines.append(line) jpayne@68: self._cur.set_payload(EMPTYSTRING.join(lines)) jpayne@68: return jpayne@68: # Make sure a valid content type was specified per RFC 2045:6.4. jpayne@68: if (str(self._cur.get('content-transfer-encoding', '8bit')).lower() jpayne@68: not in ('7bit', '8bit', 'binary')): jpayne@68: defect = errors.InvalidMultipartContentTransferEncodingDefect() jpayne@68: self.policy.handle_defect(self._cur, defect) jpayne@68: # Create a line match predicate which matches the inter-part jpayne@68: # boundary as well as the end-of-multipart boundary. Don't push jpayne@68: # this onto the input stream until we've scanned past the jpayne@68: # preamble. jpayne@68: separator = '--' + boundary jpayne@68: boundaryre = re.compile( jpayne@68: '(?P' + re.escape(separator) + jpayne@68: r')(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') jpayne@68: capturing_preamble = True jpayne@68: preamble = [] jpayne@68: linesep = False jpayne@68: close_boundary_seen = False jpayne@68: while True: jpayne@68: line = self._input.readline() jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: if line == '': jpayne@68: break jpayne@68: mo = boundaryre.match(line) jpayne@68: if mo: jpayne@68: # If we're looking at the end boundary, we're done with jpayne@68: # this multipart. If there was a newline at the end of jpayne@68: # the closing boundary, then we need to initialize the jpayne@68: # epilogue with the empty string (see below). jpayne@68: if mo.group('end'): jpayne@68: close_boundary_seen = True jpayne@68: linesep = mo.group('linesep') jpayne@68: break jpayne@68: # We saw an inter-part boundary. Were we in the preamble? jpayne@68: if capturing_preamble: jpayne@68: if preamble: jpayne@68: # According to RFC 2046, the last newline belongs jpayne@68: # to the boundary. jpayne@68: lastline = preamble[-1] jpayne@68: eolmo = NLCRE_eol.search(lastline) jpayne@68: if eolmo: jpayne@68: preamble[-1] = lastline[:-len(eolmo.group(0))] jpayne@68: self._cur.preamble = EMPTYSTRING.join(preamble) jpayne@68: capturing_preamble = False jpayne@68: self._input.unreadline(line) jpayne@68: continue jpayne@68: # We saw a boundary separating two parts. Consume any jpayne@68: # multiple boundary lines that may be following. Our jpayne@68: # interpretation of RFC 2046 BNF grammar does not produce jpayne@68: # body parts within such double boundaries. jpayne@68: while True: jpayne@68: line = self._input.readline() jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: mo = boundaryre.match(line) jpayne@68: if not mo: jpayne@68: self._input.unreadline(line) jpayne@68: break jpayne@68: # Recurse to parse this subpart; the input stream points jpayne@68: # at the subpart's first line. jpayne@68: self._input.push_eof_matcher(boundaryre.match) jpayne@68: for retval in self._parsegen(): jpayne@68: if retval is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: break jpayne@68: # Because of RFC 2046, the newline preceding the boundary jpayne@68: # separator actually belongs to the boundary, not the jpayne@68: # previous subpart's payload (or epilogue if the previous jpayne@68: # part is a multipart). jpayne@68: if self._last.get_content_maintype() == 'multipart': jpayne@68: epilogue = self._last.epilogue jpayne@68: if epilogue == '': jpayne@68: self._last.epilogue = None jpayne@68: elif epilogue is not None: jpayne@68: mo = NLCRE_eol.search(epilogue) jpayne@68: if mo: jpayne@68: end = len(mo.group(0)) jpayne@68: self._last.epilogue = epilogue[:-end] jpayne@68: else: jpayne@68: payload = self._last._payload jpayne@68: if isinstance(payload, str): jpayne@68: mo = NLCRE_eol.search(payload) jpayne@68: if mo: jpayne@68: payload = payload[:-len(mo.group(0))] jpayne@68: self._last._payload = payload jpayne@68: self._input.pop_eof_matcher() jpayne@68: self._pop_message() jpayne@68: # Set the multipart up for newline cleansing, which will jpayne@68: # happen if we're in a nested multipart. jpayne@68: self._last = self._cur jpayne@68: else: jpayne@68: # I think we must be in the preamble jpayne@68: assert capturing_preamble jpayne@68: preamble.append(line) jpayne@68: # We've seen either the EOF or the end boundary. If we're still jpayne@68: # capturing the preamble, we never saw the start boundary. Note jpayne@68: # that as a defect and store the captured text as the payload. jpayne@68: if capturing_preamble: jpayne@68: defect = errors.StartBoundaryNotFoundDefect() jpayne@68: self.policy.handle_defect(self._cur, defect) jpayne@68: self._cur.set_payload(EMPTYSTRING.join(preamble)) jpayne@68: epilogue = [] jpayne@68: for line in self._input: jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: self._cur.epilogue = EMPTYSTRING.join(epilogue) jpayne@68: return jpayne@68: # If we're not processing the preamble, then we might have seen jpayne@68: # EOF without seeing that end boundary...that is also a defect. jpayne@68: if not close_boundary_seen: jpayne@68: defect = errors.CloseBoundaryNotFoundDefect() jpayne@68: self.policy.handle_defect(self._cur, defect) jpayne@68: return jpayne@68: # Everything from here to the EOF is epilogue. If the end boundary jpayne@68: # ended in a newline, we'll need to make sure the epilogue isn't jpayne@68: # None jpayne@68: if linesep: jpayne@68: epilogue = [''] jpayne@68: else: jpayne@68: epilogue = [] jpayne@68: for line in self._input: jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: epilogue.append(line) jpayne@68: # Any CRLF at the front of the epilogue is not technically part of jpayne@68: # the epilogue. Also, watch out for an empty string epilogue, jpayne@68: # which means a single newline. jpayne@68: if epilogue: jpayne@68: firstline = epilogue[0] jpayne@68: bolmo = NLCRE_bol.match(firstline) jpayne@68: if bolmo: jpayne@68: epilogue[0] = firstline[len(bolmo.group(0)):] jpayne@68: self._cur.epilogue = EMPTYSTRING.join(epilogue) jpayne@68: return jpayne@68: # Otherwise, it's some non-multipart type, so the entire rest of the jpayne@68: # file contents becomes the payload. jpayne@68: lines = [] jpayne@68: for line in self._input: jpayne@68: if line is NeedMoreData: jpayne@68: yield NeedMoreData jpayne@68: continue jpayne@68: lines.append(line) jpayne@68: self._cur.set_payload(EMPTYSTRING.join(lines)) jpayne@68: jpayne@68: def _parse_headers(self, lines): jpayne@68: # Passed a list of lines that make up the headers for the current msg jpayne@68: lastheader = '' jpayne@68: lastvalue = [] jpayne@68: for lineno, line in enumerate(lines): jpayne@68: # Check for continuation jpayne@68: if line[0] in ' \t': jpayne@68: if not lastheader: jpayne@68: # The first line of the headers was a continuation. This jpayne@68: # is illegal, so let's note the defect, store the illegal jpayne@68: # line, and ignore it for purposes of headers. jpayne@68: defect = errors.FirstHeaderLineIsContinuationDefect(line) jpayne@68: self.policy.handle_defect(self._cur, defect) jpayne@68: continue jpayne@68: lastvalue.append(line) jpayne@68: continue jpayne@68: if lastheader: jpayne@68: self._cur.set_raw(*self.policy.header_source_parse(lastvalue)) jpayne@68: lastheader, lastvalue = '', [] jpayne@68: # Check for envelope header, i.e. unix-from jpayne@68: if line.startswith('From '): jpayne@68: if lineno == 0: jpayne@68: # Strip off the trailing newline jpayne@68: mo = NLCRE_eol.search(line) jpayne@68: if mo: jpayne@68: line = line[:-len(mo.group(0))] jpayne@68: self._cur.set_unixfrom(line) jpayne@68: continue jpayne@68: elif lineno == len(lines) - 1: jpayne@68: # Something looking like a unix-from at the end - it's jpayne@68: # probably the first line of the body, so push back the jpayne@68: # line and stop. jpayne@68: self._input.unreadline(line) jpayne@68: return jpayne@68: else: jpayne@68: # Weirdly placed unix-from line. Note this as a defect jpayne@68: # and ignore it. jpayne@68: defect = errors.MisplacedEnvelopeHeaderDefect(line) jpayne@68: self._cur.defects.append(defect) jpayne@68: continue jpayne@68: # Split the line on the colon separating field name from value. jpayne@68: # There will always be a colon, because if there wasn't the part of jpayne@68: # the parser that calls us would have started parsing the body. jpayne@68: i = line.find(':') jpayne@68: jpayne@68: # If the colon is on the start of the line the header is clearly jpayne@68: # malformed, but we might be able to salvage the rest of the jpayne@68: # message. Track the error but keep going. jpayne@68: if i == 0: jpayne@68: defect = errors.InvalidHeaderDefect("Missing header name.") jpayne@68: self._cur.defects.append(defect) jpayne@68: continue jpayne@68: jpayne@68: assert i>0, "_parse_headers fed line with no : and no leading WS" jpayne@68: lastheader = line[:i] jpayne@68: lastvalue = [line] jpayne@68: # Done with all the lines, so handle the last header. jpayne@68: if lastheader: jpayne@68: self._cur.set_raw(*self.policy.header_source_parse(lastvalue)) jpayne@68: jpayne@68: jpayne@68: class BytesFeedParser(FeedParser): jpayne@68: """Like FeedParser, but feed accepts bytes.""" jpayne@68: jpayne@68: def feed(self, data): jpayne@68: super().feed(data.decode('ascii', 'surrogateescape'))