|
jpayne@69
|
1 """Test result object"""
|
|
jpayne@69
|
2
|
|
jpayne@69
|
3 import io
|
|
jpayne@69
|
4 import sys
|
|
jpayne@69
|
5 import traceback
|
|
jpayne@69
|
6
|
|
jpayne@69
|
7 from . import util
|
|
jpayne@69
|
8 from functools import wraps
|
|
jpayne@69
|
9
|
|
jpayne@69
|
10 __unittest = True
|
|
jpayne@69
|
11
|
|
jpayne@69
|
12 def failfast(method):
|
|
jpayne@69
|
13 @wraps(method)
|
|
jpayne@69
|
14 def inner(self, *args, **kw):
|
|
jpayne@69
|
15 if getattr(self, 'failfast', False):
|
|
jpayne@69
|
16 self.stop()
|
|
jpayne@69
|
17 return method(self, *args, **kw)
|
|
jpayne@69
|
18 return inner
|
|
jpayne@69
|
19
|
|
jpayne@69
|
20 STDOUT_LINE = '\nStdout:\n%s'
|
|
jpayne@69
|
21 STDERR_LINE = '\nStderr:\n%s'
|
|
jpayne@69
|
22
|
|
jpayne@69
|
23
|
|
jpayne@69
|
24 class TestResult(object):
|
|
jpayne@69
|
25 """Holder for test result information.
|
|
jpayne@69
|
26
|
|
jpayne@69
|
27 Test results are automatically managed by the TestCase and TestSuite
|
|
jpayne@69
|
28 classes, and do not need to be explicitly manipulated by writers of tests.
|
|
jpayne@69
|
29
|
|
jpayne@69
|
30 Each instance holds the total number of tests run, and collections of
|
|
jpayne@69
|
31 failures and errors that occurred among those test runs. The collections
|
|
jpayne@69
|
32 contain tuples of (testcase, exceptioninfo), where exceptioninfo is the
|
|
jpayne@69
|
33 formatted traceback of the error that occurred.
|
|
jpayne@69
|
34 """
|
|
jpayne@69
|
35 _previousTestClass = None
|
|
jpayne@69
|
36 _testRunEntered = False
|
|
jpayne@69
|
37 _moduleSetUpFailed = False
|
|
jpayne@69
|
38 def __init__(self, stream=None, descriptions=None, verbosity=None):
|
|
jpayne@69
|
39 self.failfast = False
|
|
jpayne@69
|
40 self.failures = []
|
|
jpayne@69
|
41 self.errors = []
|
|
jpayne@69
|
42 self.testsRun = 0
|
|
jpayne@69
|
43 self.skipped = []
|
|
jpayne@69
|
44 self.expectedFailures = []
|
|
jpayne@69
|
45 self.unexpectedSuccesses = []
|
|
jpayne@69
|
46 self.shouldStop = False
|
|
jpayne@69
|
47 self.buffer = False
|
|
jpayne@69
|
48 self.tb_locals = False
|
|
jpayne@69
|
49 self._stdout_buffer = None
|
|
jpayne@69
|
50 self._stderr_buffer = None
|
|
jpayne@69
|
51 self._original_stdout = sys.stdout
|
|
jpayne@69
|
52 self._original_stderr = sys.stderr
|
|
jpayne@69
|
53 self._mirrorOutput = False
|
|
jpayne@69
|
54
|
|
jpayne@69
|
55 def printErrors(self):
|
|
jpayne@69
|
56 "Called by TestRunner after test run"
|
|
jpayne@69
|
57
|
|
jpayne@69
|
58 def startTest(self, test):
|
|
jpayne@69
|
59 "Called when the given test is about to be run"
|
|
jpayne@69
|
60 self.testsRun += 1
|
|
jpayne@69
|
61 self._mirrorOutput = False
|
|
jpayne@69
|
62 self._setupStdout()
|
|
jpayne@69
|
63
|
|
jpayne@69
|
64 def _setupStdout(self):
|
|
jpayne@69
|
65 if self.buffer:
|
|
jpayne@69
|
66 if self._stderr_buffer is None:
|
|
jpayne@69
|
67 self._stderr_buffer = io.StringIO()
|
|
jpayne@69
|
68 self._stdout_buffer = io.StringIO()
|
|
jpayne@69
|
69 sys.stdout = self._stdout_buffer
|
|
jpayne@69
|
70 sys.stderr = self._stderr_buffer
|
|
jpayne@69
|
71
|
|
jpayne@69
|
72 def startTestRun(self):
|
|
jpayne@69
|
73 """Called once before any tests are executed.
|
|
jpayne@69
|
74
|
|
jpayne@69
|
75 See startTest for a method called before each test.
|
|
jpayne@69
|
76 """
|
|
jpayne@69
|
77
|
|
jpayne@69
|
78 def stopTest(self, test):
|
|
jpayne@69
|
79 """Called when the given test has been run"""
|
|
jpayne@69
|
80 self._restoreStdout()
|
|
jpayne@69
|
81 self._mirrorOutput = False
|
|
jpayne@69
|
82
|
|
jpayne@69
|
83 def _restoreStdout(self):
|
|
jpayne@69
|
84 if self.buffer:
|
|
jpayne@69
|
85 if self._mirrorOutput:
|
|
jpayne@69
|
86 output = sys.stdout.getvalue()
|
|
jpayne@69
|
87 error = sys.stderr.getvalue()
|
|
jpayne@69
|
88 if output:
|
|
jpayne@69
|
89 if not output.endswith('\n'):
|
|
jpayne@69
|
90 output += '\n'
|
|
jpayne@69
|
91 self._original_stdout.write(STDOUT_LINE % output)
|
|
jpayne@69
|
92 if error:
|
|
jpayne@69
|
93 if not error.endswith('\n'):
|
|
jpayne@69
|
94 error += '\n'
|
|
jpayne@69
|
95 self._original_stderr.write(STDERR_LINE % error)
|
|
jpayne@69
|
96
|
|
jpayne@69
|
97 sys.stdout = self._original_stdout
|
|
jpayne@69
|
98 sys.stderr = self._original_stderr
|
|
jpayne@69
|
99 self._stdout_buffer.seek(0)
|
|
jpayne@69
|
100 self._stdout_buffer.truncate()
|
|
jpayne@69
|
101 self._stderr_buffer.seek(0)
|
|
jpayne@69
|
102 self._stderr_buffer.truncate()
|
|
jpayne@69
|
103
|
|
jpayne@69
|
104 def stopTestRun(self):
|
|
jpayne@69
|
105 """Called once after all tests are executed.
|
|
jpayne@69
|
106
|
|
jpayne@69
|
107 See stopTest for a method called after each test.
|
|
jpayne@69
|
108 """
|
|
jpayne@69
|
109
|
|
jpayne@69
|
110 @failfast
|
|
jpayne@69
|
111 def addError(self, test, err):
|
|
jpayne@69
|
112 """Called when an error has occurred. 'err' is a tuple of values as
|
|
jpayne@69
|
113 returned by sys.exc_info().
|
|
jpayne@69
|
114 """
|
|
jpayne@69
|
115 self.errors.append((test, self._exc_info_to_string(err, test)))
|
|
jpayne@69
|
116 self._mirrorOutput = True
|
|
jpayne@69
|
117
|
|
jpayne@69
|
118 @failfast
|
|
jpayne@69
|
119 def addFailure(self, test, err):
|
|
jpayne@69
|
120 """Called when an error has occurred. 'err' is a tuple of values as
|
|
jpayne@69
|
121 returned by sys.exc_info()."""
|
|
jpayne@69
|
122 self.failures.append((test, self._exc_info_to_string(err, test)))
|
|
jpayne@69
|
123 self._mirrorOutput = True
|
|
jpayne@69
|
124
|
|
jpayne@69
|
125 def addSubTest(self, test, subtest, err):
|
|
jpayne@69
|
126 """Called at the end of a subtest.
|
|
jpayne@69
|
127 'err' is None if the subtest ended successfully, otherwise it's a
|
|
jpayne@69
|
128 tuple of values as returned by sys.exc_info().
|
|
jpayne@69
|
129 """
|
|
jpayne@69
|
130 # By default, we don't do anything with successful subtests, but
|
|
jpayne@69
|
131 # more sophisticated test results might want to record them.
|
|
jpayne@69
|
132 if err is not None:
|
|
jpayne@69
|
133 if getattr(self, 'failfast', False):
|
|
jpayne@69
|
134 self.stop()
|
|
jpayne@69
|
135 if issubclass(err[0], test.failureException):
|
|
jpayne@69
|
136 errors = self.failures
|
|
jpayne@69
|
137 else:
|
|
jpayne@69
|
138 errors = self.errors
|
|
jpayne@69
|
139 errors.append((subtest, self._exc_info_to_string(err, test)))
|
|
jpayne@69
|
140 self._mirrorOutput = True
|
|
jpayne@69
|
141
|
|
jpayne@69
|
142 def addSuccess(self, test):
|
|
jpayne@69
|
143 "Called when a test has completed successfully"
|
|
jpayne@69
|
144 pass
|
|
jpayne@69
|
145
|
|
jpayne@69
|
146 def addSkip(self, test, reason):
|
|
jpayne@69
|
147 """Called when a test is skipped."""
|
|
jpayne@69
|
148 self.skipped.append((test, reason))
|
|
jpayne@69
|
149
|
|
jpayne@69
|
150 def addExpectedFailure(self, test, err):
|
|
jpayne@69
|
151 """Called when an expected failure/error occurred."""
|
|
jpayne@69
|
152 self.expectedFailures.append(
|
|
jpayne@69
|
153 (test, self._exc_info_to_string(err, test)))
|
|
jpayne@69
|
154
|
|
jpayne@69
|
155 @failfast
|
|
jpayne@69
|
156 def addUnexpectedSuccess(self, test):
|
|
jpayne@69
|
157 """Called when a test was expected to fail, but succeed."""
|
|
jpayne@69
|
158 self.unexpectedSuccesses.append(test)
|
|
jpayne@69
|
159
|
|
jpayne@69
|
160 def wasSuccessful(self):
|
|
jpayne@69
|
161 """Tells whether or not this result was a success."""
|
|
jpayne@69
|
162 # The hasattr check is for test_result's OldResult test. That
|
|
jpayne@69
|
163 # way this method works on objects that lack the attribute.
|
|
jpayne@69
|
164 # (where would such result intances come from? old stored pickles?)
|
|
jpayne@69
|
165 return ((len(self.failures) == len(self.errors) == 0) and
|
|
jpayne@69
|
166 (not hasattr(self, 'unexpectedSuccesses') or
|
|
jpayne@69
|
167 len(self.unexpectedSuccesses) == 0))
|
|
jpayne@69
|
168
|
|
jpayne@69
|
169 def stop(self):
|
|
jpayne@69
|
170 """Indicates that the tests should be aborted."""
|
|
jpayne@69
|
171 self.shouldStop = True
|
|
jpayne@69
|
172
|
|
jpayne@69
|
173 def _exc_info_to_string(self, err, test):
|
|
jpayne@69
|
174 """Converts a sys.exc_info()-style tuple of values into a string."""
|
|
jpayne@69
|
175 exctype, value, tb = err
|
|
jpayne@69
|
176 # Skip test runner traceback levels
|
|
jpayne@69
|
177 while tb and self._is_relevant_tb_level(tb):
|
|
jpayne@69
|
178 tb = tb.tb_next
|
|
jpayne@69
|
179
|
|
jpayne@69
|
180 if exctype is test.failureException:
|
|
jpayne@69
|
181 # Skip assert*() traceback levels
|
|
jpayne@69
|
182 length = self._count_relevant_tb_levels(tb)
|
|
jpayne@69
|
183 else:
|
|
jpayne@69
|
184 length = None
|
|
jpayne@69
|
185 tb_e = traceback.TracebackException(
|
|
jpayne@69
|
186 exctype, value, tb, limit=length, capture_locals=self.tb_locals)
|
|
jpayne@69
|
187 msgLines = list(tb_e.format())
|
|
jpayne@69
|
188
|
|
jpayne@69
|
189 if self.buffer:
|
|
jpayne@69
|
190 output = sys.stdout.getvalue()
|
|
jpayne@69
|
191 error = sys.stderr.getvalue()
|
|
jpayne@69
|
192 if output:
|
|
jpayne@69
|
193 if not output.endswith('\n'):
|
|
jpayne@69
|
194 output += '\n'
|
|
jpayne@69
|
195 msgLines.append(STDOUT_LINE % output)
|
|
jpayne@69
|
196 if error:
|
|
jpayne@69
|
197 if not error.endswith('\n'):
|
|
jpayne@69
|
198 error += '\n'
|
|
jpayne@69
|
199 msgLines.append(STDERR_LINE % error)
|
|
jpayne@69
|
200 return ''.join(msgLines)
|
|
jpayne@69
|
201
|
|
jpayne@69
|
202
|
|
jpayne@69
|
203 def _is_relevant_tb_level(self, tb):
|
|
jpayne@69
|
204 return '__unittest' in tb.tb_frame.f_globals
|
|
jpayne@69
|
205
|
|
jpayne@69
|
206 def _count_relevant_tb_levels(self, tb):
|
|
jpayne@69
|
207 length = 0
|
|
jpayne@69
|
208 while tb and not self._is_relevant_tb_level(tb):
|
|
jpayne@69
|
209 length += 1
|
|
jpayne@69
|
210 tb = tb.tb_next
|
|
jpayne@69
|
211 return length
|
|
jpayne@69
|
212
|
|
jpayne@69
|
213 def __repr__(self):
|
|
jpayne@69
|
214 return ("<%s run=%i errors=%i failures=%i>" %
|
|
jpayne@69
|
215 (util.strclass(self.__class__), self.testsRun, len(self.errors),
|
|
jpayne@69
|
216 len(self.failures)))
|
