jpayne@68
|
1 """TestSuite"""
|
jpayne@68
|
2
|
jpayne@68
|
3 import sys
|
jpayne@68
|
4
|
jpayne@68
|
5 from . import case
|
jpayne@68
|
6 from . import util
|
jpayne@68
|
7
|
jpayne@68
|
8 __unittest = True
|
jpayne@68
|
9
|
jpayne@68
|
10
|
jpayne@68
|
11 def _call_if_exists(parent, attr):
|
jpayne@68
|
12 func = getattr(parent, attr, lambda: None)
|
jpayne@68
|
13 func()
|
jpayne@68
|
14
|
jpayne@68
|
15
|
jpayne@68
|
16 class BaseTestSuite(object):
|
jpayne@68
|
17 """A simple test suite that doesn't provide class or module shared fixtures.
|
jpayne@68
|
18 """
|
jpayne@68
|
19 _cleanup = True
|
jpayne@68
|
20
|
jpayne@68
|
21 def __init__(self, tests=()):
|
jpayne@68
|
22 self._tests = []
|
jpayne@68
|
23 self._removed_tests = 0
|
jpayne@68
|
24 self.addTests(tests)
|
jpayne@68
|
25
|
jpayne@68
|
26 def __repr__(self):
|
jpayne@68
|
27 return "<%s tests=%s>" % (util.strclass(self.__class__), list(self))
|
jpayne@68
|
28
|
jpayne@68
|
29 def __eq__(self, other):
|
jpayne@68
|
30 if not isinstance(other, self.__class__):
|
jpayne@68
|
31 return NotImplemented
|
jpayne@68
|
32 return list(self) == list(other)
|
jpayne@68
|
33
|
jpayne@68
|
34 def __iter__(self):
|
jpayne@68
|
35 return iter(self._tests)
|
jpayne@68
|
36
|
jpayne@68
|
37 def countTestCases(self):
|
jpayne@68
|
38 cases = self._removed_tests
|
jpayne@68
|
39 for test in self:
|
jpayne@68
|
40 if test:
|
jpayne@68
|
41 cases += test.countTestCases()
|
jpayne@68
|
42 return cases
|
jpayne@68
|
43
|
jpayne@68
|
44 def addTest(self, test):
|
jpayne@68
|
45 # sanity checks
|
jpayne@68
|
46 if not callable(test):
|
jpayne@68
|
47 raise TypeError("{} is not callable".format(repr(test)))
|
jpayne@68
|
48 if isinstance(test, type) and issubclass(test,
|
jpayne@68
|
49 (case.TestCase, TestSuite)):
|
jpayne@68
|
50 raise TypeError("TestCases and TestSuites must be instantiated "
|
jpayne@68
|
51 "before passing them to addTest()")
|
jpayne@68
|
52 self._tests.append(test)
|
jpayne@68
|
53
|
jpayne@68
|
54 def addTests(self, tests):
|
jpayne@68
|
55 if isinstance(tests, str):
|
jpayne@68
|
56 raise TypeError("tests must be an iterable of tests, not a string")
|
jpayne@68
|
57 for test in tests:
|
jpayne@68
|
58 self.addTest(test)
|
jpayne@68
|
59
|
jpayne@68
|
60 def run(self, result):
|
jpayne@68
|
61 for index, test in enumerate(self):
|
jpayne@68
|
62 if result.shouldStop:
|
jpayne@68
|
63 break
|
jpayne@68
|
64 test(result)
|
jpayne@68
|
65 if self._cleanup:
|
jpayne@68
|
66 self._removeTestAtIndex(index)
|
jpayne@68
|
67 return result
|
jpayne@68
|
68
|
jpayne@68
|
69 def _removeTestAtIndex(self, index):
|
jpayne@68
|
70 """Stop holding a reference to the TestCase at index."""
|
jpayne@68
|
71 try:
|
jpayne@68
|
72 test = self._tests[index]
|
jpayne@68
|
73 except TypeError:
|
jpayne@68
|
74 # support for suite implementations that have overridden self._tests
|
jpayne@68
|
75 pass
|
jpayne@68
|
76 else:
|
jpayne@68
|
77 # Some unittest tests add non TestCase/TestSuite objects to
|
jpayne@68
|
78 # the suite.
|
jpayne@68
|
79 if hasattr(test, 'countTestCases'):
|
jpayne@68
|
80 self._removed_tests += test.countTestCases()
|
jpayne@68
|
81 self._tests[index] = None
|
jpayne@68
|
82
|
jpayne@68
|
83 def __call__(self, *args, **kwds):
|
jpayne@68
|
84 return self.run(*args, **kwds)
|
jpayne@68
|
85
|
jpayne@68
|
86 def debug(self):
|
jpayne@68
|
87 """Run the tests without collecting errors in a TestResult"""
|
jpayne@68
|
88 for test in self:
|
jpayne@68
|
89 test.debug()
|
jpayne@68
|
90
|
jpayne@68
|
91
|
jpayne@68
|
92 class TestSuite(BaseTestSuite):
|
jpayne@68
|
93 """A test suite is a composite test consisting of a number of TestCases.
|
jpayne@68
|
94
|
jpayne@68
|
95 For use, create an instance of TestSuite, then add test case instances.
|
jpayne@68
|
96 When all tests have been added, the suite can be passed to a test
|
jpayne@68
|
97 runner, such as TextTestRunner. It will run the individual test cases
|
jpayne@68
|
98 in the order in which they were added, aggregating the results. When
|
jpayne@68
|
99 subclassing, do not forget to call the base class constructor.
|
jpayne@68
|
100 """
|
jpayne@68
|
101
|
jpayne@68
|
102 def run(self, result, debug=False):
|
jpayne@68
|
103 topLevel = False
|
jpayne@68
|
104 if getattr(result, '_testRunEntered', False) is False:
|
jpayne@68
|
105 result._testRunEntered = topLevel = True
|
jpayne@68
|
106
|
jpayne@68
|
107 for index, test in enumerate(self):
|
jpayne@68
|
108 if result.shouldStop:
|
jpayne@68
|
109 break
|
jpayne@68
|
110
|
jpayne@68
|
111 if _isnotsuite(test):
|
jpayne@68
|
112 self._tearDownPreviousClass(test, result)
|
jpayne@68
|
113 self._handleModuleFixture(test, result)
|
jpayne@68
|
114 self._handleClassSetUp(test, result)
|
jpayne@68
|
115 result._previousTestClass = test.__class__
|
jpayne@68
|
116
|
jpayne@68
|
117 if (getattr(test.__class__, '_classSetupFailed', False) or
|
jpayne@68
|
118 getattr(result, '_moduleSetUpFailed', False)):
|
jpayne@68
|
119 continue
|
jpayne@68
|
120
|
jpayne@68
|
121 if not debug:
|
jpayne@68
|
122 test(result)
|
jpayne@68
|
123 else:
|
jpayne@68
|
124 test.debug()
|
jpayne@68
|
125
|
jpayne@68
|
126 if self._cleanup:
|
jpayne@68
|
127 self._removeTestAtIndex(index)
|
jpayne@68
|
128
|
jpayne@68
|
129 if topLevel:
|
jpayne@68
|
130 self._tearDownPreviousClass(None, result)
|
jpayne@68
|
131 self._handleModuleTearDown(result)
|
jpayne@68
|
132 result._testRunEntered = False
|
jpayne@68
|
133 return result
|
jpayne@68
|
134
|
jpayne@68
|
135 def debug(self):
|
jpayne@68
|
136 """Run the tests without collecting errors in a TestResult"""
|
jpayne@68
|
137 debug = _DebugResult()
|
jpayne@68
|
138 self.run(debug, True)
|
jpayne@68
|
139
|
jpayne@68
|
140 ################################
|
jpayne@68
|
141
|
jpayne@68
|
142 def _handleClassSetUp(self, test, result):
|
jpayne@68
|
143 previousClass = getattr(result, '_previousTestClass', None)
|
jpayne@68
|
144 currentClass = test.__class__
|
jpayne@68
|
145 if currentClass == previousClass:
|
jpayne@68
|
146 return
|
jpayne@68
|
147 if result._moduleSetUpFailed:
|
jpayne@68
|
148 return
|
jpayne@68
|
149 if getattr(currentClass, "__unittest_skip__", False):
|
jpayne@68
|
150 return
|
jpayne@68
|
151
|
jpayne@68
|
152 try:
|
jpayne@68
|
153 currentClass._classSetupFailed = False
|
jpayne@68
|
154 except TypeError:
|
jpayne@68
|
155 # test may actually be a function
|
jpayne@68
|
156 # so its class will be a builtin-type
|
jpayne@68
|
157 pass
|
jpayne@68
|
158
|
jpayne@68
|
159 setUpClass = getattr(currentClass, 'setUpClass', None)
|
jpayne@68
|
160 if setUpClass is not None:
|
jpayne@68
|
161 _call_if_exists(result, '_setupStdout')
|
jpayne@68
|
162 try:
|
jpayne@68
|
163 setUpClass()
|
jpayne@68
|
164 except Exception as e:
|
jpayne@68
|
165 if isinstance(result, _DebugResult):
|
jpayne@68
|
166 raise
|
jpayne@68
|
167 currentClass._classSetupFailed = True
|
jpayne@68
|
168 className = util.strclass(currentClass)
|
jpayne@68
|
169 self._createClassOrModuleLevelException(result, e,
|
jpayne@68
|
170 'setUpClass',
|
jpayne@68
|
171 className)
|
jpayne@68
|
172 finally:
|
jpayne@68
|
173 _call_if_exists(result, '_restoreStdout')
|
jpayne@68
|
174 if currentClass._classSetupFailed is True:
|
jpayne@68
|
175 currentClass.doClassCleanups()
|
jpayne@68
|
176 if len(currentClass.tearDown_exceptions) > 0:
|
jpayne@68
|
177 for exc in currentClass.tearDown_exceptions:
|
jpayne@68
|
178 self._createClassOrModuleLevelException(
|
jpayne@68
|
179 result, exc[1], 'setUpClass', className,
|
jpayne@68
|
180 info=exc)
|
jpayne@68
|
181
|
jpayne@68
|
182 def _get_previous_module(self, result):
|
jpayne@68
|
183 previousModule = None
|
jpayne@68
|
184 previousClass = getattr(result, '_previousTestClass', None)
|
jpayne@68
|
185 if previousClass is not None:
|
jpayne@68
|
186 previousModule = previousClass.__module__
|
jpayne@68
|
187 return previousModule
|
jpayne@68
|
188
|
jpayne@68
|
189
|
jpayne@68
|
190 def _handleModuleFixture(self, test, result):
|
jpayne@68
|
191 previousModule = self._get_previous_module(result)
|
jpayne@68
|
192 currentModule = test.__class__.__module__
|
jpayne@68
|
193 if currentModule == previousModule:
|
jpayne@68
|
194 return
|
jpayne@68
|
195
|
jpayne@68
|
196 self._handleModuleTearDown(result)
|
jpayne@68
|
197
|
jpayne@68
|
198
|
jpayne@68
|
199 result._moduleSetUpFailed = False
|
jpayne@68
|
200 try:
|
jpayne@68
|
201 module = sys.modules[currentModule]
|
jpayne@68
|
202 except KeyError:
|
jpayne@68
|
203 return
|
jpayne@68
|
204 setUpModule = getattr(module, 'setUpModule', None)
|
jpayne@68
|
205 if setUpModule is not None:
|
jpayne@68
|
206 _call_if_exists(result, '_setupStdout')
|
jpayne@68
|
207 try:
|
jpayne@68
|
208 setUpModule()
|
jpayne@68
|
209 except Exception as e:
|
jpayne@68
|
210 try:
|
jpayne@68
|
211 case.doModuleCleanups()
|
jpayne@68
|
212 except Exception as exc:
|
jpayne@68
|
213 self._createClassOrModuleLevelException(result, exc,
|
jpayne@68
|
214 'setUpModule',
|
jpayne@68
|
215 currentModule)
|
jpayne@68
|
216 if isinstance(result, _DebugResult):
|
jpayne@68
|
217 raise
|
jpayne@68
|
218 result._moduleSetUpFailed = True
|
jpayne@68
|
219 self._createClassOrModuleLevelException(result, e,
|
jpayne@68
|
220 'setUpModule',
|
jpayne@68
|
221 currentModule)
|
jpayne@68
|
222 finally:
|
jpayne@68
|
223 _call_if_exists(result, '_restoreStdout')
|
jpayne@68
|
224
|
jpayne@68
|
225 def _createClassOrModuleLevelException(self, result, exc, method_name,
|
jpayne@68
|
226 parent, info=None):
|
jpayne@68
|
227 errorName = f'{method_name} ({parent})'
|
jpayne@68
|
228 self._addClassOrModuleLevelException(result, exc, errorName, info)
|
jpayne@68
|
229
|
jpayne@68
|
230 def _addClassOrModuleLevelException(self, result, exception, errorName,
|
jpayne@68
|
231 info=None):
|
jpayne@68
|
232 error = _ErrorHolder(errorName)
|
jpayne@68
|
233 addSkip = getattr(result, 'addSkip', None)
|
jpayne@68
|
234 if addSkip is not None and isinstance(exception, case.SkipTest):
|
jpayne@68
|
235 addSkip(error, str(exception))
|
jpayne@68
|
236 else:
|
jpayne@68
|
237 if not info:
|
jpayne@68
|
238 result.addError(error, sys.exc_info())
|
jpayne@68
|
239 else:
|
jpayne@68
|
240 result.addError(error, info)
|
jpayne@68
|
241
|
jpayne@68
|
242 def _handleModuleTearDown(self, result):
|
jpayne@68
|
243 previousModule = self._get_previous_module(result)
|
jpayne@68
|
244 if previousModule is None:
|
jpayne@68
|
245 return
|
jpayne@68
|
246 if result._moduleSetUpFailed:
|
jpayne@68
|
247 return
|
jpayne@68
|
248
|
jpayne@68
|
249 try:
|
jpayne@68
|
250 module = sys.modules[previousModule]
|
jpayne@68
|
251 except KeyError:
|
jpayne@68
|
252 return
|
jpayne@68
|
253
|
jpayne@68
|
254 tearDownModule = getattr(module, 'tearDownModule', None)
|
jpayne@68
|
255 if tearDownModule is not None:
|
jpayne@68
|
256 _call_if_exists(result, '_setupStdout')
|
jpayne@68
|
257 try:
|
jpayne@68
|
258 tearDownModule()
|
jpayne@68
|
259 except Exception as e:
|
jpayne@68
|
260 if isinstance(result, _DebugResult):
|
jpayne@68
|
261 raise
|
jpayne@68
|
262 self._createClassOrModuleLevelException(result, e,
|
jpayne@68
|
263 'tearDownModule',
|
jpayne@68
|
264 previousModule)
|
jpayne@68
|
265 finally:
|
jpayne@68
|
266 _call_if_exists(result, '_restoreStdout')
|
jpayne@68
|
267 try:
|
jpayne@68
|
268 case.doModuleCleanups()
|
jpayne@68
|
269 except Exception as e:
|
jpayne@68
|
270 self._createClassOrModuleLevelException(result, e,
|
jpayne@68
|
271 'tearDownModule',
|
jpayne@68
|
272 previousModule)
|
jpayne@68
|
273
|
jpayne@68
|
274 def _tearDownPreviousClass(self, test, result):
|
jpayne@68
|
275 previousClass = getattr(result, '_previousTestClass', None)
|
jpayne@68
|
276 currentClass = test.__class__
|
jpayne@68
|
277 if currentClass == previousClass:
|
jpayne@68
|
278 return
|
jpayne@68
|
279 if getattr(previousClass, '_classSetupFailed', False):
|
jpayne@68
|
280 return
|
jpayne@68
|
281 if getattr(result, '_moduleSetUpFailed', False):
|
jpayne@68
|
282 return
|
jpayne@68
|
283 if getattr(previousClass, "__unittest_skip__", False):
|
jpayne@68
|
284 return
|
jpayne@68
|
285
|
jpayne@68
|
286 tearDownClass = getattr(previousClass, 'tearDownClass', None)
|
jpayne@68
|
287 if tearDownClass is not None:
|
jpayne@68
|
288 _call_if_exists(result, '_setupStdout')
|
jpayne@68
|
289 try:
|
jpayne@68
|
290 tearDownClass()
|
jpayne@68
|
291 except Exception as e:
|
jpayne@68
|
292 if isinstance(result, _DebugResult):
|
jpayne@68
|
293 raise
|
jpayne@68
|
294 className = util.strclass(previousClass)
|
jpayne@68
|
295 self._createClassOrModuleLevelException(result, e,
|
jpayne@68
|
296 'tearDownClass',
|
jpayne@68
|
297 className)
|
jpayne@68
|
298 finally:
|
jpayne@68
|
299 _call_if_exists(result, '_restoreStdout')
|
jpayne@68
|
300 previousClass.doClassCleanups()
|
jpayne@68
|
301 if len(previousClass.tearDown_exceptions) > 0:
|
jpayne@68
|
302 for exc in previousClass.tearDown_exceptions:
|
jpayne@68
|
303 className = util.strclass(previousClass)
|
jpayne@68
|
304 self._createClassOrModuleLevelException(result, exc[1],
|
jpayne@68
|
305 'tearDownClass',
|
jpayne@68
|
306 className,
|
jpayne@68
|
307 info=exc)
|
jpayne@68
|
308
|
jpayne@68
|
309
|
jpayne@68
|
310 class _ErrorHolder(object):
|
jpayne@68
|
311 """
|
jpayne@68
|
312 Placeholder for a TestCase inside a result. As far as a TestResult
|
jpayne@68
|
313 is concerned, this looks exactly like a unit test. Used to insert
|
jpayne@68
|
314 arbitrary errors into a test suite run.
|
jpayne@68
|
315 """
|
jpayne@68
|
316 # Inspired by the ErrorHolder from Twisted:
|
jpayne@68
|
317 # http://twistedmatrix.com/trac/browser/trunk/twisted/trial/runner.py
|
jpayne@68
|
318
|
jpayne@68
|
319 # attribute used by TestResult._exc_info_to_string
|
jpayne@68
|
320 failureException = None
|
jpayne@68
|
321
|
jpayne@68
|
322 def __init__(self, description):
|
jpayne@68
|
323 self.description = description
|
jpayne@68
|
324
|
jpayne@68
|
325 def id(self):
|
jpayne@68
|
326 return self.description
|
jpayne@68
|
327
|
jpayne@68
|
328 def shortDescription(self):
|
jpayne@68
|
329 return None
|
jpayne@68
|
330
|
jpayne@68
|
331 def __repr__(self):
|
jpayne@68
|
332 return "<ErrorHolder description=%r>" % (self.description,)
|
jpayne@68
|
333
|
jpayne@68
|
334 def __str__(self):
|
jpayne@68
|
335 return self.id()
|
jpayne@68
|
336
|
jpayne@68
|
337 def run(self, result):
|
jpayne@68
|
338 # could call result.addError(...) - but this test-like object
|
jpayne@68
|
339 # shouldn't be run anyway
|
jpayne@68
|
340 pass
|
jpayne@68
|
341
|
jpayne@68
|
342 def __call__(self, result):
|
jpayne@68
|
343 return self.run(result)
|
jpayne@68
|
344
|
jpayne@68
|
345 def countTestCases(self):
|
jpayne@68
|
346 return 0
|
jpayne@68
|
347
|
jpayne@68
|
348 def _isnotsuite(test):
|
jpayne@68
|
349 "A crude way to tell apart testcases and suites with duck-typing"
|
jpayne@68
|
350 try:
|
jpayne@68
|
351 iter(test)
|
jpayne@68
|
352 except TypeError:
|
jpayne@68
|
353 return True
|
jpayne@68
|
354 return False
|
jpayne@68
|
355
|
jpayne@68
|
356
|
jpayne@68
|
357 class _DebugResult(object):
|
jpayne@68
|
358 "Used by the TestSuite to hold previous class when running in debug."
|
jpayne@68
|
359 _previousTestClass = None
|
jpayne@68
|
360 _moduleSetUpFailed = False
|
jpayne@68
|
361 shouldStop = False
|