jpayne@69: """Loading unittests.""" jpayne@69: jpayne@69: import os jpayne@69: import re jpayne@69: import sys jpayne@69: import traceback jpayne@69: import types jpayne@69: import functools jpayne@69: import warnings jpayne@69: jpayne@69: from fnmatch import fnmatch, fnmatchcase jpayne@69: jpayne@69: from . import case, suite, util jpayne@69: jpayne@69: __unittest = True jpayne@69: jpayne@69: # what about .pyc (etc) jpayne@69: # we would need to avoid loading the same tests multiple times jpayne@69: # from '.py', *and* '.pyc' jpayne@69: VALID_MODULE_NAME = re.compile(r'[_a-z]\w*\.py$', re.IGNORECASE) jpayne@69: jpayne@69: jpayne@69: class _FailedTest(case.TestCase): jpayne@69: _testMethodName = None jpayne@69: jpayne@69: def __init__(self, method_name, exception): jpayne@69: self._exception = exception jpayne@69: super(_FailedTest, self).__init__(method_name) jpayne@69: jpayne@69: def __getattr__(self, name): jpayne@69: if name != self._testMethodName: jpayne@69: return super(_FailedTest, self).__getattr__(name) jpayne@69: def testFailure(): jpayne@69: raise self._exception jpayne@69: return testFailure jpayne@69: jpayne@69: jpayne@69: def _make_failed_import_test(name, suiteClass): jpayne@69: message = 'Failed to import test module: %s\n%s' % ( jpayne@69: name, traceback.format_exc()) jpayne@69: return _make_failed_test(name, ImportError(message), suiteClass, message) jpayne@69: jpayne@69: def _make_failed_load_tests(name, exception, suiteClass): jpayne@69: message = 'Failed to call load_tests:\n%s' % (traceback.format_exc(),) jpayne@69: return _make_failed_test( jpayne@69: name, exception, suiteClass, message) jpayne@69: jpayne@69: def _make_failed_test(methodname, exception, suiteClass, message): jpayne@69: test = _FailedTest(methodname, exception) jpayne@69: return suiteClass((test,)), message jpayne@69: jpayne@69: def _make_skipped_test(methodname, exception, suiteClass): jpayne@69: @case.skip(str(exception)) jpayne@69: def testSkipped(self): jpayne@69: pass jpayne@69: attrs = {methodname: testSkipped} jpayne@69: TestClass = type("ModuleSkipped", (case.TestCase,), attrs) jpayne@69: return suiteClass((TestClass(methodname),)) jpayne@69: jpayne@69: def _jython_aware_splitext(path): jpayne@69: if path.lower().endswith('$py.class'): jpayne@69: return path[:-9] jpayne@69: return os.path.splitext(path)[0] jpayne@69: jpayne@69: jpayne@69: class TestLoader(object): jpayne@69: """ jpayne@69: This class is responsible for loading tests according to various criteria jpayne@69: and returning them wrapped in a TestSuite jpayne@69: """ jpayne@69: testMethodPrefix = 'test' jpayne@69: sortTestMethodsUsing = staticmethod(util.three_way_cmp) jpayne@69: testNamePatterns = None jpayne@69: suiteClass = suite.TestSuite jpayne@69: _top_level_dir = None jpayne@69: jpayne@69: def __init__(self): jpayne@69: super(TestLoader, self).__init__() jpayne@69: self.errors = [] jpayne@69: # Tracks packages which we have called into via load_tests, to jpayne@69: # avoid infinite re-entrancy. jpayne@69: self._loading_packages = set() jpayne@69: jpayne@69: def loadTestsFromTestCase(self, testCaseClass): jpayne@69: """Return a suite of all test cases contained in testCaseClass""" jpayne@69: if issubclass(testCaseClass, suite.TestSuite): jpayne@69: raise TypeError("Test cases should not be derived from " jpayne@69: "TestSuite. Maybe you meant to derive from " jpayne@69: "TestCase?") jpayne@69: testCaseNames = self.getTestCaseNames(testCaseClass) jpayne@69: if not testCaseNames and hasattr(testCaseClass, 'runTest'): jpayne@69: testCaseNames = ['runTest'] jpayne@69: loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) jpayne@69: return loaded_suite jpayne@69: jpayne@69: # XXX After Python 3.5, remove backward compatibility hacks for jpayne@69: # use_load_tests deprecation via *args and **kws. See issue 16662. jpayne@69: def loadTestsFromModule(self, module, *args, pattern=None, **kws): jpayne@69: """Return a suite of all test cases contained in the given module""" jpayne@69: # This method used to take an undocumented and unofficial jpayne@69: # use_load_tests argument. For backward compatibility, we still jpayne@69: # accept the argument (which can also be the first position) but we jpayne@69: # ignore it and issue a deprecation warning if it's present. jpayne@69: if len(args) > 0 or 'use_load_tests' in kws: jpayne@69: warnings.warn('use_load_tests is deprecated and ignored', jpayne@69: DeprecationWarning) jpayne@69: kws.pop('use_load_tests', None) jpayne@69: if len(args) > 1: jpayne@69: # Complain about the number of arguments, but don't forget the jpayne@69: # required `module` argument. jpayne@69: complaint = len(args) + 1 jpayne@69: raise TypeError('loadTestsFromModule() takes 1 positional argument but {} were given'.format(complaint)) jpayne@69: if len(kws) != 0: jpayne@69: # Since the keyword arguments are unsorted (see PEP 468), just jpayne@69: # pick the alphabetically sorted first argument to complain about, jpayne@69: # if multiple were given. At least the error message will be jpayne@69: # predictable. jpayne@69: complaint = sorted(kws)[0] jpayne@69: raise TypeError("loadTestsFromModule() got an unexpected keyword argument '{}'".format(complaint)) jpayne@69: tests = [] jpayne@69: for name in dir(module): jpayne@69: obj = getattr(module, name) jpayne@69: if isinstance(obj, type) and issubclass(obj, case.TestCase): jpayne@69: tests.append(self.loadTestsFromTestCase(obj)) jpayne@69: jpayne@69: load_tests = getattr(module, 'load_tests', None) jpayne@69: tests = self.suiteClass(tests) jpayne@69: if load_tests is not None: jpayne@69: try: jpayne@69: return load_tests(self, tests, pattern) jpayne@69: except Exception as e: jpayne@69: error_case, error_message = _make_failed_load_tests( jpayne@69: module.__name__, e, self.suiteClass) jpayne@69: self.errors.append(error_message) jpayne@69: return error_case jpayne@69: return tests jpayne@69: jpayne@69: def loadTestsFromName(self, name, module=None): jpayne@69: """Return a suite of all test cases given a string specifier. jpayne@69: jpayne@69: The name may resolve either to a module, a test case class, a jpayne@69: test method within a test case class, or a callable object which jpayne@69: returns a TestCase or TestSuite instance. jpayne@69: jpayne@69: The method optionally resolves the names relative to a given module. jpayne@69: """ jpayne@69: parts = name.split('.') jpayne@69: error_case, error_message = None, None jpayne@69: if module is None: jpayne@69: parts_copy = parts[:] jpayne@69: while parts_copy: jpayne@69: try: jpayne@69: module_name = '.'.join(parts_copy) jpayne@69: module = __import__(module_name) jpayne@69: break jpayne@69: except ImportError: jpayne@69: next_attribute = parts_copy.pop() jpayne@69: # Last error so we can give it to the user if needed. jpayne@69: error_case, error_message = _make_failed_import_test( jpayne@69: next_attribute, self.suiteClass) jpayne@69: if not parts_copy: jpayne@69: # Even the top level import failed: report that error. jpayne@69: self.errors.append(error_message) jpayne@69: return error_case jpayne@69: parts = parts[1:] jpayne@69: obj = module jpayne@69: for part in parts: jpayne@69: try: jpayne@69: parent, obj = obj, getattr(obj, part) jpayne@69: except AttributeError as e: jpayne@69: # We can't traverse some part of the name. jpayne@69: if (getattr(obj, '__path__', None) is not None jpayne@69: and error_case is not None): jpayne@69: # This is a package (no __path__ per importlib docs), and we jpayne@69: # encountered an error importing something. We cannot tell jpayne@69: # the difference between package.WrongNameTestClass and jpayne@69: # package.wrong_module_name so we just report the jpayne@69: # ImportError - it is more informative. jpayne@69: self.errors.append(error_message) jpayne@69: return error_case jpayne@69: else: jpayne@69: # Otherwise, we signal that an AttributeError has occurred. jpayne@69: error_case, error_message = _make_failed_test( jpayne@69: part, e, self.suiteClass, jpayne@69: 'Failed to access attribute:\n%s' % ( jpayne@69: traceback.format_exc(),)) jpayne@69: self.errors.append(error_message) jpayne@69: return error_case jpayne@69: jpayne@69: if isinstance(obj, types.ModuleType): jpayne@69: return self.loadTestsFromModule(obj) jpayne@69: elif isinstance(obj, type) and issubclass(obj, case.TestCase): jpayne@69: return self.loadTestsFromTestCase(obj) jpayne@69: elif (isinstance(obj, types.FunctionType) and jpayne@69: isinstance(parent, type) and jpayne@69: issubclass(parent, case.TestCase)): jpayne@69: name = parts[-1] jpayne@69: inst = parent(name) jpayne@69: # static methods follow a different path jpayne@69: if not isinstance(getattr(inst, name), types.FunctionType): jpayne@69: return self.suiteClass([inst]) jpayne@69: elif isinstance(obj, suite.TestSuite): jpayne@69: return obj jpayne@69: if callable(obj): jpayne@69: test = obj() jpayne@69: if isinstance(test, suite.TestSuite): jpayne@69: return test jpayne@69: elif isinstance(test, case.TestCase): jpayne@69: return self.suiteClass([test]) jpayne@69: else: jpayne@69: raise TypeError("calling %s returned %s, not a test" % jpayne@69: (obj, test)) jpayne@69: else: jpayne@69: raise TypeError("don't know how to make test from: %s" % obj) jpayne@69: jpayne@69: def loadTestsFromNames(self, names, module=None): jpayne@69: """Return a suite of all test cases found using the given sequence jpayne@69: of string specifiers. See 'loadTestsFromName()'. jpayne@69: """ jpayne@69: suites = [self.loadTestsFromName(name, module) for name in names] jpayne@69: return self.suiteClass(suites) jpayne@69: jpayne@69: def getTestCaseNames(self, testCaseClass): jpayne@69: """Return a sorted sequence of method names found within testCaseClass jpayne@69: """ jpayne@69: def shouldIncludeMethod(attrname): jpayne@69: if not attrname.startswith(self.testMethodPrefix): jpayne@69: return False jpayne@69: testFunc = getattr(testCaseClass, attrname) jpayne@69: if not callable(testFunc): jpayne@69: return False jpayne@69: fullName = f'%s.%s.%s' % ( jpayne@69: testCaseClass.__module__, testCaseClass.__qualname__, attrname jpayne@69: ) jpayne@69: return self.testNamePatterns is None or \ jpayne@69: any(fnmatchcase(fullName, pattern) for pattern in self.testNamePatterns) jpayne@69: testFnNames = list(filter(shouldIncludeMethod, dir(testCaseClass))) jpayne@69: if self.sortTestMethodsUsing: jpayne@69: testFnNames.sort(key=functools.cmp_to_key(self.sortTestMethodsUsing)) jpayne@69: return testFnNames jpayne@69: jpayne@69: def discover(self, start_dir, pattern='test*.py', top_level_dir=None): jpayne@69: """Find and return all test modules from the specified start jpayne@69: directory, recursing into subdirectories to find them and return all jpayne@69: tests found within them. Only test files that match the pattern will jpayne@69: be loaded. (Using shell style pattern matching.) jpayne@69: jpayne@69: All test modules must be importable from the top level of the project. jpayne@69: If the start directory is not the top level directory then the top jpayne@69: level directory must be specified separately. jpayne@69: jpayne@69: If a test package name (directory with '__init__.py') matches the jpayne@69: pattern then the package will be checked for a 'load_tests' function. If jpayne@69: this exists then it will be called with (loader, tests, pattern) unless jpayne@69: the package has already had load_tests called from the same discovery jpayne@69: invocation, in which case the package module object is not scanned for jpayne@69: tests - this ensures that when a package uses discover to further jpayne@69: discover child tests that infinite recursion does not happen. jpayne@69: jpayne@69: If load_tests exists then discovery does *not* recurse into the package, jpayne@69: load_tests is responsible for loading all tests in the package. jpayne@69: jpayne@69: The pattern is deliberately not stored as a loader attribute so that jpayne@69: packages can continue discovery themselves. top_level_dir is stored so jpayne@69: load_tests does not need to pass this argument in to loader.discover(). jpayne@69: jpayne@69: Paths are sorted before being imported to ensure reproducible execution jpayne@69: order even on filesystems with non-alphabetical ordering like ext3/4. jpayne@69: """ jpayne@69: set_implicit_top = False jpayne@69: if top_level_dir is None and self._top_level_dir is not None: jpayne@69: # make top_level_dir optional if called from load_tests in a package jpayne@69: top_level_dir = self._top_level_dir jpayne@69: elif top_level_dir is None: jpayne@69: set_implicit_top = True jpayne@69: top_level_dir = start_dir jpayne@69: jpayne@69: top_level_dir = os.path.abspath(top_level_dir) jpayne@69: jpayne@69: if not top_level_dir in sys.path: jpayne@69: # all test modules must be importable from the top level directory jpayne@69: # should we *unconditionally* put the start directory in first jpayne@69: # in sys.path to minimise likelihood of conflicts between installed jpayne@69: # modules and development versions? jpayne@69: sys.path.insert(0, top_level_dir) jpayne@69: self._top_level_dir = top_level_dir jpayne@69: jpayne@69: is_not_importable = False jpayne@69: is_namespace = False jpayne@69: tests = [] jpayne@69: if os.path.isdir(os.path.abspath(start_dir)): jpayne@69: start_dir = os.path.abspath(start_dir) jpayne@69: if start_dir != top_level_dir: jpayne@69: is_not_importable = not os.path.isfile(os.path.join(start_dir, '__init__.py')) jpayne@69: else: jpayne@69: # support for discovery from dotted module names jpayne@69: try: jpayne@69: __import__(start_dir) jpayne@69: except ImportError: jpayne@69: is_not_importable = True jpayne@69: else: jpayne@69: the_module = sys.modules[start_dir] jpayne@69: top_part = start_dir.split('.')[0] jpayne@69: try: jpayne@69: start_dir = os.path.abspath( jpayne@69: os.path.dirname((the_module.__file__))) jpayne@69: except AttributeError: jpayne@69: # look for namespace packages jpayne@69: try: jpayne@69: spec = the_module.__spec__ jpayne@69: except AttributeError: jpayne@69: spec = None jpayne@69: jpayne@69: if spec and spec.loader is None: jpayne@69: if spec.submodule_search_locations is not None: jpayne@69: is_namespace = True jpayne@69: jpayne@69: for path in the_module.__path__: jpayne@69: if (not set_implicit_top and jpayne@69: not path.startswith(top_level_dir)): jpayne@69: continue jpayne@69: self._top_level_dir = \ jpayne@69: (path.split(the_module.__name__ jpayne@69: .replace(".", os.path.sep))[0]) jpayne@69: tests.extend(self._find_tests(path, jpayne@69: pattern, jpayne@69: namespace=True)) jpayne@69: elif the_module.__name__ in sys.builtin_module_names: jpayne@69: # builtin module jpayne@69: raise TypeError('Can not use builtin modules ' jpayne@69: 'as dotted module names') from None jpayne@69: else: jpayne@69: raise TypeError( jpayne@69: 'don\'t know how to discover from {!r}' jpayne@69: .format(the_module)) from None jpayne@69: jpayne@69: if set_implicit_top: jpayne@69: if not is_namespace: jpayne@69: self._top_level_dir = \ jpayne@69: self._get_directory_containing_module(top_part) jpayne@69: sys.path.remove(top_level_dir) jpayne@69: else: jpayne@69: sys.path.remove(top_level_dir) jpayne@69: jpayne@69: if is_not_importable: jpayne@69: raise ImportError('Start directory is not importable: %r' % start_dir) jpayne@69: jpayne@69: if not is_namespace: jpayne@69: tests = list(self._find_tests(start_dir, pattern)) jpayne@69: return self.suiteClass(tests) jpayne@69: jpayne@69: def _get_directory_containing_module(self, module_name): jpayne@69: module = sys.modules[module_name] jpayne@69: full_path = os.path.abspath(module.__file__) jpayne@69: jpayne@69: if os.path.basename(full_path).lower().startswith('__init__.py'): jpayne@69: return os.path.dirname(os.path.dirname(full_path)) jpayne@69: else: jpayne@69: # here we have been given a module rather than a package - so jpayne@69: # all we can do is search the *same* directory the module is in jpayne@69: # should an exception be raised instead jpayne@69: return os.path.dirname(full_path) jpayne@69: jpayne@69: def _get_name_from_path(self, path): jpayne@69: if path == self._top_level_dir: jpayne@69: return '.' jpayne@69: path = _jython_aware_splitext(os.path.normpath(path)) jpayne@69: jpayne@69: _relpath = os.path.relpath(path, self._top_level_dir) jpayne@69: assert not os.path.isabs(_relpath), "Path must be within the project" jpayne@69: assert not _relpath.startswith('..'), "Path must be within the project" jpayne@69: jpayne@69: name = _relpath.replace(os.path.sep, '.') jpayne@69: return name jpayne@69: jpayne@69: def _get_module_from_name(self, name): jpayne@69: __import__(name) jpayne@69: return sys.modules[name] jpayne@69: jpayne@69: def _match_path(self, path, full_path, pattern): jpayne@69: # override this method to use alternative matching strategy jpayne@69: return fnmatch(path, pattern) jpayne@69: jpayne@69: def _find_tests(self, start_dir, pattern, namespace=False): jpayne@69: """Used by discovery. Yields test suites it loads.""" jpayne@69: # Handle the __init__ in this package jpayne@69: name = self._get_name_from_path(start_dir) jpayne@69: # name is '.' when start_dir == top_level_dir (and top_level_dir is by jpayne@69: # definition not a package). jpayne@69: if name != '.' and name not in self._loading_packages: jpayne@69: # name is in self._loading_packages while we have called into jpayne@69: # loadTestsFromModule with name. jpayne@69: tests, should_recurse = self._find_test_path( jpayne@69: start_dir, pattern, namespace) jpayne@69: if tests is not None: jpayne@69: yield tests jpayne@69: if not should_recurse: jpayne@69: # Either an error occurred, or load_tests was used by the jpayne@69: # package. jpayne@69: return jpayne@69: # Handle the contents. jpayne@69: paths = sorted(os.listdir(start_dir)) jpayne@69: for path in paths: jpayne@69: full_path = os.path.join(start_dir, path) jpayne@69: tests, should_recurse = self._find_test_path( jpayne@69: full_path, pattern, namespace) jpayne@69: if tests is not None: jpayne@69: yield tests jpayne@69: if should_recurse: jpayne@69: # we found a package that didn't use load_tests. jpayne@69: name = self._get_name_from_path(full_path) jpayne@69: self._loading_packages.add(name) jpayne@69: try: jpayne@69: yield from self._find_tests(full_path, pattern, namespace) jpayne@69: finally: jpayne@69: self._loading_packages.discard(name) jpayne@69: jpayne@69: def _find_test_path(self, full_path, pattern, namespace=False): jpayne@69: """Used by discovery. jpayne@69: jpayne@69: Loads tests from a single file, or a directories' __init__.py when jpayne@69: passed the directory. jpayne@69: jpayne@69: Returns a tuple (None_or_tests_from_file, should_recurse). jpayne@69: """ jpayne@69: basename = os.path.basename(full_path) jpayne@69: if os.path.isfile(full_path): jpayne@69: if not VALID_MODULE_NAME.match(basename): jpayne@69: # valid Python identifiers only jpayne@69: return None, False jpayne@69: if not self._match_path(basename, full_path, pattern): jpayne@69: return None, False jpayne@69: # if the test file matches, load it jpayne@69: name = self._get_name_from_path(full_path) jpayne@69: try: jpayne@69: module = self._get_module_from_name(name) jpayne@69: except case.SkipTest as e: jpayne@69: return _make_skipped_test(name, e, self.suiteClass), False jpayne@69: except: jpayne@69: error_case, error_message = \ jpayne@69: _make_failed_import_test(name, self.suiteClass) jpayne@69: self.errors.append(error_message) jpayne@69: return error_case, False jpayne@69: else: jpayne@69: mod_file = os.path.abspath( jpayne@69: getattr(module, '__file__', full_path)) jpayne@69: realpath = _jython_aware_splitext( jpayne@69: os.path.realpath(mod_file)) jpayne@69: fullpath_noext = _jython_aware_splitext( jpayne@69: os.path.realpath(full_path)) jpayne@69: if realpath.lower() != fullpath_noext.lower(): jpayne@69: module_dir = os.path.dirname(realpath) jpayne@69: mod_name = _jython_aware_splitext( jpayne@69: os.path.basename(full_path)) jpayne@69: expected_dir = os.path.dirname(full_path) jpayne@69: msg = ("%r module incorrectly imported from %r. Expected " jpayne@69: "%r. Is this module globally installed?") jpayne@69: raise ImportError( jpayne@69: msg % (mod_name, module_dir, expected_dir)) jpayne@69: return self.loadTestsFromModule(module, pattern=pattern), False jpayne@69: elif os.path.isdir(full_path): jpayne@69: if (not namespace and jpayne@69: not os.path.isfile(os.path.join(full_path, '__init__.py'))): jpayne@69: return None, False jpayne@69: jpayne@69: load_tests = None jpayne@69: tests = None jpayne@69: name = self._get_name_from_path(full_path) jpayne@69: try: jpayne@69: package = self._get_module_from_name(name) jpayne@69: except case.SkipTest as e: jpayne@69: return _make_skipped_test(name, e, self.suiteClass), False jpayne@69: except: jpayne@69: error_case, error_message = \ jpayne@69: _make_failed_import_test(name, self.suiteClass) jpayne@69: self.errors.append(error_message) jpayne@69: return error_case, False jpayne@69: else: jpayne@69: load_tests = getattr(package, 'load_tests', None) jpayne@69: # Mark this package as being in load_tests (possibly ;)) jpayne@69: self._loading_packages.add(name) jpayne@69: try: jpayne@69: tests = self.loadTestsFromModule(package, pattern=pattern) jpayne@69: if load_tests is not None: jpayne@69: # loadTestsFromModule(package) has loaded tests for us. jpayne@69: return tests, False jpayne@69: return tests, True jpayne@69: finally: jpayne@69: self._loading_packages.discard(name) jpayne@69: else: jpayne@69: return None, False jpayne@69: jpayne@69: jpayne@69: defaultTestLoader = TestLoader() jpayne@69: jpayne@69: jpayne@69: def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None): jpayne@69: loader = TestLoader() jpayne@69: loader.sortTestMethodsUsing = sortUsing jpayne@69: loader.testMethodPrefix = prefix jpayne@69: loader.testNamePatterns = testNamePatterns jpayne@69: if suiteClass: jpayne@69: loader.suiteClass = suiteClass jpayne@69: return loader jpayne@69: jpayne@69: def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None): jpayne@69: return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass) jpayne@69: jpayne@69: def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp, jpayne@69: suiteClass=suite.TestSuite): jpayne@69: return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase( jpayne@69: testCaseClass) jpayne@69: jpayne@69: def findTestCases(module, prefix='test', sortUsing=util.three_way_cmp, jpayne@69: suiteClass=suite.TestSuite): jpayne@69: return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(\ jpayne@69: module)