jpayne@69: #
jpayne@69: # Code used to start processes when using the spawn or forkserver
jpayne@69: # start methods.
jpayne@69: #
jpayne@69: # multiprocessing/spawn.py
jpayne@69: #
jpayne@69: # Copyright (c) 2006-2008, R Oudkerk
jpayne@69: # Licensed to PSF under a Contributor Agreement.
jpayne@69: #
jpayne@69: 
jpayne@69: import os
jpayne@69: import sys
jpayne@69: import runpy
jpayne@69: import types
jpayne@69: 
jpayne@69: from . import get_start_method, set_start_method
jpayne@69: from . import process
jpayne@69: from .context import reduction
jpayne@69: from . import util
jpayne@69: 
jpayne@69: __all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable',
jpayne@69:            'get_preparation_data', 'get_command_line', 'import_main_path']
jpayne@69: 
jpayne@69: #
jpayne@69: # _python_exe is the assumed path to the python executable.
jpayne@69: # People embedding Python want to modify it.
jpayne@69: #
jpayne@69: 
jpayne@69: if sys.platform != 'win32':
jpayne@69:     WINEXE = False
jpayne@69:     WINSERVICE = False
jpayne@69: else:
jpayne@69:     WINEXE = getattr(sys, 'frozen', False)
jpayne@69:     WINSERVICE = sys.executable.lower().endswith("pythonservice.exe")
jpayne@69: 
jpayne@69: if WINSERVICE:
jpayne@69:     _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
jpayne@69: else:
jpayne@69:     _python_exe = sys._base_executable
jpayne@69: 
jpayne@69: def set_executable(exe):
jpayne@69:     global _python_exe
jpayne@69:     _python_exe = exe
jpayne@69: 
jpayne@69: def get_executable():
jpayne@69:     return _python_exe
jpayne@69: 
jpayne@69: #
jpayne@69: #
jpayne@69: #
jpayne@69: 
jpayne@69: def is_forking(argv):
jpayne@69:     '''
jpayne@69:     Return whether commandline indicates we are forking
jpayne@69:     '''
jpayne@69:     if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
jpayne@69:         return True
jpayne@69:     else:
jpayne@69:         return False
jpayne@69: 
jpayne@69: 
jpayne@69: def freeze_support():
jpayne@69:     '''
jpayne@69:     Run code for process object if this in not the main process
jpayne@69:     '''
jpayne@69:     if is_forking(sys.argv):
jpayne@69:         kwds = {}
jpayne@69:         for arg in sys.argv[2:]:
jpayne@69:             name, value = arg.split('=')
jpayne@69:             if value == 'None':
jpayne@69:                 kwds[name] = None
jpayne@69:             else:
jpayne@69:                 kwds[name] = int(value)
jpayne@69:         spawn_main(**kwds)
jpayne@69:         sys.exit()
jpayne@69: 
jpayne@69: 
jpayne@69: def get_command_line(**kwds):
jpayne@69:     '''
jpayne@69:     Returns prefix of command line used for spawning a child process
jpayne@69:     '''
jpayne@69:     if getattr(sys, 'frozen', False):
jpayne@69:         return ([sys.executable, '--multiprocessing-fork'] +
jpayne@69:                 ['%s=%r' % item for item in kwds.items()])
jpayne@69:     else:
jpayne@69:         prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
jpayne@69:         prog %= ', '.join('%s=%r' % item for item in kwds.items())
jpayne@69:         opts = util._args_from_interpreter_flags()
jpayne@69:         return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork']
jpayne@69: 
jpayne@69: 
jpayne@69: def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
jpayne@69:     '''
jpayne@69:     Run code specified by data received over pipe
jpayne@69:     '''
jpayne@69:     assert is_forking(sys.argv), "Not forking"
jpayne@69:     if sys.platform == 'win32':
jpayne@69:         import msvcrt
jpayne@69:         import _winapi
jpayne@69: 
jpayne@69:         if parent_pid is not None:
jpayne@69:             source_process = _winapi.OpenProcess(
jpayne@69:                 _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE,
jpayne@69:                 False, parent_pid)
jpayne@69:         else:
jpayne@69:             source_process = None
jpayne@69:         new_handle = reduction.duplicate(pipe_handle,
jpayne@69:                                          source_process=source_process)
jpayne@69:         fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
jpayne@69:         parent_sentinel = source_process
jpayne@69:     else:
jpayne@69:         from . import resource_tracker
jpayne@69:         resource_tracker._resource_tracker._fd = tracker_fd
jpayne@69:         fd = pipe_handle
jpayne@69:         parent_sentinel = os.dup(pipe_handle)
jpayne@69:     exitcode = _main(fd, parent_sentinel)
jpayne@69:     sys.exit(exitcode)
jpayne@69: 
jpayne@69: 
jpayne@69: def _main(fd, parent_sentinel):
jpayne@69:     with os.fdopen(fd, 'rb', closefd=True) as from_parent:
jpayne@69:         process.current_process()._inheriting = True
jpayne@69:         try:
jpayne@69:             preparation_data = reduction.pickle.load(from_parent)
jpayne@69:             prepare(preparation_data)
jpayne@69:             self = reduction.pickle.load(from_parent)
jpayne@69:         finally:
jpayne@69:             del process.current_process()._inheriting
jpayne@69:     return self._bootstrap(parent_sentinel)
jpayne@69: 
jpayne@69: 
jpayne@69: def _check_not_importing_main():
jpayne@69:     if getattr(process.current_process(), '_inheriting', False):
jpayne@69:         raise RuntimeError('''
jpayne@69:         An attempt has been made to start a new process before the
jpayne@69:         current process has finished its bootstrapping phase.
jpayne@69: 
jpayne@69:         This probably means that you are not using fork to start your
jpayne@69:         child processes and you have forgotten to use the proper idiom
jpayne@69:         in the main module:
jpayne@69: 
jpayne@69:             if __name__ == '__main__':
jpayne@69:                 freeze_support()
jpayne@69:                 ...
jpayne@69: 
jpayne@69:         The "freeze_support()" line can be omitted if the program
jpayne@69:         is not going to be frozen to produce an executable.''')
jpayne@69: 
jpayne@69: 
jpayne@69: def get_preparation_data(name):
jpayne@69:     '''
jpayne@69:     Return info about parent needed by child to unpickle process object
jpayne@69:     '''
jpayne@69:     _check_not_importing_main()
jpayne@69:     d = dict(
jpayne@69:         log_to_stderr=util._log_to_stderr,
jpayne@69:         authkey=process.current_process().authkey,
jpayne@69:         )
jpayne@69: 
jpayne@69:     if util._logger is not None:
jpayne@69:         d['log_level'] = util._logger.getEffectiveLevel()
jpayne@69: 
jpayne@69:     sys_path=sys.path.copy()
jpayne@69:     try:
jpayne@69:         i = sys_path.index('')
jpayne@69:     except ValueError:
jpayne@69:         pass
jpayne@69:     else:
jpayne@69:         sys_path[i] = process.ORIGINAL_DIR
jpayne@69: 
jpayne@69:     d.update(
jpayne@69:         name=name,
jpayne@69:         sys_path=sys_path,
jpayne@69:         sys_argv=sys.argv,
jpayne@69:         orig_dir=process.ORIGINAL_DIR,
jpayne@69:         dir=os.getcwd(),
jpayne@69:         start_method=get_start_method(),
jpayne@69:         )
jpayne@69: 
jpayne@69:     # Figure out whether to initialise main in the subprocess as a module
jpayne@69:     # or through direct execution (or to leave it alone entirely)
jpayne@69:     main_module = sys.modules['__main__']
jpayne@69:     main_mod_name = getattr(main_module.__spec__, "name", None)
jpayne@69:     if main_mod_name is not None:
jpayne@69:         d['init_main_from_name'] = main_mod_name
jpayne@69:     elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
jpayne@69:         main_path = getattr(main_module, '__file__', None)
jpayne@69:         if main_path is not None:
jpayne@69:             if (not os.path.isabs(main_path) and
jpayne@69:                         process.ORIGINAL_DIR is not None):
jpayne@69:                 main_path = os.path.join(process.ORIGINAL_DIR, main_path)
jpayne@69:             d['init_main_from_path'] = os.path.normpath(main_path)
jpayne@69: 
jpayne@69:     return d
jpayne@69: 
jpayne@69: #
jpayne@69: # Prepare current process
jpayne@69: #
jpayne@69: 
jpayne@69: old_main_modules = []
jpayne@69: 
jpayne@69: def prepare(data):
jpayne@69:     '''
jpayne@69:     Try to get current process ready to unpickle process object
jpayne@69:     '''
jpayne@69:     if 'name' in data:
jpayne@69:         process.current_process().name = data['name']
jpayne@69: 
jpayne@69:     if 'authkey' in data:
jpayne@69:         process.current_process().authkey = data['authkey']
jpayne@69: 
jpayne@69:     if 'log_to_stderr' in data and data['log_to_stderr']:
jpayne@69:         util.log_to_stderr()
jpayne@69: 
jpayne@69:     if 'log_level' in data:
jpayne@69:         util.get_logger().setLevel(data['log_level'])
jpayne@69: 
jpayne@69:     if 'sys_path' in data:
jpayne@69:         sys.path = data['sys_path']
jpayne@69: 
jpayne@69:     if 'sys_argv' in data:
jpayne@69:         sys.argv = data['sys_argv']
jpayne@69: 
jpayne@69:     if 'dir' in data:
jpayne@69:         os.chdir(data['dir'])
jpayne@69: 
jpayne@69:     if 'orig_dir' in data:
jpayne@69:         process.ORIGINAL_DIR = data['orig_dir']
jpayne@69: 
jpayne@69:     if 'start_method' in data:
jpayne@69:         set_start_method(data['start_method'], force=True)
jpayne@69: 
jpayne@69:     if 'init_main_from_name' in data:
jpayne@69:         _fixup_main_from_name(data['init_main_from_name'])
jpayne@69:     elif 'init_main_from_path' in data:
jpayne@69:         _fixup_main_from_path(data['init_main_from_path'])
jpayne@69: 
jpayne@69: # Multiprocessing module helpers to fix up the main module in
jpayne@69: # spawned subprocesses
jpayne@69: def _fixup_main_from_name(mod_name):
jpayne@69:     # __main__.py files for packages, directories, zip archives, etc, run
jpayne@69:     # their "main only" code unconditionally, so we don't even try to
jpayne@69:     # populate anything in __main__, nor do we make any changes to
jpayne@69:     # __main__ attributes
jpayne@69:     current_main = sys.modules['__main__']
jpayne@69:     if mod_name == "__main__" or mod_name.endswith(".__main__"):
jpayne@69:         return
jpayne@69: 
jpayne@69:     # If this process was forked, __main__ may already be populated
jpayne@69:     if getattr(current_main.__spec__, "name", None) == mod_name:
jpayne@69:         return
jpayne@69: 
jpayne@69:     # Otherwise, __main__ may contain some non-main code where we need to
jpayne@69:     # support unpickling it properly. We rerun it as __mp_main__ and make
jpayne@69:     # the normal __main__ an alias to that
jpayne@69:     old_main_modules.append(current_main)
jpayne@69:     main_module = types.ModuleType("__mp_main__")
jpayne@69:     main_content = runpy.run_module(mod_name,
jpayne@69:                                     run_name="__mp_main__",
jpayne@69:                                     alter_sys=True)
jpayne@69:     main_module.__dict__.update(main_content)
jpayne@69:     sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
jpayne@69: 
jpayne@69: 
jpayne@69: def _fixup_main_from_path(main_path):
jpayne@69:     # If this process was forked, __main__ may already be populated
jpayne@69:     current_main = sys.modules['__main__']
jpayne@69: 
jpayne@69:     # Unfortunately, the main ipython launch script historically had no
jpayne@69:     # "if __name__ == '__main__'" guard, so we work around that
jpayne@69:     # by treating it like a __main__.py file
jpayne@69:     # See https://github.com/ipython/ipython/issues/4698
jpayne@69:     main_name = os.path.splitext(os.path.basename(main_path))[0]
jpayne@69:     if main_name == 'ipython':
jpayne@69:         return
jpayne@69: 
jpayne@69:     # Otherwise, if __file__ already has the setting we expect,
jpayne@69:     # there's nothing more to do
jpayne@69:     if getattr(current_main, '__file__', None) == main_path:
jpayne@69:         return
jpayne@69: 
jpayne@69:     # If the parent process has sent a path through rather than a module
jpayne@69:     # name we assume it is an executable script that may contain
jpayne@69:     # non-main code that needs to be executed
jpayne@69:     old_main_modules.append(current_main)
jpayne@69:     main_module = types.ModuleType("__mp_main__")
jpayne@69:     main_content = runpy.run_path(main_path,
jpayne@69:                                   run_name="__mp_main__")
jpayne@69:     main_module.__dict__.update(main_content)
jpayne@69:     sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module
jpayne@69: 
jpayne@69: 
jpayne@69: def import_main_path(main_path):
jpayne@69:     '''
jpayne@69:     Set sys.modules['__main__'] to module at main_path
jpayne@69:     '''
jpayne@69:     _fixup_main_from_path(main_path)