_reloader.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. import os
  2. import sys
  3. import time
  4. import subprocess
  5. import threading
  6. from itertools import chain
  7. from werkzeug._internal import _log
  8. from werkzeug._compat import PY2, iteritems, text_type
  9. def _iter_module_files():
  10. """This iterates over all relevant Python files. It goes through all
  11. loaded files from modules, all files in folders of already loaded modules
  12. as well as all files reachable through a package.
  13. """
  14. # The list call is necessary on Python 3 in case the module
  15. # dictionary modifies during iteration.
  16. for module in list(sys.modules.values()):
  17. if module is None:
  18. continue
  19. filename = getattr(module, '__file__', None)
  20. if filename:
  21. if os.path.isdir(filename) and \
  22. os.path.exists(os.path.join(filename, "__init__.py")):
  23. filename = os.path.join(filename, "__init__.py")
  24. old = None
  25. while not os.path.isfile(filename):
  26. old = filename
  27. filename = os.path.dirname(filename)
  28. if filename == old:
  29. break
  30. else:
  31. if filename[-4:] in ('.pyc', '.pyo'):
  32. filename = filename[:-1]
  33. yield filename
  34. def _find_observable_paths(extra_files=None):
  35. """Finds all paths that should be observed."""
  36. rv = set(os.path.dirname(os.path.abspath(x))
  37. if os.path.isfile(x) else os.path.abspath(x)
  38. for x in sys.path)
  39. for filename in extra_files or ():
  40. rv.add(os.path.dirname(os.path.abspath(filename)))
  41. for module in list(sys.modules.values()):
  42. fn = getattr(module, '__file__', None)
  43. if fn is None:
  44. continue
  45. fn = os.path.abspath(fn)
  46. rv.add(os.path.dirname(fn))
  47. return _find_common_roots(rv)
  48. def _get_args_for_reloading():
  49. """Returns the executable. This contains a workaround for windows
  50. if the executable is incorrectly reported to not have the .exe
  51. extension which can cause bugs on reloading.
  52. """
  53. rv = [sys.executable]
  54. py_script = sys.argv[0]
  55. if os.name == 'nt' and not os.path.exists(py_script) and \
  56. os.path.exists(py_script + '.exe'):
  57. py_script += '.exe'
  58. if os.path.splitext(rv[0])[1] == '.exe' and os.path.splitext(py_script)[1] == '.exe':
  59. rv.pop(0)
  60. rv.append(py_script)
  61. rv.extend(sys.argv[1:])
  62. return rv
  63. def _find_common_roots(paths):
  64. """Out of some paths it finds the common roots that need monitoring."""
  65. paths = [x.split(os.path.sep) for x in paths]
  66. root = {}
  67. for chunks in sorted(paths, key=len, reverse=True):
  68. node = root
  69. for chunk in chunks:
  70. node = node.setdefault(chunk, {})
  71. node.clear()
  72. rv = set()
  73. def _walk(node, path):
  74. for prefix, child in iteritems(node):
  75. _walk(child, path + (prefix,))
  76. if not node:
  77. rv.add('/'.join(path))
  78. _walk(root, ())
  79. return rv
  80. class ReloaderLoop(object):
  81. name = None
  82. # monkeypatched by testsuite. wrapping with `staticmethod` is required in
  83. # case time.sleep has been replaced by a non-c function (e.g. by
  84. # `eventlet.monkey_patch`) before we get here
  85. _sleep = staticmethod(time.sleep)
  86. def __init__(self, extra_files=None, interval=1):
  87. self.extra_files = set(os.path.abspath(x)
  88. for x in extra_files or ())
  89. self.interval = interval
  90. def run(self):
  91. pass
  92. def restart_with_reloader(self):
  93. """Spawn a new Python interpreter with the same arguments as this one,
  94. but running the reloader thread.
  95. """
  96. while 1:
  97. _log('info', ' * Restarting with %s' % self.name)
  98. args = _get_args_for_reloading()
  99. new_environ = os.environ.copy()
  100. new_environ['WERKZEUG_RUN_MAIN'] = 'true'
  101. # a weird bug on windows. sometimes unicode strings end up in the
  102. # environment and subprocess.call does not like this, encode them
  103. # to latin1 and continue.
  104. if os.name == 'nt' and PY2:
  105. for key, value in iteritems(new_environ):
  106. if isinstance(value, text_type):
  107. new_environ[key] = value.encode('iso-8859-1')
  108. exit_code = subprocess.call(args, env=new_environ,
  109. close_fds=False)
  110. if exit_code != 3:
  111. return exit_code
  112. def trigger_reload(self, filename):
  113. self.log_reload(filename)
  114. sys.exit(3)
  115. def log_reload(self, filename):
  116. filename = os.path.abspath(filename)
  117. _log('info', ' * Detected change in %r, reloading' % filename)
  118. class StatReloaderLoop(ReloaderLoop):
  119. name = 'stat'
  120. def run(self):
  121. mtimes = {}
  122. while 1:
  123. for filename in chain(_iter_module_files(),
  124. self.extra_files):
  125. try:
  126. mtime = os.stat(filename).st_mtime
  127. except OSError:
  128. continue
  129. old_time = mtimes.get(filename)
  130. if old_time is None:
  131. mtimes[filename] = mtime
  132. continue
  133. elif mtime > old_time:
  134. self.trigger_reload(filename)
  135. self._sleep(self.interval)
  136. class WatchdogReloaderLoop(ReloaderLoop):
  137. def __init__(self, *args, **kwargs):
  138. ReloaderLoop.__init__(self, *args, **kwargs)
  139. from watchdog.observers import Observer
  140. from watchdog.events import FileSystemEventHandler
  141. self.observable_paths = set()
  142. def _check_modification(filename):
  143. if filename in self.extra_files:
  144. self.trigger_reload(filename)
  145. dirname = os.path.dirname(filename)
  146. if dirname.startswith(tuple(self.observable_paths)):
  147. if filename.endswith(('.pyc', '.pyo', '.py')):
  148. self.trigger_reload(filename)
  149. class _CustomHandler(FileSystemEventHandler):
  150. def on_created(self, event):
  151. _check_modification(event.src_path)
  152. def on_modified(self, event):
  153. _check_modification(event.src_path)
  154. def on_moved(self, event):
  155. _check_modification(event.src_path)
  156. _check_modification(event.dest_path)
  157. def on_deleted(self, event):
  158. _check_modification(event.src_path)
  159. reloader_name = Observer.__name__.lower()
  160. if reloader_name.endswith('observer'):
  161. reloader_name = reloader_name[:-8]
  162. reloader_name += ' reloader'
  163. self.name = reloader_name
  164. self.observer_class = Observer
  165. self.event_handler = _CustomHandler()
  166. self.should_reload = False
  167. def trigger_reload(self, filename):
  168. # This is called inside an event handler, which means throwing
  169. # SystemExit has no effect.
  170. # https://github.com/gorakhargosh/watchdog/issues/294
  171. self.should_reload = True
  172. self.log_reload(filename)
  173. def run(self):
  174. watches = {}
  175. observer = self.observer_class()
  176. observer.start()
  177. try:
  178. while not self.should_reload:
  179. to_delete = set(watches)
  180. paths = _find_observable_paths(self.extra_files)
  181. for path in paths:
  182. if path not in watches:
  183. try:
  184. watches[path] = observer.schedule(
  185. self.event_handler, path, recursive=True)
  186. except OSError:
  187. # Clear this path from list of watches We don't want
  188. # the same error message showing again in the next
  189. # iteration.
  190. watches[path] = None
  191. to_delete.discard(path)
  192. for path in to_delete:
  193. watch = watches.pop(path, None)
  194. if watch is not None:
  195. observer.unschedule(watch)
  196. self.observable_paths = paths
  197. self._sleep(self.interval)
  198. finally:
  199. observer.stop()
  200. observer.join()
  201. sys.exit(3)
  202. reloader_loops = {
  203. 'stat': StatReloaderLoop,
  204. 'watchdog': WatchdogReloaderLoop,
  205. }
  206. try:
  207. __import__('watchdog.observers')
  208. except ImportError:
  209. reloader_loops['auto'] = reloader_loops['stat']
  210. else:
  211. reloader_loops['auto'] = reloader_loops['watchdog']
  212. def run_with_reloader(main_func, extra_files=None, interval=1,
  213. reloader_type='auto'):
  214. """Run the given function in an independent python interpreter."""
  215. import signal
  216. reloader = reloader_loops[reloader_type](extra_files, interval)
  217. signal.signal(signal.SIGTERM, lambda *args: sys.exit(0))
  218. try:
  219. if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
  220. t = threading.Thread(target=main_func, args=())
  221. t.setDaemon(True)
  222. t.start()
  223. reloader.run()
  224. else:
  225. sys.exit(reloader.restart_with_reloader())
  226. except KeyboardInterrupt:
  227. pass