From 64b26ef76648ad2a70f64379f0f5433117450529 Mon Sep 17 00:00:00 2001 From: Mark Adams Date: Tue, 4 Oct 2016 12:08:14 -0500 Subject: [PATCH] Fix #1368 by adding InotifyReloader and 'use-inotify' configuration option Currently, '--reload' uses FS polling to find out when files have changed. For some time, the Linux kernel has had a feature called inotify that allows applications to monitor for FS events without polling. This commit adds a new 'use-inotify' configuration option that will cause gunicorn to use the new 'InotifyReloader' instead of the default 'Reloader' when 'reload' is enabled. Using inotify can result in lower CPU consumption by gunicorn especially when working with virtualized filesystems or environments with a large number of watched files / directories. --- THANKS | 1 + gunicorn/config.py | 16 +++++++++++ gunicorn/reloader.py | 62 ++++++++++++++++++++++++++++++++++++++++ gunicorn/workers/base.py | 27 ++++++++--------- 4 files changed, 93 insertions(+), 13 deletions(-) diff --git a/THANKS b/THANKS index 49cff951..c3b99cec 100644 --- a/THANKS +++ b/THANKS @@ -162,3 +162,4 @@ Wolfgang Schnerring Jason Madden Eugene Obukhov Jan-Philip Gehrcke +Mark Adams diff --git a/gunicorn/config.py b/gunicorn/config.py index ce8b41d8..02489a90 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -820,6 +820,22 @@ class Reload(Setting): application code or the reload will not work as designed. ''' +class ReloadInotify(Setting): + name = 'inotify' + section = 'Debugging' + cli = ['--use-inotify'] + validator = validate_bool + action = "store_true" + default = False + + desc = '''\ + When using the 'reload' option, use the kernel's inotify APIs to watch + files instead of polling the filesystem. On many systems this could result + in a performance improvement when using 'reload'. + + This setting must be used in conjunction with 'reload' and requires the + 'inotify' package be installed from PyPI. + ''' class Spew(Setting): name = "spew" diff --git a/gunicorn/reloader.py b/gunicorn/reloader.py index 130e3d60..b5e7679f 100644 --- a/gunicorn/reloader.py +++ b/gunicorn/reloader.py @@ -4,6 +4,7 @@ # See the NOTICE for more information. import os +import os.path import re import sys import time @@ -51,3 +52,64 @@ class Reloader(threading.Thread): if self._callback: self._callback(filename) time.sleep(self._interval) + +try: + from inotify.adapters import Inotify + import inotify.constants + has_inotify = True +except ImportError: + has_inotify = False + +class InotifyReloader(): + def __init__(self, callback=None): + raise ImportError('You must have the inotify module installed to use the INotify reloader') + +if has_inotify: + + class InotifyReloader(threading.Thread): + event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE + | inotify.constants.IN_DELETE_SELF | inotify.constants.IN_MODIFY + | inotify.constants.IN_MOVE_SELF | inotify.constants.IN_MOVED_FROM + | inotify.constants.IN_MOVED_TO) + + def __init__(self, extra_files=None, callback=None): + super(InotifyReloader, self).__init__() + self.setDaemon(True) + self._callback = callback + self._dirs = set() + self._watcher = Inotify() + + def add_extra_file(self, filename): + dirname = os.path.dirname(filename) + + if dirname in self._dirs: + return + + self._watcher.add_watch(dirname, mask=self.event_mask) + self._dirs.add(dirname) + + def get_dirs(self): + fnames = [ + os.path.dirname(re.sub('py[co]$', 'py', module.__file__)) + for module in list(sys.modules.values()) + if hasattr(module, '__file__') + ] + + return set(fnames) + + def run(self): + self._dirs = self.get_dirs() + + for dirname in self._dirs: + self._watcher.add_watch(dirname, mask=self.event_mask) + + for event in self._watcher.event_gen(): + if event is None: + continue + + types = event[1] + filename = event[3] + + self._callback(filename) + + diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 4ceb691b..887ad8d6 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -14,7 +14,7 @@ import traceback from gunicorn import util from gunicorn.workers.workertmp import WorkerTmp -from gunicorn.reloader import Reloader +from gunicorn.reloader import Reloader, InotifyReloader, has_inotify from gunicorn.http.errors import ( InvalidHeader, InvalidHeaderName, InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders, @@ -85,18 +85,6 @@ class Worker(object): loop is initiated. """ - # start the reloader - if self.cfg.reload: - def changed(fname): - self.log.info("Worker reloading: %s modified", fname) - self.alive = False - self.cfg.worker_int(self) - time.sleep(0.1) - sys.exit(0) - - self.reloader = Reloader(callback=changed) - self.reloader.start() - # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): @@ -126,6 +114,19 @@ class Worker(object): self.load_wsgi() + # start the reloader + if self.cfg.reload: + def changed(fname): + self.log.info("Worker reloading: %s modified", fname) + self.alive = False + self.cfg.worker_int(self) + time.sleep(0.1) + sys.exit(0) + + reloader_cls = Reloader if not self.cfg.inotify else InotifyReloader + self.reloader = reloader_cls(callback=changed) + self.reloader.start() + self.cfg.post_worker_init(self) # Enter main run loop