Merge pull request #1369 from mark-adams/1368-inotify-reloader

Add InotifyReloader and updated '--reload {poll,inotify}' configuration option

THANKS

@@ -162,3 +162,4 @@ Wolfgang Schnerring <wosc@wosc.de>
 Jason Madden <jason@nextthought.com>
 Eugene Obukhov <irvind25@gmail.com>
 Jan-Philip Gehrcke <jgehrcke@googlemail.com>
+Mark Adams <mark@markadams.me>

docs/source/settings.rst

@@ -255,8 +255,8 @@ Debugging
 reload
 ~~~~~~
 
-* ``--reload``
+* ``--reload RELOADER_TYPE``
-* ``False``
+* ``None``
 
 Restart workers when code changes.
 
@@ -267,6 +267,15 @@ The reloader is incompatible with application preloading. When using a
 paste configuration be sure that the server block does not import any
 application code or the reload will not work as designed.
 
+When using this option, you can optionally specify whether you would
+like to use file system polling or the kernel's inotify API to watch
+for changes. Generally, inotify should be preferred if available
+because it consumes less system resources. If no preference is given,
+inotify will attempted with a fallback to FS polling.
+
+Note: In order to use the inotify reloader, you must have the 'inotify'
+package installed.
+
 spew
 ~~~~
 

gunicorn/config.py

@@ -489,6 +489,19 @@ def validate_hostport(val):
     else:
         raise TypeError("Value must consist of: hostname:port")
 
+
+def validate_reloader(val):
+    if val is None:
+        val = 'default'
+
+    choices = ['poll', 'inotify', 'default']
+
+    if val not in choices:
+        raise ConfigError(
+            'Invalid reloader type. Must be one of: %s' % choices
+        )
+
+
 def get_default_config_file():
     config_path = os.path.join(os.path.abspath(os.getcwd()),
             'gunicorn.conf.py')
@@ -806,9 +819,12 @@ class Reload(Setting):
     name = "reload"
     section = 'Debugging'
     cli = ['--reload']
-    validator = validate_bool
+    validator = validate_reloader
-    action = 'store_true'
+    nargs = '?'
-    default = False
+    const = 'default'
+    default = None
+    meta = 'RELOADER_TYPE'
+
     desc = '''\
         Restart workers when code changes.
 
@@ -818,6 +834,15 @@ class Reload(Setting):
         The reloader is incompatible with application preloading. When using a
         paste configuration be sure that the server block does not import any
         application code or the reload will not work as designed.
+
+        When using this option, you can optionally specify whether you would
+        like to use file system polling or the kernel's inotify API to watch
+        for changes. Generally, inotify should be preferred if available
+        because it consumes less system resources. If no preference is given,
+        inotify will attempted with a fallback to FS polling.
+
+        Note: In order to use the inotify reloader, you must have the 'inotify'
+        package installed.
         '''
 
 

gunicorn/reloader.py

@@ -4,6 +4,7 @@
 # See the NOTICE for more information.
 
 import os
+import os.path
 import re
 import sys
 import time
@@ -51,3 +52,66 @@ class Reloader(threading.Thread):
                     if self._callback:
                         self._callback(filename)
             time.sleep(self._interval)
+
+try:
+    from inotify.adapters import Inotify
+    import inotify.constants
+    has_inotify = True
+except ImportError:
+    has_inotify = False
+
+
+class InotifyReloader():
+    def __init__(self, callback=None):
+        raise ImportError('You must have the inotify module installed to use '
+                          'the inotify reloader')
+
+if has_inotify:
+
+    class InotifyReloader(threading.Thread):
+        event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE
+                      | inotify.constants.IN_DELETE_SELF | inotify.constants.IN_MODIFY
+                      | inotify.constants.IN_MOVE_SELF | inotify.constants.IN_MOVED_FROM
+                      | inotify.constants.IN_MOVED_TO)
+
+        def __init__(self, extra_files=None, callback=None):
+            super(InotifyReloader, self).__init__()
+            self.setDaemon(True)
+            self._callback = callback
+            self._dirs = set()
+            self._watcher = Inotify()
+
+        def add_extra_file(self, filename):
+            dirname = os.path.dirname(filename)
+
+            if dirname in self._dirs:
+                return
+
+            self._watcher.add_watch(dirname, mask=self.event_mask)
+            self._dirs.add(dirname)
+
+        def get_dirs(self):
+            fnames = [
+                os.path.dirname(re.sub('py[co]$', 'py', module.__file__))
+                for module in list(sys.modules.values())
+                if hasattr(module, '__file__')
+            ]
+
+            return set(fnames)
+
+        def run(self):
+            self._dirs = self.get_dirs()
+
+            for dirname in self._dirs:
+                self._watcher.add_watch(dirname, mask=self.event_mask)
+
+            for event in self._watcher.event_gen():
+                if event is None:
+                    continue
+
+                filename = event[3]
+
+                self._callback(filename)
+
+
+preferred_reloader = InotifyReloader if has_inotify else Reloader

gunicorn/workers/base.py

@@ -14,7 +14,7 @@ import traceback
 
 from gunicorn import util
 from gunicorn.workers.workertmp import WorkerTmp
-from gunicorn.reloader import Reloader
+from gunicorn.reloader import preferred_reloader, Reloader, InotifyReloader
 from gunicorn.http.errors import (
     InvalidHeader, InvalidHeaderName, InvalidRequestLine, InvalidRequestMethod,
     InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders,
@@ -85,18 +85,6 @@ class Worker(object):
         loop is initiated.
         """
 
-        # start the reloader
-        if self.cfg.reload:
-            def changed(fname):
-                self.log.info("Worker reloading: %s modified", fname)
-                self.alive = False
-                self.cfg.worker_int(self)
-                time.sleep(0.1)
-                sys.exit(0)
-
-            self.reloader = Reloader(callback=changed)
-            self.reloader.start()
-
         # set environment' variables
         if self.cfg.env:
             for k, v in self.cfg.env.items():
@@ -126,6 +114,25 @@ class Worker(object):
 
         self.load_wsgi()
 
+        # start the reloader
+        if self.cfg.reload:
+            def changed(fname):
+                self.log.info("Worker reloading: %s modified", fname)
+                self.alive = False
+                self.cfg.worker_int(self)
+                time.sleep(0.1)
+                sys.exit(0)
+
+            if self.cfg.reload == 'poll':
+                reloader_cls = Reloader
+            elif self.cfg.reload == 'inotify':
+                reloader_cls = InotifyReloader
+            else:
+                reloader_cls = preferred_reloader
+
+            self.reloader = reloader_cls(callback=changed)
+            self.reloader.start()
+
         self.cfg.post_worker_init(self)
 
         # Enter main run loop