Fix #1368 by adding InotifyReloader and 'use-inotify' configuration option

Currently, '--reload' uses FS polling to find out when files have
changed. For some time, the Linux kernel has had a feature called
inotify that allows applications to monitor for FS events without
polling.

This commit adds a new 'use-inotify' configuration option that will
cause gunicorn to use the new 'InotifyReloader' instead of the default
'Reloader' when 'reload' is enabled.

Using inotify can result in lower CPU consumption by gunicorn especially
when working with virtualized filesystems or environments with a large
number of watched files / directories.

THANKS

@@ -162,3 +162,4 @@ Wolfgang Schnerring <wosc@wosc.de>
 Jason Madden <jason@nextthought.com>
 Eugene Obukhov <irvind25@gmail.com>
 Jan-Philip Gehrcke <jgehrcke@googlemail.com>
+Mark Adams <mark@markadams.me>

gunicorn/config.py

@@ -820,6 +820,22 @@ class Reload(Setting):
         application code or the reload will not work as designed.
         '''
 
+class ReloadInotify(Setting):
+    name = 'inotify'
+    section = 'Debugging'
+    cli = ['--use-inotify']
+    validator = validate_bool
+    action = "store_true"
+    default = False
+
+    desc = '''\
+        When using the 'reload' option, use the kernel's inotify APIs to watch 
+        files instead of polling the filesystem. On many systems this could result
+        in a performance improvement when using 'reload'.
+
+        This setting must be used in conjunction with 'reload' and requires the
+        'inotify' package be installed from PyPI.
+        '''
 
 class Spew(Setting):
     name = "spew"

gunicorn/reloader.py

@@ -4,6 +4,7 @@
 # See the NOTICE for more information.
 
 import os
+import os.path
 import re
 import sys
 import time
@@ -51,3 +52,64 @@ class Reloader(threading.Thread):
                     if self._callback:
                         self._callback(filename)
             time.sleep(self._interval)
+
+try:
+    from inotify.adapters import Inotify
+    import inotify.constants
+    has_inotify = True
+except ImportError:
+    has_inotify = False
+
+class InotifyReloader():
+    def __init__(self, callback=None):
+        raise ImportError('You must have the inotify module installed to use the INotify reloader')
+
+if has_inotify:
+
+    class InotifyReloader(threading.Thread):
+        event_mask = (inotify.constants.IN_CREATE | inotify.constants.IN_DELETE 
+                      | inotify.constants.IN_DELETE_SELF | inotify.constants.IN_MODIFY 
+                      | inotify.constants.IN_MOVE_SELF | inotify.constants.IN_MOVED_FROM 
+                      | inotify.constants.IN_MOVED_TO)
+        
+        def __init__(self, extra_files=None, callback=None):
+            super(InotifyReloader, self).__init__()
+            self.setDaemon(True)
+            self._callback = callback
+            self._dirs = set()
+            self._watcher = Inotify()
+
+        def add_extra_file(self, filename):
+            dirname = os.path.dirname(filename)
+
+            if dirname in self._dirs:
+                return
+
+            self._watcher.add_watch(dirname, mask=self.event_mask)
+            self._dirs.add(dirname)
+
+        def get_dirs(self):
+            fnames = [
+                os.path.dirname(re.sub('py[co]$', 'py', module.__file__))
+                for module in list(sys.modules.values())
+                if hasattr(module, '__file__')
+            ]
+
+            return set(fnames)
+
+        def run(self):
+            self._dirs = self.get_dirs()
+
+            for dirname in self._dirs:
+                self._watcher.add_watch(dirname, mask=self.event_mask)
+            
+            for event in self._watcher.event_gen():
+                if event is None:
+                    continue
+                
+                types = event[1]
+                filename = event[3]
+
+                self._callback(filename)
+
+

gunicorn/workers/base.py

@@ -14,7 +14,7 @@ import traceback
 
 from gunicorn import util
 from gunicorn.workers.workertmp import WorkerTmp
-from gunicorn.reloader import Reloader
+from gunicorn.reloader import Reloader, InotifyReloader, has_inotify
 from gunicorn.http.errors import (
     InvalidHeader, InvalidHeaderName, InvalidRequestLine, InvalidRequestMethod,
     InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders,
@@ -85,18 +85,6 @@ class Worker(object):
         loop is initiated.
         """
 
-        # start the reloader
-        if self.cfg.reload:
-            def changed(fname):
-                self.log.info("Worker reloading: %s modified", fname)
-                self.alive = False
-                self.cfg.worker_int(self)
-                time.sleep(0.1)
-                sys.exit(0)
-
-            self.reloader = Reloader(callback=changed)
-            self.reloader.start()
-
         # set environment' variables
         if self.cfg.env:
             for k, v in self.cfg.env.items():
@@ -126,6 +114,19 @@ class Worker(object):
 
         self.load_wsgi()
 
+        # start the reloader
+        if self.cfg.reload:
+            def changed(fname):
+                self.log.info("Worker reloading: %s modified", fname)
+                self.alive = False
+                self.cfg.worker_int(self)
+                time.sleep(0.1)
+                sys.exit(0)
+            
+            reloader_cls = Reloader if not self.cfg.inotify else InotifyReloader
+            self.reloader = reloader_cls(callback=changed)
+            self.reloader.start()
+
         self.cfg.post_worker_init(self)
 
         # Enter main run loop