Added support to --bind to open file descriptors (#1809)

Fixes #1107

THANKS

@@ -60,6 +60,7 @@ Eric Florenzano <floguy@gmail.com>
 Eric Shull <eric@elevenbasetwo.com>
 Eugene Obukhov <irvind25@gmail.com>
 Evan Mezeske <evan@meebo-inc.com>
+Florian Apolloner <florian@apolloner.eu>
 Gaurav Kumar <gauravkumar37@gmail.com>
 George Kollias <georgioskollias@gmail.com>
 George Notaras <gnot@g-loaded.eu>

docs/source/news.rst

@@ -2,6 +2,11 @@
 Changelog
 =========
 
+20.0 / not released
+===================
+
+- fix: Added support for binding to file descriptors (:issue:`1107`, :pr:`1809`)
+
 19.9.0 / 2018/07/03
 ===================
 

docs/source/run.rst

@@ -52,8 +52,8 @@ Commonly Used Arguments
 * ``-c CONFIG, --config=CONFIG`` - Specify a config file in the form
   ``$(PATH)``, ``file:$(PATH)``, or ``python:$(MODULE_NAME)``.
 * ``-b BIND, --bind=BIND`` - Specify a server socket to bind. Server sockets
-  can be any of ``$(HOST)``, ``$(HOST):$(PORT)``, or ``unix:$(PATH)``.
+  can be any of ``$(HOST)``, ``$(HOST):$(PORT)``, ``fd://$(FD)``, or
-  An IP is a valid ``$(HOST)``.
+  ``unix:$(PATH)``. An IP is a valid ``$(HOST)``.
 * ``-w WORKERS, --workers=WORKERS`` - The number of worker processes. This
   number should generally be between 2-4 workers per core in the server.
   Check the :ref:`faq` for ideas on tuning this parameter.

docs/source/settings.rst

@@ -1108,8 +1108,11 @@ bind
 
 The socket to bind.
 
-A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``. An IP is
+A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``,
-a valid ``HOST``.
+``fd://FD``. An IP is a valid ``HOST``.
+
+.. versionchanged:: 20.0
+   Support for ``fd://FD`` got added.
 
 Multiple addresses can be bound. ex.::
 

gunicorn/config.py

@@ -557,8 +557,11 @@ class Bind(Setting):
     desc = """\
         The socket to bind.
 
-        A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``. An IP is
+        A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``,
-        a valid ``HOST``.
+        ``fd://FD``. An IP is a valid ``HOST``.
+
+        .. versionchanged:: 20.0
+           Support for ``fd://FD`` got added.
 
         Multiple addresses can be bound. ex.::
 

gunicorn/sock.py

@@ -11,6 +11,7 @@ import sys
 import time
 
 from gunicorn import util
+from gunicorn.socketfromfd import fromfd
 
 
 class BaseSocket(object):
@@ -150,7 +151,11 @@ def create_sockets(conf, log, fds=None):
     listeners = []
 
     # get it only once
-    laddr = conf.address
+    addr = conf.address
+    fdaddr = [bind for bind in addr if isinstance(bind, int)]
+    if fds:
+        fdaddr += list(fds)
+    laddr = [bind for bind in addr if not isinstance(bind, int)]
 
     # check ssl config early to raise the error on startup
     # only the certfile is needed since it can contains the keyfile
@@ -161,9 +166,9 @@ def create_sockets(conf, log, fds=None):
         raise ValueError('keyfile "%s" does not exist' % conf.keyfile)
 
     # sockets are already bound
-    if fds is not None:
+    if fdaddr:
-        for fd in fds:
+        for fd in fdaddr:
-            sock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM)
+            sock = fromfd(fd)
             sock_name = sock.getsockname()
             sock_type = _sock_type(sock_name)
             listener = sock_type(sock_name, conf, log, fd=fd)

gunicorn/socketfromfd.py

@@ -0,0 +1,96 @@
+# Copyright (C) 2016  Christian Heimes
+"""socketfromfd -- socket.fromd() with auto-discovery
+
+ATTENTION: Do not remove this backport till the minimum required version is
+           Python 3.7. See https://bugs.python.org/issue28134 for details.
+"""
+from __future__ import print_function
+
+import ctypes
+import os
+import socket
+import sys
+from ctypes.util import find_library
+
+__all__ = ('fromfd',)
+
+SO_DOMAIN = getattr(socket, 'SO_DOMAIN', 39)
+SO_TYPE = getattr(socket, 'SO_TYPE', 3)
+SO_PROTOCOL = getattr(socket, 'SO_PROTOCOL', 38)
+
+
+_libc_name = find_library('c')
+if _libc_name is not None:
+    libc = ctypes.CDLL(_libc_name, use_errno=True)
+else:
+    raise OSError('libc not found')
+
+
+def _errcheck_errno(result, func, arguments):
+    """Raise OSError by errno for -1
+    """
+    if result == -1:
+        errno = ctypes.get_errno()
+        raise OSError(errno, os.strerror(errno))
+    return arguments
+
+
+_libc_getsockopt = libc.getsockopt
+_libc_getsockopt.argtypes = [
+    ctypes.c_int,  # int sockfd
+    ctypes.c_int,  # int level
+    ctypes.c_int,  # int optname
+    ctypes.c_void_p,  # void *optval
+    ctypes.POINTER(ctypes.c_uint32)  # socklen_t *optlen
+]
+_libc_getsockopt.restype = ctypes.c_int  # 0: ok, -1: err
+_libc_getsockopt.errcheck = _errcheck_errno
+
+
+def _raw_getsockopt(fd, level, optname):
+    """Make raw getsockopt() call for int32 optval
+
+    :param fd: socket fd
+    :param level: SOL_*
+    :param optname: SO_*
+    :return: value as int
+    """
+    optval = ctypes.c_int(0)
+    optlen = ctypes.c_uint32(4)
+    _libc_getsockopt(fd, level, optname,
+                     ctypes.byref(optval), ctypes.byref(optlen))
+    return optval.value
+
+
+def fromfd(fd, keep_fd=True):
+    """Create a socket from a file descriptor
+
+    socket domain (family), type and protocol are auto-detected. By default
+    the socket uses a dup()ed fd. The original fd can be closed.
+
+    The parameter `keep_fd` influences fd duplication. Under Python 2 the
+    fd is still duplicated but the input fd is closed. Under Python 3 and
+    with `keep_fd=True`, the new socket object uses the same fd.
+
+    :param fd: socket fd
+    :type fd: int
+    :param keep_fd: keep input fd
+    :type keep_fd: bool
+    :return: socket.socket instance
+    :raises OSError: for invalid socket fd
+    """
+    family = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_DOMAIN)
+    typ = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_TYPE)
+    proto = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_PROTOCOL)
+    if sys.version_info.major == 2:
+        # Python 2 has no fileno argument and always duplicates the fd
+        sockobj = socket.fromfd(fd, family, typ, proto)
+        sock = socket.socket(None, None, None, _sock=sockobj)
+        if not keep_fd:
+            os.close(fd)
+        return sock
+    else:
+        if keep_fd:
+            return socket.fromfd(fd, family, typ, proto)
+        else:
+            return socket.socket(family, typ, proto, fileno=fd)

gunicorn/util.py

@@ -251,6 +251,13 @@ def parse_address(netloc, default_port=8000):
     if re.match(r'unix:(//)?', netloc):
         return re.split(r'unix:(//)?', netloc)[-1]
 
+    if netloc.startswith("fd://"):
+        fd = netloc[5:]
+        try:
+            return int(fd)
+        except ValueError:
+            raise RuntimeError("%r is not a valid file descriptor." % fd) from None
+
     if netloc.startswith("tcp://"):
         netloc = netloc.split("tcp://")[1]
 

tests/test_config.py

@@ -429,3 +429,9 @@ def _test_ssl_version(options, expected):
     with AltArgs(cmdline):
         app = NoConfigApp()
     assert app.cfg.ssl_version == expected
+
+
+def test_bind_fd():
+    with AltArgs(["prog_name", "-b", "fd://42"]):
+        app = NoConfigApp()
+    assert app.cfg.bind == ["fd://42"]

tests/test_util.py

@@ -17,7 +17,8 @@ from urllib.parse import SplitResult
     ('[::1]:8000', ('::1', 8000)),
     ('localhost:8000', ('localhost', 8000)),
     ('127.0.0.1:8000', ('127.0.0.1', 8000)),
-    ('localhost', ('localhost', 8000))
+    ('localhost', ('localhost', 8000)),
+    ('fd://33', 33),
 ])
 def test_parse_address(test_input, expected):
     assert util.parse_address(test_input) == expected
@@ -29,6 +30,12 @@ def test_parse_address_invalid():
     assert "'test' is not a valid port number." in str(err)
 
 
+def test_parse_fd_invalid():
+    with pytest.raises(RuntimeError) as err:
+        util.parse_address('fd://asd')
+    assert "'asd' is not a valid file descriptor." in str(err)
+
+
 def test_http_date():
     assert util.http_date(1508607753.740316) == 'Sat, 21 Oct 2017 17:42:33 GMT'