diff --git a/THANKS b/THANKS index 8f0c944a..57ed4dab 100644 --- a/THANKS +++ b/THANKS @@ -60,6 +60,7 @@ Eric Florenzano Eric Shull Eugene Obukhov Evan Mezeske +Florian Apolloner Gaurav Kumar George Kollias George Notaras diff --git a/docs/source/news.rst b/docs/source/news.rst index e4386ab4..049e66cf 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,6 +2,11 @@ Changelog ========= +20.0 / not released +=================== + +- fix: Added support for binding to file descriptors (:issue:`1107`, :pr:`1809`) + 19.9.0 / 2018/07/03 =================== diff --git a/docs/source/run.rst b/docs/source/run.rst index 3100c526..d0799fa0 100644 --- a/docs/source/run.rst +++ b/docs/source/run.rst @@ -52,8 +52,8 @@ Commonly Used Arguments * ``-c CONFIG, --config=CONFIG`` - Specify a config file in the form ``$(PATH)``, ``file:$(PATH)``, or ``python:$(MODULE_NAME)``. * ``-b BIND, --bind=BIND`` - Specify a server socket to bind. Server sockets - can be any of ``$(HOST)``, ``$(HOST):$(PORT)``, or ``unix:$(PATH)``. - An IP is a valid ``$(HOST)``. + can be any of ``$(HOST)``, ``$(HOST):$(PORT)``, ``fd://$(FD)``, or + ``unix:$(PATH)``. An IP is a valid ``$(HOST)``. * ``-w WORKERS, --workers=WORKERS`` - The number of worker processes. This number should generally be between 2-4 workers per core in the server. Check the :ref:`faq` for ideas on tuning this parameter. diff --git a/docs/source/settings.rst b/docs/source/settings.rst index b9bd56de..bdc62f1e 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -1108,8 +1108,11 @@ bind The socket to bind. -A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``. An IP is -a valid ``HOST``. +A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, +``fd://FD``. An IP is a valid ``HOST``. + +.. versionchanged:: 20.0 + Support for ``fd://FD`` got added. Multiple addresses can be bound. ex.:: diff --git a/gunicorn/config.py b/gunicorn/config.py index e14161b6..a9b18afe 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -557,8 +557,11 @@ class Bind(Setting): desc = """\ The socket to bind. - A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``. An IP is - a valid ``HOST``. + A string of the form: ``HOST``, ``HOST:PORT``, ``unix:PATH``, + ``fd://FD``. An IP is a valid ``HOST``. + + .. versionchanged:: 20.0 + Support for ``fd://FD`` got added. Multiple addresses can be bound. ex.:: diff --git a/gunicorn/sock.py b/gunicorn/sock.py index f61443a1..8d35c4d4 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -11,6 +11,7 @@ import sys import time from gunicorn import util +from gunicorn.socketfromfd import fromfd class BaseSocket(object): @@ -150,7 +151,11 @@ def create_sockets(conf, log, fds=None): listeners = [] # get it only once - laddr = conf.address + addr = conf.address + fdaddr = [bind for bind in addr if isinstance(bind, int)] + if fds: + fdaddr += list(fds) + laddr = [bind for bind in addr if not isinstance(bind, int)] # check ssl config early to raise the error on startup # only the certfile is needed since it can contains the keyfile @@ -161,9 +166,9 @@ def create_sockets(conf, log, fds=None): raise ValueError('keyfile "%s" does not exist' % conf.keyfile) # sockets are already bound - if fds is not None: - for fd in fds: - sock = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) + if fdaddr: + for fd in fdaddr: + sock = fromfd(fd) sock_name = sock.getsockname() sock_type = _sock_type(sock_name) listener = sock_type(sock_name, conf, log, fd=fd) diff --git a/gunicorn/socketfromfd.py b/gunicorn/socketfromfd.py new file mode 100644 index 00000000..4c2847b2 --- /dev/null +++ b/gunicorn/socketfromfd.py @@ -0,0 +1,96 @@ +# Copyright (C) 2016 Christian Heimes +"""socketfromfd -- socket.fromd() with auto-discovery + +ATTENTION: Do not remove this backport till the minimum required version is + Python 3.7. See https://bugs.python.org/issue28134 for details. +""" +from __future__ import print_function + +import ctypes +import os +import socket +import sys +from ctypes.util import find_library + +__all__ = ('fromfd',) + +SO_DOMAIN = getattr(socket, 'SO_DOMAIN', 39) +SO_TYPE = getattr(socket, 'SO_TYPE', 3) +SO_PROTOCOL = getattr(socket, 'SO_PROTOCOL', 38) + + +_libc_name = find_library('c') +if _libc_name is not None: + libc = ctypes.CDLL(_libc_name, use_errno=True) +else: + raise OSError('libc not found') + + +def _errcheck_errno(result, func, arguments): + """Raise OSError by errno for -1 + """ + if result == -1: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno)) + return arguments + + +_libc_getsockopt = libc.getsockopt +_libc_getsockopt.argtypes = [ + ctypes.c_int, # int sockfd + ctypes.c_int, # int level + ctypes.c_int, # int optname + ctypes.c_void_p, # void *optval + ctypes.POINTER(ctypes.c_uint32) # socklen_t *optlen +] +_libc_getsockopt.restype = ctypes.c_int # 0: ok, -1: err +_libc_getsockopt.errcheck = _errcheck_errno + + +def _raw_getsockopt(fd, level, optname): + """Make raw getsockopt() call for int32 optval + + :param fd: socket fd + :param level: SOL_* + :param optname: SO_* + :return: value as int + """ + optval = ctypes.c_int(0) + optlen = ctypes.c_uint32(4) + _libc_getsockopt(fd, level, optname, + ctypes.byref(optval), ctypes.byref(optlen)) + return optval.value + + +def fromfd(fd, keep_fd=True): + """Create a socket from a file descriptor + + socket domain (family), type and protocol are auto-detected. By default + the socket uses a dup()ed fd. The original fd can be closed. + + The parameter `keep_fd` influences fd duplication. Under Python 2 the + fd is still duplicated but the input fd is closed. Under Python 3 and + with `keep_fd=True`, the new socket object uses the same fd. + + :param fd: socket fd + :type fd: int + :param keep_fd: keep input fd + :type keep_fd: bool + :return: socket.socket instance + :raises OSError: for invalid socket fd + """ + family = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_DOMAIN) + typ = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_TYPE) + proto = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_PROTOCOL) + if sys.version_info.major == 2: + # Python 2 has no fileno argument and always duplicates the fd + sockobj = socket.fromfd(fd, family, typ, proto) + sock = socket.socket(None, None, None, _sock=sockobj) + if not keep_fd: + os.close(fd) + return sock + else: + if keep_fd: + return socket.fromfd(fd, family, typ, proto) + else: + return socket.socket(family, typ, proto, fileno=fd) diff --git a/gunicorn/util.py b/gunicorn/util.py index 973d7ed3..899416ad 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -251,6 +251,13 @@ def parse_address(netloc, default_port=8000): if re.match(r'unix:(//)?', netloc): return re.split(r'unix:(//)?', netloc)[-1] + if netloc.startswith("fd://"): + fd = netloc[5:] + try: + return int(fd) + except ValueError: + raise RuntimeError("%r is not a valid file descriptor." % fd) from None + if netloc.startswith("tcp://"): netloc = netloc.split("tcp://")[1] diff --git a/tests/test_config.py b/tests/test_config.py index 2d009088..98420bd0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -429,3 +429,9 @@ def _test_ssl_version(options, expected): with AltArgs(cmdline): app = NoConfigApp() assert app.cfg.ssl_version == expected + + +def test_bind_fd(): + with AltArgs(["prog_name", "-b", "fd://42"]): + app = NoConfigApp() + assert app.cfg.bind == ["fd://42"] diff --git a/tests/test_util.py b/tests/test_util.py index 4d977981..3b8be0c3 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -17,7 +17,8 @@ from urllib.parse import SplitResult ('[::1]:8000', ('::1', 8000)), ('localhost:8000', ('localhost', 8000)), ('127.0.0.1:8000', ('127.0.0.1', 8000)), - ('localhost', ('localhost', 8000)) + ('localhost', ('localhost', 8000)), + ('fd://33', 33), ]) def test_parse_address(test_input, expected): assert util.parse_address(test_input) == expected @@ -29,6 +30,12 @@ def test_parse_address_invalid(): assert "'test' is not a valid port number." in str(err) +def test_parse_fd_invalid(): + with pytest.raises(RuntimeError) as err: + util.parse_address('fd://asd') + assert "'asd' is not a valid file descriptor." in str(err) + + def test_http_date(): assert util.http_date(1508607753.740316) == 'Sat, 21 Oct 2017 17:42:33 GMT'