From 20774b8331dfb76845bf4a3e4c3c3d4a9d6a8efb Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sat, 23 Nov 2019 15:36:56 -0800 Subject: [PATCH 01/11] Remove support for bdist_rpm --- docs/source/news.rst | 5 +++++ rpm/install | 16 ---------------- setup.cfg | 6 ------ 3 files changed, 5 insertions(+), 22 deletions(-) delete mode 100644 rpm/install diff --git a/docs/source/news.rst b/docs/source/news.rst index 801aa74b..7e72a4e5 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -2,6 +2,11 @@ Changelog ========= +Unreleased +========== + +- remove support for the `bdist_rpm` build + 20.0.2 / 2019/11/23 =================== diff --git a/rpm/install b/rpm/install deleted file mode 100644 index 4a8f9dc7..00000000 --- a/rpm/install +++ /dev/null @@ -1,16 +0,0 @@ -%{__python} setup.py install --skip-build --root=$RPM_BUILD_ROOT - -# Build the HTML documentation using the default theme. -%{__python} setup.py build_sphinx - -%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5) -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} -%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} -%endif - -INSTALLED_FILES="\ -%{python_sitelib}/* -%{_bindir}/* -%doc LICENSE NOTICE README.rst THANKS build/sphinx/html examples/example_config.py -" -echo "$INSTALLED_FILES" > INSTALLED_FILES diff --git a/setup.cfg b/setup.cfg index 20017519..64d0abc0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -[bdist_rpm] -build-requires = python2-devel python-setuptools python-sphinx -requires = python-setuptools >= 0.6c6 python-ctypes -install_script = rpm/install -group = System Environment/Daemons - [tool:pytest] norecursedirs = examples lib local src testpaths = tests/ From 404a7120234e2b1119f4e8a3662c542e4d8700c8 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 10:25:44 +0100 Subject: [PATCH 02/11] socketfromfd: remove python 2 compatibility --- gunicorn/socketfromfd.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/gunicorn/socketfromfd.py b/gunicorn/socketfromfd.py index 71b40d3c..a7f7b899 100644 --- a/gunicorn/socketfromfd.py +++ b/gunicorn/socketfromfd.py @@ -95,15 +95,8 @@ def fromfd(fd, keep_fd=True): family = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_DOMAIN) typ = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_TYPE) proto = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_PROTOCOL) - if sys.version_info.major == 2: - # Python 2 has no fileno argument and always duplicates the fd - sockobj = socket.fromfd(fd, family, typ, proto) - sock = socket.socket(None, None, None, _sock=sockobj) - if not keep_fd: - os.close(fd) - return sock + s + if keep_fd: + return socket.fromfd(fd, family, typ, proto) else: - if keep_fd: - return socket.fromfd(fd, family, typ, proto) - else: - return socket.socket(family, typ, proto, fileno=fd) + return socket.socket(family, typ, proto, fileno=fd) From e69e8020539989d1b5d4846f5329843c6150767d Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 11:53:02 +0100 Subject: [PATCH 03/11] socketfromfd: fix cross platform usage on osx and maybe some other systems, SO_PROTOCOL is not set, also the socket family is not detected correctly. This patch remove default values in socketfromfd and do the following: * detect proper family using getsockname instead of getsocktopt * if no type is given, default to SOCK_STREAM (we don't have any other type) * if no protocol is given, default to 0 and let the system take care of it. --- gunicorn/socketfromfd.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/gunicorn/socketfromfd.py b/gunicorn/socketfromfd.py index a7f7b899..521e0f00 100644 --- a/gunicorn/socketfromfd.py +++ b/gunicorn/socketfromfd.py @@ -16,10 +16,6 @@ from .util import find_library __all__ = ('fromfd',) -SO_DOMAIN = getattr(socket, 'SO_DOMAIN', 39) -SO_TYPE = getattr(socket, 'SO_TYPE', 3) -SO_PROTOCOL = getattr(socket, 'SO_PROTOCOL', 38) - _libc_name = find_library('c') if _libc_name is not None: if sys.platform.startswith("aix"): @@ -47,8 +43,12 @@ def _errcheck_errno(result, func, arguments): if platform.system() == 'SunOS': _libc_getsockopt = libc._so_getsockopt + _libc_getsockname = libc._so_getsockname else: _libc_getsockopt = libc.getsockopt + _libc_getsockname = libc.getsockname + + _libc_getsockopt.argtypes = [ ctypes.c_int, # int sockfd ctypes.c_int, # int level @@ -59,6 +59,17 @@ _libc_getsockopt.argtypes = [ _libc_getsockopt.restype = ctypes.c_int # 0: ok, -1: err _libc_getsockopt.errcheck = _errcheck_errno +class SockAddr(ctypes.Structure): + _fields_ = [ + ('sa_len', ctypes.c_uint8), + ('sa_family', ctypes.c_uint8), + ('sa_data', ctypes.c_char * 14) + ] +_libc_getsockname.argtypes = [ + ctypes.c_int, + ctypes.POINTER(SockAddr), + ctypes.POINTER(ctypes.c_int) +] def _raw_getsockopt(fd, level, optname): """Make raw getsockopt() call for int32 optval @@ -74,6 +85,11 @@ def _raw_getsockopt(fd, level, optname): ctypes.byref(optval), ctypes.byref(optlen)) return optval.value +def _raw_getsockname(fd): + sockaddr = SockAddr() + len = ctypes.c_int(ctypes.sizeof(sockaddr)) + _libc_getsockname(fd, sockaddr, len) + return sockaddr.sa_family def fromfd(fd, keep_fd=True): """Create a socket from a file descriptor @@ -92,10 +108,15 @@ def fromfd(fd, keep_fd=True): :return: socket.socket instance :raises OSError: for invalid socket fd """ - family = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_DOMAIN) - typ = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_TYPE) - proto = _raw_getsockopt(fd, socket.SOL_SOCKET, SO_PROTOCOL) - s + family = _raw_getsockname(fd) + if hasattr(socket, 'SO_TYPE'): + typ = _raw_getsockopt(fd, socket.SOL_SOCKET, getattr(socket, 'SO_TYPE')) + else: + typ = socket.SOCK_STREAM + if hasattr(socket, 'SO_PROTOCOL'): + proto = _raw_getsockopt(fd, socket.SOL_SOCKET, getattr(socket, 'SO_PROTOCOL')) + else: + proto = 0 if keep_fd: return socket.fromfd(fd, family, typ, proto) else: From d38f6e47ea03a4c0121036a4300fe2d0155e85c6 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 11:57:59 +0100 Subject: [PATCH 04/11] fix linting on python 3.8 --- gunicorn/socketfromfd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/socketfromfd.py b/gunicorn/socketfromfd.py index 521e0f00..ec9d26f1 100644 --- a/gunicorn/socketfromfd.py +++ b/gunicorn/socketfromfd.py @@ -87,8 +87,8 @@ def _raw_getsockopt(fd, level, optname): def _raw_getsockname(fd): sockaddr = SockAddr() - len = ctypes.c_int(ctypes.sizeof(sockaddr)) - _libc_getsockname(fd, sockaddr, len) + sockaddrlen = ctypes.c_int(ctypes.sizeof(sockaddr)) + _libc_getsockname(fd, sockaddr, sockaddrlen) return sockaddr.sa_family def fromfd(fd, keep_fd=True): From 9419d660960869438c1a19838e09b63e6dbfbac6 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 15:37:02 +0100 Subject: [PATCH 05/11] refactor module use appropriate naming for function, make _getsockname more generic and doc more useful --- NOTICE | 8 ++++++++ gunicorn/socketfromfd.py | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/NOTICE b/NOTICE index a2f4aa20..12b21ccb 100644 --- a/NOTICE +++ b/NOTICE @@ -122,3 +122,11 @@ util/unlink.py -------------- backport frop python3 Lib/test/support.py + + +gunicorn.socketfromfd +--------------------- + +Under Apache License 2 + +Copyright (C) 2016 Christian Heimes \ No newline at end of file diff --git a/gunicorn/socketfromfd.py b/gunicorn/socketfromfd.py index ec9d26f1..2bca6749 100644 --- a/gunicorn/socketfromfd.py +++ b/gunicorn/socketfromfd.py @@ -1,9 +1,20 @@ -# Copyright (C) 2016 Christian Heimes -"""socketfromfd -- socket.fromd() with auto-discovery +# -*- coding: utf-8 - +# +# This file is part of gunicorn +# See the NOTICE for more information. -ATTENTION: Do not remove this backport till the minimum required version is - Python 3.7. See https://bugs.python.org/issue28134 for details. +# Copyright (C) 2016 Christian Heimes under Apache License 2 + +# source code based on https://github.com/tiran/socketfromfd/blob/master/socketfromfd.py +# and https://github.com/python/cpython/blob/master/Modules/socketmodule.c + +"""socketfromfd -- create a socket from its file descriptor +This module detect the socket properties. + +note: Before python 3.7 auto detecting the socket was not working. +See https://bugs.python.org/issue28134 for details. """ + from __future__ import print_function import ctypes @@ -43,7 +54,7 @@ def _errcheck_errno(result, func, arguments): if platform.system() == 'SunOS': _libc_getsockopt = libc._so_getsockopt - _libc_getsockname = libc._so_getsockname + _lib_getsockname = libc._so_getsockname else: _libc_getsockopt = libc.getsockopt _libc_getsockname = libc.getsockname @@ -65,13 +76,17 @@ class SockAddr(ctypes.Structure): ('sa_family', ctypes.c_uint8), ('sa_data', ctypes.c_char * 14) ] + + _libc_getsockname.argtypes = [ ctypes.c_int, ctypes.POINTER(SockAddr), ctypes.POINTER(ctypes.c_int) ] +_libc_getsockname.restype = ctypes.c_int # 0: ok, -1: err +_libc_getsockname.errcheck = _errcheck_errno -def _raw_getsockopt(fd, level, optname): +def _getsockopt(fd, level, optname): """Make raw getsockopt() call for int32 optval :param fd: socket fd @@ -85,11 +100,11 @@ def _raw_getsockopt(fd, level, optname): ctypes.byref(optval), ctypes.byref(optlen)) return optval.value -def _raw_getsockname(fd): +def _getsockname(fd): sockaddr = SockAddr() sockaddrlen = ctypes.c_int(ctypes.sizeof(sockaddr)) _libc_getsockname(fd, sockaddr, sockaddrlen) - return sockaddr.sa_family + return sockaddr def fromfd(fd, keep_fd=True): """Create a socket from a file descriptor @@ -108,13 +123,15 @@ def fromfd(fd, keep_fd=True): :return: socket.socket instance :raises OSError: for invalid socket fd """ - family = _raw_getsockname(fd) + sockaddr = _getsockname(fd) + family = sockaddr.sa_family if hasattr(socket, 'SO_TYPE'): - typ = _raw_getsockopt(fd, socket.SOL_SOCKET, getattr(socket, 'SO_TYPE')) + typ = _getsockopt(fd, socket.SOL_SOCKET, getattr(socket, 'SO_TYPE')) else: typ = socket.SOCK_STREAM + if hasattr(socket, 'SO_PROTOCOL'): - proto = _raw_getsockopt(fd, socket.SOL_SOCKET, getattr(socket, 'SO_PROTOCOL')) + proto = _getsockopt(fd, socket.SOL_SOCKET, getattr(socket, 'SO_PROTOCOL')) else: proto = 0 if keep_fd: From 9538358511c31937529c953e98940c5da5940136 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 20:20:05 +0100 Subject: [PATCH 06/11] config file extension should be a python extension This change make it clear what is configuration file for Gunicorn. Using a filename with an extension different than a python extension was never supported. Also it gives us some room for a proper config file. --- docs/source/configure.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 9ed1a484..a3fbb635 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -51,10 +51,11 @@ isn't mentioned in the list of :ref:`settings `. Configuration File ================== -The configuration file should be a valid Python source file. It only needs to -be readable from the file system. More specifically, it does not need to be -importable. Any Python is valid. Just consider that this will be run every time -you start Gunicorn (including when you signal Gunicorn to reload). +The configuration file should be a valid Python source file with a **python +extension** (e.g. `gunicorn.conf.py`). It only needs to be readable from the +file system. More specifically, it does not need to be importable. Any Python +is valid. Just consider that this will be run every time you start Gunicorn +(including when you signal Gunicorn to reload). To set a parameter, just assign to it. There's no special syntax. The values you provide will be used for the configuration values. From 9a3e008eca1b63b27daed2404f2ff4b0e6308640 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 21:07:35 +0100 Subject: [PATCH 07/11] put back configuration file without py extension and deprecate it this change put back (and fix it) support of configuration files without ython extension and warn about its usage. --- gunicorn/app/base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index cb1bd973..9b024f15 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -3,6 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. import importlib.util +import importlib.machinery import os import sys import traceback @@ -94,9 +95,17 @@ class Application(BaseApplication): if not os.path.exists(filename): raise RuntimeError("%r doesn't exist" % filename) + ext = os.path.splitext(filename)[1] + try: module_name = '__config__' - spec = importlib.util.spec_from_file_location(module_name, filename) + if ext in [".py", ".pyc"]: + spec = importlib.util.spec_from_file_location(module_name, filename) + else: + msg = "configuration file should have a valid Python extension.\n" + util.warn(msg) + loader_ = importlib.machinery.SourceFileLoader(module_name, filename) + spec = importlib.util.spec_from_file_location(module_name, filename, loader=loader_) mod = importlib.util.module_from_spec(spec) sys.modules[module_name] = mod spec.loader.exec_module(mod) From b4e52c181ae87d93ce728fd703c37c6b6864ad2d Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 21:22:34 +0100 Subject: [PATCH 08/11] make config doc more explicit explicit what we mean by "not need to be importable". --- docs/source/configure.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index a3fbb635..df5cbe2e 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -53,9 +53,9 @@ Configuration File The configuration file should be a valid Python source file with a **python extension** (e.g. `gunicorn.conf.py`). It only needs to be readable from the -file system. More specifically, it does not need to be importable. Any Python -is valid. Just consider that this will be run every time you start Gunicorn -(including when you signal Gunicorn to reload). +file system. More specifically, it does not have to be on the module path +(sys.path, PYTHONPATH). Any Python is valid. Just consider that this will be +run every time you start Gunicorn (including when you signal Gunicorn to reload). To set a parameter, just assign to it. There's no special syntax. The values you provide will be used for the configuration values. From 8b64aaedd5b247957e89b1e06df8250922cfd439 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 22:57:00 +0100 Subject: [PATCH 09/11] changelog for the 20.0.3 --- docs/source/news.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/news.rst b/docs/source/news.rst index 7e72a4e5..7b9af4c6 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -7,6 +7,15 @@ Unreleased - remove support for the `bdist_rpm` build +20.0.3 / 2019/11/24 +=================== + +- fixed load of a config file without a Python extension +- fixed `socketfromfd.fromfd` when defaults are not set + +.. note:: we now warn when we load a config file without Python Extension +>>>>>>> 5257d92... changelog for the 20.0.3 + 20.0.2 / 2019/11/23 =================== From 2dae3ea165b7ea01f74ad96d4e2508a6055c5f50 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 22:59:12 +0100 Subject: [PATCH 10/11] bump to 20.0.3 --- gunicorn/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index d8c977b1..4d59629c 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,6 +3,6 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (20, 0, 2) +version_info = (20, 0, 3) __version__ = ".".join([str(v) for v in version_info]) SERVER_SOFTWARE = "gunicorn/%s" % __version__ From f646bdee39683cb30a6e30466ac1b49be8358ad5 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 25 Nov 2019 23:15:22 +0100 Subject: [PATCH 11/11] fix bad cherry-picking --- docs/source/news.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/news.rst b/docs/source/news.rst index 7b9af4c6..0839013d 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -14,7 +14,6 @@ Unreleased - fixed `socketfromfd.fromfd` when defaults are not set .. note:: we now warn when we load a config file without Python Extension ->>>>>>> 5257d92... changelog for the 20.0.3 20.0.2 / 2019/11/23 ===================