From 5a581c0b144c1f356c39d513d60015e878655ea4 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 10 Sep 2021 19:59:15 +0300 Subject: [PATCH 1/3] Update SSLContext handling * Change deprecated ssl.wrap_socket() to SSLContext.wrap_context(). * Add new server hook to allow user to create custom SSLContext. * Updated the documentation. Signed-off-by: Tero Saarni --- docs/source/settings.rst | 21 ++++++++++++++++++++- examples/example_config.py | 30 ++++++++++++++++++++++++++++++ gunicorn/config.py | 19 +++++++++++++++++++ gunicorn/sock.py | 20 +++++++++++++++++++- gunicorn/workers/geventlet.py | 5 ++--- gunicorn/workers/ggevent.py | 3 ++- gunicorn/workers/gthread.py | 4 ++-- gunicorn/workers/gtornado.py | 10 +++------- gunicorn/workers/sync.py | 5 ++--- 9 files changed, 99 insertions(+), 18 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 4c8e9212..bf2ad737 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -916,6 +916,26 @@ Called just before exiting Gunicorn. The callable needs to accept a single instance variable for the Arbiter. +.. _ssl-context: + +``ssl_context`` +~~~~~~~~~~~~~~~ + +**Default:** + +.. code-block:: python + + def ssl_context(config): + return None + +Called when SSLContext is needed. + +Allows fully customized SSL context to be used in place of the default +context. + +The callable needs to accept a single instance variable for the Config. +The callable needs to return SSLContext object. + Server Mechanics ---------------- @@ -1506,4 +1526,3 @@ set this to a higher value. .. note:: ``sync`` worker does not support persistent connections and will ignore this option. - diff --git a/examples/example_config.py b/examples/example_config.py index f8f3c1df..83c3ffbc 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -214,3 +214,33 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") + +def ssl_context(conf): + import ssl + + def set_defaults(context): + context.verify_mode = conf.cert_reqs + context.minimum_version = ssl.TLSVersion.TLSv1_3 + if conf.ciphers: + context.set_ciphers(conf.ciphers) + if conf.ca_certs: + context.load_verify_locations(cafile=conf.ca_certs) + + # Return different server certificate depending which hostname the client + # uses. Requires Python 3.7 or later. + def sni_callback(socket, server_hostname, context): + if server_hostname == "foo.127.0.0.1.nip.io": + new_context = ssl.SSLContext() + new_context.load_cert_chain(certfile="foo.pem", keyfile="foo-key.pem") + set_defaults(new_context) + socket.context = new_context + + context = ssl.SSLContext(conf.ssl_version) + context.sni_callback = sni_callback + set_defaults(context) + + # Load fallback certificate that will be returned when there is no match + # or client did not set TLS SNI (server_hostname == None) + context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) + + return context diff --git a/gunicorn/config.py b/gunicorn/config.py index bc24b700..3f397d12 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1967,6 +1967,25 @@ class OnExit(Setting): The callable needs to accept a single instance variable for the Arbiter. """ +class NewSSLContext(Setting): + name = "ssl_context" + section = "Server Hooks" + validator = validate_callable(1) + type = callable + + def ssl_context(config): + return None + + default = staticmethod(ssl_context) + desc = """\ + Called when SSLContext is needed. + + Allows fully customized SSL context to be used in place of the default + context. + + The callable needs to accept a single instance variable for the Config. + The callable needs to return SSLContext object. + """ class ProxyProtocol(Setting): name = "proxy_protocol" diff --git a/gunicorn/sock.py b/gunicorn/sock.py index d4586770..1481c23b 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -6,6 +6,7 @@ import errno import os import socket +import ssl import stat import sys import time @@ -203,10 +204,27 @@ def create_sockets(conf, log, fds=None): return listeners - def close_sockets(listeners, unlink=True): for sock in listeners: sock_name = sock.getsockname() sock.close() if unlink and _sock_type(sock_name) is UnixSocket: os.unlink(sock_name) + +def ssl_context(conf): + context = conf.ssl_context(conf) + if context is None: + context = ssl.SSLContext(conf.ssl_version) + context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) + context.verify_mode = conf.cert_reqs + if conf.ciphers: + context.set_ciphers(conf.ciphers) + if conf.ca_certs: + context.load_verify_locations(cafile=conf.ca_certs) + + return context + +def ssl_wrap_socket(sock, conf): + return ssl_context(conf).wrap_socket(sock, server_side=True, + suppress_ragged_eofs=conf.suppress_ragged_eofs, + do_handshake_on_connect=conf.do_handshake_on_connect) diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index ea82f3d6..844a1e06 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -21,6 +21,7 @@ import eventlet.wsgi import greenlet from gunicorn.workers.base_async import AsyncWorker +from gunicorn.sock import ssl_wrap_socket # ALREADY_HANDLED is removed in 0.30.3+ now it's `WSGI_LOCAL.already_handled: bool` # https://github.com/eventlet/eventlet/pull/544 @@ -153,9 +154,7 @@ class EventletWorker(AsyncWorker): def handle(self, listener, client, addr): if self.cfg.is_ssl: - client = eventlet.wrap_ssl(client, server_side=True, - **self.cfg.ssl_options) - + client = ssl_wrap_socket(client, self.cfg) super().handle(listener, client, addr) def run(self): diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 3941814f..81a339a8 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -24,6 +24,7 @@ from gevent import hub, monkey, socket, pywsgi import gunicorn from gunicorn.http.wsgi import base_environ +from gunicorn.sock import ssl_context from gunicorn.workers.base_async import AsyncWorker VERSION = "gevent/%s gunicorn/%s" % (gevent.__version__, gunicorn.__version__) @@ -58,7 +59,7 @@ class GeventWorker(AsyncWorker): ssl_args = {} if self.cfg.is_ssl: - ssl_args = dict(server_side=True, **self.cfg.ssl_options) + ssl_args = dict(ssl_context=ssl_context(self.cfg)) for s in self.sockets: s.setblocking(1) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index d5318115..73ea6fdf 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -27,6 +27,7 @@ from threading import RLock from . import base from .. import http from .. import util +from .. import sock from ..http import wsgi @@ -49,8 +50,7 @@ class TConn(object): if self.parser is None: # wrap the socket if needed if self.cfg.is_ssl: - self.sock = ssl.wrap_socket(self.sock, server_side=True, - **self.cfg.ssl_options) + self.sock = sock.ssl_wrap_socket(self.sock, self.cfg) # initialize the parser self.parser = http.RequestParser(self.cfg, self.sock, self.client) diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 9dd3d7bc..9da49957 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -17,6 +17,7 @@ from tornado.ioloop import IOLoop, PeriodicCallback from tornado.wsgi import WSGIContainer from gunicorn.workers.base import Worker from gunicorn import __version__ as gversion +from gunicorn.sock import ssl_context # Tornado 5.0 updated its IOLoop, and the `io_loop` arguments to many @@ -139,16 +140,11 @@ class TornadoWorker(Worker): server_class = _HTTPServer if self.cfg.is_ssl: - _ssl_opt = copy.deepcopy(self.cfg.ssl_options) - # tornado refuses initialization if ssl_options contains following - # options - del _ssl_opt["do_handshake_on_connect"] - del _ssl_opt["suppress_ragged_eofs"] if TORNADO5: - server = server_class(app, ssl_options=_ssl_opt) + server = server_class(app, ssl_options=ssl_context(self.cfg)) else: server = server_class(app, io_loop=self.ioloop, - ssl_options=_ssl_opt) + ssl_options=ssl_context(self.cfg)) else: if TORNADO5: server = server_class(app) diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index eeb7f633..4bdf0003 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -14,6 +14,7 @@ import sys import gunicorn.http as http import gunicorn.http.wsgi as wsgi +import gunicorn.sock as sock import gunicorn.util as util import gunicorn.workers.base as base @@ -128,9 +129,7 @@ class SyncWorker(base.Worker): req = None try: if self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, - **self.cfg.ssl_options) - + client = sock.ssl_wrap_socket(client, self.cfg) parser = http.RequestParser(self.cfg, client, addr) req = next(parser) self.handle_request(listener, req, client, addr) From 362a52bd8458e7cf1d89da60aaa9cb1da05d2784 Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Fri, 11 Feb 2022 19:26:56 +0200 Subject: [PATCH 2/3] Added parameter to ssl_context hook for constructing default context Signed-off-by: Tero Saarni --- examples/example_config.py | 30 ++++++++++++------------------ gunicorn/config.py | 11 +++++++---- gunicorn/sock.py | 6 +++--- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/examples/example_config.py b/examples/example_config.py index 83c3ffbc..5a399a49 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -215,32 +215,26 @@ def worker_int(worker): def worker_abort(worker): worker.log.info("worker received SIGABRT signal") -def ssl_context(conf): +def ssl_context(conf, default_ssl_context_factory): import ssl - def set_defaults(context): - context.verify_mode = conf.cert_reqs - context.minimum_version = ssl.TLSVersion.TLSv1_3 - if conf.ciphers: - context.set_ciphers(conf.ciphers) - if conf.ca_certs: - context.load_verify_locations(cafile=conf.ca_certs) + # The default SSLContext returned by the factory function is initialized + # with the TLS parameters from config, including TLS certificates and other + # parameters. + context = default_ssl_context_factory() - # Return different server certificate depending which hostname the client - # uses. Requires Python 3.7 or later. + # The SSLContext can be further customized, for example by enforcing + # minimum TLS version. + context.minimum_version = ssl.TLSVersion.TLSv1_3 + + # Server can also return different server certificate depending which + # hostname the client uses. Requires Python 3.7 or later. def sni_callback(socket, server_hostname, context): if server_hostname == "foo.127.0.0.1.nip.io": - new_context = ssl.SSLContext() + new_context = default_ssl_context_factory() new_context.load_cert_chain(certfile="foo.pem", keyfile="foo-key.pem") - set_defaults(new_context) socket.context = new_context - context = ssl.SSLContext(conf.ssl_version) context.sni_callback = sni_callback - set_defaults(context) - - # Load fallback certificate that will be returned when there is no match - # or client did not set TLS SNI (server_hostname == None) - context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) return context diff --git a/gunicorn/config.py b/gunicorn/config.py index 3f397d12..a9265374 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1970,11 +1970,11 @@ class OnExit(Setting): class NewSSLContext(Setting): name = "ssl_context" section = "Server Hooks" - validator = validate_callable(1) + validator = validate_callable(2) type = callable - def ssl_context(config): - return None + def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() default = staticmethod(ssl_context) desc = """\ @@ -1983,7 +1983,10 @@ class NewSSLContext(Setting): Allows fully customized SSL context to be used in place of the default context. - The callable needs to accept a single instance variable for the Config. + The callable needs to accept an instance variable for the Config and + a factory function that returns default SSLContext which is initialized + with certificates, private key, cert_reqs, and ciphers according to + config and can be further customized by the callable. The callable needs to return SSLContext object. """ diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 1481c23b..37631e6b 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -212,8 +212,7 @@ def close_sockets(listeners, unlink=True): os.unlink(sock_name) def ssl_context(conf): - context = conf.ssl_context(conf) - if context is None: + def default_ssl_context_factory(): context = ssl.SSLContext(conf.ssl_version) context.load_cert_chain(certfile=conf.certfile, keyfile=conf.keyfile) context.verify_mode = conf.cert_reqs @@ -221,8 +220,9 @@ def ssl_context(conf): context.set_ciphers(conf.ciphers) if conf.ca_certs: context.load_verify_locations(cafile=conf.ca_certs) + return context - return context + return conf.ssl_context(conf, default_ssl_context_factory) def ssl_wrap_socket(sock, conf): return ssl_context(conf).wrap_socket(sock, server_side=True, From ac8bc3a4553e4954d69e8080b3c937dca409a69c Mon Sep 17 00:00:00 2001 From: Tero Saarni Date: Tue, 15 Mar 2022 19:05:30 +0200 Subject: [PATCH 3/3] Regenerated rst docs Signed-off-by: Tero Saarni --- docs/source/settings.rst | 95 ++++++++++++++++++++++++++++++++++------ gunicorn/config.py | 2 + 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index a5a27026..90eed993 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -310,8 +310,6 @@ file format. ``logconfig_dict`` ~~~~~~~~~~~~~~~~~~ -**Command line:** ``--log-config-dict`` - **Default:** ``{}`` The log config dictionary to use, using the standard Python @@ -332,7 +330,7 @@ For more context you can look at the default configuration dictionary for loggin **Command line:** ``--log-syslog-to SYSLOG_ADDR`` -**Default:** ``'unix:///var/run/syslog'`` +**Default:** ``'udp://localhost:514'`` Address to send syslog messages. @@ -921,19 +919,22 @@ The callable needs to accept a single instance variable for the Arbiter. ``ssl_context`` ~~~~~~~~~~~~~~~ -**Default:** +**Default:** .. code-block:: python - def ssl_context(config): - return None + def ssl_context(config, default_ssl_context_factory): + return default_ssl_context_factory() Called when SSLContext is needed. Allows fully customized SSL context to be used in place of the default context. -The callable needs to accept a single instance variable for the Config. +The callable needs to accept an instance variable for the Config and +a factory function that returns default SSLContext which is initialized +with certificates, private key, cert_reqs, and ciphers according to +config and can be further customized by the callable. The callable needs to return SSLContext object. Server Mechanics @@ -998,9 +999,7 @@ Set the ``SO_REUSEPORT`` flag on the listening socket. **Default:** ``'.'`` -Change directory to specified directory before loading apps. - -Default is the current directory. +Change directory to specified directory before loading apps. .. _daemon: @@ -1161,10 +1160,16 @@ temporary directory. **Default:** ``{'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}`` A dictionary containing headers and values that the front-end proxy -uses to indicate HTTPS requests. These tell Gunicorn to set +uses to indicate HTTPS requests. If the source IP is permitted by +``forwarded-allow-ips`` (below), *and* at least one request header matches +a key-value pair listed in this dictionary, then Gunicorn will set ``wsgi.url_scheme`` to ``https``, so your application can tell that the request is secure. +If the other headers listed in this dictionary are not present in the request, they will be ignored, +but if the other headers are present and do not match the provided values, then +the request will fail to parse. See the note below for more detailed examples of this behaviour. + The dictionary should map upper-case header names to exact string values. The value comparisons are case-sensitive, unlike the header names, so make sure they're exactly what your front-end proxy sends @@ -1192,6 +1197,68 @@ you still trust the environment). By default, the value of the ``FORWARDED_ALLOW_IPS`` environment variable. If it is not defined, the default is ``"127.0.0.1"``. +.. note:: + + The interplay between the request headers, the value of ``forwarded_allow_ips``, and the value of + ``secure_scheme_headers`` is complex. Various scenarios are documented below to further elaborate. In each case, we + have a request from the remote address 134.213.44.18, and the default value of ``secure_scheme_headers``: + + .. code:: + + secure_scheme_headers = { + 'X-FORWARDED-PROTOCOL': 'ssl', + 'X-FORWARDED-PROTO': 'https', + 'X-FORWARDED-SSL': 'on' + } + + + .. list-table:: + :header-rows: 1 + :align: center + :widths: auto + + * - ``forwarded-allow-ips`` + - Secure Request Headers + - Result + - Explanation + * - .. code:: + + ["127.0.0.1"] + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "http" + - IP address was not allowed + * - .. code:: + + "*" + - + - .. code:: + + wsgi.url_scheme = "http" + - IP address allowed, but no secure headers provided + * - .. code:: + + "*" + - .. code:: + + X-Forwarded-Proto: https + - .. code:: + + wsgi.url_scheme = "https" + - IP address allowed, one request header matched + * - .. code:: + + ["134.213.44.18"] + - .. code:: + + X-Forwarded-Ssl: on + X-Forwarded-Proto: http + - ``InvalidSchemeHeaders()`` raised + - IP address allowed, but the two secure headers disagreed on if HTTPS was used + .. _pythonpath: ``pythonpath`` @@ -1364,8 +1431,9 @@ A positive integer generally in the ``2-4 x $(NUM_CORES)`` range. You'll want to vary this a bit to find the best for your particular application's work load. -By default, the value of the ``WEB_CONCURRENCY`` environment variable. -If it is not defined, the default is ``1``. +By default, the value of the ``WEB_CONCURRENCY`` environment variable, +which is set by some Platform-as-a-Service providers such as Heroku. If +it is not defined, the default is ``1``. .. _worker-class: @@ -1528,3 +1596,4 @@ set this to a higher value. .. note:: ``sync`` worker does not support persistent connections and will ignore this option. + diff --git a/gunicorn/config.py b/gunicorn/config.py index a9265374..392d687e 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1509,6 +1509,8 @@ class LogConfigDict(Setting): Format: https://docs.python.org/3/library/logging.config.html#logging.config.dictConfig + For more context you can look at the default configuration dictionary for logging, which can be found at ``gunicorn.glogging.CONFIG_DEFAULTS``. + .. versionadded:: 19.8 """