From 7349c4fb9a02863d9a7fc518c14ffeaf5460f7a7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 30 Mar 2014 15:27:53 +0200 Subject: [PATCH 01/67] add `--threads` param --- gunicorn/config.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/gunicorn/config.py b/gunicorn/config.py index f272bcc9..58ccc26f 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -91,11 +91,21 @@ class Config(object): @property def worker_class(self): uri = self.settings['worker_class'].get() + + ## are we using a threaded worker? + is_sync = uri.endswith('SyncWorker') or uri == 'sync' + if is_sync and self.threads > 1: + uri = "gunicorn.workers.gthread.ThreadedWorker" + worker_class = util.load_class(uri) if hasattr(worker_class, "setup"): worker_class.setup() return worker_class + @property + def threads(self): + return self.settings['threads'].get() + @property def workers(self): return self.settings['workers'].get() @@ -550,6 +560,27 @@ class WorkerClass(Setting): can also load the gevent class with ``egg:gunicorn#gevent`` """ +class WorkerThreads(Setting): + name = "threads" + section = "Worker Processes" + cli = ["--threads"] + meta = "INT" + validator = validate_pos_int + type = int + default = 1 + desc = """\ + The number of worker threads for handling requests. + + Run each worker in prethreaded mode with the specified number of + threads per worker. + + A positive integer generally in the 2-4 x $(NUM_CORES) range. You'll + want to vary this a bit to find the best for your particular + application's work load. + + If it is not defined, the default is 1. + """ + class WorkerConnections(Setting): name = "worker_connections" From 5f0a329b58cfefcbf11da0ab252917d50e3e8ae4 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 30 Mar 2014 15:28:19 +0200 Subject: [PATCH 02/67] add fdevents module This module add a new cross platform event poller to gunicorn. It allows you to listen on different fds in an efficient manner. On linux it's using epoll, bsd/darwin kqueue... --- gunicorn/fdevents.py | 467 +++++++++++++++++++++++++++++++++++++++++++ gunicorn/util.py | 4 + 2 files changed, 471 insertions(+) create mode 100644 gunicorn/fdevents.py diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py new file mode 100644 index 00000000..3c928be6 --- /dev/null +++ b/gunicorn/fdevents.py @@ -0,0 +1,467 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + + +""" module implementing Poller depending on the platform. A pollster +allows you to register an fd, and retrieve events on it. """ + +import select + +from .util import fd_, close_on_exec + + +class PollerBase(object): + + def addfd(self, fd, mode, repeat=True): + """ add a filed escriptor to poll. + + Parameters: + + * fd : file descriptor or file object + * mode: 'r' to wait for read events, 'w' to wait for write events + * repeat: true or false . to continuously wait on this event or + not (default is true). + """ + + raise NotImplementedError + + def delfd(self, fd, mode): + """ stop to poll for the event on this file descriptor + + Parameters: + + * fd : file descriptor or file object + * mode: 'r' to wait for read events, 'w' to wait for write events + """ + + raise NotImplementedError + + def waitfd(self, nsec): + """ return one event from the pollster. + + return: (fd, mode) + """ + raise NotImplementedError + + def wait(self, nsec): + """ return all events raised in the pollster when calling the + function. + + return: [(fd, mode), ....] + """ + raise NotImplementedError + + def close(self): + """ close the pollster """ + raise NotImplementedError + + +class SelectPoller(PollerBase): + + def __init__(self): + self.read_fds = {} + self.write_fds = {} + self.events = [] + + def addfd(self, fd, mode, repeat=True): + fd = fd_(fd) + + if mode == 'r': + self.read_fds[fd] = repeat + else: + self.write_fds[fd] = repeat + + def delfd(self, fd, mode): + if mode == 'r' and fd in self.read_fds: + del self.read_fds[fd] + elif fd in self.write_fds: + del self.write_fds[fd] + + def _wait(self, nsec): + read_fds = [fd for fd in self.read_fds] + write_fds = [fd for fd in self.write_fds] + + if len(self.events) == 0: + try: + r, w, e = select.select(read_fds, write_fds, [], nsec) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + events = [] + for fd in r: + if fd in self.read_fds: + if self.read_fds[fd] == False: + del self.read_fds[fd] + events.append((fd, 'r')) + + for fd in w: + if fd in self.write_fds: + if self.write_fds[fd] == False: + del self.write_fds[fd] + events.append((fd, 'w')) + + self.events.extend(events) + return self.events + + def waitfd(self, nsec): + self._wait(nsec) + if self.events: + return self.events.pop(0) + return None + + def wait(self, nsec): + events = self._wait(nsec) + self.events = [] + return events + + def close(self): + self.read_fds = [] + self.write_fds = [] + +if hasattr(selec, 'kqueue') + + class KQueuePoller(object): + + def __init__(self): + self.kq = select.kqueue() + close_on_exec(self.kq.fileno()) + self.events = [] + + def addfd(self, fd, mode, repeat=True): + if mode == 'r': + kmode = select.KQ_FILTER_READ + else: + kmode = select.KQ_FILTER_WRITE + + flags = select.KQ_EV_ADD + + if sys.platform.startswith("darwin"): + flags |= select.KQ_EV_ENABLE + + if not repeat: + flags |= select.KQ_EV_ONESHOT + + ev = select.kevent(fd_(fd), kmode, flags) + self.kq.control([ev], 0) + + def delfd(self, fd, mode): + if mode == 'r': + kmode = select.KQ_FILTER_READ + else: + kmode = select.KQ_FILTER_WRITE + + ev = select.kevent(fd_(fd), select.KQ_FILTER_READ, + select.KQ_EV_DELETE) + self.kq.control([ev], 0) + + def _wait(self, nsec=0): + if len(self.events) == 0: + try: + events = self.kq.control(None, 0, nsec) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + # process events + all_events = [] + for ev in events: + if ev.filter == select.KQ_FILTER_READ: + mode = 'r' + else: + mode = 'w' + all_events.append((fd_(ev.ident), mode)) + + self.events.extend(all_events) + + # return all events + return self.events + + def waitfd(self, nsec=0): + self._wait(nsec) + if self.events: + return self.events.pop(0) + return None + + def wait(self, nsec=0): + events = self._wait(nsec) + self.events = [] + return events + + def close(self): + self.kq.close() + +if hasattr(select, "epoll"): + class EpollPoller(object): + + def __init__(self): + self.poll = select.epoll() + close_on_exec(self.poll.fileno()) + self.fds = {} + self.events = [] + + def addfd(self, fd, mode, repeat=True): + if mode == 'r': + mode = (select.EPOLLIN, repeat) + else: + mode = (select.EPOLLOUT, repeat) + + if fd in self.fds: + modes = self.fds[fd] + if mode in self.fds[fd]: + # already registered for this mode + return + modes.append(mode) + addfd_ = self.poll.modify + else: + modes = [mode] + addfd_ = self.poll.register + + # append the new mode to fds + self.fds[fd] = modes + + mask = 0 + for mode, r in modes: + mask |= mode + + if not repeat: + mask |= select.EPOLLONESHOT + + addfd_(fd, mask) + + def delfd(self, fd, mode): + if mode == 'r': + mode = select.POLLIN | select.POLLPRI + else: + mode = select.POLLOUT + + if fd not in self.fds: + return + + modes = [] + for m, r in self.fds[fd]: + if mode != m: + modes.append((m, r)) + + if not modes: + # del the fd from the poll + self.poll.unregister(fd) + del self.fds[fd] + else: + # modify the fd in the poll + self.fds[fd] = modes + m, r = modes[0] + mask = m[0] + if r: + mask |= select.EPOLLONESHOT + + self.poll.modify(fd, mask) + + def _wait(self, nsec=0): + # wait for the events + if len(self.events) == 0: + try: + events = self.poll.poll(nsec) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + if events: + all_events = [] + fds = {} + for fd, ev in self.events: + fd = fd_(fd) + if ev == select.EPOLLIN: + mode = 'r' + else: + mode = 'w' + + all_events.append((fd, mode)) + + if fd in fds: + fds[fd].append(mode) + else: + fds[fd] = [mode] + + # eventually remove the mode from the list if repeat + # was set to False and modify the poll if needed. + modes = [] + for m, r in self.fds[fd]: + if not r: + continue + modes.append(m, r) + + if modes != self.fds[fd]: + self.fds[fd] = modes + mask = 0 + for m, r in modes: + mask |= m + self.poll.modify(fd, mask) + + self.events.extend(all_events) + + # return all events + return self.events + + def waitfd(self, nsec=0): + self._wait(nsec) + return self.events.pop(0) + + def wait(self, nsec=0): + events = self._wait(nsec) + self.events = [] + return events + + def close(self): + self.poll.close() + + +if hasattr(select, "poll") or hasattr(select, "epoll"): + + class _PollerBase(object): + + POLL_IMPL = None + + def __init__(self): + self.poll = self.POLL_IMPL() + self.fds = {} + self.events = [] + + def addfd(self, fd, mode, repeat=True): + fd = fd_(fd) + if mode == 'r': + mode = (select.POLLIN, repeat) + else: + mode = (select.POLLOUT, repeat) + + if fd in self.fds: + modes = self.fds[fd] + if mode in modes: + # already registered for this mode + return + modes.append(mode) + addfd_ = self.poll.modify + else: + modes = [mode] + addfd_ = self.poll.register + + # append the new mode to fds + self.fds[fd] = modes + + mask = 0 + for mode, r in modes: + mask |= mode + + addfd_(fd, mask) + + def delfd(self, fd, mode): + fd = fd_(fd) + + if mode == 'r': + mode = select.POLLIN | select.POLLPRI + else: + mode = select.POLLOUT + + if fd not in self.fds: + return + + modes = [] + for m, r in self.fds[fd]: + if mode != m: + modes.append((m, r)) + + if not modes: + # del the fd from the poll + self.poll.unregister(fd) + del self.fds[fd] + else: + # modify the fd in the poll + self.fds[fd] = modes + m, r = modes[0] + mask = m[0] + self.poll.modify(fd, mask) + + def _wait(self, nsec=0): + # wait for the events + if len(self.events) == 0: + try: + events = self.poll.poll(nsec) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + all_events = [] + for fd, ev in events: + fd = fd_(fd) + + if fd not in self.fds: + continue + + if ev == select.POLLIN or ev == select.POLLPRI: + mode = 'r' + else: + mode = 'w' + + # add new event to the list + all_events.append((fd, mode)) + + # eventually remove the mode from the list if repeat + # was set to False and modify the poll if needed. + modes = [] + for m, r in self.fds[fd]: + if not r: + continue + modes.append(m, r) + + if not modes: + self.poll.unregister(fd) + else: + mask = 0 + if modes != self.fds[fd]: + mask |= m + self.poll.modify(fd, mask) + + + self.events.extend(all_events) + return self.events + + def waitfd(self, nsec=0): + self._wait(nsec) + if self.events: + return self.events.pop(0) + return None + + def close(self): + for fd in self.fds: + self.poll.unregister(fd) + self.fds = [] + self.poll = None + + + if hasattr(select, "devpoll"): + + class DevPollPoller(_PollerBase): + POLL_IMPL = select.devpoll + + if hasattr(select, "poll"): + class PollPoller(_PollerBase): + POLL_IMPL = select.poll + + +# choose the best implementation depending on the platform. +if 'KqueuePoller' in globals(): + DefaultPoller = KqueuePoller +elif 'EpollPoller' in globals(): + DefaultPoller = EpollPoller +elif 'DevpollPoller' in globals(): + DefaultPoller = DevpollPoller +elif 'PollPoller' in globals(): + DefaultPoller = PollPoller +else: + DefaultPoller = SelectPoller diff --git a/gunicorn/util.py b/gunicorn/util.py index de0ef613..07db3dba 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -269,6 +269,10 @@ def set_non_blocking(fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK fcntl.fcntl(fd, fcntl.F_SETFL, flags) +def fd_(fd): + if hasattr(fd, "fileno"): + return int(fd.fileno()) + return fd def close(sock): try: From 67866f275fe673324a240148f76e240c10cb0b6a Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 10:21:48 +0200 Subject: [PATCH 03/67] add missing gthreads worker --- gunicorn/fdevents.py | 2 +- gunicorn/workers/gthread.py | 301 ++++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 gunicorn/workers/gthread.py diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py index 3c928be6..ccd5bd61 100644 --- a/gunicorn/fdevents.py +++ b/gunicorn/fdevents.py @@ -17,7 +17,7 @@ class PollerBase(object): def addfd(self, fd, mode, repeat=True): """ add a filed escriptor to poll. - Parameters: + fdevent Parameters: * fd : file descriptor or file object * mode: 'r' to wait for read events, 'w' to wait for write events diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py new file mode 100644 index 00000000..b6d6247b --- /dev/null +++ b/gunicorn/workers/gthread.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +# design: +# a threaded worker accepts connections in the main loop, accepted +# connections are are added to the thread pool as a connection job. On +# keepalive connections are put back in the loop waiting for an event. +# If no event happen after the keep alive timeout, the connectoin is +# closed. + + +import concurrent.futures as futures +from datetime import datetime +import errno +import heapq +import os +import socket +import ssl +import sys +import time + +from .. import http +from ..http import wsgi +from .. import fdevents +from .. import util +from . import base +from .. import six + + +class TConn(): + + def __init__(self, worker, listener, sock, addr): + self.listener = listener + self.sock = sock + self.addr = addr + self.when = fs.timeout + + # set the timeout + self.timeout = time.time() + worker.cfg.keepalive + + def __lt__(self, other): + return self.timeout < other.timeout + + __cmp__ = __lt__ + + +class ThreadWorker(base.worker): + + def __init__(self, *args, **kwargs): + super(ThreadWorker, self).__init__(*args, **kwargs) + # initialise the pool + self.tpool = futures.ThreadPoolExecutor(max_workers=self.cfg.threads) + self.poller = fdevents.DefaultPoller() + self.futures = set() + self._heap = [] + self.keepalived = {} + + def _wrap(self, fs, listener, client, addr): + fs.listener = listener + fs.sock = client + fs.addr = addr + + def run(self): + for s in self.sockets: + s.setblocking(False) + self.poller.add_fd(s, 'r') + + listeners = dict([(s.fileno(), s) for s in self.sockets]) + while self.alive: + + # If our parent changed then we shut down. + if self.ppid != os.getppid(): + self.log.info("Parent changed, shutting down: %s", self) + return + + # notify the arbiter we are alive + self.notify() + + events = self.poller.wait(0.1) + if events: + for (fd, mode) in events: + fs = None + client = None + if fd in listeners: + # start to accept connections + try: + client, addr = sock.accept() + + # add a job to the pool + fs = self.tpool.submit(self.handle, listeners[fd], + client, addr, False) + + self._wrap_future(fs, listemers[fd], + client, addr) + + except socket.error as e: + if e.args[0] not in (errno.EAGAIN, + errno.ECONNABORTED, errno.EWOULDBLOCK): + raise + else: + # keepalive connection + if fd in self.keepalived: + # get the client connection + client = self.keepalived[fd] + + # remove it from the heap + try: + del self._heap[operator.indexOf(self._heap, t)] + except (KeyError, IndexError): + pass + + # add a job to the pool + fs = self.tpool.submit(self.handle, client.listener, + client.sock, client.addr, True) + + self._wrap_future(fs, client.listener, + client.sock, client.addr) + + if fs is not None: + self.futures.add(fs) + + # handle jobs, we give a chance to all jobs to be executed. + if self.futures: + res = futures.wait([fs for fs in self.futures], + timeout=self.timeout, + return_when=futures.ALL_COMPLETED) + + for fs in res: + # remove the future from our list + self.futures.remove(fs) + + try: + result = fs.result() + # if the connection should be kept alived add it + # to the eventloop and record it + if result and result is not None: + # flag the socket as non blocked + fs.sock.setblocking(0) + util.close_on_exec(fs.sock) + + tconn = TConn(self, fs.listener, fs.sock, + fs.addr) + + # register the connection + heapq.heappush(self._heap, tconn) + self.keepalived[fs.sock.fileno()] = tconn + + # add the socket to the event loop + self.poller.add_fd(fs.sock.fileno(), 'r') + else: + # at this point the connection should be + # closed but we make sure it is. + util.close(fs.sock) + except: + # an exception happened, make sure to close the + # socket. + util.close(fs.sock) + + + # hanle keepalive timeouts + now = time.time() + while True: + if not len(self._heap): + continue + + conn = heapq.heappop(self._heap) + delta = t.timeout = now + if delta > 0: + heapq.heappush(self._heap, t) + break + else: + # remove the socket from the poller + self.poller.del_fd(conn.sock.fileno(), 'r') + # close the socket + conn.sock.close() + + + # shutdown the pool + self.tpool.shutdown(False) + + # wait for the workers + futures.wait([fs for fs in self.futures], + timeout=self.cfg.graceful_timeout) + + # if we have still fures running, try to close them + for fs in self.futures: + sock = fs.sock + + # the future is not running, cancel it + if not fs.done() and not fs.running(): + fs.cancel() + + # make sure we close the sockets after the graceful timeout + util.close(sock) + + + def handle(self, listener, client, addr, keepalived): + keepalive = False + try: + # wrap the connection + if not keepalived and self.cfg.is_ssl: + client = ssl.wrap_socket(client, server_side=True, + **self.cfg.ssl_options) + + client.setblocking(1) + util.close_on_exec(client) + + parser = http.RequestParser(self.cfg, sock) + req = six.next(parser) + + # handle the request + keepalive = self.handle_request(listener, req, client, addr) + except http.errors.NoMoreData as e: + self.log.debug("Ignored premature client disconnection. %s", e) + + except StopIteration as e: + self.log.debug("Closing connection. %s", e) + + except ssl.SSLError as e: + if e.args[0] == ssl.SSL_ERROR_EOF: + self.log.debug("ssl connection closed") + client.close() + else: + self.log.debug("Error processing SSL request.") + self.handle_error(req, client, addr, e) + + except socket.error as e: + if e.args[0] not in (errno.EPIPE, errno.ECONNRESET): + self.log.exception("Socket error processing request.") + else: + if e.args[0] == errno.ECONNRESET: + self.log.debug("Ignoring connection reset") + else: + self.log.debug("Ignoring EPIPE") + except Exception as e: + self.handle_error(req, client, addr, e) + finally: + if not keepalive: + util.close(client) + return keepalive + + def handle_request(self, listener, req, client, addr): + environ = {} + resp = None + try: + self.cfg.pre_request(self, req) + request_start = datetime.now() + resp, environ = wsgi.create(req, client, addr, + listener.getsockname(), self.cfg) + environ["wsgi.multithread"] = True + + self.nr += 1 + if self.nr >= self.max_requests: + self.log.info("Autorestarting worker after current request.") + self.alive = False + + if not self.cfg.keepalive: + resp.force_close() + + respiter = self.wsgi(environ, resp.start_response) + try: + if isinstance(respiter, environ['wsgi.file_wrapper']): + resp.write_file(respiter) + else: + for item in respiter: + resp.write(item) + resp.close() + request_time = datetime.now() - request_start + self.log.access(resp, req, environ, request_time) + finally: + if hasattr(respiter, "close"): + respiter.close() + + if resp.should_close(): + raise StopIteration() + + except socket.error: + exc_info = sys.exc_info() + # pass to next try-except level + six.reraise(exc_info[0], exc_info[1], exc_info[2]) + except Exception: + if resp and resp.headers_sent: + # If the requests have already been sent, we should close the + # connection to indicate the error. + self.log.exception("Error handling request") + try: + client.shutdown(socket.SHUT_RDWR) + client.close() + except socket.error: + pass + raise StopIteration() + raise + finally: + try: + self.cfg.post_request(self, req, environ, resp) + except Exception: + self.log.exception("Exception in post_request hook") + + return True From c8f6269f29701db7da7e5b6afa591c59f8076e2c Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 10:25:26 +0200 Subject: [PATCH 04/67] fix the fdevents module --- gunicorn/fdevents.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py index ccd5bd61..0bddb41e 100644 --- a/gunicorn/fdevents.py +++ b/gunicorn/fdevents.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. -""" module implementing Poller depending on the platform. A pollster +""" A module implementing Poller depending on the platform. A pollster allows you to register an fd, and retrieve events on it. """ import select @@ -70,8 +70,10 @@ class SelectPoller(PollerBase): if mode == 'r': self.read_fds[fd] = repeat - else: + elif mode == 'w': self.write_fds[fd] = repeat + else: + raise ValueError('unkown mode {0}'.format(mode)) def delfd(self, fd, mode): if mode == 'r' and fd in self.read_fds: @@ -87,9 +89,8 @@ class SelectPoller(PollerBase): try: r, w, e = select.select(read_fds, write_fds, [], nsec) except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise + if e.args[0] != errno.EINTR: + raise events = [] for fd in r: @@ -122,7 +123,7 @@ class SelectPoller(PollerBase): self.read_fds = [] self.write_fds = [] -if hasattr(selec, 'kqueue') +if hasattr(select, 'kqueue'): class KQueuePoller(object): @@ -163,9 +164,8 @@ if hasattr(selec, 'kqueue') try: events = self.kq.control(None, 0, nsec) except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise + if e.args[0] != errno.EINTR: + raise # process events all_events = [] @@ -267,9 +267,8 @@ if hasattr(select, "epoll"): try: events = self.poll.poll(nsec) except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise + if e.args[0] != errno.EINTR: + raise if events: all_events = [] @@ -392,9 +391,8 @@ if hasattr(select, "poll") or hasattr(select, "epoll"): try: events = self.poll.poll(nsec) except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise + if e.args[0] != errno.EINTR: + raise all_events = [] for fd, ev in events: From c353eaaceebea1ef03c2e2329d88e994740093c3 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 10:29:26 +0200 Subject: [PATCH 05/67] fix ThreadWorker --- gunicorn/workers/gthread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index b6d6247b..a0aebf2f 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -46,7 +46,7 @@ class TConn(): __cmp__ = __lt__ -class ThreadWorker(base.worker): +class ThreadWorker(base.Worker): def __init__(self, *args, **kwargs): super(ThreadWorker, self).__init__(*args, **kwargs) From 6aa99e44415a2a43f0f9476966a782a8e5d42a71 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 12:30:57 +0200 Subject: [PATCH 06/67] fix keepalive --- examples/test.py | 2 +- gunicorn/config.py | 2 +- gunicorn/fdevents.py | 16 +++-- gunicorn/workers/gthread.py | 121 ++++++++++++++++++------------------ 4 files changed, 74 insertions(+), 67 deletions(-) diff --git a/examples/test.py b/examples/test.py index 610cf5b1..77bef952 100644 --- a/examples/test.py +++ b/examples/test.py @@ -16,7 +16,7 @@ def app(environ, start_response): """Simplest possible application object""" errors = environ['wsgi.errors'] - pprint.pprint(('ENVIRON', environ), stream=errors) +# pprint.pprint(('ENVIRON', environ), stream=errors) data = b'Hello, World!\n' status = '200 OK' diff --git a/gunicorn/config.py b/gunicorn/config.py index 58ccc26f..cd4d1784 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -95,7 +95,7 @@ class Config(object): ## are we using a threaded worker? is_sync = uri.endswith('SyncWorker') or uri == 'sync' if is_sync and self.threads > 1: - uri = "gunicorn.workers.gthread.ThreadedWorker" + uri = "gunicorn.workers.gthread.ThreadWorker" worker_class = util.load_class(uri) if hasattr(worker_class, "setup"): diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py index 0bddb41e..1fee335a 100644 --- a/gunicorn/fdevents.py +++ b/gunicorn/fdevents.py @@ -133,6 +133,8 @@ if hasattr(select, 'kqueue'): self.events = [] def addfd(self, fd, mode, repeat=True): + fd = fd_(fd) + if mode == 'r': kmode = select.KQ_FILTER_READ else: @@ -150,6 +152,8 @@ if hasattr(select, 'kqueue'): self.kq.control([ev], 0) def delfd(self, fd, mode): + fd = fd_(fd) + if mode == 'r': kmode = select.KQ_FILTER_READ else: @@ -205,6 +209,8 @@ if hasattr(select, "epoll"): self.events = [] def addfd(self, fd, mode, repeat=True): + fd = fd_(fd) + if mode == 'r': mode = (select.EPOLLIN, repeat) else: @@ -234,6 +240,8 @@ if hasattr(select, "epoll"): addfd_(fd, mask) def delfd(self, fd, mode): + fd = fd_(fd) + if mode == 'r': mode = select.POLLIN | select.POLLPRI else: @@ -273,9 +281,9 @@ if hasattr(select, "epoll"): if events: all_events = [] fds = {} - for fd, ev in self.events: + for fd, ev in events: fd = fd_(fd) - if ev == select.EPOLLIN: + if ev & select.EPOLLIN: mode = 'r' else: mode = 'w' @@ -293,7 +301,7 @@ if hasattr(select, "epoll"): for m, r in self.fds[fd]: if not r: continue - modes.append(m, r) + modes.append((m, r)) if modes != self.fds[fd]: self.fds[fd] = modes @@ -401,7 +409,7 @@ if hasattr(select, "poll") or hasattr(select, "epoll"): if fd not in self.fds: continue - if ev == select.POLLIN or ev == select.POLLPRI: + if ev & select.POLLIN or ev & select.POLLPRI: mode = 'r' else: mode = 'w' diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index a0aebf2f..6336e6de 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -16,6 +16,7 @@ from datetime import datetime import errno import heapq import os +import operator import socket import ssl import sys @@ -31,11 +32,11 @@ from .. import six class TConn(): - def __init__(self, worker, listener, sock, addr): + def __init__(self, worker, listener, sock, addr, parser): self.listener = listener self.sock = sock self.addr = addr - self.when = fs.timeout + self.parser = parser # set the timeout self.timeout = time.time() + worker.cfg.keepalive @@ -57,15 +58,16 @@ class ThreadWorker(base.Worker): self._heap = [] self.keepalived = {} - def _wrap(self, fs, listener, client, addr): + def _wrap_future(self, fs, listener, client, addr): fs.listener = listener fs.sock = client fs.addr = addr + self.futures.add(fs) def run(self): for s in self.sockets: - s.setblocking(False) - self.poller.add_fd(s, 'r') + s.setblocking(0) + self.poller.addfd(s, 'r') listeners = dict([(s.fileno(), s) for s in self.sockets]) while self.alive: @@ -78,77 +80,70 @@ class ThreadWorker(base.Worker): # notify the arbiter we are alive self.notify() - events = self.poller.wait(0.1) + events = self.poller.wait(0.01) if events: for (fd, mode) in events: fs = None client = None if fd in listeners: + listener = listeners[fd] # start to accept connections try: - client, addr = sock.accept() + client, addr = listener.accept() # add a job to the pool - fs = self.tpool.submit(self.handle, listeners[fd], + fs = self.tpool.submit(self.handle, listener, client, addr, False) - self._wrap_future(fs, listemers[fd], - client, addr) + self._wrap_future(fs, listener, client, addr) except socket.error as e: if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise - else: - # keepalive connection - if fd in self.keepalived: - # get the client connection - client = self.keepalived[fd] + elif fd in self.keepalived: + # get the client connection + client = self.keepalived[fd] - # remove it from the heap - try: - del self._heap[operator.indexOf(self._heap, t)] - except (KeyError, IndexError): - pass + # remove it from the heap + try: + del self._heap[operator.indexOf(self._heap, client)] + except (KeyError, IndexError): + pass - # add a job to the pool - fs = self.tpool.submit(self.handle, client.listener, - client.sock, client.addr, True) + # add a job to the pool + fs = self.tpool.submit(self.handle, client.listener, + client.sock, client.addr, client.parser) - self._wrap_future(fs, client.listener, - client.sock, client.addr) - - if fs is not None: - self.futures.add(fs) + # wrap the future + self._wrap_future(fs, client.listener, client.sock, + client.addr) # handle jobs, we give a chance to all jobs to be executed. if self.futures: - res = futures.wait([fs for fs in self.futures], - timeout=self.timeout, - return_when=futures.ALL_COMPLETED) + self.notify() - for fs in res: - # remove the future from our list - self.futures.remove(fs) + res = futures.wait(self.futures, timeout=self.timeout, + return_when=futures.FIRST_COMPLETED) + for fs in res.done: try: - result = fs.result() + (keepalive, parser) = fs.result() # if the connection should be kept alived add it # to the eventloop and record it - if result and result is not None: + if keepalive: # flag the socket as non blocked fs.sock.setblocking(0) - util.close_on_exec(fs.sock) tconn = TConn(self, fs.listener, fs.sock, - fs.addr) + fs.addr, parser) # register the connection heapq.heappush(self._heap, tconn) self.keepalived[fs.sock.fileno()] = tconn # add the socket to the event loop - self.poller.add_fd(fs.sock.fileno(), 'r') + self.poller.addfd(fs.sock, 'r', False) else: # at this point the connection should be # closed but we make sure it is. @@ -157,32 +152,34 @@ class ThreadWorker(base.Worker): # an exception happened, make sure to close the # socket. util.close(fs.sock) + finally: + # remove the future from our list + self.futures.remove(fs) # hanle keepalive timeouts now = time.time() while True: if not len(self._heap): - continue + break conn = heapq.heappop(self._heap) - delta = t.timeout = now + delta = conn.timeout - now if delta > 0: - heapq.heappush(self._heap, t) + heapq.heappush(self._heap, conn) break else: # remove the socket from the poller - self.poller.del_fd(conn.sock.fileno(), 'r') + self.poller.delfd(conn.sock.fileno(), 'r') # close the socket - conn.sock.close() + util.close(conn.sock) # shutdown the pool self.tpool.shutdown(False) # wait for the workers - futures.wait([fs for fs in self.futures], - timeout=self.cfg.graceful_timeout) + futures.wait(self.futures, timeout=self.cfg.graceful_timeout) # if we have still fures running, try to close them for fs in self.futures: @@ -196,28 +193,32 @@ class ThreadWorker(base.Worker): util.close(sock) - def handle(self, listener, client, addr, keepalived): + def handle(self, listener, client, addr, parser): keepalive = False + req = None try: + client.setblocking(1) + # wrap the connection - if not keepalived and self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, - **self.cfg.ssl_options) + if not parser: + if self.cfg.is_ssl: + client = ssl.wrap_socket(client, server_side=True, + **self.cfg.ssl_options) + parser = http.RequestParser(self.cfg, client) - client.setblocking(1) - util.close_on_exec(client) - - parser = http.RequestParser(self.cfg, sock) req = six.next(parser) + if not req: + return (False, None) # handle the request keepalive = self.handle_request(listener, req, client, addr) + if keepalive: + return (keepalive, parser) except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) except StopIteration as e: self.log.debug("Closing connection. %s", e) - except ssl.SSLError as e: if e.args[0] == ssl.SSL_ERROR_EOF: self.log.debug("ssl connection closed") @@ -236,10 +237,8 @@ class ThreadWorker(base.Worker): self.log.debug("Ignoring EPIPE") except Exception as e: self.handle_error(req, client, addr, e) - finally: - if not keepalive: - util.close(client) - return keepalive + + return (False, None) def handle_request(self, listener, req, client, addr): environ = {} @@ -274,8 +273,8 @@ class ThreadWorker(base.Worker): respiter.close() if resp.should_close(): - raise StopIteration() - + self.log.debug("Closing connection.") + return False except socket.error: exc_info = sys.exc_info() # pass to next try-except level From eadc526192b28a7ed87fda5915be8e292ffd92a1 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 13:21:14 +0200 Subject: [PATCH 07/67] fix PollPoller --- gunicorn/fdevents.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py index 1fee335a..f53ef047 100644 --- a/gunicorn/fdevents.py +++ b/gunicorn/fdevents.py @@ -423,7 +423,7 @@ if hasattr(select, "poll") or hasattr(select, "epoll"): for m, r in self.fds[fd]: if not r: continue - modes.append(m, r) + modes.append((m, r)) if not modes: self.poll.unregister(fd) @@ -443,6 +443,11 @@ if hasattr(select, "poll") or hasattr(select, "epoll"): return self.events.pop(0) return None + def wait(self, nsec=0): + events = self._wait(nsec) + self.events = [] + return events + def close(self): for fd in self.fds: self.poll.unregister(fd) From 67800292e05956cbd4e3b852011495b1da3a5537 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 13:57:34 +0200 Subject: [PATCH 08/67] fix kqueue poller. this change initialise the event loop after the process has forked so we make sure to inherit from the file descriptor. Also fix the number of events we are waiting for. The python implementation requires a positive number. --- gunicorn/fdevents.py | 21 +++++++++++++++------ gunicorn/workers/gthread.py | 15 +++++++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py index f53ef047..e50f2a0d 100644 --- a/gunicorn/fdevents.py +++ b/gunicorn/fdevents.py @@ -8,6 +8,7 @@ allows you to register an fd, and retrieve events on it. """ import select +import sys from .util import fd_, close_on_exec @@ -131,10 +132,13 @@ if hasattr(select, 'kqueue'): self.kq = select.kqueue() close_on_exec(self.kq.fileno()) self.events = [] + self.max_ev = 0 def addfd(self, fd, mode, repeat=True): fd = fd_(fd) + self.max_ev += 1 + if mode == 'r': kmode = select.KQ_FILTER_READ else: @@ -148,12 +152,17 @@ if hasattr(select, 'kqueue'): if not repeat: flags |= select.KQ_EV_ONESHOT - ev = select.kevent(fd_(fd), kmode, flags) - self.kq.control([ev], 0) + ev = select.kevent(fd, kmode, flags) + self.kq.control([ev], 0, 0) def delfd(self, fd, mode): fd = fd_(fd) + self.max_ev -= 1 + + if fd < 0: + return + if mode == 'r': kmode = select.KQ_FILTER_READ else: @@ -166,7 +175,7 @@ if hasattr(select, 'kqueue'): def _wait(self, nsec=0): if len(self.events) == 0: try: - events = self.kq.control(None, 0, nsec) + events = self.kq.control(None, self.max_ev, nsec) except select.error as e: if e.args[0] != errno.EINTR: raise @@ -328,7 +337,7 @@ if hasattr(select, "epoll"): self.poll.close() -if hasattr(select, "poll") or hasattr(select, "epoll"): +if hasattr(select, "poll") or hasattr(select, "devpoll"): class _PollerBase(object): @@ -466,8 +475,8 @@ if hasattr(select, "poll") or hasattr(select, "epoll"): # choose the best implementation depending on the platform. -if 'KqueuePoller' in globals(): - DefaultPoller = KqueuePoller +if 'KQueuePoller' in globals(): + DefaultPoller = KQueuePoller elif 'EpollPoller' in globals(): DefaultPoller = EpollPoller elif 'DevpollPoller' in globals(): diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 6336e6de..f34c7867 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -52,8 +52,8 @@ class ThreadWorker(base.Worker): def __init__(self, *args, **kwargs): super(ThreadWorker, self).__init__(*args, **kwargs) # initialise the pool - self.tpool = futures.ThreadPoolExecutor(max_workers=self.cfg.threads) - self.poller = fdevents.DefaultPoller() + self.tpool = None + self.poller = None self.futures = set() self._heap = [] self.keepalived = {} @@ -64,9 +64,15 @@ class ThreadWorker(base.Worker): fs.addr = addr self.futures.add(fs) + def init_process(self): + self.tpool = futures.ThreadPoolExecutor(max_workers=self.cfg.threads) + self.poller = fdevents.DefaultPoller() + super(ThreadWorker, self).init_process() + def run(self): + # init listeners, add them to the event loop for s in self.sockets: - s.setblocking(0) + s.setblocking(False) self.poller.addfd(s, 'r') listeners = dict([(s.fileno(), s) for s in self.sockets]) @@ -170,13 +176,14 @@ class ThreadWorker(base.Worker): break else: # remove the socket from the poller - self.poller.delfd(conn.sock.fileno(), 'r') + self.poller.delfd(conn.sock, 'r') # close the socket util.close(conn.sock) # shutdown the pool self.tpool.shutdown(False) + self.poller.close() # wait for the workers futures.wait(self.futures, timeout=self.cfg.graceful_timeout) From 14f71ebf39abfd2d369d331225fc3c66af3a318c Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 May 2014 15:18:43 +0200 Subject: [PATCH 09/67] compatibility with python 2 Add support of the threaded worker on python 2.7. python 2.7 has no futures module. With this change the compatibility module is installed. --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index 050ab617..36e33853 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ import sys from gunicorn import __version__ + CLASSIFIERS = [ 'Development Status :: 4 - Beta', 'Environment :: Other Environment', @@ -61,6 +62,11 @@ class PyTest(Command): raise SystemExit(errno) +REQUIREMENTS = [] +if sys.version_info[0] == 2: + REQUIREMENTS.append('futures') + + setup( name = 'gunicorn', version = __version__, @@ -80,6 +86,8 @@ setup( tests_require = tests_require, cmdclass = {'test': PyTest}, + install_requires = REQUIREMENTS, + entry_points=""" [console_scripts] From 81810d9f04a0999e6c2ee906132bf9dc42e66518 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 15 May 2014 08:03:06 +0200 Subject: [PATCH 10/67] reuse the code --- gunicorn/fdevents.py | 58 +++++++++----------------------------------- 1 file changed, 11 insertions(+), 47 deletions(-) diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py index e50f2a0d..3a273f5d 100644 --- a/gunicorn/fdevents.py +++ b/gunicorn/fdevents.py @@ -44,7 +44,11 @@ class PollerBase(object): return: (fd, mode) """ - raise NotImplementedError + self._wait(nsec) + if self.events: + return self.events.pop(0) + return None + def wait(self, nsec): """ return all events raised in the pollster when calling the @@ -52,7 +56,9 @@ class PollerBase(object): return: [(fd, mode), ....] """ - raise NotImplementedError + events = self._wait(nsec) + self.events = [] + return events def close(self): """ close the pollster """ @@ -109,24 +115,13 @@ class SelectPoller(PollerBase): self.events.extend(events) return self.events - def waitfd(self, nsec): - self._wait(nsec) - if self.events: - return self.events.pop(0) - return None - - def wait(self, nsec): - events = self._wait(nsec) - self.events = [] - return events - def close(self): self.read_fds = [] self.write_fds = [] if hasattr(select, 'kqueue'): - class KQueuePoller(object): + class KQueuePoller(PollerBase): def __init__(self): self.kq = select.kqueue() @@ -194,22 +189,11 @@ if hasattr(select, 'kqueue'): # return all events return self.events - def waitfd(self, nsec=0): - self._wait(nsec) - if self.events: - return self.events.pop(0) - return None - - def wait(self, nsec=0): - events = self._wait(nsec) - self.events = [] - return events - def close(self): self.kq.close() if hasattr(select, "epoll"): - class EpollPoller(object): + class EpollPoller(PollerBase): def __init__(self): self.poll = select.epoll() @@ -324,22 +308,13 @@ if hasattr(select, "epoll"): # return all events return self.events - def waitfd(self, nsec=0): - self._wait(nsec) - return self.events.pop(0) - - def wait(self, nsec=0): - events = self._wait(nsec) - self.events = [] - return events - def close(self): self.poll.close() if hasattr(select, "poll") or hasattr(select, "devpoll"): - class _PollerBase(object): + class _PollerBase(PollerBase): POLL_IMPL = None @@ -446,17 +421,6 @@ if hasattr(select, "poll") or hasattr(select, "devpoll"): self.events.extend(all_events) return self.events - def waitfd(self, nsec=0): - self._wait(nsec) - if self.events: - return self.events.pop(0) - return None - - def wait(self, nsec=0): - events = self._wait(nsec) - self.events = [] - return events - def close(self): for fd in self.fds: self.poll.unregister(fd) From 7f9d745eb5919967c682d28e2a6237d62efab97e Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 30 May 2014 11:07:35 +0200 Subject: [PATCH 11/67] reuse asyncio code in the threaded worker --- gunicorn/arbiter.py | 3 +- gunicorn/config.py | 10 + gunicorn/fdevents.py | 451 --------------------------- gunicorn/selectors.py | 585 ++++++++++++++++++++++++++++++++++++ gunicorn/workers/gthread.py | 37 ++- 5 files changed, 620 insertions(+), 466 deletions(-) delete mode 100644 gunicorn/fdevents.py create mode 100644 gunicorn/selectors.py diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index bd2016d2..e0ffb225 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -133,8 +133,7 @@ class Arbiter(object): listeners_str = ",".join([str(l) for l in self.LISTENERS]) self.log.debug("Arbiter booted") self.log.info("Listening at: %s (%s)", listeners_str, self.pid) - self.log.info("Using worker: %s", - self.cfg.settings['worker_class'].get()) + self.log.info("Using worker: %s", self.cfg.worker_class_str) self.cfg.when_ready(self) diff --git a/gunicorn/config.py b/gunicorn/config.py index cd4d1784..4c28dae3 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -88,6 +88,16 @@ class Config(object): return parser + @property + def worker_class_str(self): + uri = self.settings['worker_class'].get() + + ## are we using a threaded worker? + is_sync = uri.endswith('SyncWorker') or uri == 'sync' + if is_sync and self.threads > 1: + return "threads" + return uri + @property def worker_class(self): uri = self.settings['worker_class'].get() diff --git a/gunicorn/fdevents.py b/gunicorn/fdevents.py deleted file mode 100644 index 3a273f5d..00000000 --- a/gunicorn/fdevents.py +++ /dev/null @@ -1,451 +0,0 @@ -# -*- coding: utf-8 - -# -# This file is part of gunicorn released under the MIT license. -# See the NOTICE for more information. - - -""" A module implementing Poller depending on the platform. A pollster -allows you to register an fd, and retrieve events on it. """ - -import select -import sys - -from .util import fd_, close_on_exec - - -class PollerBase(object): - - def addfd(self, fd, mode, repeat=True): - """ add a filed escriptor to poll. - - fdevent Parameters: - - * fd : file descriptor or file object - * mode: 'r' to wait for read events, 'w' to wait for write events - * repeat: true or false . to continuously wait on this event or - not (default is true). - """ - - raise NotImplementedError - - def delfd(self, fd, mode): - """ stop to poll for the event on this file descriptor - - Parameters: - - * fd : file descriptor or file object - * mode: 'r' to wait for read events, 'w' to wait for write events - """ - - raise NotImplementedError - - def waitfd(self, nsec): - """ return one event from the pollster. - - return: (fd, mode) - """ - self._wait(nsec) - if self.events: - return self.events.pop(0) - return None - - - def wait(self, nsec): - """ return all events raised in the pollster when calling the - function. - - return: [(fd, mode), ....] - """ - events = self._wait(nsec) - self.events = [] - return events - - def close(self): - """ close the pollster """ - raise NotImplementedError - - -class SelectPoller(PollerBase): - - def __init__(self): - self.read_fds = {} - self.write_fds = {} - self.events = [] - - def addfd(self, fd, mode, repeat=True): - fd = fd_(fd) - - if mode == 'r': - self.read_fds[fd] = repeat - elif mode == 'w': - self.write_fds[fd] = repeat - else: - raise ValueError('unkown mode {0}'.format(mode)) - - def delfd(self, fd, mode): - if mode == 'r' and fd in self.read_fds: - del self.read_fds[fd] - elif fd in self.write_fds: - del self.write_fds[fd] - - def _wait(self, nsec): - read_fds = [fd for fd in self.read_fds] - write_fds = [fd for fd in self.write_fds] - - if len(self.events) == 0: - try: - r, w, e = select.select(read_fds, write_fds, [], nsec) - except select.error as e: - if e.args[0] != errno.EINTR: - raise - - events = [] - for fd in r: - if fd in self.read_fds: - if self.read_fds[fd] == False: - del self.read_fds[fd] - events.append((fd, 'r')) - - for fd in w: - if fd in self.write_fds: - if self.write_fds[fd] == False: - del self.write_fds[fd] - events.append((fd, 'w')) - - self.events.extend(events) - return self.events - - def close(self): - self.read_fds = [] - self.write_fds = [] - -if hasattr(select, 'kqueue'): - - class KQueuePoller(PollerBase): - - def __init__(self): - self.kq = select.kqueue() - close_on_exec(self.kq.fileno()) - self.events = [] - self.max_ev = 0 - - def addfd(self, fd, mode, repeat=True): - fd = fd_(fd) - - self.max_ev += 1 - - if mode == 'r': - kmode = select.KQ_FILTER_READ - else: - kmode = select.KQ_FILTER_WRITE - - flags = select.KQ_EV_ADD - - if sys.platform.startswith("darwin"): - flags |= select.KQ_EV_ENABLE - - if not repeat: - flags |= select.KQ_EV_ONESHOT - - ev = select.kevent(fd, kmode, flags) - self.kq.control([ev], 0, 0) - - def delfd(self, fd, mode): - fd = fd_(fd) - - self.max_ev -= 1 - - if fd < 0: - return - - if mode == 'r': - kmode = select.KQ_FILTER_READ - else: - kmode = select.KQ_FILTER_WRITE - - ev = select.kevent(fd_(fd), select.KQ_FILTER_READ, - select.KQ_EV_DELETE) - self.kq.control([ev], 0) - - def _wait(self, nsec=0): - if len(self.events) == 0: - try: - events = self.kq.control(None, self.max_ev, nsec) - except select.error as e: - if e.args[0] != errno.EINTR: - raise - - # process events - all_events = [] - for ev in events: - if ev.filter == select.KQ_FILTER_READ: - mode = 'r' - else: - mode = 'w' - all_events.append((fd_(ev.ident), mode)) - - self.events.extend(all_events) - - # return all events - return self.events - - def close(self): - self.kq.close() - -if hasattr(select, "epoll"): - class EpollPoller(PollerBase): - - def __init__(self): - self.poll = select.epoll() - close_on_exec(self.poll.fileno()) - self.fds = {} - self.events = [] - - def addfd(self, fd, mode, repeat=True): - fd = fd_(fd) - - if mode == 'r': - mode = (select.EPOLLIN, repeat) - else: - mode = (select.EPOLLOUT, repeat) - - if fd in self.fds: - modes = self.fds[fd] - if mode in self.fds[fd]: - # already registered for this mode - return - modes.append(mode) - addfd_ = self.poll.modify - else: - modes = [mode] - addfd_ = self.poll.register - - # append the new mode to fds - self.fds[fd] = modes - - mask = 0 - for mode, r in modes: - mask |= mode - - if not repeat: - mask |= select.EPOLLONESHOT - - addfd_(fd, mask) - - def delfd(self, fd, mode): - fd = fd_(fd) - - if mode == 'r': - mode = select.POLLIN | select.POLLPRI - else: - mode = select.POLLOUT - - if fd not in self.fds: - return - - modes = [] - for m, r in self.fds[fd]: - if mode != m: - modes.append((m, r)) - - if not modes: - # del the fd from the poll - self.poll.unregister(fd) - del self.fds[fd] - else: - # modify the fd in the poll - self.fds[fd] = modes - m, r = modes[0] - mask = m[0] - if r: - mask |= select.EPOLLONESHOT - - self.poll.modify(fd, mask) - - def _wait(self, nsec=0): - # wait for the events - if len(self.events) == 0: - try: - events = self.poll.poll(nsec) - except select.error as e: - if e.args[0] != errno.EINTR: - raise - - if events: - all_events = [] - fds = {} - for fd, ev in events: - fd = fd_(fd) - if ev & select.EPOLLIN: - mode = 'r' - else: - mode = 'w' - - all_events.append((fd, mode)) - - if fd in fds: - fds[fd].append(mode) - else: - fds[fd] = [mode] - - # eventually remove the mode from the list if repeat - # was set to False and modify the poll if needed. - modes = [] - for m, r in self.fds[fd]: - if not r: - continue - modes.append((m, r)) - - if modes != self.fds[fd]: - self.fds[fd] = modes - mask = 0 - for m, r in modes: - mask |= m - self.poll.modify(fd, mask) - - self.events.extend(all_events) - - # return all events - return self.events - - def close(self): - self.poll.close() - - -if hasattr(select, "poll") or hasattr(select, "devpoll"): - - class _PollerBase(PollerBase): - - POLL_IMPL = None - - def __init__(self): - self.poll = self.POLL_IMPL() - self.fds = {} - self.events = [] - - def addfd(self, fd, mode, repeat=True): - fd = fd_(fd) - if mode == 'r': - mode = (select.POLLIN, repeat) - else: - mode = (select.POLLOUT, repeat) - - if fd in self.fds: - modes = self.fds[fd] - if mode in modes: - # already registered for this mode - return - modes.append(mode) - addfd_ = self.poll.modify - else: - modes = [mode] - addfd_ = self.poll.register - - # append the new mode to fds - self.fds[fd] = modes - - mask = 0 - for mode, r in modes: - mask |= mode - - addfd_(fd, mask) - - def delfd(self, fd, mode): - fd = fd_(fd) - - if mode == 'r': - mode = select.POLLIN | select.POLLPRI - else: - mode = select.POLLOUT - - if fd not in self.fds: - return - - modes = [] - for m, r in self.fds[fd]: - if mode != m: - modes.append((m, r)) - - if not modes: - # del the fd from the poll - self.poll.unregister(fd) - del self.fds[fd] - else: - # modify the fd in the poll - self.fds[fd] = modes - m, r = modes[0] - mask = m[0] - self.poll.modify(fd, mask) - - def _wait(self, nsec=0): - # wait for the events - if len(self.events) == 0: - try: - events = self.poll.poll(nsec) - except select.error as e: - if e.args[0] != errno.EINTR: - raise - - all_events = [] - for fd, ev in events: - fd = fd_(fd) - - if fd not in self.fds: - continue - - if ev & select.POLLIN or ev & select.POLLPRI: - mode = 'r' - else: - mode = 'w' - - # add new event to the list - all_events.append((fd, mode)) - - # eventually remove the mode from the list if repeat - # was set to False and modify the poll if needed. - modes = [] - for m, r in self.fds[fd]: - if not r: - continue - modes.append((m, r)) - - if not modes: - self.poll.unregister(fd) - else: - mask = 0 - if modes != self.fds[fd]: - mask |= m - self.poll.modify(fd, mask) - - - self.events.extend(all_events) - return self.events - - def close(self): - for fd in self.fds: - self.poll.unregister(fd) - self.fds = [] - self.poll = None - - - if hasattr(select, "devpoll"): - - class DevPollPoller(_PollerBase): - POLL_IMPL = select.devpoll - - if hasattr(select, "poll"): - class PollPoller(_PollerBase): - POLL_IMPL = select.poll - - -# choose the best implementation depending on the platform. -if 'KQueuePoller' in globals(): - DefaultPoller = KQueuePoller -elif 'EpollPoller' in globals(): - DefaultPoller = EpollPoller -elif 'DevpollPoller' in globals(): - DefaultPoller = DevpollPoller -elif 'PollPoller' in globals(): - DefaultPoller = PollPoller -else: - DefaultPoller = SelectPoller diff --git a/gunicorn/selectors.py b/gunicorn/selectors.py new file mode 100644 index 00000000..4e9ae6ec --- /dev/null +++ b/gunicorn/selectors.py @@ -0,0 +1,585 @@ +"""Selectors module. + +This module allows high-level and efficient I/O multiplexing, built upon the +`select` module primitives. +""" + + +from abc import ABCMeta, abstractmethod +from collections import namedtuple, Mapping +import math +import select +import sys + + +# generic events, that must be mapped to implementation-specific ones +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) + + +def _fileobj_to_fd(fileobj): + """Return a file descriptor from a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + corresponding file descriptor + + Raises: + ValueError if the object is invalid + """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: " + "{!r}".format(fileobj)) from None + if fd < 0: + raise ValueError("Invalid file descriptor: {}".format(fd)) + return fd + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) +"""Object used to associate a file object to its backing file descriptor, +selected event mask and attached data.""" + + +class _SelectorMapping(Mapping): + """Mapping of file objects to selector keys.""" + + def __init__(self, selector): + self._selector = selector + + def __len__(self): + return len(self._selector._fd_to_key) + + def __getitem__(self, fileobj): + try: + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key[fd] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + def __iter__(self): + return iter(self._selector._fd_to_key) + + +class BaseSelector(metaclass=ABCMeta): + """Selector abstract base class. + + A selector supports registering file objects to be monitored for specific + I/O events. + + A file object is a file descriptor or any object with a `fileno()` method. + An arbitrary object can be attached to the file object, which can be used + for example to store context information, a callback, etc. + + A selector can use various implementations (select(), poll(), epoll()...) + depending on the platform. The default `Selector` class uses the most + efficient implementation on the current platform. + """ + + @abstractmethod + def register(self, fileobj, events, data=None): + """Register a file object. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + + Raises: + ValueError if events is invalid + KeyError if fileobj is already registered + OSError if fileobj is closed or otherwise is unacceptable to + the underlying system call (if a system call is made) + + Note: + OSError may or may not be raised + """ + raise NotImplementedError + + @abstractmethod + def unregister(self, fileobj): + """Unregister a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + SelectorKey instance + + Raises: + KeyError if fileobj is not registered + + Note: + If fileobj is registered but has since been closed this does + *not* raise OSError (even if the wrapped syscall does) + """ + raise NotImplementedError + + def modify(self, fileobj, events, data=None): + """Change a registered file object monitored events or attached data. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + + Raises: + Anything that unregister() or register() raises + """ + self.unregister(fileobj) + return self.register(fileobj, events, data) + + @abstractmethod + def select(self, timeout=None): + """Perform the actual selection, until some monitored file objects are + ready or a timeout expires. + + Parameters: + timeout -- if timeout > 0, this specifies the maximum wait time, in + seconds + if timeout <= 0, the select() call won't block, and will + report the currently ready file objects + if timeout is None, select() will block until a monitored + file object becomes ready + + Returns: + list of (key, events) for ready file objects + `events` is a bitwise mask of EVENT_READ|EVENT_WRITE + """ + raise NotImplementedError + + def close(self): + """Close the selector. + + This must be called to make sure that any underlying resource is freed. + """ + pass + + def get_key(self, fileobj): + """Return the key associated to a registered file object. + + Returns: + SelectorKey for this file object + """ + mapping = self.get_map() + try: + return mapping[fileobj] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + @abstractmethod + def get_map(self): + """Return a mapping of file objects to selector keys.""" + raise NotImplementedError + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class _BaseSelectorImpl(BaseSelector): + """Base selector implementation.""" + + def __init__(self): + # this maps file descriptors to keys + self._fd_to_key = {} + # read-only mapping returned by get_map() + self._map = _SelectorMapping(self) + + def _fileobj_lookup(self, fileobj): + """Return a file descriptor from a file object. + + This wraps _fileobj_to_fd() to do an exhaustive search in case + the object is invalid but we still have it in our map. This + is used by unregister() so we can unregister an object that + was previously registered even if it is closed. It is also + used by _SelectorMapping. + """ + try: + return _fileobj_to_fd(fileobj) + except ValueError: + # Do an exhaustive search. + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + return key.fd + # Raise ValueError after all. + raise + + def register(self, fileobj, events, data=None): + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {!r}".format(events)) + + key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) + + if key.fd in self._fd_to_key: + raise KeyError("{!r} (FD {}) is already registered" + .format(fileobj, key.fd)) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + try: + key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + return key + + def modify(self, fileobj, events, data=None): + # TODO: Subclasses can probably optimize this even further. + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + if events != key.events: + self.unregister(fileobj) + key = self.register(fileobj, events, data) + elif data != key.data: + # Use a shortcut to update the data. + key = key._replace(data=data) + self._fd_to_key[key.fd] = key + return key + + def close(self): + self._fd_to_key.clear() + + def get_map(self): + return self._map + + def _key_from_fd(self, fd): + """Return the key associated to a given file descriptor. + + Parameters: + fd -- file descriptor + + Returns: + corresponding key, or None if not found + """ + try: + return self._fd_to_key[fd] + except KeyError: + return None + + +class SelectSelector(_BaseSelectorImpl): + """Select-based selector.""" + + def __init__(self): + super().__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + if sys.platform == 'win32': + def _select(self, r, w, _, timeout=None): + r, w, x = select.select(r, w, w, timeout) + return r, w + x, [] + else: + _select = select.select + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + ready = [] + try: + r, w, _ = self._select(self._readers, self._writers, [], timeout) + except InterruptedError: + return ready + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, 'poll'): + + class PollSelector(_BaseSelectorImpl): + """Poll-based selector.""" + + def __init__(self): + super().__init__() + self._poll = select.poll() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poll_events = 0 + if events & EVENT_READ: + poll_events |= select.POLLIN + if events & EVENT_WRITE: + poll_events |= select.POLLOUT + self._poll.register(key.fd, poll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def select(self, timeout=None): + if timeout is None: + timeout = None + elif timeout <= 0: + timeout = 0 + else: + # poll() has a resolution of 1 millisecond, round away from + # zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + ready = [] + try: + fd_event_list = self._poll.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.POLLIN: + events |= EVENT_WRITE + if event & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, 'epoll'): + + class EpollSelector(_BaseSelectorImpl): + """Epoll-based selector.""" + + def __init__(self): + super().__init__() + self._epoll = select.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + epoll_events = 0 + if events & EVENT_READ: + epoll_events |= select.EPOLLIN + if events & EVENT_WRITE: + epoll_events |= select.EPOLLOUT + self._epoll.register(key.fd, epoll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + try: + self._epoll.unregister(key.fd) + except OSError: + # This can happen if the FD was closed since it + # was registered. + pass + return key + + def select(self, timeout=None): + if timeout is None: + timeout = -1 + elif timeout <= 0: + timeout = 0 + else: + # epoll_wait() has a resolution of 1 millisecond, round away + # from zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) * 1e-3 + max_ev = len(self._fd_to_key) + ready = [] + try: + fd_event_list = self._epoll.poll(timeout, max_ev) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.EPOLLIN: + events |= EVENT_WRITE + if event & ~select.EPOLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._epoll.close() + super().close() + + +if hasattr(select, 'devpoll'): + + class DevpollSelector(_BaseSelectorImpl): + """Solaris /dev/poll selector.""" + + def __init__(self): + super().__init__() + self._devpoll = select.devpoll() + + def fileno(self): + return self._devpoll.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poll_events = 0 + if events & EVENT_READ: + poll_events |= select.POLLIN + if events & EVENT_WRITE: + poll_events |= select.POLLOUT + self._devpoll.register(key.fd, poll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._devpoll.unregister(key.fd) + return key + + def select(self, timeout=None): + if timeout is None: + timeout = None + elif timeout <= 0: + timeout = 0 + else: + # devpoll() has a resolution of 1 millisecond, round away from + # zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + ready = [] + try: + fd_event_list = self._devpoll.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~select.POLLIN: + events |= EVENT_WRITE + if event & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._devpoll.close() + super().close() + + +if hasattr(select, 'kqueue'): + + class KqueueSelector(_BaseSelectorImpl): + """Kqueue-based selector.""" + + def __init__(self): + super().__init__() + self._kqueue = select.kqueue() + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + if events & EVENT_READ: + kev = select.kevent(key.fd, select.KQ_FILTER_READ, + select.KQ_EV_ADD) + self._kqueue.control([kev], 0, 0) + if events & EVENT_WRITE: + kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, + select.KQ_EV_ADD) + self._kqueue.control([kev], 0, 0) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + if key.events & EVENT_READ: + kev = select.kevent(key.fd, select.KQ_FILTER_READ, + select.KQ_EV_DELETE) + try: + self._kqueue.control([kev], 0, 0) + except OSError: + # This can happen if the FD was closed since it + # was registered. + pass + if key.events & EVENT_WRITE: + kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, + select.KQ_EV_DELETE) + try: + self._kqueue.control([kev], 0, 0) + except OSError: + # See comment above. + pass + return key + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + max_ev = len(self._fd_to_key) + ready = [] + try: + kev_list = self._kqueue.control(None, max_ev, timeout) + except InterruptedError: + return ready + for kev in kev_list: + fd = kev.ident + flag = kev.filter + events = 0 + if flag == select.KQ_FILTER_READ: + events |= EVENT_READ + if flag == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._kqueue.close() + super().close() + + +# Choose the best implementation: roughly, epoll|kqueue|devpoll > poll > select. +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'KqueueSelector' in globals(): + DefaultSelector = KqueueSelector +elif 'EpollSelector' in globals(): + DefaultSelector = EpollSelector +elif 'DevpollSelector' in globals(): + DefaultSelector = DevpollSelector +elif 'PollSelector' in globals(): + DefaultSelector = PollSelector +else: + DefaultSelector = SelectSelector diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index f34c7867..3acfd40f 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -24,11 +24,18 @@ import time from .. import http from ..http import wsgi -from .. import fdevents from .. import util from . import base from .. import six +try: + from asyncio import selectors +except ImportError: + from .. import selectors + + +ACCEPT = 0 +KEEPALIVED = 1 class TConn(): @@ -66,16 +73,15 @@ class ThreadWorker(base.Worker): def init_process(self): self.tpool = futures.ThreadPoolExecutor(max_workers=self.cfg.threads) - self.poller = fdevents.DefaultPoller() + self.poller = selectors.DefaultSelector() super(ThreadWorker, self).init_process() def run(self): # init listeners, add them to the event loop for s in self.sockets: s.setblocking(False) - self.poller.addfd(s, 'r') + self.poller.register(s, selectors.EVENT_READ, ACCEPT) - listeners = dict([(s.fileno(), s) for s in self.sockets]) while self.alive: # If our parent changed then we shut down. @@ -86,13 +92,14 @@ class ThreadWorker(base.Worker): # notify the arbiter we are alive self.notify() - events = self.poller.wait(0.01) + events = self.poller.select(0.01) if events: - for (fd, mode) in events: + for key, mask in events: fs = None client = None - if fd in listeners: - listener = listeners[fd] + if key.data == ACCEPT: + listener = key.fileobj + # start to accept connections try: client, addr = listener.accept() @@ -107,16 +114,20 @@ class ThreadWorker(base.Worker): if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise - elif fd in self.keepalived: + + else: # get the client connection - client = self.keepalived[fd] + client = key.data # remove it from the heap + try: del self._heap[operator.indexOf(self._heap, client)] except (KeyError, IndexError): pass + self.poller.unregister(key.fileobj) + # add a job to the pool fs = self.tpool.submit(self.handle, client.listener, client.sock, client.addr, client.parser) @@ -146,10 +157,10 @@ class ThreadWorker(base.Worker): # register the connection heapq.heappush(self._heap, tconn) - self.keepalived[fs.sock.fileno()] = tconn # add the socket to the event loop - self.poller.addfd(fs.sock, 'r', False) + self.poller.register(fs.sock, selectors.EVENT_READ, + tconn) else: # at this point the connection should be # closed but we make sure it is. @@ -176,7 +187,7 @@ class ThreadWorker(base.Worker): break else: # remove the socket from the poller - self.poller.delfd(conn.sock, 'r') + self.poller.unregister(conn.sock) # close the socket util.close(conn.sock) From f8b415496d6a3263f8dd13559be926d841c3a3a7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 30 May 2014 15:59:47 +0200 Subject: [PATCH 12/67] refactor the gthread worker for a better usage of asyncio we have the possibility to pass a data payload to the poller when registering a file object. We are using this possibility to pass a callback. the callback will either accept or handle a connection when the read event is triggered. while I am here make the future result asynchronous so we don't block the I/O event handling. --- gunicorn/workers/gthread.py | 278 ++++++++++++++++++++---------------- 1 file changed, 151 insertions(+), 127 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 3acfd40f..dbcfdab5 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -58,32 +58,112 @@ class ThreadWorker(base.Worker): def __init__(self, *args, **kwargs): super(ThreadWorker, self).__init__(*args, **kwargs) + self.worker_connections = self.cfg.worker_connections + self.idle_workers = 0 + # initialise the pool self.tpool = None self.poller = None self.futures = set() self._heap = [] - self.keepalived = {} + self.clients = {} - def _wrap_future(self, fs, listener, client, addr): - fs.listener = listener - fs.sock = client - fs.addr = addr + + def _wrap_future(self, fs, conn): + fs.conn = conn self.futures.add(fs) + fs.add_done_callback(self.finish_request) + + def _unregister_keepalive(self, conn): + try: + del self._heap[operator.indexOf(self._heap, conn)] + except (KeyError, IndexError, ValueError): + pass def init_process(self): self.tpool = futures.ThreadPoolExecutor(max_workers=self.cfg.threads) self.poller = selectors.DefaultSelector() super(ThreadWorker, self).init_process() + + def acceptor(self, listener): + try: + client, addr = listener.accept() + client.setblocking(False) + + # wrap the socket if needed + if self.cfg.is_ssl: + client = ssl.wrap_socket(client, server_side=True, + **self.cfg.ssl_options) + + # initialise the parser + parser = http.RequestParser(self.cfg, client) + + # register the connection + tconn = TConn(self, listener, client, addr, parser) + self.clients[client] = tconn + + # wait for the read event to handle the connection + self.poller.register(client, selectors.EVENT_READ, + self.handle_client) + + except socket.error as e: + if e.args[0] not in (errno.EAGAIN, + errno.ECONNABORTED, errno.EWOULDBLOCK): + raise + + def handle_client(self, client): + # unregister the client from the poller + self.poller.unregister(client) + + try: + conn = self.clients[client] + + # maybe unregister the keepalive from the heap + self._unregister_keepalive(conn) + + # submit the connection to a worker + fs = self.tpool.submit(self.handle, conn) + self.idle_workers += 1 + self._wrap_future(fs, conn) + + except KeyError: + # no connection registered + return + + def murder_keepalived(self): + now = time.time() + while True: + if not len(self._heap): + break + + conn = heapq.heappop(self._heap) + delta = conn.timeout - now + if delta > 0: + heapq.heappush(self._heap, conn) + break + else: + # make sure the connection can't be handled + try: + del self.clients[conn.sock] + except KeyError: + pass + + # remove the socket from the poller + self.poller.unregister(conn.sock) + + # close the socket + util.close(conn.sock) + + + def run(self): # init listeners, add them to the event loop for s in self.sockets: s.setblocking(False) - self.poller.register(s, selectors.EVENT_READ, ACCEPT) + self.poller.register(s, selectors.EVENT_READ, self.acceptor) while self.alive: - # If our parent changed then we shut down. if self.ppid != os.getppid(): self.log.info("Parent changed, shutting down: %s", self) @@ -92,116 +172,32 @@ class ThreadWorker(base.Worker): # notify the arbiter we are alive self.notify() - events = self.poller.select(0.01) - if events: - for key, mask in events: - fs = None - client = None - if key.data == ACCEPT: - listener = key.fileobj - - # start to accept connections - try: - client, addr = listener.accept() - - # add a job to the pool - fs = self.tpool.submit(self.handle, listener, - client, addr, False) - - self._wrap_future(fs, listener, client, addr) - - except socket.error as e: - if e.args[0] not in (errno.EAGAIN, - errno.ECONNABORTED, errno.EWOULDBLOCK): - raise - - else: - # get the client connection - client = key.data - - # remove it from the heap - - try: - del self._heap[operator.indexOf(self._heap, client)] - except (KeyError, IndexError): - pass - - self.poller.unregister(key.fileobj) - - # add a job to the pool - fs = self.tpool.submit(self.handle, client.listener, - client.sock, client.addr, client.parser) - - # wrap the future - self._wrap_future(fs, client.listener, client.sock, - client.addr) - - # handle jobs, we give a chance to all jobs to be executed. - if self.futures: - self.notify() - - res = futures.wait(self.futures, timeout=self.timeout, - return_when=futures.FIRST_COMPLETED) - - for fs in res.done: - try: - (keepalive, parser) = fs.result() - # if the connection should be kept alived add it - # to the eventloop and record it - if keepalive: - # flag the socket as non blocked - fs.sock.setblocking(0) - - tconn = TConn(self, fs.listener, fs.sock, - fs.addr, parser) - - # register the connection - heapq.heappush(self._heap, tconn) - - # add the socket to the event loop - self.poller.register(fs.sock, selectors.EVENT_READ, - tconn) - else: - # at this point the connection should be - # closed but we make sure it is. - util.close(fs.sock) - except: - # an exception happened, make sure to close the - # socket. - util.close(fs.sock) - finally: - # remove the future from our list - self.futures.remove(fs) - + events = self.poller.select(1.0) + for key, mask in events: + callback = key.data + callback(key.fileobj) # hanle keepalive timeouts - now = time.time() - while True: - if not len(self._heap): - break + self.murder_keepalived() - conn = heapq.heappop(self._heap) - delta = conn.timeout - now - if delta > 0: - heapq.heappush(self._heap, conn) + # if we more connections than the max number of connections + # accepted on a worker, wait until some complete or exit. + if self.idle_workers >= self.worker_connections: + futures.wait(self.futures, timeout=self.cfg.timeout) + if not res: + self.log.info("max requests achieved") break - else: - # remove the socket from the poller - self.poller.unregister(conn.sock) - # close the socket - util.close(conn.sock) - # shutdown the pool - self.tpool.shutdown(False) self.poller.close() + self.tpool.shutdown(False) # wait for the workers futures.wait(self.futures, timeout=self.cfg.graceful_timeout) # if we have still fures running, try to close them for fs in self.futures: - sock = fs.sock + sock = fs.conn.sock # the future is not running, cancel it if not fs.done() and not fs.running(): @@ -211,27 +207,51 @@ class ThreadWorker(base.Worker): util.close(sock) - def handle(self, listener, client, addr, parser): + def finish_request(self, fs): + try: + (keepalive, conn) = fs.result() + # if the connection should be kept alived add it + # to the eventloop and record it + if keepalive: + # flag the socket as non blocked + conn.sock.setblocking(False) + + # register the connection + heapq.heappush(self._heap, conn) + + # add the socket to the event loop + self.poller.register(conn.sock, selectors.EVENT_READ, + self.handle_client) + else: + try: + del self.clients[conn.sock] + except KeyError: + pass + + util.close(fs.conn.sock) + except: + # an exception happened, make sure to close the + # socket. + util.close(fs.conn.sock) + finally: + # remove the future from our list + self.futures.remove(fs) + self.idle_workers -= 1 + + def handle(self, conn): keepalive = False req = None try: - client.setblocking(1) + conn.sock.setblocking(1) - # wrap the connection - if not parser: - if self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, - **self.cfg.ssl_options) - parser = http.RequestParser(self.cfg, client) - - req = six.next(parser) + req = six.next(conn.parser) if not req: return (False, None) # handle the request - keepalive = self.handle_request(listener, req, client, addr) + keepalive = self.handle_request(req, conn) if keepalive: - return (keepalive, parser) + return (keepalive, conn) except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) @@ -240,10 +260,10 @@ class ThreadWorker(base.Worker): except ssl.SSLError as e: if e.args[0] == ssl.SSL_ERROR_EOF: self.log.debug("ssl connection closed") - client.close() + conn.sock.close() else: self.log.debug("Error processing SSL request.") - self.handle_error(req, client, addr, e) + self.handle_error(req, conn.sock, conn.addr, e) except socket.error as e: if e.args[0] not in (errno.EPIPE, errno.ECONNRESET): @@ -252,27 +272,30 @@ class ThreadWorker(base.Worker): if e.args[0] == errno.ECONNRESET: self.log.debug("Ignoring connection reset") else: - self.log.debug("Ignoring EPIPE") + self.log.debug("Ignoring connection epipe") except Exception as e: - self.handle_error(req, client, addr, e) + self.handle_error(req, conn.sock, conn.addr, e) - return (False, None) + return (False, conn) - def handle_request(self, listener, req, client, addr): + def handle_request(self, req, conn): environ = {} resp = None try: self.cfg.pre_request(self, req) request_start = datetime.now() - resp, environ = wsgi.create(req, client, addr, - listener.getsockname(), self.cfg) + resp, environ = wsgi.create(req, conn.sock, conn.addr, + conn.listener.getsockname(), self.cfg) environ["wsgi.multithread"] = True self.nr += 1 - if self.nr >= self.max_requests: + + if self.alive and self.nr >= self.max_requests: self.log.info("Autorestarting worker after current request.") + resp.force_close() self.alive = False + if not self.cfg.keepalive: resp.force_close() @@ -283,6 +306,7 @@ class ThreadWorker(base.Worker): else: for item in respiter: resp.write(item) + resp.close() request_time = datetime.now() - request_start self.log.access(resp, req, environ, request_time) @@ -303,8 +327,8 @@ class ThreadWorker(base.Worker): # connection to indicate the error. self.log.exception("Error handling request") try: - client.shutdown(socket.SHUT_RDWR) - client.close() + conn.sock.shutdown(socket.SHUT_RDWR) + conn.sock.close() except socket.error: pass raise StopIteration() From c8e93a6f218eb2435a5d21091e7523940ba0c8c9 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 30 May 2014 23:26:30 +0200 Subject: [PATCH 13/67] make the code simpler and fix issue with ab --- gunicorn/workers/gthread.py | 112 ++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 62 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index dbcfdab5..417c00f9 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -10,7 +10,7 @@ # If no event happen after the keep alive timeout, the connectoin is # closed. - +from collections import deque import concurrent.futures as futures from datetime import datetime import errno @@ -39,14 +39,33 @@ KEEPALIVED = 1 class TConn(): - def __init__(self, worker, listener, sock, addr, parser): + def __init__(self, cfg, listener, sock, addr): + self.cfg = cfg self.listener = listener self.sock = sock self.addr = addr - self.parser = parser + self.timeout = None + self.parser = None + + # set the socket to non blocking + #self.sock.setblocking(False) + + + def maybe_init(self): + if self.parser is None: + # wrap the socket if needed + if self.cfg.is_ssl: + client = ssl.wrap_socket(client, server_side=True, + **self.cfg.ssl_options) + + + # initialize the parser + self.parser = http.RequestParser(self.cfg, self.sock) + + def set_timeout(self): # set the timeout - self.timeout = time.time() + worker.cfg.keepalive + self.timeout = time.time() + self.cfg.keepalive def __lt__(self, other): return self.timeout < other.timeout @@ -65,8 +84,7 @@ class ThreadWorker(base.Worker): self.tpool = None self.poller = None self.futures = set() - self._heap = [] - self.clients = {} + self._keep = deque() def _wrap_future(self, fs, conn): @@ -74,80 +92,48 @@ class ThreadWorker(base.Worker): self.futures.add(fs) fs.add_done_callback(self.finish_request) - def _unregister_keepalive(self, conn): - try: - del self._heap[operator.indexOf(self._heap, conn)] - except (KeyError, IndexError, ValueError): - pass - def init_process(self): self.tpool = futures.ThreadPoolExecutor(max_workers=self.cfg.threads) self.poller = selectors.DefaultSelector() super(ThreadWorker, self).init_process() - def acceptor(self, listener): + def accept(self, listener, *args): try: client, addr = listener.accept() - client.setblocking(False) - - # wrap the socket if needed - if self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, - **self.cfg.ssl_options) - - # initialise the parser - parser = http.RequestParser(self.cfg, client) - - # register the connection - tconn = TConn(self, listener, client, addr, parser) - self.clients[client] = tconn + conn = TConn(self.cfg, listener, client, addr) # wait for the read event to handle the connection self.poller.register(client, selectors.EVENT_READ, - self.handle_client) + (self.handle_client, (conn,))) except socket.error as e: if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise - def handle_client(self, client): + def handle_client(self, client, conn): # unregister the client from the poller self.poller.unregister(client) - try: - conn = self.clients[client] + # submit the connection to a worker + fs = self.tpool.submit(self.handle, conn) + self.idle_workers += 1 + self._wrap_future(fs, conn) - # maybe unregister the keepalive from the heap - self._unregister_keepalive(conn) - - # submit the connection to a worker - fs = self.tpool.submit(self.handle, conn) - self.idle_workers += 1 - self._wrap_future(fs, conn) - - except KeyError: - # no connection registered - return def murder_keepalived(self): now = time.time() while True: - if not len(self._heap): + if not len(self._keep): break - conn = heapq.heappop(self._heap) - delta = conn.timeout - now + delta = self._keep[0].timeout - now if delta > 0: - heapq.heappush(self._heap, conn) break else: - # make sure the connection can't be handled - try: - del self.clients[conn.sock] - except KeyError: - pass + # remove the connection from the queue + conn = self._keep.popleft() # remove the socket from the poller self.poller.unregister(conn.sock) @@ -155,13 +141,12 @@ class ThreadWorker(base.Worker): # close the socket util.close(conn.sock) - - def run(self): # init listeners, add them to the event loop for s in self.sockets: s.setblocking(False) - self.poller.register(s, selectors.EVENT_READ, self.acceptor) + self.poller.register(s, selectors.EVENT_READ, + (self.accept,())) while self.alive: # If our parent changed then we shut down. @@ -174,8 +159,8 @@ class ThreadWorker(base.Worker): events = self.poller.select(1.0) for key, mask in events: - callback = key.data - callback(key.fileobj) + (callback, args) = key.data + callback(key.fileobj, *args) # hanle keepalive timeouts self.murder_keepalived() @@ -217,17 +202,13 @@ class ThreadWorker(base.Worker): conn.sock.setblocking(False) # register the connection - heapq.heappush(self._heap, conn) + conn.set_timeout() + self._keep.append(conn) # add the socket to the event loop self.poller.register(conn.sock, selectors.EVENT_READ, - self.handle_client) + (self.handle_client, (conn,))) else: - try: - del self.clients[conn.sock] - except KeyError: - pass - util.close(fs.conn.sock) except: # an exception happened, make sure to close the @@ -239,6 +220,13 @@ class ThreadWorker(base.Worker): self.idle_workers -= 1 def handle(self, conn): + try: + self._keep.remove(conn) + except ValueError: + pass + + conn.maybe_init() + keepalive = False req = None try: From 5ba749e9cac271b6959798d519c309daec49228b Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 31 May 2014 00:17:29 +0200 Subject: [PATCH 14/67] some quick optimisations --- gunicorn/workers/gthread.py | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 417c00f9..a07da55b 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -14,7 +14,7 @@ from collections import deque import concurrent.futures as futures from datetime import datetime import errno -import heapq +from functools import partial import os import operator import socket @@ -49,10 +49,10 @@ class TConn(): self.parser = None # set the socket to non blocking - #self.sock.setblocking(False) + self.sock.setblocking(False) - def maybe_init(self): + def init(self): if self.parser is None: # wrap the socket if needed if self.cfg.is_ssl: @@ -62,6 +62,8 @@ class TConn(): # initialize the parser self.parser = http.RequestParser(self.cfg, self.sock) + return True + return False def set_timeout(self): # set the timeout @@ -78,18 +80,17 @@ class ThreadWorker(base.Worker): def __init__(self, *args, **kwargs): super(ThreadWorker, self).__init__(*args, **kwargs) self.worker_connections = self.cfg.worker_connections - self.idle_workers = 0 # initialise the pool self.tpool = None self.poller = None - self.futures = set() + self.futures = deque() self._keep = deque() def _wrap_future(self, fs, conn): fs.conn = conn - self.futures.add(fs) + self.futures.append(fs) fs.add_done_callback(self.finish_request) def init_process(self): @@ -97,28 +98,26 @@ class ThreadWorker(base.Worker): self.poller = selectors.DefaultSelector() super(ThreadWorker, self).init_process() - - def accept(self, listener, *args): + def accept(self, listener): try: client, addr = listener.accept() conn = TConn(self.cfg, listener, client, addr) # wait for the read event to handle the connection self.poller.register(client, selectors.EVENT_READ, - (self.handle_client, (conn,))) + partial(self.handle_client, conn)) except socket.error as e: if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED, errno.EWOULDBLOCK): raise - def handle_client(self, client, conn): + def handle_client(self, conn, client): # unregister the client from the poller self.poller.unregister(client) # submit the connection to a worker fs = self.tpool.submit(self.handle, conn) - self.idle_workers += 1 self._wrap_future(fs, conn) @@ -145,8 +144,7 @@ class ThreadWorker(base.Worker): # init listeners, add them to the event loop for s in self.sockets: s.setblocking(False) - self.poller.register(s, selectors.EVENT_READ, - (self.accept,())) + self.poller.register(s, selectors.EVENT_READ, self.accept) while self.alive: # If our parent changed then we shut down. @@ -157,17 +155,17 @@ class ThreadWorker(base.Worker): # notify the arbiter we are alive self.notify() - events = self.poller.select(1.0) + events = self.poller.select(0.01) for key, mask in events: - (callback, args) = key.data - callback(key.fileobj, *args) + callback = key.data + callback(key.fileobj) # hanle keepalive timeouts self.murder_keepalived() # if we more connections than the max number of connections # accepted on a worker, wait until some complete or exit. - if self.idle_workers >= self.worker_connections: + if len(self.futures) >= self.worker_connections: futures.wait(self.futures, timeout=self.cfg.timeout) if not res: self.log.info("max requests achieved") @@ -207,25 +205,27 @@ class ThreadWorker(base.Worker): # add the socket to the event loop self.poller.register(conn.sock, selectors.EVENT_READ, - (self.handle_client, (conn,))) + partial(self.handle_client, conn)) else: - util.close(fs.conn.sock) + util.close(conn.sock) except: # an exception happened, make sure to close the # socket. util.close(fs.conn.sock) finally: # remove the future from our list - self.futures.remove(fs) - self.idle_workers -= 1 + try: + self.futures.remove(fs) + except ValueError: + pass def handle(self, conn): - try: - self._keep.remove(conn) - except ValueError: - pass - - conn.maybe_init() + if not conn.init(): + # connection kept alive + try: + self._keep.remove(conn) + except ValueError: + pass keepalive = False req = None From e8e9d285a63fba36c85e9c974b7a57c183937641 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 31 May 2014 00:44:20 +0200 Subject: [PATCH 15/67] fixes --- gunicorn/workers/gthread.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index a07da55b..06a5d6e8 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -33,10 +33,6 @@ try: except ImportError: from .. import selectors - -ACCEPT = 0 -KEEPALIVED = 1 - class TConn(): def __init__(self, cfg, listener, sock, addr): @@ -166,7 +162,7 @@ class ThreadWorker(base.Worker): # if we more connections than the max number of connections # accepted on a worker, wait until some complete or exit. if len(self.futures) >= self.worker_connections: - futures.wait(self.futures, timeout=self.cfg.timeout) + res = futures.wait(self.futures, timeout=self.cfg.timeout) if not res: self.log.info("max requests achieved") break @@ -234,7 +230,7 @@ class ThreadWorker(base.Worker): req = six.next(conn.parser) if not req: - return (False, None) + return (False, conn) # handle the request keepalive = self.handle_request(req, conn) From fb53047b73b806e4e4cdd822201731d08ab137d8 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 31 May 2014 01:15:05 +0200 Subject: [PATCH 16/67] fix timeout and socket ssl wrapping --- gunicorn/workers/gthread.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 06a5d6e8..f3d52804 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -33,6 +33,7 @@ try: except ImportError: from .. import selectors + class TConn(): def __init__(self, cfg, listener, sock, addr): @@ -47,12 +48,12 @@ class TConn(): # set the socket to non blocking self.sock.setblocking(False) - def init(self): + self.sock.setblocking(True) if self.parser is None: # wrap the socket if needed if self.cfg.is_ssl: - client = ssl.wrap_socket(client, server_side=True, + self.sock = ssl.wrap_socket(client, server_side=True, **self.cfg.ssl_options) @@ -142,6 +143,8 @@ class ThreadWorker(base.Worker): s.setblocking(False) self.poller.register(s, selectors.EVENT_READ, self.accept) + timeout = self.cfg.timeout or 0.5 + while self.alive: # If our parent changed then we shut down. if self.ppid != os.getppid(): @@ -162,7 +165,7 @@ class ThreadWorker(base.Worker): # if we more connections than the max number of connections # accepted on a worker, wait until some complete or exit. if len(self.futures) >= self.worker_connections: - res = futures.wait(self.futures, timeout=self.cfg.timeout) + res = futures.wait(self.futures, timeout=timeout) if not res: self.log.info("max requests achieved") break @@ -185,7 +188,6 @@ class ThreadWorker(base.Worker): # make sure we close the sockets after the graceful timeout util.close(sock) - def finish_request(self, fs): try: (keepalive, conn) = fs.result() @@ -226,8 +228,6 @@ class ThreadWorker(base.Worker): keepalive = False req = None try: - conn.sock.setblocking(1) - req = six.next(conn.parser) if not req: return (False, conn) From d775b576e8867e2560833ee471013a7da68abd75 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 31 May 2014 01:21:05 +0200 Subject: [PATCH 17/67] improve worker shutdown --- gunicorn/workers/gthread.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index f3d52804..f47eb7fc 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -84,7 +84,6 @@ class ThreadWorker(base.Worker): self.futures = deque() self._keep = deque() - def _wrap_future(self, fs, conn): fs.conn = conn self.futures.append(fs) @@ -117,7 +116,6 @@ class ThreadWorker(base.Worker): fs = self.tpool.submit(self.handle, conn) self._wrap_future(fs, conn) - def murder_keepalived(self): now = time.time() while True: @@ -178,7 +176,12 @@ class ThreadWorker(base.Worker): futures.wait(self.futures, timeout=self.cfg.graceful_timeout) # if we have still fures running, try to close them - for fs in self.futures: + while True: + try: + fs = self.futures.popleft() + except IndexError: + break + sock = fs.conn.sock # the future is not running, cancel it @@ -279,7 +282,6 @@ class ThreadWorker(base.Worker): resp.force_close() self.alive = False - if not self.cfg.keepalive: resp.force_close() From b810a1d1a911f457d8351183ae46518fc0123d26 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 31 May 2014 07:13:36 +0200 Subject: [PATCH 18/67] fix doc --- gunicorn/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 4c28dae3..09113aa2 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -581,8 +581,7 @@ class WorkerThreads(Setting): desc = """\ The number of worker threads for handling requests. - Run each worker in prethreaded mode with the specified number of - threads per worker. + Run each worker with the specified number of threads. A positive integer generally in the 2-4 x $(NUM_CORES) range. You'll want to vary this a bit to find the best for your particular From b7cbb59bbc414eb3c03b163a8afdd296c613a4f6 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 31 May 2014 07:18:39 +0200 Subject: [PATCH 19/67] remove useless code --- gunicorn/util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index 07db3dba..9f6b79c9 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -269,11 +269,6 @@ def set_non_blocking(fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK fcntl.fcntl(fd, fcntl.F_SETFL, flags) -def fd_(fd): - if hasattr(fd, "fileno"): - return int(fd.fileno()) - return fd - def close(sock): try: sock.close() From eb17b13b1df62fc12c5eebb72bc650c61979b5ae Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sat, 31 May 2014 13:28:16 -0700 Subject: [PATCH 20/67] Guard against race condition on threads keepalive Requests after the first on a keepalive connection remove themselves from the keepalive timeout queue. This presents a race condition where the main thread might try to access the first element of the queue after it has been removed. --- gunicorn/workers/gthread.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index f47eb7fc..789d869e 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -119,10 +119,11 @@ class ThreadWorker(base.Worker): def murder_keepalived(self): now = time.time() while True: - if not len(self._keep): + try: + delta = self._keep[0].timeout - now + except IndexError: break - delta = self._keep[0].timeout - now if delta > 0: break else: From 7e699b7d51b7a17f1d2bba7ff8bacb75991c54bf Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sat, 31 May 2014 13:31:07 -0700 Subject: [PATCH 21/67] Use trollius on Py2 instead of bundling selectors --- gunicorn/selectors.py | 585 ------------------------------------ gunicorn/workers/gthread.py | 2 +- 2 files changed, 1 insertion(+), 586 deletions(-) delete mode 100644 gunicorn/selectors.py diff --git a/gunicorn/selectors.py b/gunicorn/selectors.py deleted file mode 100644 index 4e9ae6ec..00000000 --- a/gunicorn/selectors.py +++ /dev/null @@ -1,585 +0,0 @@ -"""Selectors module. - -This module allows high-level and efficient I/O multiplexing, built upon the -`select` module primitives. -""" - - -from abc import ABCMeta, abstractmethod -from collections import namedtuple, Mapping -import math -import select -import sys - - -# generic events, that must be mapped to implementation-specific ones -EVENT_READ = (1 << 0) -EVENT_WRITE = (1 << 1) - - -def _fileobj_to_fd(fileobj): - """Return a file descriptor from a file object. - - Parameters: - fileobj -- file object or file descriptor - - Returns: - corresponding file descriptor - - Raises: - ValueError if the object is invalid - """ - if isinstance(fileobj, int): - fd = fileobj - else: - try: - fd = int(fileobj.fileno()) - except (AttributeError, TypeError, ValueError): - raise ValueError("Invalid file object: " - "{!r}".format(fileobj)) from None - if fd < 0: - raise ValueError("Invalid file descriptor: {}".format(fd)) - return fd - - -SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) -"""Object used to associate a file object to its backing file descriptor, -selected event mask and attached data.""" - - -class _SelectorMapping(Mapping): - """Mapping of file objects to selector keys.""" - - def __init__(self, selector): - self._selector = selector - - def __len__(self): - return len(self._selector._fd_to_key) - - def __getitem__(self, fileobj): - try: - fd = self._selector._fileobj_lookup(fileobj) - return self._selector._fd_to_key[fd] - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None - - def __iter__(self): - return iter(self._selector._fd_to_key) - - -class BaseSelector(metaclass=ABCMeta): - """Selector abstract base class. - - A selector supports registering file objects to be monitored for specific - I/O events. - - A file object is a file descriptor or any object with a `fileno()` method. - An arbitrary object can be attached to the file object, which can be used - for example to store context information, a callback, etc. - - A selector can use various implementations (select(), poll(), epoll()...) - depending on the platform. The default `Selector` class uses the most - efficient implementation on the current platform. - """ - - @abstractmethod - def register(self, fileobj, events, data=None): - """Register a file object. - - Parameters: - fileobj -- file object or file descriptor - events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) - data -- attached data - - Returns: - SelectorKey instance - - Raises: - ValueError if events is invalid - KeyError if fileobj is already registered - OSError if fileobj is closed or otherwise is unacceptable to - the underlying system call (if a system call is made) - - Note: - OSError may or may not be raised - """ - raise NotImplementedError - - @abstractmethod - def unregister(self, fileobj): - """Unregister a file object. - - Parameters: - fileobj -- file object or file descriptor - - Returns: - SelectorKey instance - - Raises: - KeyError if fileobj is not registered - - Note: - If fileobj is registered but has since been closed this does - *not* raise OSError (even if the wrapped syscall does) - """ - raise NotImplementedError - - def modify(self, fileobj, events, data=None): - """Change a registered file object monitored events or attached data. - - Parameters: - fileobj -- file object or file descriptor - events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) - data -- attached data - - Returns: - SelectorKey instance - - Raises: - Anything that unregister() or register() raises - """ - self.unregister(fileobj) - return self.register(fileobj, events, data) - - @abstractmethod - def select(self, timeout=None): - """Perform the actual selection, until some monitored file objects are - ready or a timeout expires. - - Parameters: - timeout -- if timeout > 0, this specifies the maximum wait time, in - seconds - if timeout <= 0, the select() call won't block, and will - report the currently ready file objects - if timeout is None, select() will block until a monitored - file object becomes ready - - Returns: - list of (key, events) for ready file objects - `events` is a bitwise mask of EVENT_READ|EVENT_WRITE - """ - raise NotImplementedError - - def close(self): - """Close the selector. - - This must be called to make sure that any underlying resource is freed. - """ - pass - - def get_key(self, fileobj): - """Return the key associated to a registered file object. - - Returns: - SelectorKey for this file object - """ - mapping = self.get_map() - try: - return mapping[fileobj] - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None - - @abstractmethod - def get_map(self): - """Return a mapping of file objects to selector keys.""" - raise NotImplementedError - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - -class _BaseSelectorImpl(BaseSelector): - """Base selector implementation.""" - - def __init__(self): - # this maps file descriptors to keys - self._fd_to_key = {} - # read-only mapping returned by get_map() - self._map = _SelectorMapping(self) - - def _fileobj_lookup(self, fileobj): - """Return a file descriptor from a file object. - - This wraps _fileobj_to_fd() to do an exhaustive search in case - the object is invalid but we still have it in our map. This - is used by unregister() so we can unregister an object that - was previously registered even if it is closed. It is also - used by _SelectorMapping. - """ - try: - return _fileobj_to_fd(fileobj) - except ValueError: - # Do an exhaustive search. - for key in self._fd_to_key.values(): - if key.fileobj is fileobj: - return key.fd - # Raise ValueError after all. - raise - - def register(self, fileobj, events, data=None): - if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): - raise ValueError("Invalid events: {!r}".format(events)) - - key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) - - if key.fd in self._fd_to_key: - raise KeyError("{!r} (FD {}) is already registered" - .format(fileobj, key.fd)) - - self._fd_to_key[key.fd] = key - return key - - def unregister(self, fileobj): - try: - key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None - return key - - def modify(self, fileobj, events, data=None): - # TODO: Subclasses can probably optimize this even further. - try: - key = self._fd_to_key[self._fileobj_lookup(fileobj)] - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None - if events != key.events: - self.unregister(fileobj) - key = self.register(fileobj, events, data) - elif data != key.data: - # Use a shortcut to update the data. - key = key._replace(data=data) - self._fd_to_key[key.fd] = key - return key - - def close(self): - self._fd_to_key.clear() - - def get_map(self): - return self._map - - def _key_from_fd(self, fd): - """Return the key associated to a given file descriptor. - - Parameters: - fd -- file descriptor - - Returns: - corresponding key, or None if not found - """ - try: - return self._fd_to_key[fd] - except KeyError: - return None - - -class SelectSelector(_BaseSelectorImpl): - """Select-based selector.""" - - def __init__(self): - super().__init__() - self._readers = set() - self._writers = set() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - if events & EVENT_READ: - self._readers.add(key.fd) - if events & EVENT_WRITE: - self._writers.add(key.fd) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - self._readers.discard(key.fd) - self._writers.discard(key.fd) - return key - - if sys.platform == 'win32': - def _select(self, r, w, _, timeout=None): - r, w, x = select.select(r, w, w, timeout) - return r, w + x, [] - else: - _select = select.select - - def select(self, timeout=None): - timeout = None if timeout is None else max(timeout, 0) - ready = [] - try: - r, w, _ = self._select(self._readers, self._writers, [], timeout) - except InterruptedError: - return ready - r = set(r) - w = set(w) - for fd in r | w: - events = 0 - if fd in r: - events |= EVENT_READ - if fd in w: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - -if hasattr(select, 'poll'): - - class PollSelector(_BaseSelectorImpl): - """Poll-based selector.""" - - def __init__(self): - super().__init__() - self._poll = select.poll() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - poll_events = 0 - if events & EVENT_READ: - poll_events |= select.POLLIN - if events & EVENT_WRITE: - poll_events |= select.POLLOUT - self._poll.register(key.fd, poll_events) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - self._poll.unregister(key.fd) - return key - - def select(self, timeout=None): - if timeout is None: - timeout = None - elif timeout <= 0: - timeout = 0 - else: - # poll() has a resolution of 1 millisecond, round away from - # zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) - ready = [] - try: - fd_event_list = self._poll.poll(timeout) - except InterruptedError: - return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.POLLIN: - events |= EVENT_WRITE - if event & ~select.POLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - -if hasattr(select, 'epoll'): - - class EpollSelector(_BaseSelectorImpl): - """Epoll-based selector.""" - - def __init__(self): - super().__init__() - self._epoll = select.epoll() - - def fileno(self): - return self._epoll.fileno() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - epoll_events = 0 - if events & EVENT_READ: - epoll_events |= select.EPOLLIN - if events & EVENT_WRITE: - epoll_events |= select.EPOLLOUT - self._epoll.register(key.fd, epoll_events) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - try: - self._epoll.unregister(key.fd) - except OSError: - # This can happen if the FD was closed since it - # was registered. - pass - return key - - def select(self, timeout=None): - if timeout is None: - timeout = -1 - elif timeout <= 0: - timeout = 0 - else: - # epoll_wait() has a resolution of 1 millisecond, round away - # from zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) * 1e-3 - max_ev = len(self._fd_to_key) - ready = [] - try: - fd_event_list = self._epoll.poll(timeout, max_ev) - except InterruptedError: - return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.EPOLLIN: - events |= EVENT_WRITE - if event & ~select.EPOLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - def close(self): - self._epoll.close() - super().close() - - -if hasattr(select, 'devpoll'): - - class DevpollSelector(_BaseSelectorImpl): - """Solaris /dev/poll selector.""" - - def __init__(self): - super().__init__() - self._devpoll = select.devpoll() - - def fileno(self): - return self._devpoll.fileno() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - poll_events = 0 - if events & EVENT_READ: - poll_events |= select.POLLIN - if events & EVENT_WRITE: - poll_events |= select.POLLOUT - self._devpoll.register(key.fd, poll_events) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - self._devpoll.unregister(key.fd) - return key - - def select(self, timeout=None): - if timeout is None: - timeout = None - elif timeout <= 0: - timeout = 0 - else: - # devpoll() has a resolution of 1 millisecond, round away from - # zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) - ready = [] - try: - fd_event_list = self._devpoll.poll(timeout) - except InterruptedError: - return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.POLLIN: - events |= EVENT_WRITE - if event & ~select.POLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - def close(self): - self._devpoll.close() - super().close() - - -if hasattr(select, 'kqueue'): - - class KqueueSelector(_BaseSelectorImpl): - """Kqueue-based selector.""" - - def __init__(self): - super().__init__() - self._kqueue = select.kqueue() - - def fileno(self): - return self._kqueue.fileno() - - def register(self, fileobj, events, data=None): - key = super().register(fileobj, events, data) - if events & EVENT_READ: - kev = select.kevent(key.fd, select.KQ_FILTER_READ, - select.KQ_EV_ADD) - self._kqueue.control([kev], 0, 0) - if events & EVENT_WRITE: - kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, - select.KQ_EV_ADD) - self._kqueue.control([kev], 0, 0) - return key - - def unregister(self, fileobj): - key = super().unregister(fileobj) - if key.events & EVENT_READ: - kev = select.kevent(key.fd, select.KQ_FILTER_READ, - select.KQ_EV_DELETE) - try: - self._kqueue.control([kev], 0, 0) - except OSError: - # This can happen if the FD was closed since it - # was registered. - pass - if key.events & EVENT_WRITE: - kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, - select.KQ_EV_DELETE) - try: - self._kqueue.control([kev], 0, 0) - except OSError: - # See comment above. - pass - return key - - def select(self, timeout=None): - timeout = None if timeout is None else max(timeout, 0) - max_ev = len(self._fd_to_key) - ready = [] - try: - kev_list = self._kqueue.control(None, max_ev, timeout) - except InterruptedError: - return ready - for kev in kev_list: - fd = kev.ident - flag = kev.filter - events = 0 - if flag == select.KQ_FILTER_READ: - events |= EVENT_READ - if flag == select.KQ_FILTER_WRITE: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - def close(self): - self._kqueue.close() - super().close() - - -# Choose the best implementation: roughly, epoll|kqueue|devpoll > poll > select. -# select() also can't accept a FD > FD_SETSIZE (usually around 1024) -if 'KqueueSelector' in globals(): - DefaultSelector = KqueueSelector -elif 'EpollSelector' in globals(): - DefaultSelector = EpollSelector -elif 'DevpollSelector' in globals(): - DefaultSelector = DevpollSelector -elif 'PollSelector' in globals(): - DefaultSelector = PollSelector -else: - DefaultSelector = SelectSelector diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 789d869e..41e135f6 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -31,7 +31,7 @@ from .. import six try: from asyncio import selectors except ImportError: - from .. import selectors + from trollius import selectors class TConn(): From 3cda90a214c361507206a951d57f6ddce2e476d2 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 09:50:40 +0200 Subject: [PATCH 22/67] reduce CPU usage. --- gunicorn/workers/gthread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 41e135f6..5f660883 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -153,7 +153,7 @@ class ThreadWorker(base.Worker): # notify the arbiter we are alive self.notify() - events = self.poller.select(0.01) + events = self.poller.select(0.2) for key, mask in events: callback = key.data callback(key.fileobj) From 9ade14ae788e7d5d6f6769c7288d62a225ce877b Mon Sep 17 00:00:00 2001 From: Ben Oswald Date: Fri, 9 May 2014 01:06:24 +0200 Subject: [PATCH 23/67] add on_exit hook --- docs/source/settings.rst | 11 +++++++++++ gunicorn/arbiter.py | 1 + gunicorn/config.py | 15 +++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 476355c1..c5ac6c3d 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -782,6 +782,17 @@ two integers of number of workers after and before change. If the number of workers is set for the first time, old_value would be None. +on_exit +~~~~~~~ +* :: + + def on_exit(): + pass + +Called just before exiting gunicorn. + +The callable needs to accept a single instance variable for the Arbiter. + Server Mechanics ---------------- diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 6f4587af..3b4b05ab 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -296,6 +296,7 @@ class Arbiter(object): self.log.info("Reason: %s", reason) if self.pidfile is not None: self.pidfile.unlink() + self.cfg.on_exit(self) sys.exit(exit_status) def sleep(self): diff --git a/gunicorn/config.py b/gunicorn/config.py index 51125f2f..895049db 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1422,6 +1422,21 @@ class NumWorkersChanged(Setting): None. """ +class OnExit(Setting): + name = "on_exit" + section = "Server Hooks" + validator = validate_callable(1) + + def on_exit(server): + pass + + default = staticmethod(on_exit) + desc = """\ + Called just before exiting gunicorn. + + The callable needs to accept a single instance variable for the Arbiter. + """ + class ProxyProtocol(Setting): name = "proxy_protocol" From 5e8eeadc44cfdc818f78aae5df46c09b6b0ffe86 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 09:59:29 +0200 Subject: [PATCH 24/67] sync documentation generally speakiing people shouldn't update the settings.rst but make sure that settings.p is correctly filled. --- docs/source/settings.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index c5ac6c3d..79ca65e0 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -1,3 +1,4 @@ + .. _settings: Settings @@ -276,7 +277,7 @@ chdir ~~~~~ * ``--chdir`` -* ``/Users/benoitc/work/gunicorn_env/src/gunicorn/docs`` +* ``/home/benoitc/work/gunicorn/env_py3/src/gunicorn/docs`` Chdir to specified directory before apps loading. @@ -329,7 +330,7 @@ user ~~~~ * ``-u USER, --user USER`` -* ``501`` +* ``1000`` Switch worker processes to run as this user. @@ -341,7 +342,7 @@ group ~~~~~ * ``-g GROUP, --group GROUP`` -* ``20`` +* ``1000`` Switch worker process to run as this group. @@ -379,7 +380,7 @@ temporary directory. secure_scheme_headers ~~~~~~~~~~~~~~~~~~~~~ -* ``{'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}`` +* ``{'X-FORWARDED-SSL': 'on', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-PROTOCOL': 'ssl'}`` A dictionary containing headers and values that the front-end proxy uses to indicate HTTPS requests. These tell gunicorn to set @@ -504,7 +505,7 @@ syslog_addr ~~~~~~~~~~~ * ``--log-syslog-to SYSLOG_ADDR`` -* ``unix:///var/run/syslog`` +* ``udp://localhost:514`` Address to send syslog messages. @@ -784,9 +785,10 @@ None. on_exit ~~~~~~~ + * :: - def on_exit(): + def on_exit(server): pass Called just before exiting gunicorn. From 62f6fb2d33182d985229d873c597ecbaa9801662 Mon Sep 17 00:00:00 2001 From: bninja Date: Thu, 13 Mar 2014 22:52:41 -0700 Subject: [PATCH 25/67] save listener socket name so we can handle buffered keep-alive requests *after* listener has been closed (i.e. stopped accepting during worker teardown) --- gunicorn/workers/async.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index 29b79578..9351b999 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -32,9 +32,10 @@ class AsyncWorker(base.Worker): try: parser = http.RequestParser(self.cfg, client) try: + listener_name = listener.getsockname() if not self.cfg.keepalive: req = six.next(parser) - self.handle_request(listener, req, client, addr) + self.handle_request(listener_name, req, client, addr) else: # keepalive loop while True: @@ -78,14 +79,14 @@ class AsyncWorker(base.Worker): finally: util.close(client) - def handle_request(self, listener, req, sock, addr): + def handle_request(self, listener_name, req, sock, addr): request_start = datetime.now() environ = {} resp = None try: self.cfg.pre_request(self, req) resp, environ = wsgi.create(req, sock, addr, - listener.getsockname(), self.cfg) + listener_name, self.cfg) environ["wsgi.multithread"] = True self.nr += 1 if self.alive and self.nr >= self.max_requests: From d9b7e32d340e3ee00736e7925318af7d0b8d5910 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 11:19:32 +0200 Subject: [PATCH 26/67] try to log what happened in the worker after a timeout. fix #518 --- examples/example_config.py | 3 +++ examples/timeout.py | 21 +++++++++++++++++++++ gunicorn/arbiter.py | 8 ++++++-- gunicorn/config.py | 20 ++++++++++++++++++++ gunicorn/workers/base.py | 10 +++++++++- 5 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 examples/timeout.py diff --git a/examples/example_config.py b/examples/example_config.py index b3901a09..c939b851 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -217,3 +217,6 @@ def worker_int(worker): if line: code.append(" %s" % (line.strip())) worker.log.debug("\n".join(code)) + +def worker_abort(worker): + worker.log.info("worker received SIGABRT signal") diff --git a/examples/timeout.py b/examples/timeout.py new file mode 100644 index 00000000..cf268c35 --- /dev/null +++ b/examples/timeout.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +import sys +import time + + +def app(environ, start_response): + """Application which cooperatively pauses 10 seconds before responding""" + data = b'Hello, World!\n' + status = '200 OK' + response_headers = [ + ('Content-type','text/plain'), + ('Content-Length', str(len(data))) ] + sys.stdout.write('request will timeout') + sys.stdout.flush() + time.sleep(35) + start_response(status, response_headers) + return iter([data]) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 3b4b05ab..77002653 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -433,8 +433,12 @@ class Arbiter(object): except ValueError: continue - self.log.critical("WORKER TIMEOUT (pid:%s)", pid) - self.kill_worker(pid, signal.SIGKILL) + if not worker.aborted: + self.log.critical("WORKER TIMEOUT (pid:%s)", pid) + worker.aborted = True + self.kill_worker(pid, signal.SIGABRT) + else: + self.kill_worker(pid, signal.SIGKILL) def reap_workers(self): """\ diff --git a/gunicorn/config.py b/gunicorn/config.py index 895049db..17fef369 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1336,6 +1336,26 @@ class WorkerInt(Setting): """ +class WorkerAbort(Setting): + name = "worker_abort" + section = "Server Hooks" + validator = validate_callable(1) + type = six.callable + + def worker_abort(worker): + pass + + default = staticmethod(worker_abort) + desc = """\ + Called when a worker received the SIGABRT signal. + + This call generally happen on timeout. + + The callable needs to accept one instance variable for the initialized + Worker. + """ + + class PreExec(Setting): name = "pre_exec" section = "Server Hooks" diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 5be83efd..99c68c8f 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -23,7 +23,7 @@ from gunicorn.six import MAXSIZE class Worker(object): SIGNALS = [getattr(signal, "SIG%s" % x) \ - for x in "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] + for x in "ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] PIPE = [] @@ -40,6 +40,7 @@ class Worker(object): self.timeout = timeout self.cfg = cfg self.booted = False + self.aborted = False self.nr = 0 self.max_requests = cfg.max_requests or MAXSIZE @@ -127,6 +128,8 @@ class Worker(object): signal.signal(signal.SIGINT, self.handle_quit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) + signal.signal(signal.SIGABRT, self.handle_abort) + # Don't let SIGQUIT and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 @@ -145,6 +148,11 @@ class Worker(object): self.alive = False sys.exit(0) + def handle_abort(self, sig, frame): + self.alive = False + self.cfg.worker_abort(self) + sys.exit(1) + def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case From abac771c44f7cb889ff03e0dc0b08037202ea897 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 20:36:48 +0200 Subject: [PATCH 27/67] fix race keepalived condition by popping/appending from left --- gunicorn/workers/gthread.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 5f660883..0f486642 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -120,11 +120,13 @@ class ThreadWorker(base.Worker): now = time.time() while True: try: - delta = self._keep[0].timeout - now + conn = self._keep.popleft() except IndexError: break + delta = conn.timeout - now if delta > 0: + self._keep.appendleft(conn) break else: # remove the connection from the queue From ff6169cc20fe0e97f9db7f670d2b56ce8c5573be Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 20:44:50 +0200 Subject: [PATCH 28/67] gthreads: only check requirements for python < 3.4 --- gunicorn/workers/gthread.py | 18 ++++++++++++++++-- setup.py | 3 --- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 0f486642..aedfdd93 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -11,7 +11,6 @@ # closed. from collections import deque -import concurrent.futures as futures from datetime import datetime import errno from functools import partial @@ -28,10 +27,25 @@ from .. import util from . import base from .. import six + +try: + import concurrent.futures as futures +except ImportError: + raise RuntimeError(""" + You need 'concurrent' installed to use this worker with this python + version. + """) + try: from asyncio import selectors except ImportError: - from trollius import selectors + try: + from trollius import selectors + except ImportError: + raise RuntimeError(""" + You need 'trollius' installed to use this worker with this python + version. + """) class TConn(): diff --git a/setup.py b/setup.py index 36e33853..6f5b5b69 100644 --- a/setup.py +++ b/setup.py @@ -63,9 +63,6 @@ class PyTest(Command): REQUIREMENTS = [] -if sys.version_info[0] == 2: - REQUIREMENTS.append('futures') - setup( name = 'gunicorn', From 64fd52354c8c4f1cdb4a9a9fb66e22cec6a8be95 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 21:59:50 +0200 Subject: [PATCH 29/67] fix comment in timeout example. --- examples/timeout.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/timeout.py b/examples/timeout.py index cf268c35..ab9bf4dd 100644 --- a/examples/timeout.py +++ b/examples/timeout.py @@ -8,7 +8,8 @@ import time def app(environ, start_response): - """Application which cooperatively pauses 10 seconds before responding""" + """Application which pauses 35 seconds before responding. the worker + will timeout in default case.""" data = b'Hello, World!\n' status = '200 OK' response_headers = [ From 8436389229ea881e314687c789ccc979df1fd0ff Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 1 Jun 2014 22:30:46 +0200 Subject: [PATCH 30/67] define an object class instance. --- gunicorn/workers/gthread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index aedfdd93..122d923e 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -48,7 +48,7 @@ except ImportError: """) -class TConn(): +class TConn(object): def __init__(self, cfg, listener, sock, addr): self.cfg = cfg From dcb9464bbed505e9ab904d03410ea774f9f63018 Mon Sep 17 00:00:00 2001 From: Philipp Saveliev Date: Tue, 3 Jun 2014 03:05:27 +0400 Subject: [PATCH 31/67] fix HTTP-violating excess whitespace in write_error output --- gunicorn/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gunicorn/util.py b/gunicorn/util.py index 9f6b79c9..9ef79a3a 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -337,8 +337,7 @@ def write_error(sock, status_int, reason, mesg): Content-Type: text/html\r Content-Length: %d\r \r - %s - """) % (str(status_int), reason, len(html), html) + %s""") % (str(status_int), reason, len(html), html) write_nonblock(sock, http.encode('latin1')) From bc41bad2d6218b9695af82b7d5134f811ca63a78 Mon Sep 17 00:00:00 2001 From: Matt Billenstein Date: Wed, 4 Jun 2014 18:18:28 -0700 Subject: [PATCH 32/67] workers/async.py - handle_request takes listener_name instead of listener AFAICT - this should have been updated in 62f6fb2d --- gunicorn/workers/async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index 9351b999..d3cce5fc 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -44,7 +44,7 @@ class AsyncWorker(base.Worker): req = six.next(parser) if not req: break - self.handle_request(listener, req, client, addr) + self.handle_request(listener_name, req, client, addr) except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) except StopIteration as e: From a7eed00c2febb4a0dc43ca08764a662281bc53d1 Mon Sep 17 00:00:00 2001 From: Nikolay Kim Date: Fri, 6 Jun 2014 20:25:57 -0700 Subject: [PATCH 33/67] aiohttp worker --- gunicorn/workers/__init__.py | 1 + gunicorn/workers/gaiohttp.py | 131 +++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 gunicorn/workers/gaiohttp.py diff --git a/gunicorn/workers/__init__.py b/gunicorn/workers/__init__.py index 6c769f83..43ddc9db 100644 --- a/gunicorn/workers/__init__.py +++ b/gunicorn/workers/__init__.py @@ -7,6 +7,7 @@ SUPPORTED_WORKERS={ "sync": "gunicorn.workers.sync.SyncWorker", "eventlet": "gunicorn.workers.geventlet.EventletWorker", + "gaiohttp": "gunicorn.workers.gaiohttp.AiohttpWorker", "gevent": "gunicorn.workers.ggevent.GeventWorker", "gevent_wsgi": "gunicorn.workers.ggevent.GeventPyWSGIWorker", "gevent_pywsgi": "gunicorn.workers.ggevent.GeventPyWSGIWorker", diff --git a/gunicorn/workers/gaiohttp.py b/gunicorn/workers/gaiohttp.py new file mode 100644 index 00000000..fb109591 --- /dev/null +++ b/gunicorn/workers/gaiohttp.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. +__all__ = ['AiohttpWorker'] + +import asyncio +import functools +import os +import gunicorn.workers.base as base + +try: + from aiohttp.wsgi import WSGIServerHttpProtocol +except ImportError: + raise RuntimeError("You need aiohttp installed to use this worker.") + + +class AiohttpWorker(base.Worker): + + def __init__(self, *args, **kw): # pragma: no cover + super().__init__(*args, **kw) + + self.servers = [] + self.connections = {} + + def init_process(self): + # create new event_loop after fork + asyncio.get_event_loop().close() + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + super().init_process() + + def run(self): + self._runner = asyncio.async(self._run(), loop=self.loop) + + try: + self.loop.run_until_complete(self._runner) + finally: + self.loop.close() + + def wrap_protocol(self, proto): + proto.connection_made = _wrp( + proto, proto.connection_made, self.connections) + proto.connection_lost = _wrp( + proto, proto.connection_lost, self.connections, False) + return proto + + def factory(self, wsgi, host, port): + proto = WSGIServerHttpProtocol( + wsgi, loop=self.loop, + log=self.log, + debug=self.cfg.debug, + keep_alive=self.cfg.keepalive, + access_log=self.log.access_log, + access_log_format=self.cfg.access_log_format) + return self.wrap_protocol(proto) + + def get_factory(self, sock, host, port): + return functools.partial(self.factory, self.wsgi, host, port) + + @asyncio.coroutine + def close(self): + try: + if hasattr(self.wsgi, 'close'): + yield from self.wsgi.close() + except: + self.log.exception('Process shutdown exception') + + @asyncio.coroutine + def _run(self): + for sock in self.sockets: + factory = self.get_factory(sock.sock, *sock.cfg_addr) + self.servers.append( + (yield from self.loop.create_server(factory, sock=sock.sock))) + + # If our parent changed then we shut down. + pid = os.getpid() + try: + while self.alive or self.connections: + self.notify() + + if (self.alive and + pid == os.getpid() and self.ppid != os.getppid()): + self.log.info("Parent changed, shutting down: %s", self) + self.alive = False + + # stop accepting requests + if not self.alive: + if self.servers: + self.log.info( + "Stopping server: %s, connections: %s", + pid, len(self.connections)) + for server in self.servers: + server.close() + self.servers.clear() + + # prepare connections for closing + for conn in self.connections.values(): + if hasattr(conn, 'closing'): + conn.closing() + + yield from asyncio.sleep(1.0, loop=self.loop) + except KeyboardInterrupt: + pass + + if self.servers: + for server in self.servers: + server.close() + + yield from self.close() + + +class _wrp: + + def __init__(self, proto, meth, tracking, add=True): + self._proto = proto + self._id = id(proto) + self._meth = meth + self._tracking = tracking + self._add = add + + def __call__(self, *args): + if self._add: + self._tracking[self._id] = self._proto + elif self._id in self._tracking: + del self._tracking[self._id] + + conn = self._meth(*args) + return conn From 3665e5bb5c31d09bd91c5bd9919f332efdfe18db Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 9 Jun 2014 18:05:38 +0300 Subject: [PATCH 34/67] Register gaiohttp worker for Python 3.3+ only. --- gunicorn/workers/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gunicorn/workers/__init__.py b/gunicorn/workers/__init__.py index 43ddc9db..d580387b 100644 --- a/gunicorn/workers/__init__.py +++ b/gunicorn/workers/__init__.py @@ -3,13 +3,18 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import sys + # supported gunicorn workers. SUPPORTED_WORKERS={ "sync": "gunicorn.workers.sync.SyncWorker", "eventlet": "gunicorn.workers.geventlet.EventletWorker", - "gaiohttp": "gunicorn.workers.gaiohttp.AiohttpWorker", "gevent": "gunicorn.workers.ggevent.GeventWorker", "gevent_wsgi": "gunicorn.workers.ggevent.GeventPyWSGIWorker", "gevent_pywsgi": "gunicorn.workers.ggevent.GeventPyWSGIWorker", "tornado": "gunicorn.workers.gtornado.TornadoWorker"} + +if sys.version_info >= (3, 3): + # gaiohttp worker can be used with Python 3.3+ only. + SUPPORTED_WORKERS["gaiohttp"] = "gunicorn.workers.gaiohttp.AiohttpWorker" From e1d97f1beaef3c31ab3286a3afd21d4563d0a42f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 9 Jun 2014 18:36:46 +0300 Subject: [PATCH 35/67] Add tests for gaiohttp worker --- tests/test_009-gaiohttp.py | 181 +++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 tests/test_009-gaiohttp.py diff --git a/tests/test_009-gaiohttp.py b/tests/test_009-gaiohttp.py new file mode 100644 index 00000000..d3a37734 --- /dev/null +++ b/tests/test_009-gaiohttp.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +import t +import os +import sys +import unittest +if sys.version_info[0] < 3: + raise unittest.SkipTest("gaiohttp requires Python 3.3+") + +try: + import aiohttp +except ImportError: + raise unittest.SkipTest("gaiohttp requires aiohttp") + + +from aiohttp.wsgi import WSGIServerHttpProtocol + +import asyncio +from gunicorn.workers import gaiohttp +from gunicorn.config import Config +from unittest import mock + + +class WorkerTests(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + self.worker = gaiohttp.AiohttpWorker('age', + 'ppid', + 'sockets', + 'app', + 'timeout', + Config(), + 'log') + + def tearDown(self): + self.loop.close() + + @mock.patch('gunicorn.workers.gaiohttp.asyncio') + def test_init_process(self, m_asyncio): + try: + self.worker.init_process() + except TypeError: + # to mask incomplete initialization of AiohttWorker instance: + # we pass invalid values for ctor args + pass + + self.assertTrue(m_asyncio.get_event_loop.return_value.close.called) + self.assertTrue(m_asyncio.new_event_loop.called) + self.assertTrue(m_asyncio.set_event_loop.called) + + @mock.patch('gunicorn.workers.gaiohttp.asyncio') + def test_run(self, m_asyncio): + self.worker.loop = mock.Mock() + self.worker.run() + + self.assertTrue(m_asyncio.async.called) + self.assertTrue(self.worker.loop.run_until_complete.called) + self.assertTrue(self.worker.loop.close.called) + + def test_factory(self): + self.worker.wsgi = mock.Mock() + self.worker.loop = mock.Mock() + self.worker.log = mock.Mock() + self.worker.cfg = mock.Mock() + + f = self.worker.factory( + self.worker.wsgi, 'localhost', 8080) + self.assertIsInstance(f, WSGIServerHttpProtocol) + + @mock.patch('gunicorn.workers.gaiohttp.asyncio') + def test__run(self, m_asyncio): + self.worker.ppid = 1 + self.worker.alive = True + self.worker.servers = [] + sock = mock.Mock() + sock.cfg_addr = ('localhost', 8080) + self.worker.sockets = [sock] + self.worker.wsgi = mock.Mock() + self.worker.log = mock.Mock() + self.worker.notify = mock.Mock() + loop = self.worker.loop = mock.Mock() + loop.create_server.return_value = asyncio.Future(loop=self.loop) + loop.create_server.return_value.set_result(sock) + + self.loop.run_until_complete(self.worker._run()) + + self.assertTrue(self.worker.log.info.called) + self.assertTrue(self.worker.notify.called) + + def test__run_connections(self): + conn = mock.Mock() + self.worker.ppid = 1 + self.worker.alive = False + self.worker.servers = [mock.Mock()] + self.worker.connections = {1: conn} + self.worker.sockets = [] + self.worker.wsgi = mock.Mock() + self.worker.log = mock.Mock() + self.worker.loop = self.loop + self.worker.loop.create_server = mock.Mock() + self.worker.notify = mock.Mock() + + def _close_conns(): + self.worker.connections = {} + + self.loop.call_later(0.1, _close_conns) + self.loop.run_until_complete(self.worker._run()) + + self.assertTrue(self.worker.log.info.called) + self.assertTrue(self.worker.notify.called) + self.assertFalse(self.worker.servers) + self.assertTrue(conn.closing.called) + + @mock.patch('gunicorn.workers.gaiohttp.os') + @mock.patch('gunicorn.workers.gaiohttp.asyncio.sleep') + def test__run_exc(self, m_sleep, m_os): + m_os.getpid.return_value = 1 + m_os.getppid.return_value = 1 + + self.worker.servers = [mock.Mock()] + self.worker.ppid = 1 + self.worker.alive = True + self.worker.sockets = [] + self.worker.log = mock.Mock() + self.worker.loop = mock.Mock() + self.worker.notify = mock.Mock() + + slp = asyncio.Future(loop=self.loop) + slp.set_exception(KeyboardInterrupt) + m_sleep.return_value = slp + + self.loop.run_until_complete(self.worker._run()) + self.assertTrue(m_sleep.called) + self.assertTrue(self.worker.servers[0].close.called) + + def test_close_wsgi_app(self): + self.worker.ppid = 1 + self.worker.alive = False + self.worker.servers = [mock.Mock()] + self.worker.connections = {} + self.worker.sockets = [] + self.worker.log = mock.Mock() + self.worker.loop = self.loop + self.worker.loop.create_server = mock.Mock() + self.worker.notify = mock.Mock() + + self.worker.wsgi = mock.Mock() + self.worker.wsgi.close.return_value = asyncio.Future(loop=self.loop) + self.worker.wsgi.close.return_value.set_result(1) + + self.loop.run_until_complete(self.worker._run()) + self.assertTrue(self.worker.wsgi.close.called) + + self.worker.wsgi = mock.Mock() + self.worker.wsgi.close.return_value = asyncio.Future(loop=self.loop) + self.worker.wsgi.close.return_value.set_exception(ValueError()) + + self.loop.run_until_complete(self.worker._run()) + self.assertTrue(self.worker.wsgi.close.called) + + def test_wrp(self): + conn = object() + tracking = {} + meth = mock.Mock() + wrp = gaiohttp._wrp(conn, meth, tracking) + wrp() + + self.assertIn(id(conn), tracking) + self.assertTrue(meth.called) + + meth = mock.Mock() + wrp = gaiohttp._wrp(conn, meth, tracking, False) + wrp() + + self.assertNotIn(1, tracking) + self.assertTrue(meth.called) From e9518383da031d03a3c70f1a670bcf522cfa4d57 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 9 Jun 2014 19:31:52 +0300 Subject: [PATCH 36/67] Fix gaiohttp tests for Python 2.5 --- tests/test_009-gaiohttp.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_009-gaiohttp.py b/tests/test_009-gaiohttp.py index d3a37734..843c6cf0 100644 --- a/tests/test_009-gaiohttp.py +++ b/tests/test_009-gaiohttp.py @@ -3,17 +3,9 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import t -import os -import sys import unittest -if sys.version_info[0] < 3: - raise unittest.SkipTest("gaiohttp requires Python 3.3+") - -try: - import aiohttp -except ImportError: - raise unittest.SkipTest("gaiohttp requires aiohttp") +import pytest +aiohttp = pytest.importorskip("aiohttp") from aiohttp.wsgi import WSGIServerHttpProtocol From 4ede68c6e40b6000da5f390a5c6dce78465784f4 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Mon, 9 Jun 2014 19:41:58 +0300 Subject: [PATCH 37/67] Add gaiohttp tests for travis build --- .travis.yml | 1 + check_py3k.py | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 check_py3k.py diff --git a/.travis.yml b/.travis.yml index 2dea0409..a9fde9aa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ python: install: - "pip install -r requirements_dev.txt" - "python setup.py install" + - python check_py3k.py && pip install aiohttp || true script: py.test -x tests/ branches: only: diff --git a/check_py3k.py b/check_py3k.py new file mode 100644 index 00000000..17a9af99 --- /dev/null +++ b/check_py3k.py @@ -0,0 +1,6 @@ +import sys + +if sys.version_info[0] < 3: + sys.exit(1) +else: + sys.exit(0) From 3c841b2f58599da6eca32f85386d33c5b8e733c7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 14:38:55 +0200 Subject: [PATCH 38/67] add changelog for 19.0 --- docs/source/2014-news.rst | 143 ++++++++++++++++++++++++++++++++++++- docs/source/news.rst | 144 +++++++++++++++++++++++++++++++++++++- 2 files changed, 283 insertions(+), 4 deletions(-) diff --git a/docs/source/2014-news.rst b/docs/source/2014-news.rst index 6b3843af..66f4c149 100644 --- a/docs/source/2014-news.rst +++ b/docs/source/2014-news.rst @@ -1,8 +1,147 @@ Changelog - 2014 ================ -18.2 / unreleased +19.0 / 2014-06-12 ----------------- -- new: logging to syslog now includes the access log. +Gunicorn 19.0 is a major release with new features and fixes. This +version improve a lot the usage of Gunicorn with python 3 by adding two +new workers to it: `gthread` a fully threaded async worker using futures +and `gaiohttp` a worker using asyncio. + +Breaking Changes +~~~~~~~~~~~~~~~~ + +Switch QUIT and TERM signals +++++++++++++++++++++++++++++ + +With this change, when gunicorn receives a QUIT all the workers are +killed immediately and exit and TERM is used for the graceful shutdown. + +Note: the old behaviour was based on the NGINX but the new one is more +correct according the following doc: + +https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html + +also it is complying with the way the signals are sent by heroku: + +https://devcenter.heroku.com/articles/python-faq#what-constraints-exist-when-developing-applications-on-heroku + +Deprecations ++++++++++++++ + +`run_gunicorn`, `gunicorn_django` and `gunicorn_paster` are now +completely deprecated and will be removed in the next release. Use the +`gunicorn` command instead. + + +Changes: +~~~~~~~~ + +core +++++ + +- add aiohttp worker named `gaiohttp` using asyncio. Full async worker + on python 3. +- fix HTTP-violating excess whitespace in write_error output +- fix: try to log what happened in the worker after a timeout, add a + `worker_abort` hook on SIGABRT signal. +- fix: save listener socket name in workers so we can handle buffered + keep-alive requests after the listener has closed. +- add on_exit hook called just before exiting gunicorn. +- add support for python 3.4 +- fix: do not swallow unexpected errors when reaping +- fix: remove incompatible SSL option with python 2.6 +- add new async gthread worker and `--threads` options allows to set multiple + threads to listen on connection +- deprecate `gunicorn_django` and `gunicorn_paster` +- switch QUIT and TERM signal +- reap workers in SIGCHLD handler +- add universal wheel support +- use `email.utils.formatdate` in gunicorn.util.http_date +- deprecate the `--debug` option +- fix: log exceptions that occur after response start … +- allows loading of applications from `.pyc` files (#693) +- fix: issue #691, raw_env config file parsing +- use a dynamic timeout to wait for the optimal time. (Reduce power + usage) +- fix python3 support when notifying the arbiter +- add: honor $WEB_CONCURRENCY environment variable. Useful for heroku + setups. +- add: include tz offset in access log +- add: include access logs in the syslog handler. +- add --reload option for code reloading +- add the capability to load `gunicorn.base.Application` without the loading of the arguments of the command line. It allows you to [embed gunicorn in your own application](http://docs.gunicorn.org/en/latest/custom.html). +- improve: set wsgi.multithread to True for async workers +- fix logging: make sure to redirect wsgi.errors when needed +- add: syslog logging can now be done to a unix socket +- fix logging: don't try to redirect stdout/stderr to the logfile. +- fix logging: don't propagate log +- improve logging: file option can be overriden by the gunicorn options +`--error-logfile` and `--access-logfile` if they are given. +- fix: dont' override SERVER_* by the Host header +- fix: handle_error +- add more option to configure SSL +- fix: sendfile with SSL +- add: worker_int callback (to react on SIGTERM) +- fix: don't depend on entry point for internal classes, now absolute + modules path can be given. +- fix: Error messages are now encoded in latin1 +- fix: request line length check +- improvement: proxy_allow_ips: Allow proxy protocol if "*" specified +- fix: run worker's `setup` method before setting num_workers +- fix: FileWrapper inherit from `object` now +- fix: Error messages are now encoded in latin1 +- fix: don't spam the console on SIGWINCH. +- fix: logging -don't stringify T and D logging atoms (#621) +- add support for the latest django version +- deprecate `run_gunicorn` django option +- fix: sys imported twice + + +gevent worker ++++++++++++++ + +- fix: make sure to stop all listeners +- fix: monkey patching is now done in the worker +- fix: "global name 'hub' is not defined" +- fix: reinit `hub` on old versions of gevent +- support gevent 1.0 +- fix: add subprocess in monket patching +- fix: add support for multiple listener + +eventlet worker ++++++++++++++++ + +- fix: merge duplicate EventletWorker.init_process method (fixes #657) +- fix: missing errno import for eventlet sendfile patch +- fix: add support for multiple listener + +tornado worker +++++++++++++++ + +- add gracefull stop support + + +Breaking Changes +++++++++++++++++ + +- switch QUIT and TERM signals: + +With this change, when gunicorn receives a QUIT all the workers are +killed immediately and exit and TERM is used for the graceful shutdown. + +Note: the old behaviour was based on the NGINX but the new one is more +correct according the following doc: + +https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html + +also it is complying with the way the signals are sent by heroku: + +https://devcenter.heroku.com/articles/python-faq#what-constraints-exist-when-developing-applications-on-heroku + + +- `run_gunicorn`, `gunicorn_django` and `gunicorn_paster` are now + completely deprecated and will be removed in the next release. Use the + `gunicorn` command instead. diff --git a/docs/source/news.rst b/docs/source/news.rst index 2a11e8d1..a6df0321 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -1,10 +1,150 @@ Changelog ========= -18.2 / unreleased +19.0 / 2014-06-12 ----------------- -- new: logging to syslog now includes the access log. +Gunicorn 19.0 is a major release with new features and fixes. This +version improve a lot the usage of Gunicorn with python 3 by adding two +new workers to it: `gthread` a fully threaded async worker using futures +and `gaiohttp` a worker using asyncio. + + +Breaking Changes +~~~~~~~~~~~~~~~~ + +Switch QUIT and TERM signals +++++++++++++++++++++++++++++ + +With this change, when gunicorn receives a QUIT all the workers are +killed immediately and exit and TERM is used for the graceful shutdown. + +Note: the old behaviour was based on the NGINX but the new one is more +correct according the following doc: + +https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html + +also it is complying with the way the signals are sent by heroku: + +https://devcenter.heroku.com/articles/python-faq#what-constraints-exist-when-developing-applications-on-heroku + +Deprecations ++++++++++++++ + +`run_gunicorn`, `gunicorn_django` and `gunicorn_paster` are now +completely deprecated and will be removed in the next release. Use the +`gunicorn` command instead. + + +Changes: +~~~~~~~~ + +core +++++ + +- add aiohttp worker named `gaiohttp` using asyncio. Full async worker + on python 3. +- fix HTTP-violating excess whitespace in write_error output +- fix: try to log what happened in the worker after a timeout, add a + `worker_abort` hook on SIGABRT signal. +- fix: save listener socket name in workers so we can handle buffered + keep-alive requests after the listener has closed. +- add on_exit hook called just before exiting gunicorn. +- add support for python 3.4 +- fix: do not swallow unexpected errors when reaping +- fix: remove incompatible SSL option with python 2.6 +- add new async gthread worker and `--threads` options allows to set multiple + threads to listen on connection +- deprecate `gunicorn_django` and `gunicorn_paster` +- switch QUIT and TERM signal +- reap workers in SIGCHLD handler +- add universal wheel support +- use `email.utils.formatdate` in gunicorn.util.http_date +- deprecate the `--debug` option +- fix: log exceptions that occur after response start … +- allows loading of applications from `.pyc` files (#693) +- fix: issue #691, raw_env config file parsing +- use a dynamic timeout to wait for the optimal time. (Reduce power + usage) +- fix python3 support when notifying the arbiter +- add: honor $WEB_CONCURRENCY environment variable. Useful for heroku + setups. +- add: include tz offset in access log +- add: include access logs in the syslog handler. +- add --reload option for code reloading +- add the capability to load `gunicorn.base.Application` without the loading of the arguments of the command line. It allows you to [embed gunicorn in your own application](http://docs.gunicorn.org/en/latest/custom.html). +- improve: set wsgi.multithread to True for async workers +- fix logging: make sure to redirect wsgi.errors when needed +- add: syslog logging can now be done to a unix socket +- fix logging: don't try to redirect stdout/stderr to the logfile. +- fix logging: don't propagate log +- improve logging: file option can be overriden by the gunicorn options +`--error-logfile` and `--access-logfile` if they are given. +- fix: dont' override SERVER_* by the Host header +- fix: handle_error +- add more option to configure SSL +- fix: sendfile with SSL +- add: worker_int callback (to react on SIGTERM) +- fix: don't depend on entry point for internal classes, now absolute + modules path can be given. +- fix: Error messages are now encoded in latin1 +- fix: request line length check +- improvement: proxy_allow_ips: Allow proxy protocol if "*" specified +- fix: run worker's `setup` method before setting num_workers +- fix: FileWrapper inherit from `object` now +- fix: Error messages are now encoded in latin1 +- fix: don't spam the console on SIGWINCH. +- fix: logging -don't stringify T and D logging atoms (#621) +- add support for the latest django version +- deprecate `run_gunicorn` django option +- fix: sys imported twice + + +gevent worker ++++++++++++++ + +- fix: make sure to stop all listeners +- fix: monkey patching is now done in the worker +- fix: "global name 'hub' is not defined" +- fix: reinit `hub` on old versions of gevent +- support gevent 1.0 +- fix: add subprocess in monket patching +- fix: add support for multiple listener + +eventlet worker ++++++++++++++++ + +- fix: merge duplicate EventletWorker.init_process method (fixes #657) +- fix: missing errno import for eventlet sendfile patch +- fix: add support for multiple listener + +tornado worker +++++++++++++++ + +- add gracefull stop support + + +Breaking Changes +++++++++++++++++ + +- switch QUIT and TERM signals: + +With this change, when gunicorn receives a QUIT all the workers are +killed immediately and exit and TERM is used for the graceful shutdown. + +Note: the old behaviour was based on the NGINX but the new one is more +correct according the following doc: + +https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html + +also it is complying with the way the signals are sent by heroku: + +https://devcenter.heroku.com/articles/python-faq#what-constraints-exist-when-developing-applications-on-heroku + + +- `run_gunicorn`, `gunicorn_django` and `gunicorn_paster` are now + completely deprecated and will be removed in the next release. Use the + `gunicorn` command instead. 18.0 / 2013-08-26 From c996deaf23364a82b0aeaab2c43e6c7573c67a94 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 14:43:06 +0200 Subject: [PATCH 39/67] fix doc --- docs/source/2014-news.rst | 26 +------------------------- docs/source/news.rst | 25 +------------------------ docs/source/settings.rst | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 50 deletions(-) diff --git a/docs/source/2014-news.rst b/docs/source/2014-news.rst index 66f4c149..08307105 100644 --- a/docs/source/2014-news.rst +++ b/docs/source/2014-news.rst @@ -78,8 +78,7 @@ core - add: syslog logging can now be done to a unix socket - fix logging: don't try to redirect stdout/stderr to the logfile. - fix logging: don't propagate log -- improve logging: file option can be overriden by the gunicorn options -`--error-logfile` and `--access-logfile` if they are given. +- improve logging: file option can be overriden by the gunicorn options `--error-logfile` and `--access-logfile` if they are given. - fix: dont' override SERVER_* by the Host header - fix: handle_error - add more option to configure SSL @@ -122,26 +121,3 @@ tornado worker ++++++++++++++ - add gracefull stop support - - -Breaking Changes -++++++++++++++++ - -- switch QUIT and TERM signals: - -With this change, when gunicorn receives a QUIT all the workers are -killed immediately and exit and TERM is used for the graceful shutdown. - -Note: the old behaviour was based on the NGINX but the new one is more -correct according the following doc: - -https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html - -also it is complying with the way the signals are sent by heroku: - -https://devcenter.heroku.com/articles/python-faq#what-constraints-exist-when-developing-applications-on-heroku - - -- `run_gunicorn`, `gunicorn_django` and `gunicorn_paster` are now - completely deprecated and will be removed in the next release. Use the - `gunicorn` command instead. diff --git a/docs/source/news.rst b/docs/source/news.rst index a6df0321..1d571619 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -123,30 +123,6 @@ tornado worker - add gracefull stop support - -Breaking Changes -++++++++++++++++ - -- switch QUIT and TERM signals: - -With this change, when gunicorn receives a QUIT all the workers are -killed immediately and exit and TERM is used for the graceful shutdown. - -Note: the old behaviour was based on the NGINX but the new one is more -correct according the following doc: - -https://www.gnu.org/software/libc/manual/html_node/Termination-Signals.html - -also it is complying with the way the signals are sent by heroku: - -https://devcenter.heroku.com/articles/python-faq#what-constraints-exist-when-developing-applications-on-heroku - - -- `run_gunicorn`, `gunicorn_django` and `gunicorn_paster` are now - completely deprecated and will be removed in the next release. Use the - `gunicorn` command instead. - - 18.0 / 2013-08-26 ----------------- @@ -366,6 +342,7 @@ History .. toctree:: :titlesonly: + 2014-news 2013-news 2012-news 2011-news diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 79ca65e0..8c4ee5f5 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -104,6 +104,22 @@ alternative syntax will load the gevent class: ``gunicorn.workers.ggevent.GeventWorker``. Alternatively the syntax can also load the gevent class with ``egg:gunicorn#gevent`` +threads +~~~~~~~ + +* ``--threads INT`` +* ``1`` + +The number of worker threads for handling requests. + +Run each worker with the specified number of threads. + +A positive integer generally in the 2-4 x $(NUM_CORES) range. You'll +want to vary this a bit to find the best for your particular +application's work load. + +If it is not defined, the default is 1. + worker_connections ~~~~~~~~~~~~~~~~~~ @@ -380,7 +396,7 @@ temporary directory. secure_scheme_headers ~~~~~~~~~~~~~~~~~~~~~ -* ``{'X-FORWARDED-SSL': 'on', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-PROTOCOL': 'ssl'}`` +* ``{'X-FORWARDED-SSL': 'on', 'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https'}`` A dictionary containing headers and values that the front-end proxy uses to indicate HTTPS requests. These tell gunicorn to set @@ -716,6 +732,21 @@ Called just after a worker exited on SIGINT or SIGTERM. The callable needs to accept one instance variable for the initialized Worker. +worker_abort +~~~~~~~~~~~~ + +* :: + + def worker_abort(worker): + pass + +Called when a worker received the SIGABRT signal. + +This call generally happen on timeout. + +The callable needs to accept one instance variable for the initialized +Worker. + pre_exec ~~~~~~~~ From 0fee3e5db23239d8678eb75b4c4963ff1f6cb993 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 14:50:22 +0200 Subject: [PATCH 40/67] update doc for the new workers --- docs/source/design.rst | 13 +++++++++++++ docs/source/settings.rst | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index fd895d2e..fea69b0f 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -49,6 +49,19 @@ There's also a Tornado worker class. It can be used to write applications using the Tornado framework. Although the Tornado workers are capable of serving a WSGI application, this is not a recommended configuration. +AsyncIO Workers +--------------- + +These workers are compatible with python3. You have two kind of workers. + +The worker `gthread` is a threaded worker. It accepts connections in the +main loop, accepted connections are are added to the thread pool as a +connection job. On keepalive connections are put back in the loop +waiting for an event. If no event happen after the keep alive timeout, +the connection is closed. + +The worker `gaiohttp` is a full asyncio worker using [aiohttp](https://github.com/KeepSafe/aiohttp). + Choosing a Worker Type ====================== diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 8c4ee5f5..58e179fb 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -396,7 +396,7 @@ temporary directory. secure_scheme_headers ~~~~~~~~~~~~~~~~~~~~~ -* ``{'X-FORWARDED-SSL': 'on', 'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https'}`` +* ``{'X-FORWARDED-SSL': 'on', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-PROTOCOL': 'ssl'}`` A dictionary containing headers and values that the front-end proxy uses to indicate HTTPS requests. These tell gunicorn to set From d2e4d071a3d15b1c86cca4b4c9089ea82fd61228 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 14:53:44 +0200 Subject: [PATCH 41/67] fix aiohttp link --- docs/source/design.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index fea69b0f..bac34da9 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -60,7 +60,7 @@ connection job. On keepalive connections are put back in the loop waiting for an event. If no event happen after the keep alive timeout, the connection is closed. -The worker `gaiohttp` is a full asyncio worker using [aiohttp](https://github.com/KeepSafe/aiohttp). +The worker `gaiohttp` is a full asyncio worker using aiohttp_ Choosing a Worker Type ====================== @@ -111,3 +111,4 @@ of the entire system. .. _Eventlet: http://eventlet.net .. _Gevent: http://gevent.org .. _Slowloris: http://ha.ckers.org/slowloris/ +.. _aiohttp: https://github.com/KeepSafe/aiohttp From 49a868f2a7a8960c12a99efd435b37f714afbf64 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 14:55:57 +0200 Subject: [PATCH 42/67] dot because we need one. --- docs/source/design.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index bac34da9..dff78d30 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -60,7 +60,7 @@ connection job. On keepalive connections are put back in the loop waiting for an event. If no event happen after the keep alive timeout, the connection is closed. -The worker `gaiohttp` is a full asyncio worker using aiohttp_ +The worker `gaiohttp` is a full asyncio worker using aiohttp_. Choosing a Worker Type ====================== From c21c932343b33f79363a849dd338561d2d337eaa Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 15:09:37 +0200 Subject: [PATCH 43/67] asyncio workers doc link --- docs/source/news.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/news.rst b/docs/source/news.rst index 1d571619..caa5b3c1 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -5,8 +5,8 @@ Changelog ----------------- Gunicorn 19.0 is a major release with new features and fixes. This -version improve a lot the usage of Gunicorn with python 3 by adding two -new workers to it: `gthread` a fully threaded async worker using futures +version improve a lot the usage of Gunicorn with python 3 by adding `two +new workers `_ to it: `gthread` a fully threaded async worker using futures and `gaiohttp` a worker using asyncio. @@ -51,7 +51,7 @@ core keep-alive requests after the listener has closed. - add on_exit hook called just before exiting gunicorn. - add support for python 3.4 -- fix: do not swallow unexpected errors when reaping +- fix: do not swallow unexpected errors when reaping - fix: remove incompatible SSL option with python 2.6 - add new async gthread worker and `--threads` options allows to set multiple threads to listen on connection @@ -61,7 +61,7 @@ core - add universal wheel support - use `email.utils.formatdate` in gunicorn.util.http_date - deprecate the `--debug` option -- fix: log exceptions that occur after response start … +- fix: log exceptions that occur after response start … - allows loading of applications from `.pyc` files (#693) - fix: issue #691, raw_env config file parsing - use a dynamic timeout to wait for the optimal time. (Reduce power @@ -80,7 +80,7 @@ core - fix logging: don't propagate log - improve logging: file option can be overriden by the gunicorn options `--error-logfile` and `--access-logfile` if they are given. -- fix: dont' override SERVER_* by the Host header +- fix: dont' override SERVER_* by the Host header - fix: handle_error - add more option to configure SSL - fix: sendfile with SSL @@ -90,7 +90,7 @@ core - fix: Error messages are now encoded in latin1 - fix: request line length check - improvement: proxy_allow_ips: Allow proxy protocol if "*" specified -- fix: run worker's `setup` method before setting num_workers +- fix: run worker's `setup` method before setting num_workers - fix: FileWrapper inherit from `object` now - fix: Error messages are now encoded in latin1 - fix: don't spam the console on SIGWINCH. @@ -104,7 +104,7 @@ gevent worker +++++++++++++ - fix: make sure to stop all listeners -- fix: monkey patching is now done in the worker +- fix: monkey patching is now done in the worker - fix: "global name 'hub' is not defined" - fix: reinit `hub` on old versions of gevent - support gevent 1.0 From c904ae9ff31a0b9f579611d0e1b768e4dcce47fd Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 15:32:10 +0200 Subject: [PATCH 44/67] bump the website to 19.0 --- MAINTAINERS | 3 +++ docs/site/index.html | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 2dce9ee5..f16bb982 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3,3 +3,6 @@ Paul J. Davis Randall Leeds Konstantin Kapustin Kenneth Reitz +Nikolay Kim +Andrew Svetlov +Stéphane Wirtel diff --git a/docs/site/index.html b/docs/site/index.html index f3f1bfce..dcd4a376 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -16,7 +16,7 @@
Latest version: 18.0 + href="http://docs.gunicorn.org/en/19.0/news.html#id1">19.0
From 2945f7995948ea16789dd44a4ca9485db362e48b Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 15:35:29 +0200 Subject: [PATCH 45/67] update the readme. --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e0a06b5b..52ea6e6e 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Or from Pypi:: You may also want to install Eventlet_ or Gevent_ if you expect that your application code may need to pause for extended periods of time during -request processing. Check out the FAQ_ for more information on when you'll +request processing. If you're on Python 3 you may also consider one othe asyncio workers. Check out the FAQ_ for more information on when you'll want to consider one of the alternate worker types. To install eventlet:: @@ -63,7 +63,7 @@ Commonly Used Arguments to run. You'll definitely want to read the `production page`_ for the implications of this parameter. You can set this to ``egg:gunicorn#$(NAME)`` where ``$(NAME)`` is one of ``sync``, ``eventlet``, ``gevent``, or - ``tornado``. ``sync`` is the default. + ``tornado``, ``gthread``, ``gaiohttp`. ``sync`` is the default. * ``-n APP_NAME, --name=APP_NAME`` - If setproctitle_ is installed you can adjust the name of Gunicorn process as they appear in the process system table (which affects tools like ``ps`` and ``top``). From b1d5d9cbe5735c38d6ce440cb701bb08f1a9d138 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 15:45:04 +0200 Subject: [PATCH 46/67] link to the asyncio doc --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 52ea6e6e..1df658e1 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ Or from Pypi:: You may also want to install Eventlet_ or Gevent_ if you expect that your application code may need to pause for extended periods of time during -request processing. If you're on Python 3 you may also consider one othe asyncio workers. Check out the FAQ_ for more information on when you'll +request processing. If you're on Python 3 you may also consider one othe Asyncio_ workers. Check out the FAQ_ for more information on when you'll want to consider one of the alternate worker types. To install eventlet:: @@ -153,6 +153,7 @@ details. .. _freenode: http://freenode.net .. _Eventlet: http://eventlet.net .. _Gevent: http://gevent.org +.. _Asyncio: https://docs.python.org/3/library/asyncio.html .. _FAQ: http://docs.gunicorn.org/en/latest/faq.html .. _libev: http://software.schmorp.de/pkg/libev.html .. _libevent: http://monkey.org/~provos/libevent From b00e875cc475b861b50fc69b38ad8427ece62db0 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 12 Jun 2014 16:45:44 +0200 Subject: [PATCH 47/67] =?UTF-8?q?add=20Berker=20Peksa=C3=84=C2=9F=20to=20t?= =?UTF-8?q?he=20maintainers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index f16bb982..1799675e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6,3 +6,4 @@ Kenneth Reitz Nikolay Kim Andrew Svetlov Stéphane Wirtel +Berker Peksağ From ab3428ec38e04a68c772e4eac82f31c1fbfaf444 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 12 Jun 2014 17:55:09 +0300 Subject: [PATCH 48/67] Use TRAVIS_PYTHON_VERSION to install aiohttp on Python 3. --- .travis.yml | 2 +- check_py3k.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 check_py3k.py diff --git a/.travis.yml b/.travis.yml index a9fde9aa..6c2324ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ python: install: - "pip install -r requirements_dev.txt" - "python setup.py install" - - python check_py3k.py && pip install aiohttp || true + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then pip install aiohttp; fi script: py.test -x tests/ branches: only: diff --git a/check_py3k.py b/check_py3k.py deleted file mode 100644 index 17a9af99..00000000 --- a/check_py3k.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - -if sys.version_info[0] < 3: - sys.exit(1) -else: - sys.exit(0) From fd5c2e99fb6e7359309437f776898b0d69ca2a93 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 13 Jun 2014 13:22:18 +0200 Subject: [PATCH 49/67] fix #785: handle binary type address given to a client socket address --- gunicorn/http/wsgi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 303b890e..045c90c3 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -159,6 +159,8 @@ def create(req, sock, client, server, cfg): # http://www.ietf.org/rfc/rfc3875 if isinstance(client, string_types): environ['REMOTE_ADDR'] = client + elif isinstance(client, binary_type): + environ['REMOTE_ADDR'] = str(client) else: environ['REMOTE_ADDR'] = client[0] environ['REMOTE_PORT'] = str(client[1]) From 1c68516fd783642aa0c77486b489348693c4e517 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 12 Jun 2014 20:30:55 +0300 Subject: [PATCH 50/67] Update custom application example. It's running on both Python 2 and 3 now. --- docs/source/custom.rst | 51 +++----------------------------------- examples/standalone_app.py | 30 ++++++++++------------ 2 files changed, 16 insertions(+), 65 deletions(-) diff --git a/docs/source/custom.rst b/docs/source/custom.rst index 88f5eaf2..35c7be24 100644 --- a/docs/source/custom.rst +++ b/docs/source/custom.rst @@ -10,52 +10,7 @@ Sometimes, you want to integrate Gunicorn with your WSGI application. In this case, you can inherit from :class:`gunicorn.app.base.BaseApplication`. Here is a small example where we create a very small WSGI app and load it with a -custom Application:: +custom Application: - #!/usr/bin/env python - import gunicorn.app.base - - def handler_app(environ, start_response): - response_body = 'Works fine' - status = '200 OK' - - response_headers = [ - ('Content-Type', 'text/plain'), - ('Content-Length', str(len(response_body))) - ] - - start_response(status, response_headers) - - return [response_body] - - class StandaloneApplication(gunicorn.app.base.BaseApplication): - def __init__(self, app, options=None): - self.options = dict(options or {}) - self.application = app - super(StandaloneApplication, self).__init__() - - def load_config(self): - tmp_config = map( - lambda item: (item[0].lower(), item[1]), - self.options.iteritems() - ) - - config = dict( - (key, value) - for key, value in tmp_config - if key in self.cfg.settings and value is not None - ) - - for key, value in config.iteritems(): - self.cfg.set(key.lower(), value) - - def load(self): - return self.application - - if __name__ == '__main__': - options = { - 'bind': '%s:%s' % ('127.0.0.1', '8080'), - 'workers': 4, - # 'pidfile': pidfile, - } - StandaloneApplication(handler_app, options).run() +.. literalinclude:: ../../examples/standalone_app.py + :lines: 11-60 diff --git a/examples/standalone_app.py b/examples/standalone_app.py index a9a7267a..07c98956 100644 --- a/examples/standalone_app.py +++ b/examples/standalone_app.py @@ -8,21 +8,25 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import gunicorn -import gunicorn.app.base +from __future__ import unicode_literals + import multiprocessing +import gunicorn.app.base + +from gunicorn.six import iteritems + + def number_of_workers(): return (multiprocessing.cpu_count() * 2) + 1 def handler_app(environ, start_response): - response_body = 'Works fine' + response_body = b'Works fine' status = '200 OK' response_headers = [ ('Content-Type', 'text/plain'), - ('Content-Lenfth', str(len(response_body))), ] start_response(status, response_headers) @@ -31,24 +35,16 @@ def handler_app(environ, start_response): class StandaloneApplication(gunicorn.app.base.BaseApplication): + def __init__(self, app, options=None): - self.options = dict(options or {}) + self.options = options or {} self.application = app super(StandaloneApplication, self).__init__() def load_config(self): - tmp_config = map( - lambda item: (item[0].lower(), item[1]), - self.options.iteritems() - ) - - config = dict( - (key, value) - for key, value in tmp_config - if key in self.cfg.settings and value is not None - ) - - for key, value in config.iteritems(): + config = dict([(key, value) for key, value in iteritems(self.options) + if key in self.cfg.settings and value is not None]) + for key, value in iteritems(config): self.cfg.set(key.lower(), value) def load(self): From 9bf6aec443c095ffc5bf327e66f57e0147686821 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 13 Jun 2014 13:27:50 +0200 Subject: [PATCH 51/67] add latest fix to changes. note: it should come with the commit ... --- docs/source/news.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/news.rst b/docs/source/news.rst index caa5b3c1..cbb03bd3 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -1,6 +1,16 @@ Changelog ========= +19.1 / unreleased + +Changes +~~~~~~~ + +Core +++++ + +- fix #785: handle binary type address given to a client socket address + 19.0 / 2014-06-12 ----------------- From ed2d2a8ab3a96d673b0ba138ed71bd6ab69ba1cd Mon Sep 17 00:00:00 2001 From: Andrew Burdo Date: Fri, 13 Jun 2014 14:32:11 +0300 Subject: [PATCH 52/67] Change the rest of QUIT and TERM signals. This will restore graceful shutdown of workers by master. Also worker_int callback is moved from handle_exit (INT and TERM before switching) to handle_quit(INT and QUIT). --- docs/source/deploy.rst | 2 +- docs/source/settings.rst | 2 +- docs/source/signals.rst | 4 ++-- examples/example_config.py | 2 +- examples/when_ready.conf.py | 2 +- gunicorn/arbiter.py | 6 +++--- gunicorn/config.py | 2 +- gunicorn/workers/base.py | 10 +++++----- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index 152f1309..adfa7073 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -262,7 +262,7 @@ systemd: WorkingDirectory=/home/urban/gunicorn/bin ExecStart=/home/someuser/gunicorn/bin/gunicorn -p /home/urban/gunicorn/gunicorn.pid- test:app ExecReload=/bin/kill -s HUP $MAINPID - ExecStop=/bin/kill -s QUIT $MAINPID + ExecStop=/bin/kill -s TERM $MAINPID PrivateTmp=true **gunicorn.socket**:: diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 58e179fb..9eee8b2e 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -727,7 +727,7 @@ worker_int def worker_int(worker): pass -Called just after a worker exited on SIGINT or SIGTERM. +Called just after a worker exited on SIGINT or SIGQUIT. The callable needs to accept one instance variable for the initialized Worker. diff --git a/docs/source/signals.rst b/docs/source/signals.rst index c37a34f3..c041a364 100644 --- a/docs/source/signals.rst +++ b/docs/source/signals.rst @@ -22,7 +22,7 @@ Master process - **TTIN**: Increment the number of processes by one - **TTOU**: Decrement the nunber of processes by one - **USR1**: Reopen the log files -- **USR2**: Upgrade the Gunicorn on the fly. A separate **QUIT** signal should +- **USR2**: Upgrade the Gunicorn on the fly. A separate **TERM** signal should be used to kill the old process. This signal can also be used to use the new versions of pre-loaded applications. - **WINCH**: Gracefully shutdown the worker processes when gunicorn is @@ -91,7 +91,7 @@ incoming requests together. To phase the old instance out, you have to send **WINCH** signal to the old master process, and its worker processes will start to gracefully shut down. -t this point you can still revert to the old server because it hasn't closed its listen sockets yet, by following these steps: +At this point you can still revert to the old server because it hasn't closed its listen sockets yet, by following these steps: - Send HUP signal to the old master process - it will start the worker processes without reloading a configuration file - Send TERM signal to the new master process to gracefully shut down its worker processes diff --git a/examples/example_config.py b/examples/example_config.py index c939b851..7341447b 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -202,7 +202,7 @@ def when_ready(server): server.log.info("Server is ready. Spwawning workers") def worker_int(worker): - worker.log.info("worker received INT or TERM signal") + worker.log.info("worker received INT or QUIT signal") ## get traceback info import threading, sys, traceback diff --git a/examples/when_ready.conf.py b/examples/when_ready.conf.py index 61a04df3..578caacb 100644 --- a/examples/when_ready.conf.py +++ b/examples/when_ready.conf.py @@ -29,7 +29,7 @@ class MemoryWatch(threading.Thread): if self.memory_usage(pid) > self.max_mem: self.server.log.info("Pid %s killed (memory usage > %s)", pid, self.max_mem) - self.server.kill_worker(pid, signal.SIGQUIT) + self.server.kill_worker(pid, signal.SIGTERM) time.sleep(self.timeout) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index a05750ea..4489b7df 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -229,7 +229,7 @@ class Arbiter(object): raise StopIteration def handle_quit(self): - "SIGTERM handling" + "SIGQUIT handling" self.stop(False) raise StopIteration @@ -273,7 +273,7 @@ class Arbiter(object): if self.cfg.daemon: self.log.info("graceful stop of workers") self.num_workers = 0 - self.kill_workers(signal.SIGQUIT) + self.kill_workers(signal.SIGTERM) else: self.log.debug("SIGWINCH ignored. Not daemonized") @@ -480,7 +480,7 @@ class Arbiter(object): workers = sorted(workers, key=lambda w: w[1].age) while len(workers) > self.num_workers: (pid, _) = workers.pop(0) - self.kill_worker(pid, signal.SIGQUIT) + self.kill_worker(pid, signal.SIGTERM) def spawn_worker(self): self.worker_age += 1 diff --git a/gunicorn/config.py b/gunicorn/config.py index 5a165724..d940fa19 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -1369,7 +1369,7 @@ class WorkerInt(Setting): default = staticmethod(worker_int) desc = """\ - Called just after a worker exited on SIGINT or SIGTERM. + Called just after a worker exited on SIGINT or SIGQUIT. The callable needs to accept one instance variable for the initialized Worker. diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 99c68c8f..b0bc50ab 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -83,7 +83,7 @@ class Worker(object): if self.cfg.reload: def changed(fname): self.log.info("Worker reloading: %s modified", fname) - os.kill(self.pid, signal.SIGTERM) + os.kill(self.pid, signal.SIGQUIT) raise SystemExit() Reloader(callback=changed).start() @@ -130,10 +130,10 @@ class Worker(object): signal.signal(signal.SIGUSR1, self.handle_usr1) signal.signal(signal.SIGABRT, self.handle_abort) - # Don't let SIGQUIT and SIGUSR1 disturb active requests + # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 - signal.siginterrupt(signal.SIGQUIT, False) + signal.siginterrupt(signal.SIGTERM, False) signal.siginterrupt(signal.SIGUSR1, False) def handle_usr1(self, sig, frame): @@ -141,11 +141,11 @@ class Worker(object): def handle_exit(self, sig, frame): self.alive = False - # worker_int callback - self.cfg.worker_int(self) def handle_quit(self, sig, frame): self.alive = False + # worker_int callback + self.cfg.worker_int(self) sys.exit(0) def handle_abort(self, sig, frame): From 26df0651bcf1655ef63ac3b968344d737d3a7bfc Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 13 Jun 2014 22:45:37 +0200 Subject: [PATCH 53/67] bump to 19.1 --- gunicorn/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index a68c6c0d..4e7b4c65 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,6 +3,6 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (19, 0, 0) +version_info = (19, 1, 0) __version__ = ".".join([str(v) for v in version_info]) SERVER_SOFTWARE = "gunicorn/%s" % __version__ From 39a1c3adadee37eeed4be6bd7f1ef8f2c6568894 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 13 Jun 2014 22:47:36 +0200 Subject: [PATCH 54/67] add the change to the news --- docs/source/news.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/news.rst b/docs/source/news.rst index cbb03bd3..aaef2e24 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -10,6 +10,8 @@ Core ++++ - fix #785: handle binary type address given to a client socket address +- fix graceful shutdown. make sure QUIT and TERMS signals are switched + everywhere. 19.0 / 2014-06-12 ----------------- From 412840215f15c4207a4ed219f42610c4908eb0cc Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Fri, 13 Jun 2014 18:17:35 -0700 Subject: [PATCH 55/67] Add multithread information to the design docs Relates to #784 --- docs/source/design.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/design.rst b/docs/source/design.rst index dff78d30..4dc3f5d2 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -107,6 +107,22 @@ Always remember, there is such a thing as too many workers. After a point your worker processes will start thrashing system resources decreasing the throughput of the entire system. +How Many Threads? +=================== + +Since Gunicorn 19, a threads option can be used to process requests in multiple +threads. Using threads assumes use of the sync worker. One benefit from threads +is that requests can take longer than the worker timeout while notifying the +master process that it is not frozen and should not be killed. Depending on the +system, using multiple threads, multiple worker processes, or some mixture, may +yield the best results. For example, CPython may not perform as well as Jython +when using threads, as threading is implemented differently by each. Using +threads instead of processes is a good way to reduce the memory footprint of +Gunicorn, while still allowing for application upgrades using the reload signal, +as the application code will be shared among workers but loaded only in the +worker processes (unlike when using the preload setting, which loads the code in +the master process). + .. _Greenlets: https://github.com/python-greenlet/greenlet .. _Eventlet: http://eventlet.net .. _Gevent: http://gevent.org From 18355eff3061bf11b8d8a735a16ecb28551d45d6 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Fri, 13 Jun 2014 18:22:20 -0700 Subject: [PATCH 56/67] Add thundering herd question to FAQ Close #784 --- docs/source/faq.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 6bf842c8..240ff5e1 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -82,6 +82,16 @@ To decrease the worker count by one:: $ kill -TTOU $masterpid +Does Gunicorn suffer from the thundering herd problem? +------------------------------------------------------ + +The thundering herd problem occurs when many sleeping request handlers, which +may be either threads or processes, wake up at the same time to handle an new +request. Since only one handler will receive the request, the others will have +been awakened for no reaon, wasting CPU cycles. At this time, Gunicorn does not +implement any IPC solution for coordinating between worker processes. You may +experience high load due to this problem when using many workers or threads. + .. _worker_class: configure.html#worker-class .. _`number of workers`: design.html#how-many-workers From 7b902a2378f73f36d85d3dd4f38026f18cf31d75 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 06:58:13 +0200 Subject: [PATCH 57/67] fix #783 fix tornado worker with missing option Gunicorn doesn't ovveride the WSGI headers with the X heaaders anymore. --- gunicorn/workers/gtornado.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gunicorn/workers/gtornado.py b/gunicorn/workers/gtornado.py index 2572587e..8b6eee3f 100644 --- a/gunicorn/workers/gtornado.py +++ b/gunicorn/workers/gtornado.py @@ -93,7 +93,6 @@ class TornadoWorker(Worker): server._sockets[s.fileno()] = s server.no_keep_alive = self.cfg.keepalive <= 0 - server.xheaders = bool(self.cfg.x_forwarded_for_header) server.start(num_processes=1) self.ioloop.start() From 94ee1de00cca87172ee29fb7e1428d674b265890 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 07:13:05 +0200 Subject: [PATCH 58/67] update news --- docs/source/news.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/source/news.rst b/docs/source/news.rst index aaef2e24..b2e026c4 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -13,6 +13,11 @@ Core - fix graceful shutdown. make sure QUIT and TERMS signals are switched everywhere. +Tornado worker +++++++++++++++ + +- fix x_headers error + 19.0 / 2014-06-12 ----------------- From 1a3ab6490d091b6978b18e424aa0650cd9c7f987 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 07:29:55 +0200 Subject: [PATCH 59/67] fix #783 : link to the corresponding discussion in the changelog --- docs/source/news.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/source/news.rst b/docs/source/news.rst index b2e026c4..dba5c4b6 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -16,7 +16,12 @@ Core Tornado worker ++++++++++++++ -- fix x_headers error +- fix #783: x_headers error. The x-forwarded-headers option has been removed + in `c4873681299212d6082cd9902740eef18c2f14f1 + `_. The discussion is + available on `#633 `_. + + 19.0 / 2014-06-12 ----------------- From 31f465d27163494ac7f0704f9fe182c71f06bfbb Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 10:08:55 +0200 Subject: [PATCH 60/67] fix #787 check if we load a pyc file or not. --- examples/example_config | 222 ++++++++++++++++++++++++++++++++++++++++ gunicorn/six.py | 10 +- 2 files changed, 230 insertions(+), 2 deletions(-) create mode 100644 examples/example_config diff --git a/examples/example_config b/examples/example_config new file mode 100644 index 00000000..7341447b --- /dev/null +++ b/examples/example_config @@ -0,0 +1,222 @@ +# Sample Gunicorn configuration file. + +# +# Server socket +# +# bind - The socket to bind. +# +# A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'. +# An IP is a valid HOST. +# +# backlog - The number of pending connections. This refers +# to the number of clients that can be waiting to be +# served. Exceeding this number results in the client +# getting an error when attempting to connect. It should +# only affect servers under significant load. +# +# Must be a positive integer. Generally set in the 64-2048 +# range. +# + +bind = '127.0.0.1:8000' +backlog = 2048 + +# +# Worker processes +# +# workers - The number of worker processes that this server +# should keep alive for handling requests. +# +# A positive integer generally in the 2-4 x $(NUM_CORES) +# range. You'll want to vary this a bit to find the best +# for your particular application's work load. +# +# worker_class - The type of workers to use. The default +# async class should handle most 'normal' types of work +# loads. You'll want to read http://gunicorn/deployment.hml +# for information on when you might want to choose one +# of the other worker classes. +# +# An string referring to a 'gunicorn.workers' entry point +# or a python path to a subclass of +# gunicorn.workers.base.Worker. The default provided values +# are: +# +# egg:gunicorn#sync +# egg:gunicorn#eventlet - Requires eventlet >= 0.9.7 +# egg:gunicorn#gevent - Requires gevent >= 0.12.2 (?) +# egg:gunicorn#tornado - Requires tornado >= 0.2 +# +# worker_connections - For the eventlet and gevent worker classes +# this limits the maximum number of simultaneous clients that +# a single process can handle. +# +# A positive integer generally set to around 1000. +# +# timeout - If a worker does not notify the master process in this +# number of seconds it is killed and a new worker is spawned +# to replace it. +# +# Generally set to thirty seconds. Only set this noticeably +# higher if you're sure of the repercussions for sync workers. +# For the non sync workers it just means that the worker +# process is still communicating and is not tied to the length +# of time required to handle a single request. +# +# keepalive - The number of seconds to wait for the next request +# on a Keep-Alive HTTP connection. +# +# A positive integer. Generally set in the 1-5 seconds range. +# + +workers = 1 +worker_class = 'sync' +worker_connections = 1000 +timeout = 30 +keepalive = 2 + +# +# Debugging +# +# debug - Turn on debugging in the server. This limits the number of +# worker processes to 1 and changes some error handling that's +# sent to clients. +# +# True or False +# +# spew - Install a trace function that spews every line of Python +# that is executed when running the server. This is the +# nuclear option. +# +# True or False +# + +debug = False +spew = False + +# +# Server mechanics +# +# daemon - Detach the main Gunicorn process from the controlling +# terminal with a standard fork/fork sequence. +# +# True or False +# +# pidfile - The path to a pid file to write +# +# A path string or None to not write a pid file. +# +# user - Switch worker processes to run as this user. +# +# A valid user id (as an integer) or the name of a user that +# can be retrieved with a call to pwd.getpwnam(value) or None +# to not change the worker process user. +# +# group - Switch worker process to run as this group. +# +# A valid group id (as an integer) or the name of a user that +# can be retrieved with a call to pwd.getgrnam(value) or None +# to change the worker processes group. +# +# umask - A mask for file permissions written by Gunicorn. Note that +# this affects unix socket permissions. +# +# A valid value for the os.umask(mode) call or a string +# compatible with int(value, 0) (0 means Python guesses +# the base, so values like "0", "0xFF", "0022" are valid +# for decimal, hex, and octal representations) +# +# tmp_upload_dir - A directory to store temporary request data when +# requests are read. This will most likely be disappearing soon. +# +# A path to a directory where the process owner can write. Or +# None to signal that Python should choose one on its own. +# + +daemon = False +pidfile = None +umask = 0 +user = None +group = None +tmp_upload_dir = None + +# +# Logging +# +# logfile - The path to a log file to write to. +# +# A path string. "-" means log to stdout. +# +# loglevel - The granularity of log output +# +# A string of "debug", "info", "warning", "error", "critical" +# + +errorlog = '-' +loglevel = 'info' +accesslog = '-' + +# +# Process naming +# +# proc_name - A base to use with setproctitle to change the way +# that Gunicorn processes are reported in the system process +# table. This affects things like 'ps' and 'top'. If you're +# going to be running more than one instance of Gunicorn you'll +# probably want to set a name to tell them apart. This requires +# that you install the setproctitle module. +# +# A string or None to choose a default of something like 'gunicorn'. +# + +proc_name = None + +# +# Server hooks +# +# post_fork - Called just after a worker has been forked. +# +# A callable that takes a server and worker instance +# as arguments. +# +# pre_fork - Called just prior to forking the worker subprocess. +# +# A callable that accepts the same arguments as after_fork +# +# pre_exec - Called just prior to forking off a secondary +# master process during things like config reloading. +# +# A callable that takes a server instance as the sole argument. +# + +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + +def pre_fork(server, worker): + pass + +def pre_exec(server): + server.log.info("Forked child, re-executing.") + +def when_ready(server): + server.log.info("Server is ready. Spwawning workers") + +def worker_int(worker): + worker.log.info("worker received INT or QUIT signal") + + ## get traceback info + import threading, sys, traceback + id2name = dict([(th.ident, th.name) for th in threading.enumerate()]) + code = [] + for threadId, stack in sys._current_frames().items(): + code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""), + threadId)) + for filename, lineno, name, line in traceback.extract_stack(stack): + code.append('File: "%s", line %d, in %s' % (filename, + lineno, name)) + if line: + code.append(" %s" % (line.strip())) + worker.log.debug("\n".join(code)) + +def worker_abort(worker): + worker.log.info("worker received SIGABRT signal") diff --git a/gunicorn/six.py b/gunicorn/six.py index f4fb9b86..bdc17c94 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -357,7 +357,11 @@ if PY3: print_ = getattr(builtins, "print") def execfile_(fname, *args): - return exec_(_get_codeobj(fname), *args) + if fname.endswith(".pyc"): + code = _get_codeobj(fname) + else: + code = compile(open(fname, 'rb').read(), fname, 'exec') + return exec_(code, *args) del builtins @@ -382,7 +386,9 @@ else: def execfile_(fname, *args): """ Overriding PY2 execfile() implementation to support .pyc files """ - return exec_(_get_codeobj(fname), *args) + if fname.endswith(".pyc"): + return exec_(_get_codeobj(fname), *args) + return execfile(fname, *args) def print_(*args, **kwargs): From f3824a06854a722c140cc84b21dc89f722cfcc4e Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 10:16:04 +0200 Subject: [PATCH 61/67] this example shouldn't have been comitted. --- examples/example_config | 222 ---------------------------------------- 1 file changed, 222 deletions(-) delete mode 100644 examples/example_config diff --git a/examples/example_config b/examples/example_config deleted file mode 100644 index 7341447b..00000000 --- a/examples/example_config +++ /dev/null @@ -1,222 +0,0 @@ -# Sample Gunicorn configuration file. - -# -# Server socket -# -# bind - The socket to bind. -# -# A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'. -# An IP is a valid HOST. -# -# backlog - The number of pending connections. This refers -# to the number of clients that can be waiting to be -# served. Exceeding this number results in the client -# getting an error when attempting to connect. It should -# only affect servers under significant load. -# -# Must be a positive integer. Generally set in the 64-2048 -# range. -# - -bind = '127.0.0.1:8000' -backlog = 2048 - -# -# Worker processes -# -# workers - The number of worker processes that this server -# should keep alive for handling requests. -# -# A positive integer generally in the 2-4 x $(NUM_CORES) -# range. You'll want to vary this a bit to find the best -# for your particular application's work load. -# -# worker_class - The type of workers to use. The default -# async class should handle most 'normal' types of work -# loads. You'll want to read http://gunicorn/deployment.hml -# for information on when you might want to choose one -# of the other worker classes. -# -# An string referring to a 'gunicorn.workers' entry point -# or a python path to a subclass of -# gunicorn.workers.base.Worker. The default provided values -# are: -# -# egg:gunicorn#sync -# egg:gunicorn#eventlet - Requires eventlet >= 0.9.7 -# egg:gunicorn#gevent - Requires gevent >= 0.12.2 (?) -# egg:gunicorn#tornado - Requires tornado >= 0.2 -# -# worker_connections - For the eventlet and gevent worker classes -# this limits the maximum number of simultaneous clients that -# a single process can handle. -# -# A positive integer generally set to around 1000. -# -# timeout - If a worker does not notify the master process in this -# number of seconds it is killed and a new worker is spawned -# to replace it. -# -# Generally set to thirty seconds. Only set this noticeably -# higher if you're sure of the repercussions for sync workers. -# For the non sync workers it just means that the worker -# process is still communicating and is not tied to the length -# of time required to handle a single request. -# -# keepalive - The number of seconds to wait for the next request -# on a Keep-Alive HTTP connection. -# -# A positive integer. Generally set in the 1-5 seconds range. -# - -workers = 1 -worker_class = 'sync' -worker_connections = 1000 -timeout = 30 -keepalive = 2 - -# -# Debugging -# -# debug - Turn on debugging in the server. This limits the number of -# worker processes to 1 and changes some error handling that's -# sent to clients. -# -# True or False -# -# spew - Install a trace function that spews every line of Python -# that is executed when running the server. This is the -# nuclear option. -# -# True or False -# - -debug = False -spew = False - -# -# Server mechanics -# -# daemon - Detach the main Gunicorn process from the controlling -# terminal with a standard fork/fork sequence. -# -# True or False -# -# pidfile - The path to a pid file to write -# -# A path string or None to not write a pid file. -# -# user - Switch worker processes to run as this user. -# -# A valid user id (as an integer) or the name of a user that -# can be retrieved with a call to pwd.getpwnam(value) or None -# to not change the worker process user. -# -# group - Switch worker process to run as this group. -# -# A valid group id (as an integer) or the name of a user that -# can be retrieved with a call to pwd.getgrnam(value) or None -# to change the worker processes group. -# -# umask - A mask for file permissions written by Gunicorn. Note that -# this affects unix socket permissions. -# -# A valid value for the os.umask(mode) call or a string -# compatible with int(value, 0) (0 means Python guesses -# the base, so values like "0", "0xFF", "0022" are valid -# for decimal, hex, and octal representations) -# -# tmp_upload_dir - A directory to store temporary request data when -# requests are read. This will most likely be disappearing soon. -# -# A path to a directory where the process owner can write. Or -# None to signal that Python should choose one on its own. -# - -daemon = False -pidfile = None -umask = 0 -user = None -group = None -tmp_upload_dir = None - -# -# Logging -# -# logfile - The path to a log file to write to. -# -# A path string. "-" means log to stdout. -# -# loglevel - The granularity of log output -# -# A string of "debug", "info", "warning", "error", "critical" -# - -errorlog = '-' -loglevel = 'info' -accesslog = '-' - -# -# Process naming -# -# proc_name - A base to use with setproctitle to change the way -# that Gunicorn processes are reported in the system process -# table. This affects things like 'ps' and 'top'. If you're -# going to be running more than one instance of Gunicorn you'll -# probably want to set a name to tell them apart. This requires -# that you install the setproctitle module. -# -# A string or None to choose a default of something like 'gunicorn'. -# - -proc_name = None - -# -# Server hooks -# -# post_fork - Called just after a worker has been forked. -# -# A callable that takes a server and worker instance -# as arguments. -# -# pre_fork - Called just prior to forking the worker subprocess. -# -# A callable that accepts the same arguments as after_fork -# -# pre_exec - Called just prior to forking off a secondary -# master process during things like config reloading. -# -# A callable that takes a server instance as the sole argument. -# - -def post_fork(server, worker): - server.log.info("Worker spawned (pid: %s)", worker.pid) - -def pre_fork(server, worker): - pass - -def pre_exec(server): - server.log.info("Forked child, re-executing.") - -def when_ready(server): - server.log.info("Server is ready. Spwawning workers") - -def worker_int(worker): - worker.log.info("worker received INT or QUIT signal") - - ## get traceback info - import threading, sys, traceback - id2name = dict([(th.ident, th.name) for th in threading.enumerate()]) - code = [] - for threadId, stack in sys._current_frames().items(): - code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""), - threadId)) - for filename, lineno, name, line in traceback.extract_stack(stack): - code.append('File: "%s", line %d, in %s' % (filename, - lineno, name)) - if line: - code.append(" %s" % (line.strip())) - worker.log.debug("\n".join(code)) - -def worker_abort(worker): - worker.log.info("worker received SIGABRT signal") From 4ac7e55161553503817dfbd01ccc5f263d163e69 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 11:59:32 +0200 Subject: [PATCH 62/67] check the python requirements for the gaiohttp server fix #788 --- docs/source/news.rst | 3 ++- gunicorn/workers/gaiohttp.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source/news.rst b/docs/source/news.rst index dba5c4b6..de246eeb 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -9,6 +9,7 @@ Changes Core ++++ +- improvement: fix #788 check the python requirements for the gaiohttp server - fix #785: handle binary type address given to a client socket address - fix graceful shutdown. make sure QUIT and TERMS signals are switched everywhere. @@ -19,7 +20,7 @@ Tornado worker - fix #783: x_headers error. The x-forwarded-headers option has been removed in `c4873681299212d6082cd9902740eef18c2f14f1 `_. The discussion is - available on `#633 `_. + available on `#633 `_. diff --git a/gunicorn/workers/gaiohttp.py b/gunicorn/workers/gaiohttp.py index fb109591..80c7bfd2 100644 --- a/gunicorn/workers/gaiohttp.py +++ b/gunicorn/workers/gaiohttp.py @@ -7,7 +7,12 @@ __all__ = ['AiohttpWorker'] import asyncio import functools import os -import gunicorn.workers.base as base +import sys + +from . import base + +if sys.version_info < (3, 3, 0): + raise RuntimeError("Python 3.3 and later is needed to run this worker") try: from aiohttp.wsgi import WSGIServerHttpProtocol From 68cd2b92c8d5160f02236899a86c1249fd926469 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 12:05:00 +0200 Subject: [PATCH 63/67] Revert "check the python requirements for the gaiohttp server" This reverts commit 4ac7e55161553503817dfbd01ccc5f263d163e69. --- docs/source/news.rst | 3 +-- gunicorn/workers/gaiohttp.py | 7 +------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/source/news.rst b/docs/source/news.rst index de246eeb..dba5c4b6 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -9,7 +9,6 @@ Changes Core ++++ -- improvement: fix #788 check the python requirements for the gaiohttp server - fix #785: handle binary type address given to a client socket address - fix graceful shutdown. make sure QUIT and TERMS signals are switched everywhere. @@ -20,7 +19,7 @@ Tornado worker - fix #783: x_headers error. The x-forwarded-headers option has been removed in `c4873681299212d6082cd9902740eef18c2f14f1 `_. The discussion is - available on `#633 `_. + available on `#633 `_. diff --git a/gunicorn/workers/gaiohttp.py b/gunicorn/workers/gaiohttp.py index 80c7bfd2..fb109591 100644 --- a/gunicorn/workers/gaiohttp.py +++ b/gunicorn/workers/gaiohttp.py @@ -7,12 +7,7 @@ __all__ = ['AiohttpWorker'] import asyncio import functools import os -import sys - -from . import base - -if sys.version_info < (3, 3, 0): - raise RuntimeError("Python 3.3 and later is needed to run this worker") +import gunicorn.workers.base as base try: from aiohttp.wsgi import WSGIServerHttpProtocol From f41f86c3dac1310d9e5b61341585896724734de8 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 14 Jun 2014 21:46:35 +0200 Subject: [PATCH 64/67] StopIteration shouldn't be catched at this level. fix #790 --- gunicorn/workers/async.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index d3cce5fc..4cfa7a86 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -114,6 +114,8 @@ class AsyncWorker(base.Worker): respiter.close() if resp.should_close(): raise StopIteration() + except StopIteration: + raise except Exception: if resp and resp.headers_sent: # If the requests have already been sent, we should close the From c8b4f00266b9175d174b1dd0d8bc97ba843e8f23 Mon Sep 17 00:00:00 2001 From: Rhys Powell Date: Mon, 16 Jun 2014 07:57:07 +0100 Subject: [PATCH 65/67] Fixed spelling mistake fixed spelling in TTOU description --- docs/source/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/signals.rst b/docs/source/signals.rst index c041a364..68e40742 100644 --- a/docs/source/signals.rst +++ b/docs/source/signals.rst @@ -20,7 +20,7 @@ Master process not preloaded (using the ``--preload`` option), Gunicorn will also load the new version. - **TTIN**: Increment the number of processes by one -- **TTOU**: Decrement the nunber of processes by one +- **TTOU**: Decrement the number of processes by one - **USR1**: Reopen the log files - **USR2**: Upgrade the Gunicorn on the fly. A separate **TERM** signal should be used to kill the old process. This signal can also be used to use the new From 83d97c25ca84e8ac66fb5b3942562ff16056dbc7 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 16 Jun 2014 14:01:43 +0300 Subject: [PATCH 66/67] Fix a markup error in news.rst. This commit also silences a couple of Sphinx warnings e.g. WARNING: Bullet list ends without a blank line; unexpected unindent. --- docs/source/news.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/news.rst b/docs/source/news.rst index dba5c4b6..2838a3f1 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -94,14 +94,16 @@ core - add: include tz offset in access log - add: include access logs in the syslog handler. - add --reload option for code reloading -- add the capability to load `gunicorn.base.Application` without the loading of the arguments of the command line. It allows you to [embed gunicorn in your own application](http://docs.gunicorn.org/en/latest/custom.html). +- add the capability to load `gunicorn.base.Application` without the loading of + the arguments of the command line. It allows you to :ref:`embed gunicorn in + your own application `. - improve: set wsgi.multithread to True for async workers - fix logging: make sure to redirect wsgi.errors when needed - add: syslog logging can now be done to a unix socket - fix logging: don't try to redirect stdout/stderr to the logfile. - fix logging: don't propagate log - improve logging: file option can be overriden by the gunicorn options -`--error-logfile` and `--access-logfile` if they are given. + `--error-logfile` and `--access-logfile` if they are given. - fix: dont' override SERVER_* by the Host header - fix: handle_error - add more option to configure SSL From 944e224d36df49e772cdfd1152d8abf80f02f55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Wirtel?= Date: Mon, 16 Jun 2014 17:51:34 +0200 Subject: [PATCH 67/67] Fix typo in the examples --- examples/example_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_config.py b/examples/example_config.py index 7341447b..5d2ef60c 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -199,7 +199,7 @@ def pre_exec(server): server.log.info("Forked child, re-executing.") def when_ready(server): - server.log.info("Server is ready. Spwawning workers") + server.log.info("Server is ready. Spawning workers") def worker_int(worker): worker.log.info("worker received INT or QUIT signal")