From 55de904e9dcc0ef6b137ca7ed5b0e86fcf2f425d Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sun, 18 Sep 2011 03:02:12 -0700 Subject: [PATCH 1/2] add access logging for gevent_* workers --- gunicorn/glogging.py | 2 +- gunicorn/http/message.py | 14 +++++++------- gunicorn/http/wsgi.py | 20 ++++++++++---------- gunicorn/workers/ggevent.py | 14 ++++++++++---- gunicorn/workers/ggevent_wsgi.py | 18 +++++++++++++++--- 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index a5268cf2..f1e5a0d4 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -93,7 +93,7 @@ class Logger(object): 'r': "%s %s %s" % (environ['REQUEST_METHOD'], environ['RAW_URI'], environ["SERVER_PROTOCOL"]), 's': status, - 'b': str(resp.clength) or '-', + 'b': str(resp.response_length) or '-', 'f': environ.get('HTTP_REFERER', '-'), 'a': environ.get('HTTP_USER_AGENT', '-'), 'T': str(request_time.seconds), diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index e67930a2..d2b97bf9 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -61,25 +61,25 @@ class Message(object): def set_body_reader(self): chunked = False - clength = None + response_length = None for (name, value) in self.headers: if name == "CONTENT-LENGTH": try: - clength = int(value) + response_length = int(value) except ValueError: - clength = None + response_length = None elif name == "TRANSFER-ENCODING": chunked = value.lower() == "chunked" elif name == "SEC-WEBSOCKET-KEY1": - clength = 8 + response_length = 8 - if clength is not None or chunked: + if response_length is not None or chunked: break if chunked: self.body = Body(ChunkedReader(self, self.unreader)) - elif clength is not None: - self.body = Body(LengthReader(self.unreader, clength)) + elif response_length is not None: + self.body = Body(LengthReader(self.unreader, response_length)) else: self.body = Body(EOFReader(self.unreader)) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 7872104a..c6a0ef70 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -154,7 +154,7 @@ class Response(object): self.must_close = False self.headers = [] self.headers_sent = False - self.clength = None + self.response_length = None self.sent = 0 def force_close(self): @@ -163,7 +163,7 @@ class Response(object): def should_close(self): if self.must_close or self.req.should_close(): return True - if self.clength is not None or self.chunked: + if self.response_length is not None or self.chunked: return False return True @@ -187,7 +187,7 @@ class Response(object): assert isinstance(name, basestring), "%r is not a string" % name lname = name.lower().strip() if lname == "content-length": - self.clength = int(value) + self.response_length = int(value) elif util.is_hoppish(name): if lname == "connection": # handle websocket @@ -203,7 +203,7 @@ class Response(object): # Only use chunked responses when the client is # speaking HTTP/1.1 or newer and there was # no Content-Length header set. - if self.clength is not None: + if self.response_length is not None: return False elif self.req.version <= (1,0): return False @@ -242,12 +242,12 @@ class Response(object): arglen = len(arg) tosend = arglen - if self.clength is not None: - if self.sent >= self.clength: - # Never write more than self.clength bytes + if self.response_length is not None: + if self.sent >= self.response_length: + # Never write more than self.response_length bytes return - tosend = min(self.clength - self.sent, tosend) + tosend = min(self.response_length - self.sent, tosend) if tosend < arglen: arg = arg[:tosend] @@ -287,8 +287,8 @@ class Response(object): fo_offset = respiter.filelike.tell() nbytes = max(os.fstat(fileno).st_size - fo_offset, 0) - if self.clength: - nbytes = min(nbytes, self.clength) + if self.response_length: + nbytes = min(nbytes, self.response_length) if nbytes == 0: return diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 586d8ec1..24974f14 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -7,6 +7,7 @@ from __future__ import with_statement import os import sys +from datetime import datetime # workaround on osx, disable kqueue if sys.platform == "darwin": @@ -131,8 +132,9 @@ class GeventBaseWorker(Worker): self.socket.setblocking(1) pool = Pool(self.worker_connections) self.server_class.base_env['wsgi.multiprocess'] = (self.cfg.workers > 1) - server = self.server_class(self.socket, application=self.wsgi, - spawn=pool, handler_class=self.wsgi_handler) + server = self.server_class( + self.socket, application=self.wsgi, spawn=pool, log=self.log, + handler_class=self.wsgi_handler) server.start() try: while self.alive: @@ -165,8 +167,12 @@ class GeventBaseWorker(Worker): class PyWSGIHandler(pywsgi.WSGIHandler): - def log_request(self, *args): - pass + + def log_request(self): + start = datetime.fromtimestamp(self.time_start) + finish = datetime.fromtimestamp(self.time_finish) + response_time = finish - start + self.server.log.access(self, self.environ, response_time) def get_environ(self): env = super(PyWSGIHandler, self).get_environ() diff --git a/gunicorn/workers/ggevent_wsgi.py b/gunicorn/workers/ggevent_wsgi.py index 9bf79055..fc9b1185 100644 --- a/gunicorn/workers/ggevent_wsgi.py +++ b/gunicorn/workers/ggevent_wsgi.py @@ -8,14 +8,26 @@ from gevent import wsgi class WSGIHandler(wsgi.WSGIHandler): - def log_request(self, *args): - pass + + @property + def status(self): + return ' '.join([str(self.code), self.reason]) + + def log_request(self, length): + self.response_length = length + response_time = datetime.now() - self.time_start + self.server.log.access(self, self.environ, response_time) def prepare_env(self): env = super(WSGIHandler, self).prepare_env() env['RAW_URI'] = self.request.uri + self.environ = env return env - + + def handle(self): + self.time_start = datetime.now() + super(WSGIHandler, self).handle() + class WSGIServer(wsgi.WSGIServer): base_env = BASE_WSGI_ENV From d6915942f78d7d871380773e90d59e45a692c36b Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sun, 18 Sep 2011 03:06:29 -0700 Subject: [PATCH 2/2] factor out duplicate gevent-based worker code --- gunicorn/workers/ggevent.py | 85 +++----------------------------- gunicorn/workers/ggevent_wsgi.py | 6 ++- 2 files changed, 11 insertions(+), 80 deletions(-) diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 24974f14..3643903e 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -37,19 +37,6 @@ BASE_WSGI_ENV = { 'wsgi.run_once': False } - -class GGeventServer(StreamServer): - def __init__(self, listener, handle, spawn='default', worker=None): - StreamServer.__init__(self, listener, spawn=spawn) - self.handle_func = handle - self.worker = worker - - def stop(self, timeout=None): - super(GGeventServer, self).stop(timeout=timeout) - - def handle(self, sock, addr): - self.handle_func(sock, addr) - class GeventWorker(AsyncWorker): @classmethod @@ -66,8 +53,12 @@ class GeventWorker(AsyncWorker): self.socket.setblocking(1) pool = Pool(self.worker_connections) - server = GGeventServer(self.socket, self.handle, spawn=pool, - worker=self) + if hasattr(self, 'server_class'): + server = self.server_class( + self.socket, application=self.wsgi, spawn=pool, log=self.log, + handler_class=self.wsgi_handler) + else: + server = StreamServer(self.socket, handle=self.handle, spawn=pool) server.start() try: @@ -104,68 +95,6 @@ class GeventWorker(AsyncWorker): gevent.core.dns_init() super(GeventWorker, self).init_process() - -class GeventBaseWorker(Worker): - """\ - This base class is used for the two variants of workers that use - Gevent's two different WSGI workers. ``gevent_wsgi`` worker uses - the libevent HTTP parser but does not support streaming response - bodies or Keep-Alive. The ``gevent_pywsgi`` worker uses an - alternative Gevent WSGI server that supports streaming and Keep- - Alive but does not use the libevent HTTP parser. - """ - server_class = None - wsgi_handler = None - - def __init__(self, *args, **kwargs): - super(GeventBaseWorker, self).__init__(*args, **kwargs) - self.worker_connections = self.cfg.worker_connections - - @classmethod - def setup(cls): - from gevent import monkey - monkey.noisy = False - monkey.patch_all() - - - def run(self): - self.socket.setblocking(1) - pool = Pool(self.worker_connections) - self.server_class.base_env['wsgi.multiprocess'] = (self.cfg.workers > 1) - server = self.server_class( - self.socket, application=self.wsgi, spawn=pool, log=self.log, - handler_class=self.wsgi_handler) - server.start() - try: - while self.alive: - self.notify() - - if self.ppid != os.getppid(): - self.log.info("Parent changed, shutting down: %s", self) - break - - gevent.sleep(1.0) - - except KeyboardInterrupt: - pass - - # try to stop the connections - try: - self.notify() - server.stop(timeout=self.timeout) - except: - pass - - if hasattr(gevent.core, 'dns_shutdown'): - - def init_process(self): - #gevent 0.13 and older doesn't reinitialize dns for us after forking - #here's the workaround - gevent.core.dns_shutdown(fail_requests=1) - gevent.core.dns_init() - super(GeventBaseWorker, self).init_process() - - class PyWSGIHandler(pywsgi.WSGIHandler): def log_request(self): @@ -183,7 +112,7 @@ class PyWSGIHandler(pywsgi.WSGIHandler): class PyWSGIServer(pywsgi.WSGIServer): base_env = BASE_WSGI_ENV -class GeventPyWSGIWorker(GeventBaseWorker): +class GeventPyWSGIWorker(GeventWorker): "The Gevent StreamServer based workers." server_class = PyWSGIServer wsgi_handler = PyWSGIHandler diff --git a/gunicorn/workers/ggevent_wsgi.py b/gunicorn/workers/ggevent_wsgi.py index fc9b1185..bab05a02 100644 --- a/gunicorn/workers/ggevent_wsgi.py +++ b/gunicorn/workers/ggevent_wsgi.py @@ -3,7 +3,9 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from gunicorn.workers.ggevent import BASE_WSGI_ENV, GeventBaseWorker +from datetime import datetime + +from gunicorn.workers.ggevent import BASE_WSGI_ENV, GeventWorker from gevent import wsgi @@ -32,7 +34,7 @@ class WSGIHandler(wsgi.WSGIHandler): class WSGIServer(wsgi.WSGIServer): base_env = BASE_WSGI_ENV -class GeventWSGIWorker(GeventBaseWorker): +class GeventWSGIWorker(GeventWorker): "The Gevent StreamServer based workers." server_class = WSGIServer wsgi_handler = WSGIHandler