From 883c36862ddfa3266cd1cfa6153b3a73f58d5df9 Mon Sep 17 00:00:00 2001 From: zakdances Date: Sat, 29 Sep 2012 17:45:35 -0500 Subject: [PATCH 01/47] Update README.rst Updated readme with script example for Gunicorn Paster. --- README.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.rst b/README.rst index c2111eb8..c6ef7948 100644 --- a/README.rst +++ b/README.rst @@ -153,6 +153,28 @@ And then as per usual:: $ cd yourpasteproject $ paster serve development.ini workers=2 +**Gunicorn paster from script** + +If you'd like to run Gunicorn paster from a script instead of the command line (for example: a runapp.py to start a Pyramid app), +you can use this example to help get you started:: + + import os + import multiprocessing + + from paste.deploy import appconfig, loadapp + from gunicorn.app.pasterapp import paste_server + + if __name__ == "__main__": + + iniFile = 'config:development.ini' + port = int(os.environ.get("PORT", 5000)) + workers = multiprocessing.cpu_count() * 2 + 1 + worker_class = 'gevent' + + app = loadapp(iniFile, relative_to='.') + paste_server(app, host='0.0.0.0', port=port, workers=workers, worker_class=worker_class) + + LICENSE ------- From fd6c712dd432f6cbbadd53bb59e7c5ce7b07e0cb Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Mon, 15 Oct 2012 14:07:43 -0700 Subject: [PATCH 02/47] fix gevent graceful timeout for real `server.kill()` is too aggressive. It sends a GreenletExit exception to all the pool workers, causing them to exit immediately. A simple one line fix is to use `server.stop()`. In my testing, it appears that `server.stop_accepting()` will make the server stop listening, but pending connections already in the `accept()` backlog are still handled. With `server.stop()` the accept backlog is not handled, the listener is closed in the worker, but existing requests are allowed to exit gracefully. --- gunicorn/workers/ggevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index b89cd4c2..90d5c262 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -79,7 +79,7 @@ class GeventWorker(AsyncWorker): try: # Stop accepting requests - server.kill() + server.close() # Handle current requests until graceful_timeout ts = time.time() From 68b5abc88137e2eb53c94121a1161d499bbc604c Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 11:33:25 +0200 Subject: [PATCH 03/47] some setup enhancements preparing the python 3 release --- setup.py | 51 +++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 567260cf..37feeb61 100644 --- a/setup.py +++ b/setup.py @@ -10,38 +10,42 @@ import sys from gunicorn import __version__ +CLASSIFIERS = [ + 'Development Status :: 4 - Beta', + 'Environment :: Other Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX', + 'Programming Language :: Python', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet', + 'Topic :: Utilities', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: WSGI', + 'Topic :: Internet :: WWW/HTTP :: WSGI :: Server', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content'] + +# read long description +with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: + long_description = f.read() + setup( name = 'gunicorn', version = __version__, description = 'WSGI HTTP Server for UNIX', - long_description = file( - os.path.join( - os.path.dirname(__file__), - 'README.rst' - ) - ).read(), + long_description = long_description, author = 'Benoit Chesneau', author_email = 'benoitc@e-engura.com', license = 'MIT', url = 'http://gunicorn.org', - classifiers = [ - 'Development Status :: 4 - Beta', - 'Environment :: Other Environment', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX', - 'Programming Language :: Python', - 'Topic :: Internet', - 'Topic :: Utilities', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Internet :: WWW/HTTP :: WSGI', - 'Topic :: Internet :: WWW/HTTP :: WSGI :: Server', - 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', - ], + classifiers = CLASSIFIERS, zip_safe = False, packages = find_packages(exclude=['examples', 'tests']), include_package_data = True, @@ -66,6 +70,5 @@ setup( [paste.server_runner] main=gunicorn.app.pasterapp:paste_server - """, - test_suite = 'nose.collector', + """ ) From e984008010a1e7bf2fa130be2f7666e910c1ce18 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 11:44:42 +0200 Subject: [PATCH 04/47] add rss feed to the homepage. fix #146 --- docs/site/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/site/index.html b/docs/site/index.html index 8aef473b..6a8380a2 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -6,6 +6,10 @@ + +
From 53ce50bc7b089e5eb2d90e78bf9fa2df2cef52c9 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 12:11:15 +0200 Subject: [PATCH 05/47] obvious syntax fixes preparing python3 support --- .../django/djangotest/testing/views.py | 24 +++++++++---------- .../testing/apps/someapp/middleware.py | 2 +- .../testing/testing/apps/someapp/views.py | 6 ++--- examples/longpoll.py | 8 ++++--- examples/multiapp.py | 6 ++--- examples/slowclient.py | 9 +++---- examples/test.py | 3 +-- gunicorn/app/base.py | 6 ++--- gunicorn/app/pasterapp.py | 2 +- gunicorn/arbiter.py | 14 +++++------ gunicorn/debug.py | 4 ++-- gunicorn/http/message.py | 2 +- gunicorn/logging_config.py | 2 +- gunicorn/pidfile.py | 4 ++-- gunicorn/sock.py | 6 ++--- gunicorn/util.py | 4 ++-- gunicorn/workers/async.py | 14 +++++------ 17 files changed, 59 insertions(+), 57 deletions(-) diff --git a/examples/frameworks/django/djangotest/testing/views.py b/examples/frameworks/django/djangotest/testing/views.py index 780cfdc2..482571cf 100755 --- a/examples/frameworks/django/djangotest/testing/views.py +++ b/examples/frameworks/django/djangotest/testing/views.py @@ -12,18 +12,18 @@ class MsgForm(forms.Form): subject = forms.CharField(max_length=100) message = forms.CharField() f = forms.FileField() - + def home(request): from django.conf import settings - print settings.SOME_VALUE + print(settings.SOME_VALUE) subject = None message = None size = 0 - print request.META + print(request.META) if request.POST: form = MsgForm(request.POST, request.FILES) - print request.FILES + print(request.FILES) if form.is_valid(): subject = form.cleaned_data['subject'] message = form.cleaned_data['message'] @@ -31,29 +31,29 @@ def home(request): size = int(os.fstat(f.fileno())[6]) else: form = MsgForm() - - + + return render_to_response('home.html', { 'form': form, 'subject': subject, 'message': message, 'size': size }, RequestContext(request)) - - + + def acsv(request): rows = [ {'a': 1, 'b': 2}, {'a': 3, 'b': 3} ] - + response = HttpResponse(mimetype='text/csv') response['Content-Disposition'] = 'attachment; filename=report.csv' - + writer = csv.writer(response) writer.writerow(['a', 'b']) - + for r in rows: writer.writerow([r['a'], r['b']]) - + return response diff --git a/examples/frameworks/django/testing/testing/apps/someapp/middleware.py b/examples/frameworks/django/testing/testing/apps/someapp/middleware.py index 3fe5a825..c346e54f 100644 --- a/examples/frameworks/django/testing/testing/apps/someapp/middleware.py +++ b/examples/frameworks/django/testing/testing/apps/someapp/middleware.py @@ -4,7 +4,7 @@ import gevent def child_process(queue): while True: - print queue.get() + print(queue.get()) requests.get('http://requestb.in/15s95oz1') class GunicornSubProcessTestMiddleware(object): diff --git a/examples/frameworks/django/testing/testing/apps/someapp/views.py b/examples/frameworks/django/testing/testing/apps/someapp/views.py index 9ad0046d..d685b5bc 100755 --- a/examples/frameworks/django/testing/testing/apps/someapp/views.py +++ b/examples/frameworks/django/testing/testing/apps/someapp/views.py @@ -16,14 +16,14 @@ class MsgForm(forms.Form): def home(request): from django.conf import settings - print settings.SOME_VALUE + print(settings.SOME_VALUE) subject = None message = None size = 0 - print request.META + print(request.META) if request.POST: form = MsgForm(request.POST, request.FILES) - print request.FILES + print(request.FILES) if form.is_valid(): subject = form.cleaned_data['subject'] message = form.cleaned_data['message'] diff --git a/examples/longpoll.py b/examples/longpoll.py index 0bbbb283..19559d1b 100644 --- a/examples/longpoll.py +++ b/examples/longpoll.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import sys import time class TestIter(object): - + def __iter__(self): lines = ['line 1\n', 'line 2\n'] for line in lines: @@ -22,6 +23,7 @@ def app(environ, start_response): ('Content-type','text/plain'), ('Transfer-Encoding', "chunked"), ] - print 'request received' + sys.stdout.write('request received') + sys.stdout.flush() start_response(status, response_headers) return TestIter() diff --git a/examples/multiapp.py b/examples/multiapp.py index 340fa7cc..e48a253d 100644 --- a/examples/multiapp.py +++ b/examples/multiapp.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. # # Run this application with: @@ -17,7 +17,7 @@ try: from routes import Mapper except: - print "This example requires Routes to be installed" + print("This example requires Routes to be installed") # Obviously you'd import your app callables # from different places... @@ -55,4 +55,4 @@ class Application(object): start_response('404 Not Found', headers) return [html] -app = Application() \ No newline at end of file +app = Application() diff --git a/examples/slowclient.py b/examples/slowclient.py index b4bc2019..6f612bea 100644 --- a/examples/slowclient.py +++ b/examples/slowclient.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. - +import sys import time @@ -14,7 +14,8 @@ def app(environ, start_response): response_headers = [ ('Content-type','text/plain'), ('Content-Length', str(len(data))) ] - print 'request received, pausing 10 seconds' + sys.stdout.write('request received, pausing 10 seconds') + sys.stdout.flush() time.sleep(10) start_response(status, response_headers) - return iter([data]) \ No newline at end of file + return iter([data]) diff --git a/examples/test.py b/examples/test.py index cf55bf0d..8972f68d 100644 --- a/examples/test.py +++ b/examples/test.py @@ -14,8 +14,7 @@ def app(environ, start_response): """Simplest possible application object""" data = 'Hello, World!\n' status = '200 OK' -# print("print to stdout in test app") -# sys.stderr.write("stderr, print to stderr in test app\n") + response_headers = [ ('Content-type','text/plain'), ('Content-Length', str(len(data))), diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index d6917e72..443bf894 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -31,7 +31,7 @@ class Application(object): def do_load_config(self): try: self.load_config() - except Exception, e: + except Exception as e: sys.stderr.write("\nError: %s\n" % str(e)) sys.stderr.flush() sys.exit(1) @@ -64,7 +64,7 @@ class Application(object): try: execfile(opts.config, cfg, cfg) except Exception: - print "Failed to read config file: %s" % opts.config + print("Failed to read config file: %s" % opts.config) traceback.print_exc() sys.exit(1) @@ -122,7 +122,7 @@ class Application(object): try: Arbiter(self).run() - except RuntimeError, e: + except RuntimeError as e: sys.stderr.write("\nError: %s\n\n" % e) sys.stderr.flush() sys.exit(1) diff --git a/gunicorn/app/pasterapp.py b/gunicorn/app/pasterapp.py index df014723..2c7a2cdb 100644 --- a/gunicorn/app/pasterapp.py +++ b/gunicorn/app/pasterapp.py @@ -118,7 +118,7 @@ class PasterServerApplication(PasterBaseApplication): for k, v in cfg.items(): if k.lower() in self.cfg.settings and v is not None: self.cfg.set(k.lower(), v) - except Exception, e: + except Exception as e: sys.stderr.write("\nConfig error: %s\n" % str(e)) sys.stderr.flush() sys.exit(1) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 1e7d829a..4e0d665e 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -181,7 +181,7 @@ class Arbiter(object): self.halt() except KeyboardInterrupt: self.halt() - except HaltServer, inst: + except HaltServer as inst: self.halt(reason=inst.reason, exit_status=inst.exit_status) except SystemExit: raise @@ -271,7 +271,7 @@ class Arbiter(object): """ try: os.write(self.PIPE[1], '.') - except IOError, e: + except IOError as e: if e.errno not in [errno.EAGAIN, errno.EINTR]: raise @@ -296,10 +296,10 @@ class Arbiter(object): return while os.read(self.PIPE[0], 1): pass - except select.error, e: - if e[0] not in [errno.EAGAIN, errno.EINTR]: + except select.error as e: + if e.args[0] not in [errno.EAGAIN, errno.EINTR]: raise - except OSError, e: + except OSError as e: if e.errno not in [errno.EAGAIN, errno.EINTR]: raise except KeyboardInterrupt: @@ -423,7 +423,7 @@ class Arbiter(object): if not worker: continue worker.tmp.close() - except OSError, e: + except OSError as e: if e.errno == errno.ECHILD: pass @@ -504,7 +504,7 @@ class Arbiter(object): """ try: os.kill(pid, sig) - except OSError, e: + except OSError as e: if e.errno == errno.ESRCH: try: worker = self.WORKERS.pop(pid) diff --git a/gunicorn/debug.py b/gunicorn/debug.py index 3f342448..52cda798 100644 --- a/gunicorn/debug.py +++ b/gunicorn/debug.py @@ -43,7 +43,7 @@ class Spew(object): line = 'Unknown code named [%s]. VM instruction #%d' % ( frame.f_code.co_name, frame.f_lasti) if self.trace_names is None or name in self.trace_names: - print '%s:%s: %s' % (name, lineno, line.rstrip()) + print('%s:%s: %s' % (name, lineno, line.rstrip())) if not self.show_values: return self details = [] @@ -54,7 +54,7 @@ class Spew(object): if tok in frame.f_locals: details.append('%s=%r' % (tok, frame.f_locals[tok])) if details: - print "\t%s" % ' '.join(details) + print("\t%s" % ' '.join(details)) return self diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index ab3575fc..d24ea5d0 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -256,7 +256,7 @@ class Request(Message): try: remote_host = self.unreader.sock.getpeername()[0] except socket.error as e: - if e[0] == ENOTCONN: + if e.args[0] == ENOTCONN: raise ForbiddenProxyRequest("UNKNOW") raise if remote_host not in self.cfg.proxy_allow_ips: diff --git a/gunicorn/logging_config.py b/gunicorn/logging_config.py index f1dd6dd8..14e7fe63 100644 --- a/gunicorn/logging_config.py +++ b/gunicorn/logging_config.py @@ -286,7 +286,7 @@ def listen(port=DEFAULT_LOGGING_CONFIG_PORT): except: traceback.print_exc() os.remove(file) - except socket.error, e: + except socket.error as e: if type(e.args) != types.TupleType: raise else: diff --git a/gunicorn/pidfile.py b/gunicorn/pidfile.py index d04f941e..5fa38995 100644 --- a/gunicorn/pidfile.py +++ b/gunicorn/pidfile.py @@ -76,11 +76,11 @@ class Pidfile(object): try: os.kill(wpid, 0) return wpid - except OSError, e: + except OSError as e: if e[0] == errno.ESRCH: return raise - except IOError, e: + except IOError as e: if e[0] == errno.ENOENT: return raise diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 9f0cc480..e172ceb2 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -44,7 +44,7 @@ class BaseSocket(object): def close(self): try: self.sock.close() - except socket.error, e: + except socket.error as e: self.log.info("Error while closing socket %s", str(e)) time.sleep(0.3) del self.sock @@ -117,7 +117,7 @@ def create_socket(conf, log): fd = int(os.environ.pop('GUNICORN_FD')) try: return sock_type(conf, log, fd=fd) - except socket.error, e: + except socket.error as e: if e[0] == errno.ENOTCONN: log.error("GUNICORN_FD should refer to an open socket.") else: @@ -130,7 +130,7 @@ def create_socket(conf, log): for i in range(5): try: return sock_type(conf, log) - except socket.error, e: + except socket.error as e: if e[0] == errno.EADDRINUSE: log.error("Connection in use: %s", str(addr)) if e[0] == errno.EADDRNOTAVAIL: diff --git a/gunicorn/util.py b/gunicorn/util.py index e919d53c..e6fdb5ed 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -125,7 +125,7 @@ def load_class(uri, default="sync", section="gunicorn.workers"): return pkg_resources.load_entry_point("gunicorn", section, uri) - except ImportError, e: + except ImportError as e: raise RuntimeError("class uri invalid or not found: " + "[%s]" % str(e)) klass = components.pop(-1) @@ -350,6 +350,6 @@ def seed(): def check_is_writeable(path): try: f = open(path, 'a') - except IOError, e: + except IOError as e: raise RuntimeError("Error: '%s' isn't writable [%r]" % (path, e)) f.close() diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index 65b16981..08fed0ee 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -43,25 +43,25 @@ class AsyncWorker(base.Worker): if not req: break self.handle_request(req, client, addr) - except http.errors.NoMoreData, e: + except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) - except StopIteration, e: + except StopIteration as e: self.log.debug("Closing connection. %s", e) except socket.error: raise # pass to next try-except level - except Exception, e: + except Exception as e: self.handle_error(req, client, addr, e) except socket.timeout as e: self.handle_error(req, client, addr, e) - except socket.error, e: - if e[0] not in (errno.EPIPE, errno.ECONNRESET): + except socket.error as e: + if e.args[0] not in (errno.EPIPE, errno.ECONNRESET): self.log.exception("Socket error processing request.") else: - if e[0] == errno.ECONNRESET: + if e.args[0] == errno.ECONNRESET: self.log.debug("Ignoring connection reset") else: self.log.debug("Ignoring EPIPE") - except Exception, e: + except Exception as e: self.handle_error(req, client, addr, e) finally: util.close(client) From 7c579f6ca4674745de2a81c785540c483a3c2538 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 12:12:52 +0200 Subject: [PATCH 06/47] support >= 2.6 we don't need anymore this port --- gunicorn/glogging.py | 6 +- gunicorn/logging_config.py | 346 ------------------------------------- 2 files changed, 1 insertion(+), 351 deletions(-) delete mode 100644 gunicorn/logging_config.py diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 027c048d..d554a207 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -11,11 +11,7 @@ import sys import traceback import threading -try: - from logging.config import fileConfig -except ImportError: - from gunicorn.logging_config import fileConfig - +from logging.config import fileConfig from gunicorn import util CONFIG_DEFAULTS = dict( diff --git a/gunicorn/logging_config.py b/gunicorn/logging_config.py deleted file mode 100644 index 14e7fe63..00000000 --- a/gunicorn/logging_config.py +++ /dev/null @@ -1,346 +0,0 @@ -# -*- coding: utf-8 - -# -# This file is part of gunicorn released under the MIT license. -# See the NOTICE for more information. -# -# Copyright 2001-2005 by Vinay Sajip. All Rights Reserved. -# - -""" -Configuration functions for the logging package for Python. The core package -is based on PEP 282 and comments thereto in comp.lang.python, and influenced -by Apache's log4j system. - -Should work under Python versions >= 1.5.2, except that source line -information is not available unless 'sys._getframe()' is. - -Copyright (C) 2001-2004 Vinay Sajip. All Rights Reserved. - -To use, simply 'import logging' and log away! -""" - -import sys, logging, logging.handlers, string, socket, struct, os, traceback, types - -try: - import thread - import threading -except ImportError: - thread = None - -from SocketServer import ThreadingTCPServer, StreamRequestHandler - - -DEFAULT_LOGGING_CONFIG_PORT = 9030 - -if sys.platform == "win32": - RESET_ERROR = 10054 #WSAECONNRESET -else: - RESET_ERROR = 104 #ECONNRESET - -# -# The following code implements a socket listener for on-the-fly -# reconfiguration of logging. -# -# _listener holds the server object doing the listening -_listener = None - -def fileConfig(fname, defaults=None): - """ - Read the logging configuration from a ConfigParser-format file. - - This can be called several times from an application, allowing an end user - the ability to select from various pre-canned configurations (if the - developer provides a mechanism to present the choices and load the chosen - configuration). - In versions of ConfigParser which have the readfp method [typically - shipped in 2.x versions of Python], you can pass in a file-like object - rather than a filename, in which case the file-like object will be read - using readfp. - """ - import ConfigParser - - cp = ConfigParser.ConfigParser(defaults) - if hasattr(cp, 'readfp') and hasattr(fname, 'readline'): - cp.readfp(fname) - else: - cp.read(fname) - - formatters = _create_formatters(cp) - - # critical section - logging._acquireLock() - try: - logging._handlers.clear() - if hasattr(logging, '_handlerList'): - del logging._handlerList[:] - # Handlers add themselves to logging._handlers - handlers = _install_handlers(cp, formatters) - _install_loggers(cp, handlers) - finally: - logging._releaseLock() - - -def _resolve(name): - """Resolve a dotted name to a global object.""" - name = string.split(name, '.') - used = name.pop(0) - found = __import__(used) - for n in name: - used = used + '.' + n - try: - found = getattr(found, n) - except AttributeError: - __import__(used) - found = getattr(found, n) - return found - - -def _create_formatters(cp): - """Create and return formatters""" - flist = cp.get("formatters", "keys") - if not len(flist): - return {} - flist = string.split(flist, ",") - formatters = {} - for form in flist: - form = string.strip(form) - sectname = "formatter_%s" % form - opts = cp.options(sectname) - if "format" in opts: - fs = cp.get(sectname, "format", 1) - else: - fs = None - if "datefmt" in opts: - dfs = cp.get(sectname, "datefmt", 1) - else: - dfs = None - c = logging.Formatter - if "class" in opts: - class_name = cp.get(sectname, "class") - if class_name: - c = _resolve(class_name) - f = c(fs, dfs) - formatters[form] = f - return formatters - - -def _install_handlers(cp, formatters): - """Install and return handlers""" - hlist = cp.get("handlers", "keys") - if not len(hlist): - return {} - hlist = string.split(hlist, ",") - handlers = {} - fixups = [] #for inter-handler references - for hand in hlist: - hand = string.strip(hand) - sectname = "handler_%s" % hand - klass = cp.get(sectname, "class") - opts = cp.options(sectname) - if "formatter" in opts: - fmt = cp.get(sectname, "formatter") - else: - fmt = "" - try: - klass = eval(klass, vars(logging)) - except (AttributeError, NameError): - klass = _resolve(klass) - args = cp.get(sectname, "args") - args = eval(args, vars(logging)) - h = apply(klass, args) - if "level" in opts: - level = cp.get(sectname, "level") - h.setLevel(logging._levelNames[level]) - if len(fmt): - h.setFormatter(formatters[fmt]) - #temporary hack for FileHandler and MemoryHandler. - if klass == logging.handlers.MemoryHandler: - if "target" in opts: - target = cp.get(sectname,"target") - else: - target = "" - if len(target): #the target handler may not be loaded yet, so keep for later... - fixups.append((h, target)) - handlers[hand] = h - #now all handlers are loaded, fixup inter-handler references... - for h, t in fixups: - h.setTarget(handlers[t]) - return handlers - - -def _install_loggers(cp, handlers): - """Create and install loggers""" - - # configure the root first - llist = cp.get("loggers", "keys") - llist = string.split(llist, ",") - llist = map(lambda x: string.strip(x), llist) - llist.remove("root") - sectname = "logger_root" - root = logging.root - log = root - opts = cp.options(sectname) - if "level" in opts: - level = cp.get(sectname, "level") - log.setLevel(logging._levelNames[level]) - for h in root.handlers[:]: - root.removeHandler(h) - hlist = cp.get(sectname, "handlers") - if len(hlist): - hlist = string.split(hlist, ",") - for hand in hlist: - log.addHandler(handlers[string.strip(hand)]) - - #and now the others... - #we don't want to lose the existing loggers, - #since other threads may have pointers to them. - #existing is set to contain all existing loggers, - #and as we go through the new configuration we - #remove any which are configured. At the end, - #what's left in existing is the set of loggers - #which were in the previous configuration but - #which are not in the new configuration. - existing = root.manager.loggerDict.keys() - #now set up the new ones... - for log in llist: - sectname = "logger_%s" % log - qn = cp.get(sectname, "qualname") - opts = cp.options(sectname) - if "propagate" in opts: - propagate = cp.getint(sectname, "propagate") - else: - propagate = 1 - logger = logging.getLogger(qn) - if qn in existing: - existing.remove(qn) - if "level" in opts: - level = cp.get(sectname, "level") - logger.setLevel(logging._levelNames[level]) - for h in logger.handlers[:]: - logger.removeHandler(h) - logger.propagate = propagate - logger.disabled = 0 - hlist = cp.get(sectname, "handlers") - if len(hlist): - hlist = string.split(hlist, ",") - for hand in hlist: - logger.addHandler(handlers[string.strip(hand)]) - - #Disable any old loggers. There's no point deleting - #them as other threads may continue to hold references - #and by disabling them, you stop them doing any logging. - for log in existing: - root.manager.loggerDict[log].disabled = 1 - - -def listen(port=DEFAULT_LOGGING_CONFIG_PORT): - """ - Start up a socket server on the specified port, and listen for new - configurations. - - These will be sent as a file suitable for processing by fileConfig(). - Returns a Thread object on which you can call start() to start the server, - and which you can join() when appropriate. To stop the server, call - stopListening(). - """ - if not thread: - raise NotImplementedError, "listen() needs threading to work" - - class ConfigStreamHandler(StreamRequestHandler): - """ - Handler for a logging configuration request. - - It expects a completely new logging configuration and uses fileConfig - to install it. - """ - def handle(self): - """ - Handle a request. - - Each request is expected to be a 4-byte length, packed using - struct.pack(">L", n), followed by the config file. - Uses fileConfig() to do the grunt work. - """ - import tempfile - try: - conn = self.connection - chunk = conn.recv(4) - if len(chunk) == 4: - slen = struct.unpack(">L", chunk)[0] - chunk = self.connection.recv(slen) - while len(chunk) < slen: - chunk = chunk + conn.recv(slen - len(chunk)) - #Apply new configuration. We'd like to be able to - #create a StringIO and pass that in, but unfortunately - #1.5.2 ConfigParser does not support reading file - #objects, only actual files. So we create a temporary - #file and remove it later. - file = tempfile.mktemp(".ini") - f = open(file, "w") - f.write(chunk) - f.close() - try: - fileConfig(file) - except (KeyboardInterrupt, SystemExit): - raise - except: - traceback.print_exc() - os.remove(file) - except socket.error as e: - if type(e.args) != types.TupleType: - raise - else: - errcode = e.args[0] - if errcode != RESET_ERROR: - raise - - class ConfigSocketReceiver(ThreadingTCPServer): - """ - A simple TCP socket-based logging config receiver. - """ - - allow_reuse_address = 1 - - def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, - handler=None): - ThreadingTCPServer.__init__(self, (host, port), handler) - logging._acquireLock() - self.abort = 0 - logging._releaseLock() - self.timeout = 1 - - def serve_until_stopped(self): - import select - abort = 0 - while not abort: - rd, wr, ex = select.select([self.socket.fileno()], - [], [], - self.timeout) - if rd: - self.handle_request() - logging._acquireLock() - abort = self.abort - logging._releaseLock() - - def serve(rcvr, hdlr, port): - server = rcvr(port=port, handler=hdlr) - global _listener - logging._acquireLock() - _listener = server - logging._releaseLock() - server.serve_until_stopped() - - return threading.Thread(target=serve, - args=(ConfigSocketReceiver, - ConfigStreamHandler, port)) - -def stopListening(): - """ - Stop the listening server which was created with a call to listen(). - """ - global _listener - if _listener: - logging._acquireLock() - _listener.abort = 1 - _listener = None - logging._releaseLock() From 5759d59f086a34ca815b38ed37c65029f362dbbb Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 12:18:16 +0200 Subject: [PATCH 07/47] add six modules inside gunicorn --- gunicorn/six.py | 366 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 366 insertions(+) create mode 100644 gunicorn/six.py diff --git a/gunicorn/six.py b/gunicorn/six.py new file mode 100644 index 00000000..44b80a44 --- /dev/null +++ b/gunicorn/six.py @@ -0,0 +1,366 @@ +"""Utilities for writing code that runs on Python 2 and 3""" + +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.2.0" + + +# True if we are running on Python 3. +PY3 = sys.version_info[0] == 3 + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform == "java": + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) + # This is a bit ugly, but it avoids running this again. + delattr(tp, self.name) + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + + +class _MovedItems(types.ModuleType): + """Lazy loading of moved objects""" + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("reload_module", "__builtin__", "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("winreg", "_winreg"), +] +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) +del attr + +moves = sys.modules["gunicorn.six.moves"] = _MovedItems("moves") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_code = "__code__" + _func_defaults = "__defaults__" + + _iterkeys = "keys" + _itervalues = "values" + _iteritems = "items" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_code = "func_code" + _func_defaults = "func_defaults" + + _iterkeys = "iterkeys" + _itervalues = "itervalues" + _iteritems = "iteritems" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +if PY3: + def get_unbound_function(unbound): + return unbound + + Iterator = object + + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) +else: + def get_unbound_function(unbound): + return unbound.im_func + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) + + +def iterkeys(d): + """Return an iterator over the keys of a dictionary.""" + return iter(getattr(d, _iterkeys)()) + +def itervalues(d): + """Return an iterator over the values of a dictionary.""" + return iter(getattr(d, _itervalues)()) + +def iteritems(d): + """Return an iterator over the (key, value) pairs of a dictionary.""" + return iter(getattr(d, _iteritems)()) + + +if PY3: + def b(s): + return s.encode("latin-1") + def u(s): + return s + if sys.version_info[1] <= 1: + def int2byte(i): + return bytes((i,)) + else: + # This is about 2x faster than the implementation above on 3.2+ + int2byte = operator.methodcaller("to_bytes", 1, "big") + import io + StringIO = io.StringIO + BytesIO = io.BytesIO +else: + def b(s): + return s + def u(s): + return unicode(s, "unicode_escape") + int2byte = chr + import StringIO + StringIO = BytesIO = StringIO.StringIO +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +if PY3: + import builtins + exec_ = getattr(builtins, "exec") + + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + + print_ = getattr(builtins, "print") + del builtins + +else: + def exec_(code, globs=None, locs=None): + """Execute code in a namespace.""" + if globs is None: + frame = sys._getframe(1) + globs = frame.f_globals + if locs is None: + locs = frame.f_locals + del frame + elif locs is None: + locs = globs + exec("""exec code in globs, locs""") + + + exec_("""def reraise(tp, value, tb=None): + raise tp, value, tb +""") + + + def print_(*args, **kwargs): + """The new-style print function.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + def write(data): + if not isinstance(data, basestring): + data = str(data) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) + +_add_doc(reraise, """Reraise an exception.""") + + +def with_metaclass(meta, base=object): + """Create a base class with a metaclass.""" + return meta("NewBase", (base,), {}) From 003c474fe2270fe67393a630516e2db659cd3918 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 13:51:35 +0200 Subject: [PATCH 08/47] handle bytes & native strings This patch makes sure that we now handle correctly bytes and native strings on python 3: - In python 3, sockets are now taking and returning bytes. - according to PEP3333, headers should be native strings and body in bytes. --- examples/longpoll.py | 2 +- examples/multiapp.py | 2 +- examples/multidomainapp.py | 8 +++---- examples/slowclient.py | 2 +- examples/test.py | 2 +- gunicorn/http/body.py | 46 +++++++++++++++++--------------------- gunicorn/http/message.py | 27 +++++++++------------- gunicorn/http/wsgi.py | 19 +++++++++++----- gunicorn/six.py | 34 ++++++++++++++++++++++++++++ gunicorn/util.py | 10 ++++++--- 10 files changed, 94 insertions(+), 58 deletions(-) diff --git a/examples/longpoll.py b/examples/longpoll.py index 19559d1b..97d6647f 100644 --- a/examples/longpoll.py +++ b/examples/longpoll.py @@ -17,7 +17,7 @@ class TestIter(object): def app(environ, start_response): """Application which cooperatively pauses 20 seconds (needed to surpass normal timeouts) before responding""" - data = 'Hello, World!\n' + data = b'Hello, World!\n' status = '200 OK' response_headers = [ ('Content-type','text/plain'), diff --git a/examples/multiapp.py b/examples/multiapp.py index e48a253d..c6a4c90b 100644 --- a/examples/multiapp.py +++ b/examples/multiapp.py @@ -38,7 +38,7 @@ class Application(object): return match[0]['app'](environ, start_response) def error404(self, environ, start_response): - html = """\ + html = b"""\ 404 - Not Found diff --git a/examples/multidomainapp.py b/examples/multidomainapp.py index 89e59e2b..948a5359 100644 --- a/examples/multidomainapp.py +++ b/examples/multidomainapp.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. import re @@ -20,15 +20,15 @@ class SubDomainApp: return app(environ, start_response) else: start_response("404 Not Found", []) - return [""] + return [b""] def hello(environ, start_response): start_response("200 OK", [("Content-Type", "text/plain")]) - return ["Hello, world\n"] + return [b"Hello, world\n"] def bye(environ, start_response): start_response("200 OK", [("Content-Type", "text/plain")]) - return ["Goodbye!\n"] + return [b"Goodbye!\n"] app = SubDomainApp([ ("localhost", hello), diff --git a/examples/slowclient.py b/examples/slowclient.py index 6f612bea..15729d93 100644 --- a/examples/slowclient.py +++ b/examples/slowclient.py @@ -9,7 +9,7 @@ import time def app(environ, start_response): """Application which cooperatively pauses 10 seconds before responding""" - data = 'Hello, World!\n' + data = b'Hello, World!\n' status = '200 OK' response_headers = [ ('Content-type','text/plain'), diff --git a/examples/test.py b/examples/test.py index 8972f68d..c61f672e 100644 --- a/examples/test.py +++ b/examples/test.py @@ -12,7 +12,7 @@ from gunicorn import __version__ #@validator def app(environ, start_response): """Simplest possible application object""" - data = 'Hello, World!\n' + data = b'Hello, World!\n' status = '200 OK' response_headers = [ diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index b331f292..82797e4c 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -5,13 +5,9 @@ import sys -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -from gunicorn.http.errors import NoMoreData, ChunkMissingTerminator, \ -InvalidChunkSize +from gunicorn.http.errors import (NoMoreData, ChunkMissingTerminator, + InvalidChunkSize) +from gunicorn.six import StringIO, bytes_to_str class ChunkedReader(object): def __init__(self, req, unreader): @@ -25,7 +21,7 @@ class ChunkedReader(object): if size <= 0: raise ValueError("Size must be positive.") if size == 0: - return "" + return b"" if self.parser: while self.buf.tell() < size: @@ -45,16 +41,17 @@ class ChunkedReader(object): buf = StringIO() buf.write(data) - idx = buf.getvalue().find("\r\n\r\n") - done = buf.getvalue()[:2] == "\r\n" + idx = buf.getvalue().find(b"\r\n\r\n") + done = buf.getvalue()[:2] == b"\r\n" while idx < 0 and not done: self.get_data(unreader, buf) - idx = buf.getvalue().find("\r\n\r\n") - done = buf.getvalue()[:2] == "\r\n" + idx = buf.getvalue().find(b"\r\n\r\n") + done = buf.getvalue()[:2] == b"\r\n" if done: unreader.unread(buf.getvalue()[2:]) - return "" - self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx]) + return b"" + self.req.trailers = self.req.parse_headers( + bytes_to_str(buf.getvalue()[:idx])) unreader.unread(buf.getvalue()[idx+4:]) def parse_chunked(self, unreader): @@ -71,7 +68,7 @@ class ChunkedReader(object): rest = rest[size:] while len(rest) < 2: rest += unreader.read() - if rest[:2] != '\r\n': + if rest[:2] != b'\r\n': raise ChunkMissingTerminator(rest[:2]) (size, rest) = self.parse_chunk_size(unreader, data=rest[2:]) @@ -80,15 +77,15 @@ class ChunkedReader(object): if data is not None: buf.write(data) - idx = buf.getvalue().find("\r\n") + idx = buf.getvalue().find(b"\r\n") while idx < 0: self.get_data(unreader, buf) - idx = buf.getvalue().find("\r\n") + idx = buf.getvalue().find(b"\r\n") data = buf.getvalue() line, rest_chunk = data[:idx], data[idx+2:] - chunk_size = line.split(";", 1)[0].strip() + chunk_size = line.split(b";", 1)[0].strip() try: chunk_size = int(chunk_size, 16) except ValueError: @@ -121,7 +118,7 @@ class LengthReader(object): if size < 0: raise ValueError("Size must be positive.") if size == 0: - return "" + return b"" buf = StringIO() @@ -201,7 +198,7 @@ class Body(object): def read(self, size=None): size = self.getsize(size) if size == 0: - return "" + return b"" if size < self.buf.tell(): data = self.buf.getvalue() @@ -225,7 +222,7 @@ class Body(object): def readline(self, size=None): size = self.getsize(size) if size == 0: - return "" + return b"" line = self.buf.getvalue() self.buf.truncate(0) @@ -234,7 +231,7 @@ class Body(object): extra_buf_data = line[size:] line = line[:size] - idx = line.find("\n") + idx = line.find(b"\n") if idx >= 0: ret = line[:idx+1] self.buf.write(line[idx+1:]) @@ -247,12 +244,11 @@ class Body(object): ret = [] data = self.read() while len(data): - pos = data.find("\n") + pos = data.find(b"\n") if pos < 0: ret.append(data) - data = "" + data = b"" else: line, data = data[:pos+1], data[pos+1:] ret.append(line) return ret - diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index d24ea5d0..b32dbc2b 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -4,21 +4,16 @@ # See the NOTICE for more information. import re -import urlparse import socket from errno import ENOTCONN -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - from gunicorn.http.unreader import SocketUnreader from gunicorn.http.body import ChunkedReader, LengthReader, EOFReader, Body from gunicorn.http.errors import InvalidHeader, InvalidHeaderName, NoMoreData, \ InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, \ LimitRequestLine, LimitRequestHeaders from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest +from gunicorn.six import StringIO, urlsplit, bytes_to_str MAX_REQUEST_LINE = 8190 MAX_HEADERS = 32768 @@ -61,7 +56,7 @@ class Message(object): headers = [] # Split lines on \r\n keeping the \r\n on each line - lines = [line + "\r\n" for line in data.split("\r\n")] + lines = [bytes_to_str(line) + "\r\n" for line in data.split(b"\r\n")] # Parse headers into key/value pairs paying attention # to continuation lines. @@ -173,24 +168,24 @@ class Request(Message): line, rbuf = self.read_line(unreader, buf, self.limit_request_line) # proxy protocol - if self.proxy_protocol(line): + if self.proxy_protocol(bytes_to_str(line)): # get next request line buf = StringIO() buf.write(rbuf) line, rbuf = self.read_line(unreader, buf, self.limit_request_line) - self.parse_request_line(line) + self.parse_request_line(bytes_to_str(line)) buf = StringIO() buf.write(rbuf) # Headers data = buf.getvalue() - idx = data.find("\r\n\r\n") + idx = data.find(b"\r\n\r\n") - done = data[:2] == "\r\n" + done = data[:2] == b"\r\n" while True: - idx = data.find("\r\n\r\n") - done = data[:2] == "\r\n" + idx = data.find(b"\r\n\r\n") + done = data[:2] == b"\r\n" if idx < 0 and not done: self.get_data(unreader, buf) @@ -202,7 +197,7 @@ class Request(Message): if done: self.unreader.unread(data[2:]) - return "" + return b"" self.headers = self.parse_headers(data[:idx]) @@ -214,7 +209,7 @@ class Request(Message): data = buf.getvalue() while True: - idx = data.find("\r\n") + idx = data.find(b"\r\n") if idx >= 0: # check if the request line is too large if idx > limit > 0: @@ -328,7 +323,7 @@ class Request(Message): else: self.uri = bits[1] - parts = urlparse.urlsplit(self.uri) + parts = urlsplit(self.uri) self.path = parts.path or "" self.query = parts.query or "" self.fragment = parts.fragment or "" diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 228ed941..026a9c98 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -7,8 +7,8 @@ import logging import os import re import sys -from urllib import unquote +from gunicorn.six import unquote, string_types, binary_type from gunicorn import SERVER_SOFTWARE import gunicorn.util as util @@ -265,12 +265,18 @@ class Response(object): return tosend = self.default_headers() tosend.extend(["%s: %s\r\n" % (n, v) for n, v in self.headers]) - util.write(self.sock, "%s\r\n" % "".join(tosend)) + + header_str = "%s\r\n" % "".join(tosend) + util.write(self.sock, header_str.encode('latin1')) self.headers_sent = True def write(self, arg): self.send_headers() - assert isinstance(arg, basestring), "%r is not a string." % arg + + if isinstance(arg, text_type): + arg = arg.decode('utf-8') + + assert isinstance(arg, binary_type), "%r is not a byte." % arg arglen = len(arg) tosend = arglen @@ -328,12 +334,13 @@ class Response(object): self.send_headers() if self.is_chunked(): - self.sock.sendall("%X\r\n" % nbytes) + chunk_size = "%X\r\n" % nbytes + self.sock.sendall(chunk_size.encode('utf-8')) self.sendfile_all(fileno, self.sock.fileno(), fo_offset, nbytes) if self.is_chunked(): - self.sock.sendall("\r\n") + self.sock.sendall(b"\r\n") os.lseek(fileno, fd_offset, os.SEEK_SET) else: @@ -344,4 +351,4 @@ class Response(object): if not self.headers_sent: self.send_headers() if self.chunked: - util.write_chunk(self.sock, "") + util.write_chunk(self.sock, b"") diff --git a/gunicorn/six.py b/gunicorn/six.py index 44b80a44..e82ddce2 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -364,3 +364,37 @@ _add_doc(reraise, """Reraise an exception.""") def with_metaclass(meta, base=object): """Create a base class with a metaclass.""" return meta("NewBase", (base,), {}) + + +# specific to gunicorn +if PY3: + import io + StringIO = io.BytesIO + + def bytes_to_str(b): + return str(b, 'latin1') + + import urllib.parse + + unquote = urllib.parse.unquote + urlsplit = urllib.parse.urlsplit + +else: + try: + import cStringIO as StringIO + except ImportError: + import StringIO + + StringIO = StringIO + + + def bytestring(s): + if isinstance(s, unicode): + return s.encode('utf-8') + return s + + import urlparse as orig_urlparse + urlsplit = orig_urlparse.urlsplit + + import urllib + urlunquote = urllib.unquote diff --git a/gunicorn/util.py b/gunicorn/util.py index e6fdb5ed..11731327 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -25,6 +25,7 @@ import textwrap import time import inspect +from gunicorn.six import text_type MAXFD = 1024 if (hasattr(os, "devnull")): @@ -223,7 +224,10 @@ except ImportError: pass def write_chunk(sock, data): - chunk = "".join(("%X\r\n" % len(data), data, "\r\n")) + if instance(data, text_type): + data = data.decode('utf-8') + chunk_size = "%X\r\n" % len(data) + chunk = b"".join([chunk_size.decode('utf-8'), data, b"\r\n"]) sock.sendall(chunk) def write(sock, data, chunked=False): @@ -259,7 +263,7 @@ def write_error(sock, status_int, reason, mesg): """) % {"reason": reason, "mesg": mesg} - http = textwrap.dedent("""\ + headers = textwrap.dedent("""\ HTTP/1.1 %s %s\r Connection: close\r Content-Type: text/html\r @@ -267,7 +271,7 @@ def write_error(sock, status_int, reason, mesg): \r %s """) % (str(status_int), reason, len(html), html) - write_nonblock(sock, http) + write_nonblock(sock, http.encode('latin1')) def normalize_name(name): return "-".join([w.lower().capitalize() for w in name.split("-")]) From 64371a085842e14602a478c6c5ff73ed54f24daf Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 14:01:04 +0200 Subject: [PATCH 09/47] s/raise/reraise --- gunicorn/http/wsgi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 026a9c98..9981389f 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -8,7 +8,7 @@ import os import re import sys -from gunicorn.six import unquote, string_types, binary_type +from gunicorn.six import unquote, string_types, binary_type, reraise from gunicorn import SERVER_SOFTWARE import gunicorn.util as util @@ -195,7 +195,7 @@ class Response(object): if exc_info: try: if self.status and self.headers_sent: - raise exc_info[0], exc_info[1], exc_info[2] + reraise(exc_info[0], exc_info[1], exc_info[2]) finally: exc_info = None elif self.status is not None: From 039bf47c3d14f1f95001a5b7002bd90cc03b1899 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 14:03:18 +0200 Subject: [PATCH 10/47] fix exceptions --- gunicorn/workers/sync.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index baaa8952..bd375d94 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -40,8 +40,8 @@ class SyncWorker(base.Worker): # process. continue - except socket.error, e: - if e[0] not in (errno.EAGAIN, errno.ECONNABORTED): + except socket.error as e: + if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED): raise # If our parent changed then we shut down. @@ -54,10 +54,10 @@ class SyncWorker(base.Worker): ret = select.select([self.socket], [], self.PIPE, self.timeout) if ret[0]: continue - except select.error, e: - if e[0] == errno.EINTR: + except select.error as e: + if e.args[0] == errno.EINTR: continue - if e[0] == errno.EBADF: + if e.args[0] == errno.EBADF: if self.nr < 0: continue else: @@ -71,18 +71,18 @@ class SyncWorker(base.Worker): parser = http.RequestParser(self.cfg, client) req = parser.next() self.handle_request(req, client, addr) - except http.errors.NoMoreData, e: + except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) - except StopIteration, e: + except StopIteration as e: self.log.debug("Closing connection. %s", e) except socket.timeout as e: self.handle_error(req, client, addr, e) - except socket.error, e: - if e[0] != errno.EPIPE: + except socket.error as e: + if e.args[0] != errno.EPIPE: self.log.exception("Error processing request.") else: self.log.debug("Ignoring EPIPE") - except Exception, e: + except Exception as e: self.handle_error(req, client, addr, e) finally: util.close(client) @@ -117,7 +117,7 @@ class SyncWorker(base.Worker): respiter.close() except socket.error: raise - except Exception, e: + except Exception as e: # Only send back traceback in HTTP in debug mode. self.handle_error(req, client, addr, e) return From 60644b12aff0cf494a4a6115ed3e4ad1b21bc98c Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 14:24:19 +0200 Subject: [PATCH 11/47] miscellaneous fixes --- gunicorn/app/base.py | 9 ++------- gunicorn/app/django_wsgi.py | 4 +++- gunicorn/app/djangoapp.py | 2 +- gunicorn/app/pasterapp.py | 6 +++++- gunicorn/config.py | 7 ++++--- gunicorn/glogging.py | 6 ++++-- gunicorn/http/body.py | 10 +++++----- gunicorn/http/unreader.py | 7 ++----- gunicorn/http/wsgi.py | 9 +++++---- gunicorn/six.py | 5 +++++ gunicorn/sock.py | 4 ++-- gunicorn/util.py | 12 ++++++------ 12 files changed, 44 insertions(+), 37 deletions(-) diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index 443bf894..e5d4c996 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -3,17 +3,15 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import errno import os import sys import traceback - -from gunicorn.glogging import Logger from gunicorn import util from gunicorn.arbiter import Arbiter from gunicorn.config import Config from gunicorn import debug +from gunicorn.six import execfile_ class Application(object): """\ @@ -62,7 +60,7 @@ class Application(object): "__package__": None } try: - execfile(opts.config, cfg, cfg) + execfile_(opts.config, cfg, cfg) except Exception: print("Failed to read config file: %s" % opts.config) traceback.print_exc() @@ -104,15 +102,12 @@ class Application(object): def run(self): if self.cfg.check_config: try: - self.load() except: sys.stderr.write("\nError while loading the application:\n\n") traceback.print_exc() - sys.stderr.flush() sys.exit(1) - sys.exit(0) if self.cfg.spew: diff --git a/gunicorn/app/django_wsgi.py b/gunicorn/app/django_wsgi.py index 910dd265..7ec41bf3 100644 --- a/gunicorn/app/django_wsgi.py +++ b/gunicorn/app/django_wsgi.py @@ -10,10 +10,12 @@ import re import sys import time try: - from cStringIO import StringIO + from io import StringIO + from imp import reload except ImportError: from StringIO import StringIO + from django.conf import settings from django.core.management.validation import get_validation_errors from django.utils import translation diff --git a/gunicorn/app/djangoapp.py b/gunicorn/app/djangoapp.py index 0d20da4e..6eb5d574 100644 --- a/gunicorn/app/djangoapp.py +++ b/gunicorn/app/djangoapp.py @@ -49,7 +49,7 @@ def make_default_env(cfg): sys.path.insert(0, pythonpath) try: - _ = os.environ['DJANGO_SETTINGS_MODULE'] + os.environ['DJANGO_SETTINGS_MODULE'] except KeyError: # not settings env set, try to build one. project_path, settings_name = find_settings_module(os.getcwd()) diff --git a/gunicorn/app/pasterapp.py b/gunicorn/app/pasterapp.py index 2c7a2cdb..9a2e132d 100644 --- a/gunicorn/app/pasterapp.py +++ b/gunicorn/app/pasterapp.py @@ -6,7 +6,11 @@ import os import pkg_resources import sys -import ConfigParser + +try: + import configparser as ConfigParser +except ImportError: + import ConfigParser from paste.deploy import loadapp, loadwsgi SERVER = loadwsgi.SERVER diff --git a/gunicorn/config.py b/gunicorn/config.py index c9045cb0..fa3d15bb 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -15,6 +15,7 @@ import types from gunicorn import __version__ from gunicorn.errors import ConfigError from gunicorn import util +from gunicorn.six import string_types KNOWN_SETTINGS = [] @@ -181,7 +182,7 @@ class Setting(object): def validate_bool(val): if isinstance(val, types.BooleanType): return val - if not isinstance(val, basestring): + if not isinstance(val, string_types): raise TypeError("Invalid type for casting: %s" % val) if val.lower().strip() == "true": return True @@ -208,7 +209,7 @@ def validate_pos_int(val): def validate_string(val): if val is None: return None - if not isinstance(val, basestring): + if not isinstance(val, string_types): raise TypeError("Not a string: %s" % val) return val.strip() @@ -229,7 +230,7 @@ def validate_class(val): def validate_callable(arity): def _validate_callable(val): - if isinstance(val, basestring): + if isinstance(val, string_types): try: mod_name, obj_name = val.rsplit(".", 1) except ValueError: diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index d554a207..eaa82708 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -6,13 +6,15 @@ import datetime import logging logging.Logger.manager.emittedNoHandlerWarning = 1 +from logging.config import fileConfig import os import sys import traceback import threading -from logging.config import fileConfig + from gunicorn import util +from gunicorn.six import string_types CONFIG_DEFAULTS = dict( version = 1, @@ -175,7 +177,7 @@ class Logger(object): self.error_log.exception(msg, *args) def log(self, lvl, msg, *args, **kwargs): - if isinstance(lvl, basestring): + if isinstance(lvl, string_types): lvl = self.LOG_LEVELS.get(lvl.lower(), logging.INFO) self.error_log.log(lvl, msg, *args, **kwargs) diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index 82797e4c..e81e655a 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -7,7 +7,7 @@ import sys from gunicorn.http.errors import (NoMoreData, ChunkMissingTerminator, InvalidChunkSize) -from gunicorn.six import StringIO, bytes_to_str +from gunicorn.six import StringIO, bytes_to_str, integer_types class ChunkedReader(object): def __init__(self, req, unreader): @@ -16,7 +16,7 @@ class ChunkedReader(object): self.buf = StringIO() def read(self, size): - if not isinstance(size, (int, long)): + if not isinstance(size, integer_types): raise TypeError("size must be an integral type") if size <= 0: raise ValueError("Size must be positive.") @@ -111,7 +111,7 @@ class LengthReader(object): self.length = length def read(self, size): - if not isinstance(size, (int, long)): + if not isinstance(size, integer_types): raise TypeError("size must be an integral type") size = min(self.length, size) @@ -142,7 +142,7 @@ class EOFReader(object): self.finished = False def read(self, size): - if not isinstance(size, (int, long)): + if not isinstance(size, integer_types): raise TypeError("size must be an integral type") if size < 0: raise ValueError("Size must be positive.") @@ -189,7 +189,7 @@ class Body(object): def getsize(self, size): if size is None: return sys.maxint - elif not isinstance(size, (int, long)): + elif not isinstance(size, integer_types): raise TypeError("size must be an integral type") elif size < 0: return sys.maxint diff --git a/gunicorn/http/unreader.py b/gunicorn/http/unreader.py index be1ca3dc..1631a24a 100644 --- a/gunicorn/http/unreader.py +++ b/gunicorn/http/unreader.py @@ -5,10 +5,7 @@ import os -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from gunicorn.six import integer_types, StringIO # Classes that can undo reading data from # a given type of data source. @@ -21,7 +18,7 @@ class Unreader(object): raise NotImplementedError() def read(self, size=None): - if size is not None and not isinstance(size, (int, long)): + if size is not None and not isinstance(size, integer_types): raise TypeError("size parameter must be an int or long.") if size == 0: return "" diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 9981389f..5ea54f28 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -8,7 +8,8 @@ import os import re import sys -from gunicorn.six import unquote, string_types, binary_type, reraise +from gunicorn.six import (unquote, string_types, binary_type, reraise, + text_type) from gunicorn import SERVER_SOFTWARE import gunicorn.util as util @@ -117,7 +118,7 @@ def create(req, sock, client, server, cfg): environ['wsgi.url_scheme'] = url_scheme - if isinstance(forward, basestring): + if isinstance(forward, string_types): # we only took the last one # http://en.wikipedia.org/wiki/X-Forwarded-For if forward.find(",") >= 0: @@ -144,7 +145,7 @@ def create(req, sock, client, server, cfg): environ['REMOTE_ADDR'] = remote[0] environ['REMOTE_PORT'] = str(remote[1]) - if isinstance(server, basestring): + if isinstance(server, string_types): server = server.split(":") if len(server) == 1: if url_scheme == "http": @@ -208,7 +209,7 @@ class Response(object): def process_headers(self, headers): for name, value in headers: - assert isinstance(name, basestring), "%r is not a string" % name + assert isinstance(name, string_types), "%r is not a string" % name lname = name.lower().strip() if lname == "content-length": self.response_length = int(value) diff --git a/gunicorn/six.py b/gunicorn/six.py index e82ddce2..76c15530 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -294,6 +294,10 @@ if PY3: print_ = getattr(builtins, "print") del builtins + def execfile_(file, globals=globals(), locals=locals()): + with open(file, "r") as fh: + exec_(fh.read()+"\n", globals, locals) + else: def exec_(code, globs=None, locs=None): """Execute code in a namespace.""" @@ -312,6 +316,7 @@ else: raise tp, value, tb """) + execfile_ = execfile def print_(*args, **kwargs): """The new-style print function.""" diff --git a/gunicorn/sock.py b/gunicorn/sock.py index e172ceb2..11dcd75e 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -10,7 +10,7 @@ import sys import time from gunicorn import util - +from gunicorn.six import string_types class BaseSocket(object): @@ -108,7 +108,7 @@ def create_socket(conf, log): sock_type = TCP6Socket else: sock_type = TCPSocket - elif isinstance(addr, basestring): + elif isinstance(addr, string_types): sock_type = UnixSocket else: raise TypeError("Unable to create socket from: %r" % addr) diff --git a/gunicorn/util.py b/gunicorn/util.py index 11731327..0f6895a7 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -25,7 +25,7 @@ import textwrap import time import inspect -from gunicorn.six import text_type +from gunicorn.six import text_type, string_types MAXFD = 1024 if (hasattr(os, "devnull")): @@ -75,7 +75,7 @@ except ImportError: if not hasattr(package, 'rindex'): raise ValueError("'package' not set to a string") dot = len(package) - for x in xrange(level, 1, -1): + for x in range(level, 1, -1): try: dot = package.rindex('.', 0, dot) except ValueError: @@ -217,14 +217,14 @@ try: except ImportError: def closerange(fd_low, fd_high): # Iterate through and close all file descriptors. - for fd in xrange(fd_low, fd_high): + for fd in range(fd_low, fd_high): try: os.close(fd) except OSError: # ERROR, fd wasn't open to begin with (ignored) pass def write_chunk(sock, data): - if instance(data, text_type): + if isinstance(data, text_type): data = data.decode('utf-8') chunk_size = "%X\r\n" % len(data) chunk = b"".join([chunk_size.decode('utf-8'), data, b"\r\n"]) @@ -263,7 +263,7 @@ def write_error(sock, status_int, reason, mesg): """) % {"reason": reason, "mesg": mesg} - headers = textwrap.dedent("""\ + http = textwrap.dedent("""\ HTTP/1.1 %s %s\r Connection: close\r Content-Type: text/html\r @@ -313,7 +313,7 @@ def http_date(timestamp=None): def to_bytestring(s): """ convert to bytestring an unicode """ - if not isinstance(s, basestring): + if not isinstance(s, string_types): return s if isinstance(s, unicode): return s.encode('utf-8') From 8d453fb341eceb13e4cb99e24844d198bda73391 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 22:07:35 +0200 Subject: [PATCH 12/47] all tests pass under python 3 --- gunicorn/app/django_wsgi.py | 6 +- gunicorn/arbiter.py | 7 ++- gunicorn/config.py | 21 +++++-- gunicorn/errors.py | 4 +- gunicorn/http/body.py | 48 +++++++-------- gunicorn/http/errors.py | 3 +- gunicorn/http/message.py | 11 ++-- gunicorn/http/parser.py | 5 +- gunicorn/http/unreader.py | 31 +++++----- gunicorn/six.py | 27 +++----- gunicorn/util.py | 9 --- gunicorn/workers/async.py | 5 +- gunicorn/workers/base.py | 3 +- gunicorn/workers/sync.py | 3 +- tests/004-test-http-body.py | 61 ------------------- tests/requests/valid/001.py | 4 +- tests/requests/valid/002.py | 4 +- tests/requests/valid/003.py | 4 +- tests/requests/valid/004.py | 4 +- tests/requests/valid/005.py | 4 +- tests/requests/valid/006.py | 4 +- tests/requests/valid/007.py | 4 +- tests/requests/valid/008.py | 4 +- tests/requests/valid/009.py | 4 +- tests/requests/valid/010.py | 4 +- tests/requests/valid/011.py | 4 +- tests/requests/valid/012.py | 4 +- tests/requests/valid/013.py | 4 +- tests/requests/valid/014.py | 4 +- tests/requests/valid/015.py | 4 +- tests/requests/valid/016.py | 4 +- tests/requests/valid/017.py | 2 +- tests/requests/valid/018.py | 4 +- tests/requests/valid/019.py | 4 +- tests/requests/valid/020.py | 4 +- tests/requests/valid/021.py | 4 +- tests/requests/valid/022.py | 6 +- tests/requests/valid/023.py | 6 +- tests/requests/valid/025.py | 4 +- tests/requests/valid/pp_01.py | 2 +- tests/requests/valid/pp_02.py | 4 +- tests/t.py | 35 ++++++----- ...requests.py => test_001-valid-requests.py} | 4 +- ...quests.py => test_002-invalid-requests.py} | 15 ++--- ...{003-test-config.py => test_003-config.py} | 57 ++++------------- tests/test_004-http-body.py | 61 +++++++++++++++++++ tests/treq.py | 49 ++++++++------- 47 files changed, 276 insertions(+), 293 deletions(-) delete mode 100644 tests/004-test-http-body.py rename tests/{001-test-valid-requests.py => test_001-valid-requests.py} (90%) rename tests/{002-test-invalid-requests.py => test_002-invalid-requests.py} (63%) rename tests/{003-test-config.py => test_003-config.py} (81%) create mode 100644 tests/test_004-http-body.py diff --git a/gunicorn/app/django_wsgi.py b/gunicorn/app/django_wsgi.py index 7ec41bf3..2372e7b2 100644 --- a/gunicorn/app/django_wsgi.py +++ b/gunicorn/app/django_wsgi.py @@ -74,10 +74,10 @@ def reload_django_settings(): app_mod = util.import_module(app[:-2]) appdir = os.path.dirname(app_mod.__file__) app_subdirs = os.listdir(appdir) - app_subdirs.sort() name_pattern = re.compile(r'[a-zA-Z]\w*') - for d in app_subdirs: - if name_pattern.match(d) and os.path.isdir(os.path.join(appdir, d)): + for d in sorted(app_subdirs): + if (name_pattern.match(d) and + os.path.isdir(os.path.join(appdir, d))): new_installed_apps.append('%s.%s' % (app[:-2], d)) else: new_installed_apps.append(app) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 4e0d665e..af0209e1 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -99,7 +99,10 @@ class Arbiter(object): if self.cfg.debug: self.log.debug("Current configuration:") - for config, value in sorted(self.cfg.settings.iteritems()): + + + for config, value in sorted(self.cfg.settings.items(), + key=lambda setting: setting[1]): self.log.debug(" %s: %s", config, value.value) if self.cfg.preload_app: @@ -436,7 +439,7 @@ class Arbiter(object): self.spawn_workers() workers = self.WORKERS.items() - workers.sort(key=lambda w: w[1].age) + workers = sorted(workers, key=lambda w: w[1].age) while len(workers) > self.num_workers: (pid, _) = workers.pop(0) self.kill_worker(pid, signal.SIGQUIT) diff --git a/gunicorn/config.py b/gunicorn/config.py index fa3d15bb..e8f7fc32 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -15,7 +15,7 @@ import types from gunicorn import __version__ from gunicorn.errors import ConfigError from gunicorn import util -from gunicorn.six import string_types +from gunicorn.six import string_types, integer_types, bytes_to_str KNOWN_SETTINGS = [] @@ -62,10 +62,12 @@ class Config(object): } parser = optparse.OptionParser(**kwargs) - keys = self.settings.keys() + keys = list(self.settings) def sorter(k): return (self.settings[k].section, self.settings[k].order) - keys.sort(key=sorter) + + + keys = sorted(self.settings, key=self.settings.__getitem__) for k in keys: self.settings[k].add_option(parser) return parser @@ -85,7 +87,7 @@ class Config(object): @property def address(self): bind = self.settings['bind'].get() - return util.parse_address(util.to_bytestring(bind)) + return util.parse_address(bytes_to_str(bind)) @property def uid(self): @@ -179,8 +181,15 @@ class Setting(object): assert callable(self.validator), "Invalid validator: %s" % self.name self.value = self.validator(val) + def __lt__(self, other): + return (self.section == other.section and + self.order < other.order) + __cmp__ = __lt__ + +Setting = SettingMeta('Setting', (Setting,), {}) + def validate_bool(val): - if isinstance(val, types.BooleanType): + if isinstance(val, bool): return val if not isinstance(val, string_types): raise TypeError("Invalid type for casting: %s" % val) @@ -197,7 +206,7 @@ def validate_dict(val): return val def validate_pos_int(val): - if not isinstance(val, (types.IntType, types.LongType)): + if not isinstance(val, integer_types): val = int(val, 0) else: # Booleans are ints! diff --git a/gunicorn/errors.py b/gunicorn/errors.py index e4fbcdd3..9669f5ed 100644 --- a/gunicorn/errors.py +++ b/gunicorn/errors.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. -class HaltServer(Exception): +class HaltServer(BaseException): def __init__(self, reason, exit_status=1): self.reason = reason self.exit_status = exit_status @@ -12,5 +12,5 @@ class HaltServer(Exception): def __str__(self): return "" % (self.reason, self.exit_status) -class ConfigError(Exception): +class ConfigError(BaseException): """ Exception raised on config error """ diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index e81e655a..9f685ac0 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -7,16 +7,16 @@ import sys from gunicorn.http.errors import (NoMoreData, ChunkMissingTerminator, InvalidChunkSize) -from gunicorn.six import StringIO, bytes_to_str, integer_types +from gunicorn import six class ChunkedReader(object): def __init__(self, req, unreader): self.req = req self.parser = self.parse_chunked(unreader) - self.buf = StringIO() + self.buf = six.BytesIO() def read(self, size): - if not isinstance(size, integer_types): + if not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") if size <= 0: raise ValueError("Size must be positive.") @@ -26,19 +26,19 @@ class ChunkedReader(object): if self.parser: while self.buf.tell() < size: try: - self.buf.write(self.parser.next()) + self.buf.write(six.next(self.parser)) except StopIteration: self.parser = None break data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret def parse_trailers(self, unreader, data): - buf = StringIO() + buf = six.BytesIO() buf.write(data) idx = buf.getvalue().find(b"\r\n\r\n") @@ -50,8 +50,7 @@ class ChunkedReader(object): if done: unreader.unread(buf.getvalue()[2:]) return b"" - self.req.trailers = self.req.parse_headers( - bytes_to_str(buf.getvalue()[:idx])) + self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx]) unreader.unread(buf.getvalue()[idx+4:]) def parse_chunked(self, unreader): @@ -73,7 +72,7 @@ class ChunkedReader(object): (size, rest) = self.parse_chunk_size(unreader, data=rest[2:]) def parse_chunk_size(self, unreader, data=None): - buf = StringIO() + buf = six.BytesIO() if data is not None: buf.write(data) @@ -111,7 +110,7 @@ class LengthReader(object): self.length = length def read(self, size): - if not isinstance(size, integer_types): + if not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") size = min(self.length, size) @@ -121,7 +120,7 @@ class LengthReader(object): return b"" - buf = StringIO() + buf = six.BytesIO() data = self.unreader.read() while data: buf.write(data) @@ -138,21 +137,21 @@ class LengthReader(object): class EOFReader(object): def __init__(self, unreader): self.unreader = unreader - self.buf = StringIO() + self.buf = six.BytesIO() self.finished = False def read(self, size): - if not isinstance(size, integer_types): + if not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") if size < 0: raise ValueError("Size must be positive.") if size == 0: - return "" + return b"" if self.finished: data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret @@ -168,31 +167,32 @@ class EOFReader(object): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret class Body(object): def __init__(self, reader): self.reader = reader - self.buf = StringIO() + self.buf = six.BytesIO() def __iter__(self): return self - def next(self): + def __next__(self): ret = self.readline() if not ret: raise StopIteration() return ret + next = __next__ def getsize(self, size): if size is None: - return sys.maxint - elif not isinstance(size, integer_types): + return six.MAXSIZE + elif not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") elif size < 0: - return sys.maxint + return six.MAXSIZE return size def read(self, size=None): @@ -203,7 +203,7 @@ class Body(object): if size < self.buf.tell(): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret @@ -215,7 +215,7 @@ class Body(object): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret @@ -225,7 +225,7 @@ class Body(object): return b"" line = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() if len(line) < size: line += self.reader.read(size - len(line)) extra_buf_data = line[size:] diff --git a/gunicorn/http/errors.py b/gunicorn/http/errors.py index 7ba4949f..5baf5b69 100644 --- a/gunicorn/http/errors.py +++ b/gunicorn/http/errors.py @@ -3,12 +3,13 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -class ParseException(Exception): +class ParseException(BaseException): pass class NoMoreData(IOError): def __init__(self, buf=None): self.buf = buf + def __str__(self): return "No more data after: %r" % self.buf diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index b32dbc2b..1d1ff588 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -13,7 +13,7 @@ from gunicorn.http.errors import InvalidHeader, InvalidHeaderName, NoMoreData, \ InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, \ LimitRequestLine, LimitRequestHeaders from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest -from gunicorn.six import StringIO, urlsplit, bytes_to_str +from gunicorn.six import BytesIO, urlsplit, bytes_to_str MAX_REQUEST_LINE = 8190 MAX_HEADERS = 32768 @@ -148,7 +148,6 @@ class Request(Message): self.req_number = req_number self.proxy_protocol_info = None - super(Request, self).__init__(cfg, unreader) @@ -161,7 +160,7 @@ class Request(Message): buf.write(data) def parse(self, unreader): - buf = StringIO() + buf = BytesIO() self.get_data(unreader, buf, stop=True) # get request line @@ -170,12 +169,12 @@ class Request(Message): # proxy protocol if self.proxy_protocol(bytes_to_str(line)): # get next request line - buf = StringIO() + buf = BytesIO() buf.write(rbuf) line, rbuf = self.read_line(unreader, buf, self.limit_request_line) self.parse_request_line(bytes_to_str(line)) - buf = StringIO() + buf = BytesIO() buf.write(rbuf) # Headers @@ -202,7 +201,7 @@ class Request(Message): self.headers = self.parse_headers(data[:idx]) ret = data[idx+4:] - buf = StringIO() + buf = BytesIO() return ret def read_line(self, unreader, buf, limit=0): diff --git a/gunicorn/http/parser.py b/gunicorn/http/parser.py index 54bbddea..77e959e5 100644 --- a/gunicorn/http/parser.py +++ b/gunicorn/http/parser.py @@ -22,7 +22,7 @@ class Parser(object): def __iter__(self): return self - def next(self): + def __next__(self): # Stop if HTTP dictates a stop. if self.mesg and self.mesg.should_close(): raise StopIteration() @@ -33,6 +33,7 @@ class Parser(object): while data: data = self.mesg.body.read(8192) + # Parse the next request self.req_count += 1 self.mesg = self.mesg_class(self.cfg, self.unreader, self.req_count) @@ -40,6 +41,8 @@ class Parser(object): raise StopIteration() return self.mesg + next = __next__ + class RequestParser(Parser): def __init__(self, *args, **kwargs): super(RequestParser, self).__init__(Request, *args, **kwargs) diff --git a/gunicorn/http/unreader.py b/gunicorn/http/unreader.py index 1631a24a..2cc2133e 100644 --- a/gunicorn/http/unreader.py +++ b/gunicorn/http/unreader.py @@ -5,44 +5,47 @@ import os -from gunicorn.six import integer_types, StringIO +from gunicorn import six # Classes that can undo reading data from # a given type of data source. class Unreader(object): def __init__(self): - self.buf = StringIO() + self.buf = six.BytesIO() def chunk(self): raise NotImplementedError() def read(self, size=None): - if size is not None and not isinstance(size, integer_types): + if size is not None and not isinstance(size, six.integer_types): raise TypeError("size parameter must be an int or long.") - if size == 0: - return "" - if size < 0: - size = None + + if size is not None: + if size == 0: + return b"" + if size < 0: + size = None self.buf.seek(0, os.SEEK_END) if size is None and self.buf.tell(): ret = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() return ret if size is None: - return self.chunk() + d = self.chunk() + return d while self.buf.tell() < size: chunk = self.chunk() if not len(chunk): ret = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() return ret self.buf.write(chunk) data = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(data[size:]) return data[:size] @@ -66,9 +69,9 @@ class IterUnreader(Unreader): def chunk(self): if not self.iter: - return "" + return b"" try: - return self.iter.next() + return six.next(self.iter) except StopIteration: self.iter = None - return "" + return b"" diff --git a/gunicorn/six.py b/gunicorn/six.py index 76c15530..9070e400 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -281,10 +281,14 @@ _add_doc(u, """Text literal""") if PY3: + + def execfile_(fname, *args): + return exec(compile(open(fname, 'rb').read(), fname, 'exec'), *args) + + import builtins exec_ = getattr(builtins, "exec") - def reraise(tp, value, tb=None): if value.__traceback__ is not tb: raise value.with_traceback(tb) @@ -294,10 +298,6 @@ if PY3: print_ = getattr(builtins, "print") del builtins - def execfile_(file, globals=globals(), locals=locals()): - with open(file, "r") as fh: - exec_(fh.read()+"\n", globals, locals) - else: def exec_(code, globs=None, locs=None): """Execute code in a namespace.""" @@ -373,33 +373,26 @@ def with_metaclass(meta, base=object): # specific to gunicorn if PY3: - import io - StringIO = io.BytesIO - def bytes_to_str(b): + if isinstance(b, text_type): + return b return str(b, 'latin1') import urllib.parse unquote = urllib.parse.unquote urlsplit = urllib.parse.urlsplit + urlparse = urllib.parse.urlparse else: - try: - import cStringIO as StringIO - except ImportError: - import StringIO - - StringIO = StringIO - - - def bytestring(s): + def bytes_to_str(s): if isinstance(s, unicode): return s.encode('utf-8') return s import urlparse as orig_urlparse urlsplit = orig_urlparse.urlsplit + urlparse = orig_urlparse.urlparse import urllib urlunquote = urllib.unquote diff --git a/gunicorn/util.py b/gunicorn/util.py index 0f6895a7..142f6e10 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -311,15 +311,6 @@ def http_date(timestamp=None): hh, mm, ss) return s -def to_bytestring(s): - """ convert to bytestring an unicode """ - if not isinstance(s, string_types): - return s - if isinstance(s, unicode): - return s.encode('utf-8') - else: - return s - def is_hoppish(header): return header.lower().strip() in hop_headers diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index 08fed0ee..ecdab864 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -13,6 +13,7 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi import gunicorn.util as util import gunicorn.workers.base as base +from gunicorn import six ALREADY_HANDLED = object() @@ -32,14 +33,14 @@ class AsyncWorker(base.Worker): parser = http.RequestParser(self.cfg, client) try: if not self.cfg.keepalive: - req = parser.next() + req = six.next(parser) self.handle_request(req, client, addr) else: # keepalive loop while True: req = None with self.timeout_ctx(): - req = parser.next() + req = six.next(parser) if not req: break self.handle_request(req, client, addr) diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 474cf3d3..bdc069f0 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -18,6 +18,7 @@ InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, \ LimitRequestLine, LimitRequestHeaders from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest from gunicorn.http.wsgi import default_environ, Response +from gunicorn.six import MAXSIZE class Worker(object): @@ -43,7 +44,7 @@ class Worker(object): self.booted = False self.nr = 0 - self.max_requests = cfg.max_requests or sys.maxint + self.max_requests = cfg.max_requests or MAXSIZE self.alive = True self.log = log self.debug = cfg.debug diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index bd375d94..d18af1a6 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -14,6 +14,7 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi import gunicorn.util as util import gunicorn.workers.base as base +from gunicorn import six class SyncWorker(base.Worker): @@ -69,7 +70,7 @@ class SyncWorker(base.Worker): try: client.settimeout(self.cfg.timeout) parser = http.RequestParser(self.cfg, client) - req = parser.next() + req = six.next(parser) self.handle_request(req, client, addr) except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) diff --git a/tests/004-test-http-body.py b/tests/004-test-http-body.py deleted file mode 100644 index 22357478..00000000 --- a/tests/004-test-http-body.py +++ /dev/null @@ -1,61 +0,0 @@ -from StringIO import StringIO - -import t -from gunicorn.http.body import Body - - -def assert_readline(payload, size, expected): - body = Body(StringIO(payload)) - t.eq(body.readline(size), expected) - - -def test_readline_empty_body(): - assert_readline("", None, "") - assert_readline("", 1, "") - - -def test_readline_zero_size(): - assert_readline("abc", 0, "") - assert_readline("\n", 0, "") - - -def test_readline_new_line_before_size(): - body = Body(StringIO("abc\ndef")) - t.eq(body.readline(4), "abc\n") - t.eq(body.readline(), "def") - - -def test_readline_new_line_after_size(): - body = Body(StringIO("abc\ndef")) - t.eq(body.readline(2), "ab") - t.eq(body.readline(), "c\n") - - -def test_readline_no_new_line(): - body = Body(StringIO("abcdef")) - t.eq(body.readline(), "abcdef") - body = Body(StringIO("abcdef")) - t.eq(body.readline(2), "ab") - t.eq(body.readline(2), "cd") - t.eq(body.readline(2), "ef") - - -def test_readline_buffer_loaded(): - reader = StringIO("abc\ndef") - body = Body(reader) - body.read(1) # load internal buffer - reader.write("g\nhi") - reader.seek(7) - t.eq(body.readline(), "bc\n") - t.eq(body.readline(), "defg\n") - t.eq(body.readline(), "hi") - - -def test_readline_buffer_loaded_with_size(): - body = Body(StringIO("abc\ndef")) - body.read(1) # load internal buffer - t.eq(body.readline(2), "bc") - t.eq(body.readline(2), "\n") - t.eq(body.readline(2), "de") - t.eq(body.readline(2), "f") - diff --git a/tests/requests/valid/001.py b/tests/requests/valid/001.py index 4b3109eb..1eee7b5e 100644 --- a/tests/requests/valid/001.py +++ b/tests/requests/valid/001.py @@ -7,5 +7,5 @@ request = { ("CONTENT-TYPE", "application/json"), ("CONTENT-LENGTH", "14") ], - "body": '{"nom": "nom"}' -} \ No newline at end of file + "body": b'{"nom": "nom"}' +} diff --git a/tests/requests/valid/002.py b/tests/requests/valid/002.py index 3bde4643..b511ffb6 100644 --- a/tests/requests/valid/002.py +++ b/tests/requests/valid/002.py @@ -7,5 +7,5 @@ request = { ("HOST", "0.0.0.0=5000"), ("ACCEPT", "*/*") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/003.py b/tests/requests/valid/003.py index d54bc7de..8ce87f80 100644 --- a/tests/requests/valid/003.py +++ b/tests/requests/valid/003.py @@ -12,5 +12,5 @@ request = { ("KEEP-ALIVE", "300"), ("CONNECTION", "keep-alive") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/004.py b/tests/requests/valid/004.py index 5840e522..6006f144 100644 --- a/tests/requests/valid/004.py +++ b/tests/requests/valid/004.py @@ -5,5 +5,5 @@ request = { "headers": [ ("AAAAAAAAAAAAA", "++++++++++") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/005.py b/tests/requests/valid/005.py index 5312de11..a7e54472 100644 --- a/tests/requests/valid/005.py +++ b/tests/requests/valid/005.py @@ -3,5 +3,5 @@ request = { "uri": uri("/forums/1/topics/2375?page=1#posts-17408"), "version": (1, 1), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/006.py b/tests/requests/valid/006.py index aea28729..01a4be17 100644 --- a/tests/requests/valid/006.py +++ b/tests/requests/valid/006.py @@ -3,5 +3,5 @@ request = { "uri": uri("/get_no_headers_no_body/world"), "version": (1, 1), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/007.py b/tests/requests/valid/007.py index fb531635..f5c2c798 100644 --- a/tests/requests/valid/007.py +++ b/tests/requests/valid/007.py @@ -5,5 +5,5 @@ request = { "headers": [ ("ACCEPT", "*/*") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/008.py b/tests/requests/valid/008.py index f9b77c56..379f9a2b 100644 --- a/tests/requests/valid/008.py +++ b/tests/requests/valid/008.py @@ -5,5 +5,5 @@ request = { "headers": [ ("CONTENT-LENGTH", "5") ], - "body": "HELLO" -} \ No newline at end of file + "body": b"HELLO" +} diff --git a/tests/requests/valid/009.py b/tests/requests/valid/009.py index 5cf51472..7e1f52dd 100644 --- a/tests/requests/valid/009.py +++ b/tests/requests/valid/009.py @@ -7,5 +7,5 @@ request = { ("TRANSFER-ENCODING", "identity"), ("CONTENT-LENGTH", "5") ], - "body": "World" -} \ No newline at end of file + "body": b"World" +} diff --git a/tests/requests/valid/010.py b/tests/requests/valid/010.py index 9fc566ba..996ef381 100644 --- a/tests/requests/valid/010.py +++ b/tests/requests/valid/010.py @@ -5,5 +5,5 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked"), ], - "body": "all your base are belong to us" -} \ No newline at end of file + "body": b"all your base are belong to us" +} diff --git a/tests/requests/valid/011.py b/tests/requests/valid/011.py index 3b7f6c08..05555adc 100644 --- a/tests/requests/valid/011.py +++ b/tests/requests/valid/011.py @@ -5,5 +5,5 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" -} \ No newline at end of file + "body": b"hello world" +} diff --git a/tests/requests/valid/012.py b/tests/requests/valid/012.py index 8f14e4c9..af071e5b 100644 --- a/tests/requests/valid/012.py +++ b/tests/requests/valid/012.py @@ -5,9 +5,9 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world", + "body": b"hello world", "trailers": [ ("VARY", "*"), ("CONTENT-TYPE", "text/plain") ] -} \ No newline at end of file +} diff --git a/tests/requests/valid/013.py b/tests/requests/valid/013.py index f17ec601..75ae982d 100644 --- a/tests/requests/valid/013.py +++ b/tests/requests/valid/013.py @@ -5,5 +5,5 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" -} \ No newline at end of file + "body": b"hello world" +} diff --git a/tests/requests/valid/014.py b/tests/requests/valid/014.py index 22d380c5..0b1b0d2b 100644 --- a/tests/requests/valid/014.py +++ b/tests/requests/valid/014.py @@ -3,5 +3,5 @@ request = { "uri": uri('/with_"quotes"?foo="bar"'), "version": (1, 1), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/015.py b/tests/requests/valid/015.py index 4c36ba48..cd2b14f5 100644 --- a/tests/requests/valid/015.py +++ b/tests/requests/valid/015.py @@ -7,5 +7,5 @@ request = { ("USER-AGENT", "ApacheBench/2.3"), ("ACCEPT", "*/*") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/016.py b/tests/requests/valid/016.py index 3f3fa7ba..139b2700 100644 --- a/tests/requests/valid/016.py +++ b/tests/requests/valid/016.py @@ -36,5 +36,5 @@ request = { "uri": uri("/"), "version": (1, 1), "headers": [("X-SSL-CERT", certificate)], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/017.py b/tests/requests/valid/017.py index 638fd332..5fe13e86 100644 --- a/tests/requests/valid/017.py +++ b/tests/requests/valid/017.py @@ -6,5 +6,5 @@ request = { ("IF-MATCH", "bazinga!"), ("IF-MATCH", "large-sound") ], - "body": "" + "body": b"" } diff --git a/tests/requests/valid/018.py b/tests/requests/valid/018.py index f11c9461..fa10a96c 100644 --- a/tests/requests/valid/018.py +++ b/tests/requests/valid/018.py @@ -3,7 +3,7 @@ req1 = { "uri": uri("/first"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } req2 = { @@ -11,7 +11,7 @@ req2 = { "uri": uri("/second"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } request = [req1, req2] diff --git a/tests/requests/valid/019.py b/tests/requests/valid/019.py index 76d941d5..6fabd151 100644 --- a/tests/requests/valid/019.py +++ b/tests/requests/valid/019.py @@ -3,5 +3,5 @@ request = { "uri": uri("/first"), "version": (1, 0), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/020.py b/tests/requests/valid/020.py index 56702278..f5cfa21d 100644 --- a/tests/requests/valid/020.py +++ b/tests/requests/valid/020.py @@ -3,5 +3,5 @@ request = { "uri": uri("/first"), "version": (1, 0), "headers": [('CONTENT-LENGTH', '24')], - "body": "GET /second HTTP/1.1\r\n\r\n" -} \ No newline at end of file + "body": b"GET /second HTTP/1.1\r\n\r\n" +} diff --git a/tests/requests/valid/021.py b/tests/requests/valid/021.py index 4815a1c5..992dd308 100644 --- a/tests/requests/valid/021.py +++ b/tests/requests/valid/021.py @@ -3,5 +3,5 @@ request = { "uri": uri("/first"), "version": (1, 1), "headers": [("CONNECTION", "Close")], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/022.py b/tests/requests/valid/022.py index a51a6f74..9b87e5e5 100644 --- a/tests/requests/valid/022.py +++ b/tests/requests/valid/022.py @@ -3,7 +3,7 @@ req1 = { "uri": uri("/first"), "version": (1, 0), "headers": [("CONNECTION", "Keep-Alive")], - "body": "" + "body": b"" } req2 = { @@ -11,7 +11,7 @@ req2 = { "uri": uri("/second"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } -request = [req1, req2] \ No newline at end of file +request = [req1, req2] diff --git a/tests/requests/valid/023.py b/tests/requests/valid/023.py index 0fa05f1a..f9a0ef5d 100644 --- a/tests/requests/valid/023.py +++ b/tests/requests/valid/023.py @@ -5,7 +5,7 @@ req1 = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" + "body": b"hello world" } req2 = { @@ -13,7 +13,7 @@ req2 = { "uri": uri("/second"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } -request = [req1, req2] \ No newline at end of file +request = [req1, req2] diff --git a/tests/requests/valid/025.py b/tests/requests/valid/025.py index 7e8f1826..12ea9ab7 100644 --- a/tests/requests/valid/025.py +++ b/tests/requests/valid/025.py @@ -6,7 +6,7 @@ req1 = { ("CONTENT-LENGTH", "-1"), ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" + "body": b"hello world" } req2 = { @@ -17,7 +17,7 @@ req2 = { ("TRANSFER-ENCODING", "chunked"), ("CONTENT-LENGTH", "-1"), ], - "body": "hello world" + "body": b"hello world" } request = [req1, req2] diff --git a/tests/requests/valid/pp_01.py b/tests/requests/valid/pp_01.py index 2e5b85a3..8f112506 100644 --- a/tests/requests/valid/pp_01.py +++ b/tests/requests/valid/pp_01.py @@ -12,5 +12,5 @@ request = { ("CONTENT-TYPE", "application/json"), ("CONTENT-LENGTH", "14") ], - "body": '{"nom": "nom"}' + "body": b'{"nom": "nom"}' } diff --git a/tests/requests/valid/pp_02.py b/tests/requests/valid/pp_02.py index f756b526..701ff2a7 100644 --- a/tests/requests/valid/pp_02.py +++ b/tests/requests/valid/pp_02.py @@ -13,7 +13,7 @@ req1 = { ("CONTENT-LENGTH", "14"), ("CONNECTION", "keep-alive") ], - "body": '{"nom": "nom"}' + "body": b'{"nom": "nom"}' } @@ -24,7 +24,7 @@ req2 = { "headers": [ ("TRANSFER-ENCODING", "chunked"), ], - "body": "all your base are belong to us" + "body": b"all your base are belong to us" } request = [req1, req2] diff --git a/tests/t.py b/tests/t.py index 5f776170..6f1d044a 100644 --- a/tests/t.py +++ b/tests/t.py @@ -1,43 +1,43 @@ # -*- coding: utf-8 - # Copyright 2009 Paul J. Davis # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. from __future__ import with_statement import array import os -from StringIO import StringIO import tempfile dirname = os.path.dirname(__file__) from gunicorn.http.parser import RequestParser from gunicorn.config import Config +from gunicorn.six import BytesIO def data_source(fname): - buf = StringIO() + buf = BytesIO() with open(fname) as handle: for line in handle: line = line.rstrip("\n").replace("\\r\\n", "\r\n") - buf.write(line) + buf.write(line.encode('latin1')) return buf class request(object): def __init__(self, name): self.fname = os.path.join(dirname, "requests", name) - + def __call__(self, func): def run(): src = data_source(self.fname) func(src, RequestParser(src)) run.func_name = func.func_name return run - - + + class FakeSocket(object): - + def __init__(self, data): self.tmp = tempfile.TemporaryFile() if data: @@ -47,32 +47,32 @@ class FakeSocket(object): def fileno(self): return self.tmp.fileno() - + def len(self): return self.tmp.len - + def recv(self, length=None): return self.tmp.read() - + def recv_into(self, buf, length): tmp_buffer = self.tmp.read(length) v = len(tmp_buffer) for i, c in enumerate(tmp_buffer): buf[i] = c return v - + def send(self, data): self.tmp.write(data) self.tmp.flush() - + def seek(self, offset, whence=0): self.tmp.seek(offset, whence) - - + + class http_request(object): def __init__(self, name): self.fname = os.path.join(dirname, "requests", name) - + def __call__(self, func): def run(): fsock = FakeSocket(data_source(self.fname)) @@ -80,7 +80,7 @@ class http_request(object): func(req) run.func_name = func.func_name return run - + def eq(a, b): assert a == b, "%r != %r" % (a, b) @@ -117,4 +117,3 @@ def raises(exctype, func, *args, **kwargs): func_name = getattr(func, "func_name", "") raise AssertionError("Function %s did not raise %s" % ( func_name, exctype.__name__)) - diff --git a/tests/001-test-valid-requests.py b/tests/test_001-valid-requests.py similarity index 90% rename from tests/001-test-valid-requests.py rename to tests/test_001-valid-requests.py index 6499d029..132eb9b8 100644 --- a/tests/001-test-valid-requests.py +++ b/tests/test_001-valid-requests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. import t @@ -9,6 +9,8 @@ import treq import glob import os dirname = os.path.dirname(__file__) + +from py.test import skip reqdir = os.path.join(dirname, "requests", "valid") def a_case(fname): diff --git a/tests/002-test-invalid-requests.py b/tests/test_002-invalid-requests.py similarity index 63% rename from tests/002-test-invalid-requests.py rename to tests/test_002-invalid-requests.py index 15308d25..d5f6e765 100644 --- a/tests/002-test-invalid-requests.py +++ b/tests/test_002-invalid-requests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. import t @@ -8,7 +8,8 @@ import treq import glob import os -from nose.tools import raises + +import pytest dirname = os.path.dirname(__file__) reqdir = os.path.join(dirname, "requests", "invalid") @@ -17,12 +18,12 @@ reqdir = os.path.join(dirname, "requests", "invalid") def test_http_parser(): for fname in glob.glob(os.path.join(reqdir, "*.http")): env = treq.load_py(os.path.splitext(fname)[0] + ".py") + expect = env['request'] cfg = env['cfg'] req = treq.badrequest(fname) - @raises(expect) - def check(fname): - return req.check(cfg) - - yield check, fname # fname is pass so that we know which test failed + with pytest.raises(expect): + def f(fname): + return req.check(cfg) + f(fname) diff --git a/tests/003-test-config.py b/tests/test_003-config.py similarity index 81% rename from tests/003-test-config.py rename to tests/test_003-config.py index 555ba9f8..e5639b22 100644 --- a/tests/003-test-config.py +++ b/tests/test_003-config.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. from __future__ import with_statement -from nose.plugins.skip import SkipTest - import t import functools @@ -23,14 +21,6 @@ def cfg_file(): def paster_ini(): return os.path.join(dirname, "..", "examples", "frameworks", "pylonstest", "nose.ini") -def PasterApp(): - try: - from paste.deploy import loadapp, loadwsgi - except ImportError: - raise SkipTest() - from gunicorn.app.pasterapp import PasterApplication - return PasterApplication("no_usage") - class AltArgs(object): def __init__(self, args=None): self.args = args or [] @@ -38,17 +28,17 @@ class AltArgs(object): def __enter__(self): sys.argv = self.args - + def __exit__(self, exc_type, exc_inst, traceback): sys.argv = self.orig class NoConfigApp(Application): def __init__(self): super(NoConfigApp, self).__init__("no_usage") - + def init(self, parser, opts, args): pass - + def load(self): pass @@ -63,25 +53,25 @@ def test_property_access(): c = config.Config() for s in config.KNOWN_SETTINGS: getattr(c, s.name) - + # Class was loaded t.eq(c.worker_class, SyncWorker) - + # Debug affects workers t.eq(c.workers, 1) c.set("workers", 3) t.eq(c.workers, 3) - + # Address is parsed t.eq(c.address, ("127.0.0.1", 8000)) - + # User and group defaults t.eq(os.geteuid(), c.uid) t.eq(os.getegid(), c.gid) - + # Proc name t.eq("gunicorn", c.proc_name) - + # Not a config property t.raises(AttributeError, getattr, c, "foo") # Force to be not an error @@ -93,10 +83,10 @@ def test_property_access(): # Attempt to set a cfg not via c.set t.raises(AttributeError, setattr, c, "proc_name", "baz") - + # No setting for name t.raises(AttributeError, c.set, "baz", "bar") - + def test_bool_validation(): c = config.Config() t.eq(c.debug, False) @@ -196,30 +186,9 @@ def test_load_config(): t.eq(app.cfg.bind, "unix:/tmp/bar/baz") t.eq(app.cfg.workers, 3) t.eq(app.cfg.proc_name, "fooey") - + def test_cli_overrides_config(): with AltArgs(["prog_name", "-c", cfg_file(), "-b", "blarney"]): app = NoConfigApp() t.eq(app.cfg.bind, "blarney") t.eq(app.cfg.proc_name, "fooey") - -def test_paster_config(): - with AltArgs(["prog_name", paster_ini()]): - app = PasterApp() - t.eq(app.cfg.bind, "192.168.0.1:80") - t.eq(app.cfg.proc_name, "brim") - t.eq("ignore_me" in app.cfg.settings, False) - -def test_cfg_over_paster(): - with AltArgs(["prog_name", "-c", cfg_file(), paster_ini()]): - app = PasterApp() - t.eq(app.cfg.bind, "unix:/tmp/bar/baz") - t.eq(app.cfg.proc_name, "fooey") - t.eq(app.cfg.default_proc_name, "blurgh") - -def test_cli_cfg_paster(): - with AltArgs(["prog_name", "-c", cfg_file(), "-b", "whee", paster_ini()]): - app = PasterApp() - t.eq(app.cfg.bind, "whee") - t.eq(app.cfg.proc_name, "fooey") - t.eq(app.cfg.default_proc_name, "blurgh") diff --git a/tests/test_004-http-body.py b/tests/test_004-http-body.py new file mode 100644 index 00000000..351b66bd --- /dev/null +++ b/tests/test_004-http-body.py @@ -0,0 +1,61 @@ +import t +from gunicorn.http.body import Body +from gunicorn.six import BytesIO + + +def assert_readline(payload, size, expected): + body = Body(BytesIO(payload)) + t.eq(body.readline(size), expected) + + +def test_readline_empty_body(): + assert_readline(b"", None, b"") + assert_readline(b"", 1, b"") + + +def test_readline_zero_size(): + assert_readline(b"abc", 0, b"") + assert_readline(b"\n", 0, b"") + + +def test_readline_new_line_before_size(): + body = Body(BytesIO(b"abc\ndef")) + t.eq(body.readline(4), b"abc\n") + t.eq(body.readline(), b"def") + + +def test_readline_new_line_after_size(): + body = Body(BytesIO(b"abc\ndef")) + t.eq(body.readline(2), b"ab") + t.eq(body.readline(), b"c\n") + + +def test_readline_no_new_line(): + body = Body(BytesIO(b"abcdef")) + t.eq(body.readline(), b"abcdef") + body = Body(BytesIO(b"abcdef")) + t.eq(body.readline(2), b"ab") + t.eq(body.readline(2), b"cd") + t.eq(body.readline(2), b"ef") + + +def test_readline_buffer_loaded(): + reader = BytesIO(b"abc\ndef") + body = Body(reader) + body.read(1) # load internal buffer + reader.write(b"g\nhi") + reader.seek(7) + print(reader.getvalue()) + t.eq(body.readline(), b"bc\n") + t.eq(body.readline(), b"defg\n") + t.eq(body.readline(), b"hi") + + +def test_readline_buffer_loaded_with_size(): + body = Body(BytesIO(b"abc\ndef")) + body.read(1) # load internal buffer + t.eq(body.readline(2), b"bc") + t.eq(body.readline(2), b"\n") + t.eq(body.readline(2), b"de") + t.eq(body.readline(2), b"f") + diff --git a/tests/treq.py b/tests/treq.py index 23a8eea5..5ec54be3 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -10,18 +10,19 @@ import t import inspect import os import random -import urlparse from gunicorn.config import Config from gunicorn.http.errors import ParseException from gunicorn.http.parser import RequestParser +from gunicorn.six import urlparse, execfile_ +from gunicorn import six dirname = os.path.dirname(__file__) random.seed() def uri(data): ret = {"raw": data} - parts = urlparse.urlparse(data) + parts = urlparse(data) ret["scheme"] = parts.scheme or '' ret["host"] = parts.netloc.rsplit(":", 1)[0] or None ret["port"] = parts.port or 80 @@ -42,7 +43,7 @@ def load_py(fname): config = globals().copy() config["uri"] = uri config["cfg"] = Config() - execfile(fname, config) + execfile_(fname, config) return config class request(object): @@ -54,10 +55,10 @@ class request(object): if not isinstance(self.expect, list): self.expect = [self.expect] - with open(self.fname) as handle: + with open(self.fname, 'rb') as handle: self.data = handle.read() - self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n") - self.data = self.data.replace("\\0", "\000") + self.data = self.data.replace(b"\n", b"").replace(b"\\r\\n", b"\r\n") + self.data = self.data.replace(b"\\0", b"\000") # Functions for sending data to the parser. # These functions mock out reading from a @@ -69,20 +70,20 @@ class request(object): def send_lines(self): lines = self.data - pos = lines.find("\r\n") + pos = lines.find(b"\r\n") while pos > 0: yield lines[:pos+2] lines = lines[pos+2:] - pos = lines.find("\r\n") + pos = lines.find(b"\r\n") if len(lines): yield lines def send_bytes(self): - for d in self.data: - yield d + for d in str(self.data, "latin1"): + yield bytes(d, "latin1") def send_random(self): - maxs = len(self.data) / 10 + maxs = round(len(self.data) / 10) read = 0 while read < len(self.data): chunk = random.randint(1, maxs) @@ -143,7 +144,7 @@ class request(object): while len(body): if body[:len(data)] != data: raise AssertionError("Invalid data read: %r" % data) - if '\n' in data[:-1]: + if b'\n' in data[:-1]: raise AssertionError("Embedded new line: %r" % data) body = body[len(data):] data = self.szread(req.body.readline, sizes) @@ -165,7 +166,7 @@ class request(object): """ data = req.body.readlines() for line in data: - if '\n' in line[:-1]: + if b'\n' in line[:-1]: raise AssertionError("Embedded new line: %r" % line) if line != body[:len(line)]: raise AssertionError("Invalid body data read: %r != %r" % ( @@ -182,7 +183,7 @@ class request(object): This skips sizes because there's its not part of the iter api. """ for line in req.body: - if '\n' in line[:-1]: + if b'\n' in line[:-1]: raise AssertionError("Embedded new line: %r" % line) if line != body[:len(line)]: raise AssertionError("Invalid body data read: %r != %r" % ( @@ -191,7 +192,7 @@ class request(object): if len(body): raise AssertionError("Failed to read entire body: %r" % body) try: - data = iter(req.body).next() + data = six.next(iter(req.body)) raise AssertionError("Read data after body finished: %r" % data) except StopIteration: pass @@ -214,9 +215,15 @@ class request(object): ret = [] for (mt, sz, sn) in cfgs: - mtn = mt.func_name[6:] - szn = sz.func_name[5:] - snn = sn.func_name[5:] + if hasattr(mt, 'funcname'): + mtn = mt.func_name[6:] + szn = sz.func_name[5:] + snn = sn.func_name[5:] + else: + mtn = mt.__name__[6:] + szn = sz.__name__[5:] + snn = sn.__name__[5:] + def test_req(sn, sz, mt): self.check(cfg, sn, sz, mt) desc = "%s: MT: %s SZ: %s SN: %s" % (self.name, mtn, szn, snn) @@ -251,9 +258,10 @@ class badrequest(object): self.data = handle.read() self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n") self.data = self.data.replace("\\0", "\000") + self.data = self.data.encode('latin1') def send(self): - maxs = len(self.data) / 10 + maxs = round(len(self.data) / 10) read = 0 while read < len(self.data): chunk = random.randint(1, maxs) @@ -262,5 +270,4 @@ class badrequest(object): def check(self, cfg): p = RequestParser(cfg, self.send()) - [req for req in p] - + six.next(p) From bb00d41ff9423f4f1ebe34481d2475d93764a401 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 23:08:15 +0200 Subject: [PATCH 13/47] fixes for py27 --- gunicorn/config.py | 2 -- gunicorn/six.py | 11 ++++++----- tests/treq.py | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index e8f7fc32..41f176b0 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -137,8 +137,6 @@ class SettingMeta(type): setattr(cls, "short", desc.splitlines()[0]) class Setting(object): - __metaclass__ = SettingMeta - name = None value = None section = None diff --git a/gunicorn/six.py b/gunicorn/six.py index 9070e400..8f698d07 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -282,10 +282,6 @@ _add_doc(u, """Text literal""") if PY3: - def execfile_(fname, *args): - return exec(compile(open(fname, 'rb').read(), fname, 'exec'), *args) - - import builtins exec_ = getattr(builtins, "exec") @@ -296,6 +292,11 @@ if PY3: print_ = getattr(builtins, "print") + + def execfile_(fname, *args): + return exec_(compile(open(fname, 'rb').read(), fname, 'exec'), *args) + + del builtins else: @@ -395,4 +396,4 @@ else: urlparse = orig_urlparse.urlparse import urllib - urlunquote = urllib.unquote + unquote = urllib.unquote diff --git a/tests/treq.py b/tests/treq.py index 5ec54be3..072f0ae0 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -79,8 +79,8 @@ class request(object): yield lines def send_bytes(self): - for d in str(self.data, "latin1"): - yield bytes(d, "latin1") + for d in str(self.data.decode("latin1")): + yield bytes(d.encode("latin1")) def send_random(self): maxs = round(len(self.data) / 10) From f6dee213bd131ddc67527bacf389a4761487999c Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 23:22:18 +0200 Subject: [PATCH 14/47] we are now using py.test for testing --- requirements_dev.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements_dev.txt diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000..e079f8a6 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1 @@ +pytest From e12f520cea4a13511b94ebc1c9b90c0acca140d7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 25 Oct 2012 07:14:21 +0200 Subject: [PATCH 15/47] use args for the socket errno --- gunicorn/sock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 11dcd75e..6f1f6d7e 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -118,7 +118,7 @@ def create_socket(conf, log): try: return sock_type(conf, log, fd=fd) except socket.error as e: - if e[0] == errno.ENOTCONN: + if e.args[0] == errno.ENOTCONN: log.error("GUNICORN_FD should refer to an open socket.") else: raise @@ -131,9 +131,9 @@ def create_socket(conf, log): try: return sock_type(conf, log) except socket.error as e: - if e[0] == errno.EADDRINUSE: + if e.args[0] == errno.EADDRINUSE: log.error("Connection in use: %s", str(addr)) - if e[0] == errno.EADDRNOTAVAIL: + if e.args[0] == errno.EADDRNOTAVAIL: log.error("Invalid address: %s", str(addr)) sys.exit(1) if i < 5: From d218ba54c652adb9d772e1b12d179167f21681a0 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 25 Oct 2012 10:14:05 +0200 Subject: [PATCH 16/47] use args as well. --- gunicorn/pidfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gunicorn/pidfile.py b/gunicorn/pidfile.py index 5fa38995..1d3911a2 100644 --- a/gunicorn/pidfile.py +++ b/gunicorn/pidfile.py @@ -77,10 +77,10 @@ class Pidfile(object): os.kill(wpid, 0) return wpid except OSError as e: - if e[0] == errno.ESRCH: + if e.args[0] == errno.ESRCH: return raise except IOError as e: - if e[0] == errno.ENOENT: + if e.args[0] == errno.ENOENT: return raise From f0deed152924ea8f43fd923a5a29d0abe9582d52 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 25 Oct 2012 20:57:23 +0200 Subject: [PATCH 17/47] fix from @sirkonst feedback --- gunicorn/http/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/http/errors.py b/gunicorn/http/errors.py index 5baf5b69..7719f285 100644 --- a/gunicorn/http/errors.py +++ b/gunicorn/http/errors.py @@ -3,7 +3,7 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -class ParseException(BaseException): +class ParseException(Exception): pass class NoMoreData(IOError): From fa5af28cf6fa957c991ad7c1c82e12d00b55cb1e Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 26 Oct 2012 21:07:27 +0200 Subject: [PATCH 18/47] update tox & makefile for py.test --- Makefile | 8 +++----- requirements_dev.txt | 1 + tox.ini | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 8f17f19b..99b60a6d 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,13 @@ build: virtualenv --no-site-packages . bin/python setup.py develop - bin/pip install coverage - bin/pip install nose + bin/pip install -r requirements_dev.txt test: - bin/nosetests + ./bin/py.test tests/ coverage: - bin/nosetests --with-coverage --cover-html --cover-html-dir=html \ - --cover-package=gunicorn + ./bin/py.test --cov gunicorn tests/ clean: @rm -rf .Python bin lib include man build html diff --git a/requirements_dev.txt b/requirements_dev.txt index e079f8a6..9955decc 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1 +1,2 @@ pytest +pytest-cov diff --git a/tox.ini b/tox.ini index b92db304..8f28d7c0 100644 --- a/tox.ini +++ b/tox.ini @@ -4,8 +4,8 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27, pypy +envlist = py26, py27, py33, pypy [testenv] -commands = nosetests -deps = nose +commands = py.test tests/ +deps = pytest From cd601a466a4271494159dbd1b2b7205b8c393f6e Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 26 Oct 2012 21:09:23 +0200 Subject: [PATCH 19/47] update travis.xml to est on py3 --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc4d06bc..52b3f5e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,15 @@ language: python python: - 2.6 - 2.7 + - 3.2 + - 3.3 - pypy -install: python setup.py install -script: nosetests +install: + - pip install -r requirements_dev.txt --use-mirrors + - python setup.py install + +script: py.test tests/ branches: only: From e372a26342da49c9b0276f34fb19db93ddafb024 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sat, 27 Oct 2012 12:30:27 +0200 Subject: [PATCH 20/47] Revert "fix gevent graceful timeout for real" This reverts commit fd6c712dd432f6cbbadd53bb59e7c5ce7b07e0cb. --- gunicorn/workers/ggevent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index 90d5c262..b89cd4c2 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -79,7 +79,7 @@ class GeventWorker(AsyncWorker): try: # Stop accepting requests - server.close() + server.kill() # Handle current requests until graceful_timeout ts = time.time() From f7b9a08c9c8abc14a2d52844f9071d4c536e860d Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Sat, 27 Oct 2012 22:11:32 -0700 Subject: [PATCH 21/47] resolve py3 bytes issue for django apps --- gunicorn/arbiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index af0209e1..340ea299 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -273,7 +273,7 @@ class Arbiter(object): Wake up the arbiter by writing to the PIPE """ try: - os.write(self.PIPE[1], '.') + os.write(self.PIPE[1], b'.') except IOError as e: if e.errno not in [errno.EAGAIN, errno.EINTR]: raise From e4fbc805b6aa4afe80a08f50eb12af665a6af621 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 28 Oct 2012 06:56:00 +0100 Subject: [PATCH 22/47] fix error spotted by @andrewsg --- .../django/testing/testing/apps/someapp/views.py | 8 +++++++- gunicorn/http/wsgi.py | 2 +- gunicorn/util.py | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/frameworks/django/testing/testing/apps/someapp/views.py b/examples/frameworks/django/testing/testing/apps/someapp/views.py index d685b5bc..714abaaf 100755 --- a/examples/frameworks/django/testing/testing/apps/someapp/views.py +++ b/examples/frameworks/django/testing/testing/apps/someapp/views.py @@ -1,6 +1,7 @@ # Create your views here. import csv +import io import os from django import forms from django.http import HttpResponse @@ -28,10 +29,15 @@ def home(request): subject = form.cleaned_data['subject'] message = form.cleaned_data['message'] f = request.FILES['f'] + + if not hasattr(f, "fileno"): size = len(f.read()) else: - size = int(os.fstat(f.fileno())[6]) + try: + size = int(os.fstat(f.fileno())[6]) + except io.UnsupportedOperation: + size = len(f.read()) else: form = MsgForm() diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 5ea54f28..a32c2573 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -275,7 +275,7 @@ class Response(object): self.send_headers() if isinstance(arg, text_type): - arg = arg.decode('utf-8') + arg = arg.encode('utf-8') assert isinstance(arg, binary_type), "%r is not a byte." % arg diff --git a/gunicorn/util.py b/gunicorn/util.py index 142f6e10..f367d2e3 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -225,9 +225,9 @@ except ImportError: def write_chunk(sock, data): if isinstance(data, text_type): - data = data.decode('utf-8') + data = data.encode('utf-8') chunk_size = "%X\r\n" % len(data) - chunk = b"".join([chunk_size.decode('utf-8'), data, b"\r\n"]) + chunk = b"".join([chunk_size.encode('utf-8'), data, b"\r\n"]) sock.sendall(chunk) def write(sock, data, chunked=False): From 1d0eed5205986e80c81810d16d20d93c24f4f984 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 28 Oct 2012 07:04:11 +0100 Subject: [PATCH 23/47] remove examples from pytest dirs --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index af975ae6..e8361c22 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,3 +2,6 @@ build-requires = python2-devel python-setuptools requires = python-setuptools >= 0.6c6 python-ctypes install_script = rpm/install + +[pytest] +norecursedirs = examples From ee3946fba6fc2b88eb058981be53b1cdd49ab0d5 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 28 Oct 2012 07:11:10 +0100 Subject: [PATCH 24/47] add @andrewsg to thanks. --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index e2af3149..23e7ef64 100644 --- a/THANKS +++ b/THANKS @@ -45,3 +45,4 @@ Caleb Brown Marc Abramowitz Vangelis Koukis Prateek Singh Paudel +Andrew Gorcester From 674c1ac8026b76ad0f794895407a8b50a2347548 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Sun, 28 Oct 2012 22:59:19 -0700 Subject: [PATCH 25/47] skip virtualenv dirs when looking for tests --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e8361c22..f070c8c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,4 +4,4 @@ requires = python-setuptools >= 0.6c6 python-ctypes install_script = rpm/install [pytest] -norecursedirs = examples +norecursedirs = examples lib local src From 1505e29248d82311c99b8f4924293420c4b21ec2 Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Fri, 26 Oct 2012 14:06:29 -0700 Subject: [PATCH 26/47] integrate pytest with setup.py --- Makefile | 6 +++--- setup.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 99b60a6d..43908092 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ build: virtualenv --no-site-packages . bin/python setup.py develop - bin/pip install -r requirements_dev.txt + bin/pip install -r requirements_dev.txt test: - ./bin/py.test tests/ + bin/python setup.py test coverage: - ./bin/py.test --cov gunicorn tests/ + bin/python setup.py test --cov clean: @rm -rf .Python bin lib include man build html diff --git a/setup.py b/setup.py index 37feeb61..2b379e2d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import os -from setuptools import setup, find_packages +from setuptools import setup, find_packages, Command import sys from gunicorn import __version__ @@ -34,6 +34,31 @@ CLASSIFIERS = [ with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: long_description = f.read() +# read dev requirements +fname = os.path.join(os.path.dirname(__file__), 'requirements_dev.txt') +with open(fname) as f: + tests_require = list(map(lambda l: l.strip(), f.readlines())) + +class PyTest(Command): + user_options = [ + ("cov", None, "measure coverage") + ] + + def initialize_options(self): + self.cov = None + + def finalize_options(self): + pass + + def run(self): + import sys,subprocess + basecmd = [sys.executable, '-m', 'py.test'] + if self.cov: + basecmd += ['--cov', 'gunicorn'] + errno = subprocess.call(basecmd + ['tests']) + raise SystemExit(errno) + + setup( name = 'gunicorn', version = __version__, @@ -50,6 +75,9 @@ setup( packages = find_packages(exclude=['examples', 'tests']), include_package_data = True, + tests_require = tests_require, + cmdclass = {'test': PyTest}, + entry_points=""" [console_scripts] From daa04fc931098f96f88db728e080c77bf649e1a7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Thu, 1 Nov 2012 10:46:44 +0100 Subject: [PATCH 27/47] add a note about `--check-config` --- docs/source/configure.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 3d403520..d2d2418a 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -20,6 +20,17 @@ Once again, in order of least to most authoritative: 2. Configuration File 3. Command Line + +.. note:: + + To check your configuration when using the command line or the + configuration file you can run the following command:: + + $ gunicorn --check-config + + It will also allows you to know if your applican can be launched. + + Framework Settings ================== From bb9ddb4ca97c92150b677dc073c9f0bf85b3a602 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 6 Nov 2012 05:33:29 +0100 Subject: [PATCH 28/47] Revert "Set timeout for client socket (slow client DoS)." This changes introduced an issue with websockets support (#432) and is probably related to #428 & #416 . It is safer for now to revert it. This reverts commit aa22115cfc9c2b76c818ce231089b01c690052b6. Conflicts: gunicorn/workers/async.py gunicorn/workers/sync.py --- gunicorn/workers/async.py | 5 ----- gunicorn/workers/base.py | 5 ----- gunicorn/workers/sync.py | 3 --- 3 files changed, 13 deletions(-) diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index ecdab864..fc6a7dd7 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -29,7 +29,6 @@ class AsyncWorker(base.Worker): def handle(self, client, addr): req = None try: - client.settimeout(self.cfg.timeout) parser = http.RequestParser(self.cfg, client) try: if not self.cfg.keepalive: @@ -48,12 +47,8 @@ class AsyncWorker(base.Worker): self.log.debug("Ignored premature client disconnection. %s", e) except StopIteration as e: self.log.debug("Closing connection. %s", e) - except socket.error: - raise # pass to next try-except level except Exception as e: self.handle_error(req, client, addr, e) - except socket.timeout as e: - self.handle_error(req, client, addr, e) except socket.error as e: if e.args[0] not in (errno.EPIPE, errno.ECONNRESET): self.log.exception("Socket error processing request.") diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index bdc069f0..136791cb 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -8,7 +8,6 @@ import os import signal import sys import traceback -import socket from gunicorn import util @@ -165,10 +164,6 @@ class Worker(object): error=str(exc), ) ) - elif isinstance(exc, socket.timeout): - status_int = 408 - reason = "Request Timeout" - mesg = "

The server timed out handling for the request

" else: self.log.exception("Error handling request") diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index d18af1a6..0eb837de 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -68,7 +68,6 @@ class SyncWorker(base.Worker): def handle(self, client, addr): req = None try: - client.settimeout(self.cfg.timeout) parser = http.RequestParser(self.cfg, client) req = six.next(parser) self.handle_request(req, client, addr) @@ -76,8 +75,6 @@ class SyncWorker(base.Worker): self.log.debug("Ignored premature client disconnection. %s", e) except StopIteration as e: self.log.debug("Closing connection. %s", e) - except socket.timeout as e: - self.handle_error(req, client, addr, e) except socket.error as e: if e.args[0] != errno.EPIPE: self.log.exception("Error processing request.") From d06380d1f0143ce50a6bafcd2b229b203390bbe9 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 6 Nov 2012 05:51:57 +0100 Subject: [PATCH 29/47] fix module detection. fix #322 Gunicorn has now the possibility to directly pass the settings env as argument but it was breaking the old way to do it when giving a path to the settings file instead. --- gunicorn/app/djangoapp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gunicorn/app/djangoapp.py b/gunicorn/app/djangoapp.py index 6eb5d574..7fe44cff 100644 --- a/gunicorn/app/djangoapp.py +++ b/gunicorn/app/djangoapp.py @@ -71,7 +71,8 @@ class DjangoApplication(Application): def init(self, parser, opts, args): if args: - if "." in args[0]: + if ("." in args[0] and not (os.path.isfile(args[0]) + or os.path.isdir(args[0]))): self.cfg.set("django_settings", args[0]) else: # not settings env set, try to build one. From 88fd29db2f91b673b80b70327bce46d3d375bc6b Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 6 Nov 2012 06:02:47 +0100 Subject: [PATCH 30/47] add a direct link to latest changes info. fix #424 --- docs/site/css/style.css | 11 ++++++++++- docs/site/index.html | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/site/css/style.css b/docs/site/css/style.css index 7803d57d..65c18261 100644 --- a/docs/site/css/style.css +++ b/docs/site/css/style.css @@ -60,6 +60,15 @@ a:hover { border-bottom: 1px solid #2A8729; } +.latest { + width: 150px; + top: 0; + display: block; + float: right; + font-weight: bold; +} + + .logo-div { width: 1000px; margin: 0 auto; @@ -389,4 +398,4 @@ pre { .footer-wp a:hover { color: #1D692D; -} \ No newline at end of file +} diff --git a/docs/site/index.html b/docs/site/index.html index 6a8380a2..c9bcd7a3 100644 --- a/docs/site/index.html +++ b/docs/site/index.html @@ -14,6 +14,11 @@
+
+ Latest version: 0.15.0 +
+
From 23f66c23899136d5293c8c0332e3b04f9229c157 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 6 Nov 2012 06:06:20 +0100 Subject: [PATCH 31/47] fix latest link color --- docs/site/css/style.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/site/css/style.css b/docs/site/css/style.css index 65c18261..1a4842cb 100644 --- a/docs/site/css/style.css +++ b/docs/site/css/style.css @@ -292,12 +292,14 @@ a:hover { margin: 0 0 9px; } -.tab-box a { +.tab-box a, +.latest a { color: #3F3F27; text-decoration: underline; } -.tab-box a:hover { +.tab-box a:hover, +.latest a:hover { color: #1D692D; } From 9fb0d9669c3a66857ee09c9dbe82607295009ad7 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 7 Nov 2012 09:52:49 +0100 Subject: [PATCH 32/47] expose --pythonpath command to all modes . fix #433 --pythonpath may also be useful in other commands, so expose it to all. --- docs/source/settings.rst | 3 +++ gunicorn/app/base.py | 8 ++++++++ gunicorn/app/djangoapp.py | 8 +++++--- gunicorn/config.py | 4 ++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/source/settings.rst b/docs/source/settings.rst index e0b16424..66dfebd6 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -482,6 +482,9 @@ The Python path to a Django settings module. e.g. 'myproject.settings.main'. If this isn't provided, the DJANGO_SETTINGS_MODULE environment variable will be used. +Server Mechanics +---------------- + pythonpath ~~~~~~~~~~ diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index e5d4c996..b980daf5 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -115,6 +115,14 @@ class Application(object): if self.cfg.daemon: util.daemonize() + # set python paths + if self.cfg.pythonpath and self.cfg.pythonpath is not None: + paths = self.cfg.pythonpath.split(",") + for path in paths: + pythonpath = os.path.abspath(self.cfg.pythonpath) + if pythonpath not in sys.path: + sys.path.insert(0, pythonpath) + try: Arbiter(self).run() except RuntimeError as e: diff --git a/gunicorn/app/djangoapp.py b/gunicorn/app/djangoapp.py index 7fe44cff..de403578 100644 --- a/gunicorn/app/djangoapp.py +++ b/gunicorn/app/djangoapp.py @@ -44,9 +44,11 @@ def make_default_env(cfg): os.environ['DJANGO_SETTINGS_MODULE'] = cfg.django_settings if cfg.pythonpath and cfg.pythonpath is not None: - pythonpath = os.path.abspath(cfg.pythonpath) - if pythonpath not in sys.path: - sys.path.insert(0, pythonpath) + paths = cfg.pythonpath.split(",") + for path in paths: + pythonpath = os.path.abspath(cfg.pythonpath) + if pythonpath not in sys.path: + sys.path.insert(0, pythonpath) try: os.environ['DJANGO_SETTINGS_MODULE'] diff --git a/gunicorn/config.py b/gunicorn/config.py index 41f176b0..1901d492 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -867,9 +867,9 @@ class DjangoSettings(Setting): DJANGO_SETTINGS_MODULE environment variable will be used. """ -class DjangoPythonPath(Setting): +class PythonPath(Setting): name = "pythonpath" - section = "Django" + section = "Server Mechanics" cli = ["--pythonpath"] meta = "STRING" validator = validate_string From 6f726e0ec9a7aab14d489ee50b2a8f15868ea619 Mon Sep 17 00:00:00 2001 From: benoitc Date: Tue, 13 Nov 2012 09:52:37 +0100 Subject: [PATCH 33/47] make sure to catch EPIPE and ECONNRESET error --- gunicorn/workers/async.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index fc6a7dd7..c616248a 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -47,6 +47,8 @@ class AsyncWorker(base.Worker): self.log.debug("Ignored premature client disconnection. %s", e) except StopIteration as e: self.log.debug("Closing connection. %s", e) + except socket.error: + raise # pass to next try-except level except Exception as e: self.handle_error(req, client, addr, e) except socket.error as e: From 402f003ca22e524021f53fd46421714f2d0b7d05 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 16 Nov 2012 03:20:25 +0100 Subject: [PATCH 34/47] Honor $PORT environment variable. --- gunicorn/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gunicorn/config.py b/gunicorn/config.py index 1901d492..35d7ae22 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -319,7 +319,12 @@ class Bind(Setting): cli = ["-b", "--bind"] meta = "ADDRESS" validator = validate_string - default = "127.0.0.1:8000" + + if 'PORT' in os.environ: + default = '0.0.0.0:{0}'.format(os.environ.get('PORT')) + else: + default = '127.0.0.1:8000' + desc = """\ The socket to bind. From 044732f7bcbfb0decde6748e53e2ccd536378a52 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 07:38:30 +0100 Subject: [PATCH 35/47] add Kenneth Reitz to THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 23e7ef64..b33cdde9 100644 --- a/THANKS +++ b/THANKS @@ -46,3 +46,4 @@ Marc Abramowitz Vangelis Koukis Prateek Singh Paudel Andrew Gorcester +Kenneth Reitz From e58f8b59b79f0419cb471cb7430df98530872567 Mon Sep 17 00:00:00 2001 From: Eric Shull Date: Thu, 8 Nov 2012 13:36:33 -0500 Subject: [PATCH 36/47] Add isatty method to LazyWriter. --- gunicorn/glogging.py | 3 +++ tests/test_005-lazywriter-isatty.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/test_005-lazywriter-isatty.py diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index eaa82708..d15925fd 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -87,6 +87,9 @@ class LazyWriter(object): def flush(self): self.open().flush() + def isatty(self): + return bool(self.fileobj and self.fileobj.isatty()) + class SafeAtoms(dict): def __init__(self, atoms): diff --git a/tests/test_005-lazywriter-isatty.py b/tests/test_005-lazywriter-isatty.py new file mode 100644 index 00000000..50cc71c9 --- /dev/null +++ b/tests/test_005-lazywriter-isatty.py @@ -0,0 +1,13 @@ +import sys + +from gunicorn.glogging import LazyWriter + + +def test_lazywriter_isatty(): + orig = sys.stdout + sys.stdout = LazyWriter('test.log') + try: + sys.stdout.isatty() + except AttributeError: + raise AssertionError("LazyWriter has no attribute 'isatty'") + sys.stdout = orig From f240b78fd3a71ac6f1b24807041615f27c439e5b Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 07:57:19 +0100 Subject: [PATCH 37/47] add Eric Shull to THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index b33cdde9..20bf199c 100644 --- a/THANKS +++ b/THANKS @@ -47,3 +47,4 @@ Vangelis Koukis Prateek Singh Paudel Andrew Gorcester Kenneth Reitz +Eric Shull From 91e7d138dcff4226aaf76b31905bff5cebcf6d40 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 09:57:35 +0100 Subject: [PATCH 38/47] fix header encoding --- examples/test.py | 3 ++- gunicorn/http/wsgi.py | 9 +++++---- gunicorn/util.py | 7 +++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/examples/test.py b/examples/test.py index c61f672e..44283565 100644 --- a/examples/test.py +++ b/examples/test.py @@ -18,7 +18,8 @@ def app(environ, start_response): response_headers = [ ('Content-type','text/plain'), ('Content-Length', str(len(data))), - ('X-Gunicorn-Version', __version__) + ('X-Gunicorn-Version', __version__), + ("Test", "test ั‚ะตัั‚"), ] start_response(status, response_headers) return iter([data]) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index a32c2573..bb3c3f24 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -210,6 +210,7 @@ class Response(object): def process_headers(self, headers): for name, value in headers: assert isinstance(name, string_types), "%r is not a string" % name + lname = name.lower().strip() if lname == "content-length": self.response_length = int(value) @@ -220,11 +221,11 @@ class Response(object): self.upgrade = True elif lname == "upgrade": if value.lower().strip() == "websocket": - self.headers.append((name.strip(), str(value).strip())) + self.headers.append((name.strip(), value.strip())) # ignore hopbyhop headers continue - self.headers.append((name.strip(), str(value).strip())) + self.headers.append((name.strip(), str(value.strip()))) def is_chunked(self): @@ -265,10 +266,10 @@ class Response(object): if self.headers_sent: return tosend = self.default_headers() - tosend.extend(["%s: %s\r\n" % (n, v) for n, v in self.headers]) + tosend.extend(["%s: %s\r\n" % (k,v) for k, v in self.headers]) header_str = "%s\r\n" % "".join(tosend) - util.write(self.sock, header_str.encode('latin1')) + util.write(self.sock, util.to_bytestring(header_str)) self.headers_sent = True def write(self, arg): diff --git a/gunicorn/util.py b/gunicorn/util.py index f367d2e3..cf976b6e 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -348,3 +348,10 @@ def check_is_writeable(path): except IOError as e: raise RuntimeError("Error: '%s' isn't writable [%r]" % (path, e)) f.close() + +def to_bytestring(value): + """Converts a string argument to a byte string""" + if isinstance(value, bytes): + return value + assert isinstance(value, text_type) + return value.encode("utf-8") From d9faae01db0bacde86c82c73808fea593ee00507 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 10:23:45 +0100 Subject: [PATCH 39/47] bump to 0.16 --- gunicorn/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index e71c6ecc..8aa0d2d4 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -3,6 +3,6 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -version_info = (0, 15, 0) +version_info = (0, 16, 0) __version__ = ".".join(map(str, version_info)) SERVER_SOFTWARE = "gunicorn/%s" % __version__ From efdc99dd91e67b4e03a764c136154146f597d28a Mon Sep 17 00:00:00 2001 From: Christos Stavrakakis Date: Thu, 15 Nov 2012 14:43:15 +0200 Subject: [PATCH 40/47] Reopen stdout & stderr if redirected to error log To use the logrotate utility, a USR1 signal is sent, and the corresponding handler reopens the log files. However, sys.stdout and sys.stderr, which may be redirected to the error log file, are not updated. This commit fixes this, by closing the fileobj of the LazyWriter object. There is no need to reopen it, since the LazyWriter will open it when needed. --- gunicorn/glogging.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index d15925fd..1c5c603b 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -74,6 +74,16 @@ class LazyWriter(object): self.lock.release() return self.fileobj + def close(self): + if self.fileobj: + self.lock.acquire() + try: + if self.fileobj: + self.fileobj.close() + self.fileobj = None + finally: + self.lock.release() + def write(self, text): fileobj = self.open() fileobj.write(text) @@ -239,6 +249,10 @@ class Logger(object): def reopen_files(self): + if self.cfg.errorlog != "-": + # Close stderr & stdout if they are redirected to error log file + sys.stderr.close() + sys.stdout.close() for log in loggers(): for handler in log.handlers: if isinstance(handler, logging.FileHandler): From 98b2114199b04678cd41e25deb9a3478e0f76e45 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 10:51:51 +0100 Subject: [PATCH 41/47] say hello to python 3.3 --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 2b379e2d..aef666b5 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,10 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', 'Topic :: Internet', 'Topic :: Utilities', 'Topic :: Software Development :: Libraries :: Python Modules', From 46e041b9f1d05e14afb34f4f0cc6580a1af225b4 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 11:08:02 +0100 Subject: [PATCH 42/47] don't use map --- gunicorn/arbiter.py | 21 +++++++++++++-------- gunicorn/http/wsgi.py | 2 +- gunicorn/workers/base.py | 15 ++++++++------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 340ea299..4ee751bf 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -41,10 +41,8 @@ class Arbiter(object): # I love dynamic languages SIG_QUEUE = [] - SIGNALS = map( - lambda x: getattr(signal, "SIG%s" % x), - "HUP QUIT INT TERM TTIN TTOU USR1 USR2 WINCH".split() - ) + SIGNALS = [getattr(signal, "SIG%s" % x) \ + for x in "HUP QUIT INT TERM TTIN TTOU USR1 USR2 WINCH".split()] SIG_NAMES = dict( (getattr(signal, name), name[3:].lower()) for name in dir(signal) if name[:3] == "SIG" and name[3] != "_" @@ -138,13 +136,20 @@ class Arbiter(object): Initialize master signal handling. Most of the signals are queued. Child signals only wake up the master. """ + # close old PIPE if self.PIPE: - map(os.close, self.PIPE) + [os.close(p) for p in self.PIPE] + + # initialize the pipe self.PIPE = pair = os.pipe() - map(util.set_non_blocking, pair) - map(util.close_on_exec, pair) + for p in pair: + util.set_non_blocking(p) + util.close_on_exec(p) + self.log.close_on_exec() - map(lambda s: signal.signal(s, self.signal), self.SIGNALS) + + # intialiatze all signals + [signal.signal(s, self.signal) for s in self.SIGNALS] signal.signal(signal.SIGCHLD, self.handle_chld) def signal(self, sig, frame): diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index bb3c3f24..5dec1c09 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -55,7 +55,7 @@ def default_environ(req, sock, cfg): "REQUEST_METHOD": req.method, "QUERY_STRING": req.query, "RAW_URI": req.uri, - "SERVER_PROTOCOL": "HTTP/%s" % ".".join(map(str, req.version)) + "SERVER_PROTOCOL": "HTTP/%s" % ".".join([str(v) for v in req.version]) } def proxy_environ(req): diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 136791cb..44195259 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -21,10 +21,8 @@ from gunicorn.six import MAXSIZE class Worker(object): - SIGNALS = map( - lambda x: getattr(signal, "SIG%s" % x), - "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() - ) + SIGNALS = [getattr(signal, "SIG%s" % x) \ + for x in "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] PIPE = [] @@ -87,8 +85,9 @@ class Worker(object): # For waking ourselves up self.PIPE = os.pipe() - map(util.set_non_blocking, self.PIPE) - map(util.close_on_exec, self.PIPE) + for p in self.PIPE: + util.set_non_blocking(p) + util.close_on_exec(p) # Prevent fd inherientence util.close_on_exec(self.socket) @@ -105,7 +104,9 @@ class Worker(object): self.run() def init_signals(self): - map(lambda s: signal.signal(s, signal.SIG_DFL), self.SIGNALS) + # reset signaling + [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] + # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) From 4da8f8067d048a6ad1fc8bdacd9554bd8861ce96 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 11:12:57 +0100 Subject: [PATCH 43/47] str should be applied first. change based on @sirkonst feedback --- gunicorn/http/wsgi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index 5dec1c09..48c34498 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -211,6 +211,7 @@ class Response(object): for name, value in headers: assert isinstance(name, string_types), "%r is not a string" % name + value = str(value).strip() lname = name.lower().strip() if lname == "content-length": self.response_length = int(value) @@ -221,11 +222,11 @@ class Response(object): self.upgrade = True elif lname == "upgrade": if value.lower().strip() == "websocket": - self.headers.append((name.strip(), value.strip())) + self.headers.append((name.strip(), value)) # ignore hopbyhop headers continue - self.headers.append((name.strip(), str(value.strip()))) + self.headers.append((name.strip(), value)) def is_chunked(self): From 155b3c3823cacf4aca08387a0d5a427d33118b57 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 11:16:19 +0100 Subject: [PATCH 44/47] don't use map at all --- gunicorn/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn/__init__.py b/gunicorn/__init__.py index 8aa0d2d4..f6ef7ba6 100644 --- a/gunicorn/__init__.py +++ b/gunicorn/__init__.py @@ -4,5 +4,5 @@ # See the NOTICE for more information. version_info = (0, 16, 0) -__version__ = ".".join(map(str, version_info)) +__version__ = ".".join([str(v) for v in version_info]) SERVER_SOFTWARE = "gunicorn/%s" % __version__ From 95a2e9e61075852897f8d1525e612a1836ad1a56 Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 11:22:21 +0100 Subject: [PATCH 45/47] what's new in 0.16.0 --- docs/source/news.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/news.rst b/docs/source/news.rst index 8ee33174..0827c048 100644 --- a/docs/source/news.rst +++ b/docs/source/news.rst @@ -1,6 +1,18 @@ Changelog ========= +0.16.0 / develop +---------------- + +- **Added support for Python 3.2 & 3.3** +- Expose --pythonpath command to all gunicorn commands +- Honor $PORT environment variable, useful for deployement on heroku +- Removed support for Python 2.5 +- Make sure we reopen teh logs on the console +- Fix django settings module detection from path +- Reverted timeout for client socket. Fix issue on blocking issues. +- Fixed gevent worker + 0.15.0 / 2012-10-18 ------------------- From 594a189cc15f43622653f51cbec587a26e4ca0cc Mon Sep 17 00:00:00 2001 From: benoitc Date: Fri, 16 Nov 2012 11:53:30 +0100 Subject: [PATCH 46/47] add Christos Stavrakakis to THANKS --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 20bf199c..7058add0 100644 --- a/THANKS +++ b/THANKS @@ -48,3 +48,4 @@ Prateek Singh Paudel Andrew Gorcester Kenneth Reitz Eric Shull +Christos Stavrakakis From f724c53e172f5cf7ef358f1cc9cce150ab2f23c1 Mon Sep 17 00:00:00 2001 From: benoitc Date: Mon, 19 Nov 2012 08:31:54 +0100 Subject: [PATCH 47/47] make sure to add the current dir to sys.path as well. close #322 --- gunicorn/app/djangoapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gunicorn/app/djangoapp.py b/gunicorn/app/djangoapp.py index de403578..8811b748 100644 --- a/gunicorn/app/djangoapp.py +++ b/gunicorn/app/djangoapp.py @@ -80,6 +80,8 @@ class DjangoApplication(Application): # not settings env set, try to build one. project_path, settings_name = find_settings_module( os.path.abspath(args[0])) + if project_path not in sys.path: + sys.path.insert(0, project_path) if not project_path: raise RuntimeError("django project not found")