From 8d453fb341eceb13e4cb99e24844d198bda73391 Mon Sep 17 00:00:00 2001 From: benoitc Date: Wed, 24 Oct 2012 22:07:35 +0200 Subject: [PATCH] all tests pass under python 3 --- gunicorn/app/django_wsgi.py | 6 +- gunicorn/arbiter.py | 7 ++- gunicorn/config.py | 21 +++++-- gunicorn/errors.py | 4 +- gunicorn/http/body.py | 48 +++++++-------- gunicorn/http/errors.py | 3 +- gunicorn/http/message.py | 11 ++-- gunicorn/http/parser.py | 5 +- gunicorn/http/unreader.py | 31 +++++----- gunicorn/six.py | 27 +++----- gunicorn/util.py | 9 --- gunicorn/workers/async.py | 5 +- gunicorn/workers/base.py | 3 +- gunicorn/workers/sync.py | 3 +- tests/004-test-http-body.py | 61 ------------------- tests/requests/valid/001.py | 4 +- tests/requests/valid/002.py | 4 +- tests/requests/valid/003.py | 4 +- tests/requests/valid/004.py | 4 +- tests/requests/valid/005.py | 4 +- tests/requests/valid/006.py | 4 +- tests/requests/valid/007.py | 4 +- tests/requests/valid/008.py | 4 +- tests/requests/valid/009.py | 4 +- tests/requests/valid/010.py | 4 +- tests/requests/valid/011.py | 4 +- tests/requests/valid/012.py | 4 +- tests/requests/valid/013.py | 4 +- tests/requests/valid/014.py | 4 +- tests/requests/valid/015.py | 4 +- tests/requests/valid/016.py | 4 +- tests/requests/valid/017.py | 2 +- tests/requests/valid/018.py | 4 +- tests/requests/valid/019.py | 4 +- tests/requests/valid/020.py | 4 +- tests/requests/valid/021.py | 4 +- tests/requests/valid/022.py | 6 +- tests/requests/valid/023.py | 6 +- tests/requests/valid/025.py | 4 +- tests/requests/valid/pp_01.py | 2 +- tests/requests/valid/pp_02.py | 4 +- tests/t.py | 35 ++++++----- ...requests.py => test_001-valid-requests.py} | 4 +- ...quests.py => test_002-invalid-requests.py} | 15 ++--- ...{003-test-config.py => test_003-config.py} | 57 ++++------------- tests/test_004-http-body.py | 61 +++++++++++++++++++ tests/treq.py | 49 ++++++++------- 47 files changed, 276 insertions(+), 293 deletions(-) delete mode 100644 tests/004-test-http-body.py rename tests/{001-test-valid-requests.py => test_001-valid-requests.py} (90%) rename tests/{002-test-invalid-requests.py => test_002-invalid-requests.py} (63%) rename tests/{003-test-config.py => test_003-config.py} (81%) create mode 100644 tests/test_004-http-body.py diff --git a/gunicorn/app/django_wsgi.py b/gunicorn/app/django_wsgi.py index 7ec41bf3..2372e7b2 100644 --- a/gunicorn/app/django_wsgi.py +++ b/gunicorn/app/django_wsgi.py @@ -74,10 +74,10 @@ def reload_django_settings(): app_mod = util.import_module(app[:-2]) appdir = os.path.dirname(app_mod.__file__) app_subdirs = os.listdir(appdir) - app_subdirs.sort() name_pattern = re.compile(r'[a-zA-Z]\w*') - for d in app_subdirs: - if name_pattern.match(d) and os.path.isdir(os.path.join(appdir, d)): + for d in sorted(app_subdirs): + if (name_pattern.match(d) and + os.path.isdir(os.path.join(appdir, d))): new_installed_apps.append('%s.%s' % (app[:-2], d)) else: new_installed_apps.append(app) diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 4e0d665e..af0209e1 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -99,7 +99,10 @@ class Arbiter(object): if self.cfg.debug: self.log.debug("Current configuration:") - for config, value in sorted(self.cfg.settings.iteritems()): + + + for config, value in sorted(self.cfg.settings.items(), + key=lambda setting: setting[1]): self.log.debug(" %s: %s", config, value.value) if self.cfg.preload_app: @@ -436,7 +439,7 @@ class Arbiter(object): self.spawn_workers() workers = self.WORKERS.items() - workers.sort(key=lambda w: w[1].age) + workers = sorted(workers, key=lambda w: w[1].age) while len(workers) > self.num_workers: (pid, _) = workers.pop(0) self.kill_worker(pid, signal.SIGQUIT) diff --git a/gunicorn/config.py b/gunicorn/config.py index fa3d15bb..e8f7fc32 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -15,7 +15,7 @@ import types from gunicorn import __version__ from gunicorn.errors import ConfigError from gunicorn import util -from gunicorn.six import string_types +from gunicorn.six import string_types, integer_types, bytes_to_str KNOWN_SETTINGS = [] @@ -62,10 +62,12 @@ class Config(object): } parser = optparse.OptionParser(**kwargs) - keys = self.settings.keys() + keys = list(self.settings) def sorter(k): return (self.settings[k].section, self.settings[k].order) - keys.sort(key=sorter) + + + keys = sorted(self.settings, key=self.settings.__getitem__) for k in keys: self.settings[k].add_option(parser) return parser @@ -85,7 +87,7 @@ class Config(object): @property def address(self): bind = self.settings['bind'].get() - return util.parse_address(util.to_bytestring(bind)) + return util.parse_address(bytes_to_str(bind)) @property def uid(self): @@ -179,8 +181,15 @@ class Setting(object): assert callable(self.validator), "Invalid validator: %s" % self.name self.value = self.validator(val) + def __lt__(self, other): + return (self.section == other.section and + self.order < other.order) + __cmp__ = __lt__ + +Setting = SettingMeta('Setting', (Setting,), {}) + def validate_bool(val): - if isinstance(val, types.BooleanType): + if isinstance(val, bool): return val if not isinstance(val, string_types): raise TypeError("Invalid type for casting: %s" % val) @@ -197,7 +206,7 @@ def validate_dict(val): return val def validate_pos_int(val): - if not isinstance(val, (types.IntType, types.LongType)): + if not isinstance(val, integer_types): val = int(val, 0) else: # Booleans are ints! diff --git a/gunicorn/errors.py b/gunicorn/errors.py index e4fbcdd3..9669f5ed 100644 --- a/gunicorn/errors.py +++ b/gunicorn/errors.py @@ -4,7 +4,7 @@ # See the NOTICE for more information. -class HaltServer(Exception): +class HaltServer(BaseException): def __init__(self, reason, exit_status=1): self.reason = reason self.exit_status = exit_status @@ -12,5 +12,5 @@ class HaltServer(Exception): def __str__(self): return "" % (self.reason, self.exit_status) -class ConfigError(Exception): +class ConfigError(BaseException): """ Exception raised on config error """ diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index e81e655a..9f685ac0 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -7,16 +7,16 @@ import sys from gunicorn.http.errors import (NoMoreData, ChunkMissingTerminator, InvalidChunkSize) -from gunicorn.six import StringIO, bytes_to_str, integer_types +from gunicorn import six class ChunkedReader(object): def __init__(self, req, unreader): self.req = req self.parser = self.parse_chunked(unreader) - self.buf = StringIO() + self.buf = six.BytesIO() def read(self, size): - if not isinstance(size, integer_types): + if not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") if size <= 0: raise ValueError("Size must be positive.") @@ -26,19 +26,19 @@ class ChunkedReader(object): if self.parser: while self.buf.tell() < size: try: - self.buf.write(self.parser.next()) + self.buf.write(six.next(self.parser)) except StopIteration: self.parser = None break data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret def parse_trailers(self, unreader, data): - buf = StringIO() + buf = six.BytesIO() buf.write(data) idx = buf.getvalue().find(b"\r\n\r\n") @@ -50,8 +50,7 @@ class ChunkedReader(object): if done: unreader.unread(buf.getvalue()[2:]) return b"" - self.req.trailers = self.req.parse_headers( - bytes_to_str(buf.getvalue()[:idx])) + self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx]) unreader.unread(buf.getvalue()[idx+4:]) def parse_chunked(self, unreader): @@ -73,7 +72,7 @@ class ChunkedReader(object): (size, rest) = self.parse_chunk_size(unreader, data=rest[2:]) def parse_chunk_size(self, unreader, data=None): - buf = StringIO() + buf = six.BytesIO() if data is not None: buf.write(data) @@ -111,7 +110,7 @@ class LengthReader(object): self.length = length def read(self, size): - if not isinstance(size, integer_types): + if not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") size = min(self.length, size) @@ -121,7 +120,7 @@ class LengthReader(object): return b"" - buf = StringIO() + buf = six.BytesIO() data = self.unreader.read() while data: buf.write(data) @@ -138,21 +137,21 @@ class LengthReader(object): class EOFReader(object): def __init__(self, unreader): self.unreader = unreader - self.buf = StringIO() + self.buf = six.BytesIO() self.finished = False def read(self, size): - if not isinstance(size, integer_types): + if not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") if size < 0: raise ValueError("Size must be positive.") if size == 0: - return "" + return b"" if self.finished: data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret @@ -168,31 +167,32 @@ class EOFReader(object): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret class Body(object): def __init__(self, reader): self.reader = reader - self.buf = StringIO() + self.buf = six.BytesIO() def __iter__(self): return self - def next(self): + def __next__(self): ret = self.readline() if not ret: raise StopIteration() return ret + next = __next__ def getsize(self, size): if size is None: - return sys.maxint - elif not isinstance(size, integer_types): + return six.MAXSIZE + elif not isinstance(size, six.integer_types): raise TypeError("size must be an integral type") elif size < 0: - return sys.maxint + return six.MAXSIZE return size def read(self, size=None): @@ -203,7 +203,7 @@ class Body(object): if size < self.buf.tell(): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret @@ -215,7 +215,7 @@ class Body(object): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(rest) return ret @@ -225,7 +225,7 @@ class Body(object): return b"" line = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() if len(line) < size: line += self.reader.read(size - len(line)) extra_buf_data = line[size:] diff --git a/gunicorn/http/errors.py b/gunicorn/http/errors.py index 7ba4949f..5baf5b69 100644 --- a/gunicorn/http/errors.py +++ b/gunicorn/http/errors.py @@ -3,12 +3,13 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -class ParseException(Exception): +class ParseException(BaseException): pass class NoMoreData(IOError): def __init__(self, buf=None): self.buf = buf + def __str__(self): return "No more data after: %r" % self.buf diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index b32dbc2b..1d1ff588 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -13,7 +13,7 @@ from gunicorn.http.errors import InvalidHeader, InvalidHeaderName, NoMoreData, \ InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, \ LimitRequestLine, LimitRequestHeaders from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest -from gunicorn.six import StringIO, urlsplit, bytes_to_str +from gunicorn.six import BytesIO, urlsplit, bytes_to_str MAX_REQUEST_LINE = 8190 MAX_HEADERS = 32768 @@ -148,7 +148,6 @@ class Request(Message): self.req_number = req_number self.proxy_protocol_info = None - super(Request, self).__init__(cfg, unreader) @@ -161,7 +160,7 @@ class Request(Message): buf.write(data) def parse(self, unreader): - buf = StringIO() + buf = BytesIO() self.get_data(unreader, buf, stop=True) # get request line @@ -170,12 +169,12 @@ class Request(Message): # proxy protocol if self.proxy_protocol(bytes_to_str(line)): # get next request line - buf = StringIO() + buf = BytesIO() buf.write(rbuf) line, rbuf = self.read_line(unreader, buf, self.limit_request_line) self.parse_request_line(bytes_to_str(line)) - buf = StringIO() + buf = BytesIO() buf.write(rbuf) # Headers @@ -202,7 +201,7 @@ class Request(Message): self.headers = self.parse_headers(data[:idx]) ret = data[idx+4:] - buf = StringIO() + buf = BytesIO() return ret def read_line(self, unreader, buf, limit=0): diff --git a/gunicorn/http/parser.py b/gunicorn/http/parser.py index 54bbddea..77e959e5 100644 --- a/gunicorn/http/parser.py +++ b/gunicorn/http/parser.py @@ -22,7 +22,7 @@ class Parser(object): def __iter__(self): return self - def next(self): + def __next__(self): # Stop if HTTP dictates a stop. if self.mesg and self.mesg.should_close(): raise StopIteration() @@ -33,6 +33,7 @@ class Parser(object): while data: data = self.mesg.body.read(8192) + # Parse the next request self.req_count += 1 self.mesg = self.mesg_class(self.cfg, self.unreader, self.req_count) @@ -40,6 +41,8 @@ class Parser(object): raise StopIteration() return self.mesg + next = __next__ + class RequestParser(Parser): def __init__(self, *args, **kwargs): super(RequestParser, self).__init__(Request, *args, **kwargs) diff --git a/gunicorn/http/unreader.py b/gunicorn/http/unreader.py index 1631a24a..2cc2133e 100644 --- a/gunicorn/http/unreader.py +++ b/gunicorn/http/unreader.py @@ -5,44 +5,47 @@ import os -from gunicorn.six import integer_types, StringIO +from gunicorn import six # Classes that can undo reading data from # a given type of data source. class Unreader(object): def __init__(self): - self.buf = StringIO() + self.buf = six.BytesIO() def chunk(self): raise NotImplementedError() def read(self, size=None): - if size is not None and not isinstance(size, integer_types): + if size is not None and not isinstance(size, six.integer_types): raise TypeError("size parameter must be an int or long.") - if size == 0: - return "" - if size < 0: - size = None + + if size is not None: + if size == 0: + return b"" + if size < 0: + size = None self.buf.seek(0, os.SEEK_END) if size is None and self.buf.tell(): ret = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() return ret if size is None: - return self.chunk() + d = self.chunk() + return d while self.buf.tell() < size: chunk = self.chunk() if not len(chunk): ret = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() return ret self.buf.write(chunk) data = self.buf.getvalue() - self.buf.truncate(0) + self.buf = six.BytesIO() self.buf.write(data[size:]) return data[:size] @@ -66,9 +69,9 @@ class IterUnreader(Unreader): def chunk(self): if not self.iter: - return "" + return b"" try: - return self.iter.next() + return six.next(self.iter) except StopIteration: self.iter = None - return "" + return b"" diff --git a/gunicorn/six.py b/gunicorn/six.py index 76c15530..9070e400 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -281,10 +281,14 @@ _add_doc(u, """Text literal""") if PY3: + + def execfile_(fname, *args): + return exec(compile(open(fname, 'rb').read(), fname, 'exec'), *args) + + import builtins exec_ = getattr(builtins, "exec") - def reraise(tp, value, tb=None): if value.__traceback__ is not tb: raise value.with_traceback(tb) @@ -294,10 +298,6 @@ if PY3: print_ = getattr(builtins, "print") del builtins - def execfile_(file, globals=globals(), locals=locals()): - with open(file, "r") as fh: - exec_(fh.read()+"\n", globals, locals) - else: def exec_(code, globs=None, locs=None): """Execute code in a namespace.""" @@ -373,33 +373,26 @@ def with_metaclass(meta, base=object): # specific to gunicorn if PY3: - import io - StringIO = io.BytesIO - def bytes_to_str(b): + if isinstance(b, text_type): + return b return str(b, 'latin1') import urllib.parse unquote = urllib.parse.unquote urlsplit = urllib.parse.urlsplit + urlparse = urllib.parse.urlparse else: - try: - import cStringIO as StringIO - except ImportError: - import StringIO - - StringIO = StringIO - - - def bytestring(s): + def bytes_to_str(s): if isinstance(s, unicode): return s.encode('utf-8') return s import urlparse as orig_urlparse urlsplit = orig_urlparse.urlsplit + urlparse = orig_urlparse.urlparse import urllib urlunquote = urllib.unquote diff --git a/gunicorn/util.py b/gunicorn/util.py index 0f6895a7..142f6e10 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -311,15 +311,6 @@ def http_date(timestamp=None): hh, mm, ss) return s -def to_bytestring(s): - """ convert to bytestring an unicode """ - if not isinstance(s, string_types): - return s - if isinstance(s, unicode): - return s.encode('utf-8') - else: - return s - def is_hoppish(header): return header.lower().strip() in hop_headers diff --git a/gunicorn/workers/async.py b/gunicorn/workers/async.py index 08fed0ee..ecdab864 100644 --- a/gunicorn/workers/async.py +++ b/gunicorn/workers/async.py @@ -13,6 +13,7 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi import gunicorn.util as util import gunicorn.workers.base as base +from gunicorn import six ALREADY_HANDLED = object() @@ -32,14 +33,14 @@ class AsyncWorker(base.Worker): parser = http.RequestParser(self.cfg, client) try: if not self.cfg.keepalive: - req = parser.next() + req = six.next(parser) self.handle_request(req, client, addr) else: # keepalive loop while True: req = None with self.timeout_ctx(): - req = parser.next() + req = six.next(parser) if not req: break self.handle_request(req, client, addr) diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 474cf3d3..bdc069f0 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -18,6 +18,7 @@ InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, \ LimitRequestLine, LimitRequestHeaders from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest from gunicorn.http.wsgi import default_environ, Response +from gunicorn.six import MAXSIZE class Worker(object): @@ -43,7 +44,7 @@ class Worker(object): self.booted = False self.nr = 0 - self.max_requests = cfg.max_requests or sys.maxint + self.max_requests = cfg.max_requests or MAXSIZE self.alive = True self.log = log self.debug = cfg.debug diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index bd375d94..d18af1a6 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -14,6 +14,7 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi import gunicorn.util as util import gunicorn.workers.base as base +from gunicorn import six class SyncWorker(base.Worker): @@ -69,7 +70,7 @@ class SyncWorker(base.Worker): try: client.settimeout(self.cfg.timeout) parser = http.RequestParser(self.cfg, client) - req = parser.next() + req = six.next(parser) self.handle_request(req, client, addr) except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) diff --git a/tests/004-test-http-body.py b/tests/004-test-http-body.py deleted file mode 100644 index 22357478..00000000 --- a/tests/004-test-http-body.py +++ /dev/null @@ -1,61 +0,0 @@ -from StringIO import StringIO - -import t -from gunicorn.http.body import Body - - -def assert_readline(payload, size, expected): - body = Body(StringIO(payload)) - t.eq(body.readline(size), expected) - - -def test_readline_empty_body(): - assert_readline("", None, "") - assert_readline("", 1, "") - - -def test_readline_zero_size(): - assert_readline("abc", 0, "") - assert_readline("\n", 0, "") - - -def test_readline_new_line_before_size(): - body = Body(StringIO("abc\ndef")) - t.eq(body.readline(4), "abc\n") - t.eq(body.readline(), "def") - - -def test_readline_new_line_after_size(): - body = Body(StringIO("abc\ndef")) - t.eq(body.readline(2), "ab") - t.eq(body.readline(), "c\n") - - -def test_readline_no_new_line(): - body = Body(StringIO("abcdef")) - t.eq(body.readline(), "abcdef") - body = Body(StringIO("abcdef")) - t.eq(body.readline(2), "ab") - t.eq(body.readline(2), "cd") - t.eq(body.readline(2), "ef") - - -def test_readline_buffer_loaded(): - reader = StringIO("abc\ndef") - body = Body(reader) - body.read(1) # load internal buffer - reader.write("g\nhi") - reader.seek(7) - t.eq(body.readline(), "bc\n") - t.eq(body.readline(), "defg\n") - t.eq(body.readline(), "hi") - - -def test_readline_buffer_loaded_with_size(): - body = Body(StringIO("abc\ndef")) - body.read(1) # load internal buffer - t.eq(body.readline(2), "bc") - t.eq(body.readline(2), "\n") - t.eq(body.readline(2), "de") - t.eq(body.readline(2), "f") - diff --git a/tests/requests/valid/001.py b/tests/requests/valid/001.py index 4b3109eb..1eee7b5e 100644 --- a/tests/requests/valid/001.py +++ b/tests/requests/valid/001.py @@ -7,5 +7,5 @@ request = { ("CONTENT-TYPE", "application/json"), ("CONTENT-LENGTH", "14") ], - "body": '{"nom": "nom"}' -} \ No newline at end of file + "body": b'{"nom": "nom"}' +} diff --git a/tests/requests/valid/002.py b/tests/requests/valid/002.py index 3bde4643..b511ffb6 100644 --- a/tests/requests/valid/002.py +++ b/tests/requests/valid/002.py @@ -7,5 +7,5 @@ request = { ("HOST", "0.0.0.0=5000"), ("ACCEPT", "*/*") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/003.py b/tests/requests/valid/003.py index d54bc7de..8ce87f80 100644 --- a/tests/requests/valid/003.py +++ b/tests/requests/valid/003.py @@ -12,5 +12,5 @@ request = { ("KEEP-ALIVE", "300"), ("CONNECTION", "keep-alive") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/004.py b/tests/requests/valid/004.py index 5840e522..6006f144 100644 --- a/tests/requests/valid/004.py +++ b/tests/requests/valid/004.py @@ -5,5 +5,5 @@ request = { "headers": [ ("AAAAAAAAAAAAA", "++++++++++") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/005.py b/tests/requests/valid/005.py index 5312de11..a7e54472 100644 --- a/tests/requests/valid/005.py +++ b/tests/requests/valid/005.py @@ -3,5 +3,5 @@ request = { "uri": uri("/forums/1/topics/2375?page=1#posts-17408"), "version": (1, 1), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/006.py b/tests/requests/valid/006.py index aea28729..01a4be17 100644 --- a/tests/requests/valid/006.py +++ b/tests/requests/valid/006.py @@ -3,5 +3,5 @@ request = { "uri": uri("/get_no_headers_no_body/world"), "version": (1, 1), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/007.py b/tests/requests/valid/007.py index fb531635..f5c2c798 100644 --- a/tests/requests/valid/007.py +++ b/tests/requests/valid/007.py @@ -5,5 +5,5 @@ request = { "headers": [ ("ACCEPT", "*/*") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/008.py b/tests/requests/valid/008.py index f9b77c56..379f9a2b 100644 --- a/tests/requests/valid/008.py +++ b/tests/requests/valid/008.py @@ -5,5 +5,5 @@ request = { "headers": [ ("CONTENT-LENGTH", "5") ], - "body": "HELLO" -} \ No newline at end of file + "body": b"HELLO" +} diff --git a/tests/requests/valid/009.py b/tests/requests/valid/009.py index 5cf51472..7e1f52dd 100644 --- a/tests/requests/valid/009.py +++ b/tests/requests/valid/009.py @@ -7,5 +7,5 @@ request = { ("TRANSFER-ENCODING", "identity"), ("CONTENT-LENGTH", "5") ], - "body": "World" -} \ No newline at end of file + "body": b"World" +} diff --git a/tests/requests/valid/010.py b/tests/requests/valid/010.py index 9fc566ba..996ef381 100644 --- a/tests/requests/valid/010.py +++ b/tests/requests/valid/010.py @@ -5,5 +5,5 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked"), ], - "body": "all your base are belong to us" -} \ No newline at end of file + "body": b"all your base are belong to us" +} diff --git a/tests/requests/valid/011.py b/tests/requests/valid/011.py index 3b7f6c08..05555adc 100644 --- a/tests/requests/valid/011.py +++ b/tests/requests/valid/011.py @@ -5,5 +5,5 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" -} \ No newline at end of file + "body": b"hello world" +} diff --git a/tests/requests/valid/012.py b/tests/requests/valid/012.py index 8f14e4c9..af071e5b 100644 --- a/tests/requests/valid/012.py +++ b/tests/requests/valid/012.py @@ -5,9 +5,9 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world", + "body": b"hello world", "trailers": [ ("VARY", "*"), ("CONTENT-TYPE", "text/plain") ] -} \ No newline at end of file +} diff --git a/tests/requests/valid/013.py b/tests/requests/valid/013.py index f17ec601..75ae982d 100644 --- a/tests/requests/valid/013.py +++ b/tests/requests/valid/013.py @@ -5,5 +5,5 @@ request = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" -} \ No newline at end of file + "body": b"hello world" +} diff --git a/tests/requests/valid/014.py b/tests/requests/valid/014.py index 22d380c5..0b1b0d2b 100644 --- a/tests/requests/valid/014.py +++ b/tests/requests/valid/014.py @@ -3,5 +3,5 @@ request = { "uri": uri('/with_"quotes"?foo="bar"'), "version": (1, 1), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/015.py b/tests/requests/valid/015.py index 4c36ba48..cd2b14f5 100644 --- a/tests/requests/valid/015.py +++ b/tests/requests/valid/015.py @@ -7,5 +7,5 @@ request = { ("USER-AGENT", "ApacheBench/2.3"), ("ACCEPT", "*/*") ], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/016.py b/tests/requests/valid/016.py index 3f3fa7ba..139b2700 100644 --- a/tests/requests/valid/016.py +++ b/tests/requests/valid/016.py @@ -36,5 +36,5 @@ request = { "uri": uri("/"), "version": (1, 1), "headers": [("X-SSL-CERT", certificate)], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/017.py b/tests/requests/valid/017.py index 638fd332..5fe13e86 100644 --- a/tests/requests/valid/017.py +++ b/tests/requests/valid/017.py @@ -6,5 +6,5 @@ request = { ("IF-MATCH", "bazinga!"), ("IF-MATCH", "large-sound") ], - "body": "" + "body": b"" } diff --git a/tests/requests/valid/018.py b/tests/requests/valid/018.py index f11c9461..fa10a96c 100644 --- a/tests/requests/valid/018.py +++ b/tests/requests/valid/018.py @@ -3,7 +3,7 @@ req1 = { "uri": uri("/first"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } req2 = { @@ -11,7 +11,7 @@ req2 = { "uri": uri("/second"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } request = [req1, req2] diff --git a/tests/requests/valid/019.py b/tests/requests/valid/019.py index 76d941d5..6fabd151 100644 --- a/tests/requests/valid/019.py +++ b/tests/requests/valid/019.py @@ -3,5 +3,5 @@ request = { "uri": uri("/first"), "version": (1, 0), "headers": [], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/020.py b/tests/requests/valid/020.py index 56702278..f5cfa21d 100644 --- a/tests/requests/valid/020.py +++ b/tests/requests/valid/020.py @@ -3,5 +3,5 @@ request = { "uri": uri("/first"), "version": (1, 0), "headers": [('CONTENT-LENGTH', '24')], - "body": "GET /second HTTP/1.1\r\n\r\n" -} \ No newline at end of file + "body": b"GET /second HTTP/1.1\r\n\r\n" +} diff --git a/tests/requests/valid/021.py b/tests/requests/valid/021.py index 4815a1c5..992dd308 100644 --- a/tests/requests/valid/021.py +++ b/tests/requests/valid/021.py @@ -3,5 +3,5 @@ request = { "uri": uri("/first"), "version": (1, 1), "headers": [("CONNECTION", "Close")], - "body": "" -} \ No newline at end of file + "body": b"" +} diff --git a/tests/requests/valid/022.py b/tests/requests/valid/022.py index a51a6f74..9b87e5e5 100644 --- a/tests/requests/valid/022.py +++ b/tests/requests/valid/022.py @@ -3,7 +3,7 @@ req1 = { "uri": uri("/first"), "version": (1, 0), "headers": [("CONNECTION", "Keep-Alive")], - "body": "" + "body": b"" } req2 = { @@ -11,7 +11,7 @@ req2 = { "uri": uri("/second"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } -request = [req1, req2] \ No newline at end of file +request = [req1, req2] diff --git a/tests/requests/valid/023.py b/tests/requests/valid/023.py index 0fa05f1a..f9a0ef5d 100644 --- a/tests/requests/valid/023.py +++ b/tests/requests/valid/023.py @@ -5,7 +5,7 @@ req1 = { "headers": [ ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" + "body": b"hello world" } req2 = { @@ -13,7 +13,7 @@ req2 = { "uri": uri("/second"), "version": (1, 1), "headers": [], - "body": "" + "body": b"" } -request = [req1, req2] \ No newline at end of file +request = [req1, req2] diff --git a/tests/requests/valid/025.py b/tests/requests/valid/025.py index 7e8f1826..12ea9ab7 100644 --- a/tests/requests/valid/025.py +++ b/tests/requests/valid/025.py @@ -6,7 +6,7 @@ req1 = { ("CONTENT-LENGTH", "-1"), ("TRANSFER-ENCODING", "chunked") ], - "body": "hello world" + "body": b"hello world" } req2 = { @@ -17,7 +17,7 @@ req2 = { ("TRANSFER-ENCODING", "chunked"), ("CONTENT-LENGTH", "-1"), ], - "body": "hello world" + "body": b"hello world" } request = [req1, req2] diff --git a/tests/requests/valid/pp_01.py b/tests/requests/valid/pp_01.py index 2e5b85a3..8f112506 100644 --- a/tests/requests/valid/pp_01.py +++ b/tests/requests/valid/pp_01.py @@ -12,5 +12,5 @@ request = { ("CONTENT-TYPE", "application/json"), ("CONTENT-LENGTH", "14") ], - "body": '{"nom": "nom"}' + "body": b'{"nom": "nom"}' } diff --git a/tests/requests/valid/pp_02.py b/tests/requests/valid/pp_02.py index f756b526..701ff2a7 100644 --- a/tests/requests/valid/pp_02.py +++ b/tests/requests/valid/pp_02.py @@ -13,7 +13,7 @@ req1 = { ("CONTENT-LENGTH", "14"), ("CONNECTION", "keep-alive") ], - "body": '{"nom": "nom"}' + "body": b'{"nom": "nom"}' } @@ -24,7 +24,7 @@ req2 = { "headers": [ ("TRANSFER-ENCODING", "chunked"), ], - "body": "all your base are belong to us" + "body": b"all your base are belong to us" } request = [req1, req2] diff --git a/tests/t.py b/tests/t.py index 5f776170..6f1d044a 100644 --- a/tests/t.py +++ b/tests/t.py @@ -1,43 +1,43 @@ # -*- coding: utf-8 - # Copyright 2009 Paul J. Davis # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. from __future__ import with_statement import array import os -from StringIO import StringIO import tempfile dirname = os.path.dirname(__file__) from gunicorn.http.parser import RequestParser from gunicorn.config import Config +from gunicorn.six import BytesIO def data_source(fname): - buf = StringIO() + buf = BytesIO() with open(fname) as handle: for line in handle: line = line.rstrip("\n").replace("\\r\\n", "\r\n") - buf.write(line) + buf.write(line.encode('latin1')) return buf class request(object): def __init__(self, name): self.fname = os.path.join(dirname, "requests", name) - + def __call__(self, func): def run(): src = data_source(self.fname) func(src, RequestParser(src)) run.func_name = func.func_name return run - - + + class FakeSocket(object): - + def __init__(self, data): self.tmp = tempfile.TemporaryFile() if data: @@ -47,32 +47,32 @@ class FakeSocket(object): def fileno(self): return self.tmp.fileno() - + def len(self): return self.tmp.len - + def recv(self, length=None): return self.tmp.read() - + def recv_into(self, buf, length): tmp_buffer = self.tmp.read(length) v = len(tmp_buffer) for i, c in enumerate(tmp_buffer): buf[i] = c return v - + def send(self, data): self.tmp.write(data) self.tmp.flush() - + def seek(self, offset, whence=0): self.tmp.seek(offset, whence) - - + + class http_request(object): def __init__(self, name): self.fname = os.path.join(dirname, "requests", name) - + def __call__(self, func): def run(): fsock = FakeSocket(data_source(self.fname)) @@ -80,7 +80,7 @@ class http_request(object): func(req) run.func_name = func.func_name return run - + def eq(a, b): assert a == b, "%r != %r" % (a, b) @@ -117,4 +117,3 @@ def raises(exctype, func, *args, **kwargs): func_name = getattr(func, "func_name", "") raise AssertionError("Function %s did not raise %s" % ( func_name, exctype.__name__)) - diff --git a/tests/001-test-valid-requests.py b/tests/test_001-valid-requests.py similarity index 90% rename from tests/001-test-valid-requests.py rename to tests/test_001-valid-requests.py index 6499d029..132eb9b8 100644 --- a/tests/001-test-valid-requests.py +++ b/tests/test_001-valid-requests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. import t @@ -9,6 +9,8 @@ import treq import glob import os dirname = os.path.dirname(__file__) + +from py.test import skip reqdir = os.path.join(dirname, "requests", "valid") def a_case(fname): diff --git a/tests/002-test-invalid-requests.py b/tests/test_002-invalid-requests.py similarity index 63% rename from tests/002-test-invalid-requests.py rename to tests/test_002-invalid-requests.py index 15308d25..d5f6e765 100644 --- a/tests/002-test-invalid-requests.py +++ b/tests/test_002-invalid-requests.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. import t @@ -8,7 +8,8 @@ import treq import glob import os -from nose.tools import raises + +import pytest dirname = os.path.dirname(__file__) reqdir = os.path.join(dirname, "requests", "invalid") @@ -17,12 +18,12 @@ reqdir = os.path.join(dirname, "requests", "invalid") def test_http_parser(): for fname in glob.glob(os.path.join(reqdir, "*.http")): env = treq.load_py(os.path.splitext(fname)[0] + ".py") + expect = env['request'] cfg = env['cfg'] req = treq.badrequest(fname) - @raises(expect) - def check(fname): - return req.check(cfg) - - yield check, fname # fname is pass so that we know which test failed + with pytest.raises(expect): + def f(fname): + return req.check(cfg) + f(fname) diff --git a/tests/003-test-config.py b/tests/test_003-config.py similarity index 81% rename from tests/003-test-config.py rename to tests/test_003-config.py index 555ba9f8..e5639b22 100644 --- a/tests/003-test-config.py +++ b/tests/test_003-config.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 - # -# This file is part of gunicorn released under the MIT license. +# This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. from __future__ import with_statement -from nose.plugins.skip import SkipTest - import t import functools @@ -23,14 +21,6 @@ def cfg_file(): def paster_ini(): return os.path.join(dirname, "..", "examples", "frameworks", "pylonstest", "nose.ini") -def PasterApp(): - try: - from paste.deploy import loadapp, loadwsgi - except ImportError: - raise SkipTest() - from gunicorn.app.pasterapp import PasterApplication - return PasterApplication("no_usage") - class AltArgs(object): def __init__(self, args=None): self.args = args or [] @@ -38,17 +28,17 @@ class AltArgs(object): def __enter__(self): sys.argv = self.args - + def __exit__(self, exc_type, exc_inst, traceback): sys.argv = self.orig class NoConfigApp(Application): def __init__(self): super(NoConfigApp, self).__init__("no_usage") - + def init(self, parser, opts, args): pass - + def load(self): pass @@ -63,25 +53,25 @@ def test_property_access(): c = config.Config() for s in config.KNOWN_SETTINGS: getattr(c, s.name) - + # Class was loaded t.eq(c.worker_class, SyncWorker) - + # Debug affects workers t.eq(c.workers, 1) c.set("workers", 3) t.eq(c.workers, 3) - + # Address is parsed t.eq(c.address, ("127.0.0.1", 8000)) - + # User and group defaults t.eq(os.geteuid(), c.uid) t.eq(os.getegid(), c.gid) - + # Proc name t.eq("gunicorn", c.proc_name) - + # Not a config property t.raises(AttributeError, getattr, c, "foo") # Force to be not an error @@ -93,10 +83,10 @@ def test_property_access(): # Attempt to set a cfg not via c.set t.raises(AttributeError, setattr, c, "proc_name", "baz") - + # No setting for name t.raises(AttributeError, c.set, "baz", "bar") - + def test_bool_validation(): c = config.Config() t.eq(c.debug, False) @@ -196,30 +186,9 @@ def test_load_config(): t.eq(app.cfg.bind, "unix:/tmp/bar/baz") t.eq(app.cfg.workers, 3) t.eq(app.cfg.proc_name, "fooey") - + def test_cli_overrides_config(): with AltArgs(["prog_name", "-c", cfg_file(), "-b", "blarney"]): app = NoConfigApp() t.eq(app.cfg.bind, "blarney") t.eq(app.cfg.proc_name, "fooey") - -def test_paster_config(): - with AltArgs(["prog_name", paster_ini()]): - app = PasterApp() - t.eq(app.cfg.bind, "192.168.0.1:80") - t.eq(app.cfg.proc_name, "brim") - t.eq("ignore_me" in app.cfg.settings, False) - -def test_cfg_over_paster(): - with AltArgs(["prog_name", "-c", cfg_file(), paster_ini()]): - app = PasterApp() - t.eq(app.cfg.bind, "unix:/tmp/bar/baz") - t.eq(app.cfg.proc_name, "fooey") - t.eq(app.cfg.default_proc_name, "blurgh") - -def test_cli_cfg_paster(): - with AltArgs(["prog_name", "-c", cfg_file(), "-b", "whee", paster_ini()]): - app = PasterApp() - t.eq(app.cfg.bind, "whee") - t.eq(app.cfg.proc_name, "fooey") - t.eq(app.cfg.default_proc_name, "blurgh") diff --git a/tests/test_004-http-body.py b/tests/test_004-http-body.py new file mode 100644 index 00000000..351b66bd --- /dev/null +++ b/tests/test_004-http-body.py @@ -0,0 +1,61 @@ +import t +from gunicorn.http.body import Body +from gunicorn.six import BytesIO + + +def assert_readline(payload, size, expected): + body = Body(BytesIO(payload)) + t.eq(body.readline(size), expected) + + +def test_readline_empty_body(): + assert_readline(b"", None, b"") + assert_readline(b"", 1, b"") + + +def test_readline_zero_size(): + assert_readline(b"abc", 0, b"") + assert_readline(b"\n", 0, b"") + + +def test_readline_new_line_before_size(): + body = Body(BytesIO(b"abc\ndef")) + t.eq(body.readline(4), b"abc\n") + t.eq(body.readline(), b"def") + + +def test_readline_new_line_after_size(): + body = Body(BytesIO(b"abc\ndef")) + t.eq(body.readline(2), b"ab") + t.eq(body.readline(), b"c\n") + + +def test_readline_no_new_line(): + body = Body(BytesIO(b"abcdef")) + t.eq(body.readline(), b"abcdef") + body = Body(BytesIO(b"abcdef")) + t.eq(body.readline(2), b"ab") + t.eq(body.readline(2), b"cd") + t.eq(body.readline(2), b"ef") + + +def test_readline_buffer_loaded(): + reader = BytesIO(b"abc\ndef") + body = Body(reader) + body.read(1) # load internal buffer + reader.write(b"g\nhi") + reader.seek(7) + print(reader.getvalue()) + t.eq(body.readline(), b"bc\n") + t.eq(body.readline(), b"defg\n") + t.eq(body.readline(), b"hi") + + +def test_readline_buffer_loaded_with_size(): + body = Body(BytesIO(b"abc\ndef")) + body.read(1) # load internal buffer + t.eq(body.readline(2), b"bc") + t.eq(body.readline(2), b"\n") + t.eq(body.readline(2), b"de") + t.eq(body.readline(2), b"f") + diff --git a/tests/treq.py b/tests/treq.py index 23a8eea5..5ec54be3 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -10,18 +10,19 @@ import t import inspect import os import random -import urlparse from gunicorn.config import Config from gunicorn.http.errors import ParseException from gunicorn.http.parser import RequestParser +from gunicorn.six import urlparse, execfile_ +from gunicorn import six dirname = os.path.dirname(__file__) random.seed() def uri(data): ret = {"raw": data} - parts = urlparse.urlparse(data) + parts = urlparse(data) ret["scheme"] = parts.scheme or '' ret["host"] = parts.netloc.rsplit(":", 1)[0] or None ret["port"] = parts.port or 80 @@ -42,7 +43,7 @@ def load_py(fname): config = globals().copy() config["uri"] = uri config["cfg"] = Config() - execfile(fname, config) + execfile_(fname, config) return config class request(object): @@ -54,10 +55,10 @@ class request(object): if not isinstance(self.expect, list): self.expect = [self.expect] - with open(self.fname) as handle: + with open(self.fname, 'rb') as handle: self.data = handle.read() - self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n") - self.data = self.data.replace("\\0", "\000") + self.data = self.data.replace(b"\n", b"").replace(b"\\r\\n", b"\r\n") + self.data = self.data.replace(b"\\0", b"\000") # Functions for sending data to the parser. # These functions mock out reading from a @@ -69,20 +70,20 @@ class request(object): def send_lines(self): lines = self.data - pos = lines.find("\r\n") + pos = lines.find(b"\r\n") while pos > 0: yield lines[:pos+2] lines = lines[pos+2:] - pos = lines.find("\r\n") + pos = lines.find(b"\r\n") if len(lines): yield lines def send_bytes(self): - for d in self.data: - yield d + for d in str(self.data, "latin1"): + yield bytes(d, "latin1") def send_random(self): - maxs = len(self.data) / 10 + maxs = round(len(self.data) / 10) read = 0 while read < len(self.data): chunk = random.randint(1, maxs) @@ -143,7 +144,7 @@ class request(object): while len(body): if body[:len(data)] != data: raise AssertionError("Invalid data read: %r" % data) - if '\n' in data[:-1]: + if b'\n' in data[:-1]: raise AssertionError("Embedded new line: %r" % data) body = body[len(data):] data = self.szread(req.body.readline, sizes) @@ -165,7 +166,7 @@ class request(object): """ data = req.body.readlines() for line in data: - if '\n' in line[:-1]: + if b'\n' in line[:-1]: raise AssertionError("Embedded new line: %r" % line) if line != body[:len(line)]: raise AssertionError("Invalid body data read: %r != %r" % ( @@ -182,7 +183,7 @@ class request(object): This skips sizes because there's its not part of the iter api. """ for line in req.body: - if '\n' in line[:-1]: + if b'\n' in line[:-1]: raise AssertionError("Embedded new line: %r" % line) if line != body[:len(line)]: raise AssertionError("Invalid body data read: %r != %r" % ( @@ -191,7 +192,7 @@ class request(object): if len(body): raise AssertionError("Failed to read entire body: %r" % body) try: - data = iter(req.body).next() + data = six.next(iter(req.body)) raise AssertionError("Read data after body finished: %r" % data) except StopIteration: pass @@ -214,9 +215,15 @@ class request(object): ret = [] for (mt, sz, sn) in cfgs: - mtn = mt.func_name[6:] - szn = sz.func_name[5:] - snn = sn.func_name[5:] + if hasattr(mt, 'funcname'): + mtn = mt.func_name[6:] + szn = sz.func_name[5:] + snn = sn.func_name[5:] + else: + mtn = mt.__name__[6:] + szn = sz.__name__[5:] + snn = sn.__name__[5:] + def test_req(sn, sz, mt): self.check(cfg, sn, sz, mt) desc = "%s: MT: %s SZ: %s SN: %s" % (self.name, mtn, szn, snn) @@ -251,9 +258,10 @@ class badrequest(object): self.data = handle.read() self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n") self.data = self.data.replace("\\0", "\000") + self.data = self.data.encode('latin1') def send(self): - maxs = len(self.data) / 10 + maxs = round(len(self.data) / 10) read = 0 while read < len(self.data): chunk = random.randint(1, maxs) @@ -262,5 +270,4 @@ class badrequest(object): def check(self, cfg): p = RequestParser(cfg, self.send()) - [req for req in p] - + six.next(p)