diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index 293c5359..a9170e8c 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -3,6 +3,8 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import sys + try: from cStringIO import StringIO except ImportError: @@ -177,50 +179,69 @@ class Body(object): if not ret: raise StopIteration() return ret + + def getsize(self, size): + if size is None: + return sys.maxint + elif not isinstance(size, (int, long)): + raise TypeError("size must be an integral type") + elif size < 0: + return sys.maxint + return size def read(self, size=None): - if size is not None and not isinstance(size, (int, long)): - raise TypeError("size must be an integral type") + size = self.getsize(size) + if size == 0: + return "" - if size is not None and size < self.buf.tell(): + if size < self.buf.tell(): data = self.buf.getvalue() ret, rest = data[:size], data[size:] self.buf.truncate(0) self.buf.write(rest) return ret - if size > 0: - size -= self.buf.tell() - else: - size = None - - ret = self.buf.getvalue() + self.reader.read(size=size) + while size > self.buf.tell(): + data = self.reader.read(1024) + if not len(data): + break + self.buf.write(data) + + data = self.buf.getvalue() + ret, rest = data[:size], data[size:] self.buf.truncate(0) + self.buf.write(rest) return ret def readline(self, size=None): + size = self.getsize(size) if size == 0: return "" - if size < 0: - size = None - idx = -1 + idx = self.buf.getvalue().find("\n") while idx < 0: data = self.reader.read(1024) if not len(data): break self.buf.write(data) - if size is not None and self.buf.tell() > size: + idx = self.buf.getvalue().find("\n") + if size < self.buf.tell(): break - idx = self.buf.getvalue().find("\r\n") + + # If we didn't find it, and we got here, we've + # exceeded size or run out of data. + if idx < 0: + rlen = min(size, self.buf.tell()) + else: + rlen = idx + 1 - if idx < 0 and size is not None: - idx = size - elif idx < 0: - idx = self.buf.tell() + # If rlen is beyond our size threshold, trim back + if rlen > size: + rlen = size data = self.buf.getvalue() - ret, rest = data[:idx], data[idx:] + ret, rest = data[:rlen], data[rlen:] + self.buf.truncate(0) self.buf.write(rest) return ret diff --git a/tests/001-test-parser.py b/tests/001-test-parser.py deleted file mode 100644 index a306d04e..00000000 --- a/tests/001-test-parser.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 - -# -# This file is part of gunicorn released under the MIT license. -# See the NOTICE for more information. - -import t -import treq - -import glob -import os -dirname = os.path.dirname(__file__) -reqdir = os.path.join(dirname, "requests") - -def load_py(fname): - config = globals().copy() - config["uri"] = treq.uri - execfile(fname, config) - return config["request"] - -def test_http_parser(): - for fname in glob.glob(os.path.join(reqdir, "*.http")): - expect = load_py(os.path.splitext(fname)[0] + ".py") - req = treq.request(fname, expect) - for case in req.gen_cases(): - yield case diff --git a/tests/001-test-valid-requests.py b/tests/001-test-valid-requests.py new file mode 100644 index 00000000..97d7ce48 --- /dev/null +++ b/tests/001-test-valid-requests.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +import t +import treq + +import glob +import os +dirname = os.path.dirname(__file__) +reqdir = os.path.join(dirname, "requests", "valid") + +def load_py(fname): + config = globals().copy() + config["uri"] = treq.uri + execfile(fname, config) + return config["request"] + +def a_case(fname): + expect = load_py(os.path.splitext(fname)[0] + ".py") + req = treq.request(fname, expect) + for case in req.gen_cases(): + case[0](*case[1:]) + +def test_http_parser(): + for fname in glob.glob(os.path.join(reqdir, "*.http")): + if os.getenv("GUNS_BLAZING"): + expect = load_py(os.path.splitext(fname)[0] + ".py") + req = treq.request(fname, expect) + for case in req.gen_cases(): + yield case + else: + yield (a_case, fname) diff --git a/tests/requests/018.http b/tests/requests/invalid/001.http similarity index 100% rename from tests/requests/018.http rename to tests/requests/invalid/001.http diff --git a/tests/requests/018.py b/tests/requests/invalid/001.py similarity index 100% rename from tests/requests/018.py rename to tests/requests/invalid/001.py diff --git a/tests/requests/019.http b/tests/requests/invalid/002.http similarity index 100% rename from tests/requests/019.http rename to tests/requests/invalid/002.http diff --git a/tests/requests/019.py b/tests/requests/invalid/002.py similarity index 100% rename from tests/requests/019.py rename to tests/requests/invalid/002.py diff --git a/tests/requests/020.http b/tests/requests/invalid/003.http similarity index 100% rename from tests/requests/020.http rename to tests/requests/invalid/003.http diff --git a/tests/requests/020.py b/tests/requests/invalid/003.py similarity index 100% rename from tests/requests/020.py rename to tests/requests/invalid/003.py diff --git a/tests/requests/021.http b/tests/requests/invalid/004.http similarity index 100% rename from tests/requests/021.http rename to tests/requests/invalid/004.http diff --git a/tests/requests/021.py b/tests/requests/invalid/004.py similarity index 100% rename from tests/requests/021.py rename to tests/requests/invalid/004.py diff --git a/tests/requests/022.http b/tests/requests/invalid/005.http similarity index 100% rename from tests/requests/022.http rename to tests/requests/invalid/005.http diff --git a/tests/requests/022.py b/tests/requests/invalid/005.py similarity index 100% rename from tests/requests/022.py rename to tests/requests/invalid/005.py diff --git a/tests/requests/001.http b/tests/requests/valid/001.http similarity index 100% rename from tests/requests/001.http rename to tests/requests/valid/001.http diff --git a/tests/requests/001.py b/tests/requests/valid/001.py similarity index 100% rename from tests/requests/001.py rename to tests/requests/valid/001.py diff --git a/tests/requests/002.http b/tests/requests/valid/002.http similarity index 100% rename from tests/requests/002.http rename to tests/requests/valid/002.http diff --git a/tests/requests/002.py b/tests/requests/valid/002.py similarity index 100% rename from tests/requests/002.py rename to tests/requests/valid/002.py diff --git a/tests/requests/003.http b/tests/requests/valid/003.http similarity index 100% rename from tests/requests/003.http rename to tests/requests/valid/003.http diff --git a/tests/requests/003.py b/tests/requests/valid/003.py similarity index 100% rename from tests/requests/003.py rename to tests/requests/valid/003.py diff --git a/tests/requests/004.http b/tests/requests/valid/004.http similarity index 100% rename from tests/requests/004.http rename to tests/requests/valid/004.http diff --git a/tests/requests/004.py b/tests/requests/valid/004.py similarity index 100% rename from tests/requests/004.py rename to tests/requests/valid/004.py diff --git a/tests/requests/005.http b/tests/requests/valid/005.http similarity index 100% rename from tests/requests/005.http rename to tests/requests/valid/005.http diff --git a/tests/requests/005.py b/tests/requests/valid/005.py similarity index 100% rename from tests/requests/005.py rename to tests/requests/valid/005.py diff --git a/tests/requests/006.http b/tests/requests/valid/006.http similarity index 100% rename from tests/requests/006.http rename to tests/requests/valid/006.http diff --git a/tests/requests/006.py b/tests/requests/valid/006.py similarity index 100% rename from tests/requests/006.py rename to tests/requests/valid/006.py diff --git a/tests/requests/007.http b/tests/requests/valid/007.http similarity index 100% rename from tests/requests/007.http rename to tests/requests/valid/007.http diff --git a/tests/requests/007.py b/tests/requests/valid/007.py similarity index 100% rename from tests/requests/007.py rename to tests/requests/valid/007.py diff --git a/tests/requests/008.http b/tests/requests/valid/008.http similarity index 100% rename from tests/requests/008.http rename to tests/requests/valid/008.http diff --git a/tests/requests/008.py b/tests/requests/valid/008.py similarity index 100% rename from tests/requests/008.py rename to tests/requests/valid/008.py diff --git a/tests/requests/009.http b/tests/requests/valid/009.http similarity index 100% rename from tests/requests/009.http rename to tests/requests/valid/009.http diff --git a/tests/requests/009.py b/tests/requests/valid/009.py similarity index 100% rename from tests/requests/009.py rename to tests/requests/valid/009.py diff --git a/tests/requests/010.http b/tests/requests/valid/010.http similarity index 100% rename from tests/requests/010.http rename to tests/requests/valid/010.http diff --git a/tests/requests/010.py b/tests/requests/valid/010.py similarity index 100% rename from tests/requests/010.py rename to tests/requests/valid/010.py diff --git a/tests/requests/011.http b/tests/requests/valid/011.http similarity index 100% rename from tests/requests/011.http rename to tests/requests/valid/011.http diff --git a/tests/requests/011.py b/tests/requests/valid/011.py similarity index 100% rename from tests/requests/011.py rename to tests/requests/valid/011.py diff --git a/tests/requests/012.http b/tests/requests/valid/012.http similarity index 100% rename from tests/requests/012.http rename to tests/requests/valid/012.http diff --git a/tests/requests/012.py b/tests/requests/valid/012.py similarity index 100% rename from tests/requests/012.py rename to tests/requests/valid/012.py diff --git a/tests/requests/013.http b/tests/requests/valid/013.http similarity index 100% rename from tests/requests/013.http rename to tests/requests/valid/013.http diff --git a/tests/requests/013.py b/tests/requests/valid/013.py similarity index 100% rename from tests/requests/013.py rename to tests/requests/valid/013.py diff --git a/tests/requests/014.http b/tests/requests/valid/014.http similarity index 100% rename from tests/requests/014.http rename to tests/requests/valid/014.http diff --git a/tests/requests/014.py b/tests/requests/valid/014.py similarity index 100% rename from tests/requests/014.py rename to tests/requests/valid/014.py diff --git a/tests/requests/015.http b/tests/requests/valid/015.http similarity index 100% rename from tests/requests/015.http rename to tests/requests/valid/015.http diff --git a/tests/requests/015.py b/tests/requests/valid/015.py similarity index 100% rename from tests/requests/015.py rename to tests/requests/valid/015.py diff --git a/tests/requests/016.http b/tests/requests/valid/016.http similarity index 100% rename from tests/requests/016.http rename to tests/requests/valid/016.http diff --git a/tests/requests/016.py b/tests/requests/valid/016.py similarity index 100% rename from tests/requests/016.py rename to tests/requests/valid/016.py diff --git a/tests/requests/017.http b/tests/requests/valid/017.http similarity index 100% rename from tests/requests/017.http rename to tests/requests/valid/017.http diff --git a/tests/requests/017.py b/tests/requests/valid/017.py similarity index 100% rename from tests/requests/017.py rename to tests/requests/valid/017.py diff --git a/tests/requests/valid/018.http b/tests/requests/valid/018.http new file mode 100644 index 00000000..05bb2180 --- /dev/null +++ b/tests/requests/valid/018.http @@ -0,0 +1,4 @@ +GET /first HTTP/1.1\r\n +\r\n +GET /second HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/018.py b/tests/requests/valid/018.py new file mode 100644 index 00000000..f11c9461 --- /dev/null +++ b/tests/requests/valid/018.py @@ -0,0 +1,17 @@ +req1 = { + "method": "GET", + "uri": uri("/first"), + "version": (1, 1), + "headers": [], + "body": "" +} + +req2 = { + "method": "GET", + "uri": uri("/second"), + "version": (1, 1), + "headers": [], + "body": "" +} + +request = [req1, req2] diff --git a/tests/requests/valid/019.http b/tests/requests/valid/019.http new file mode 100644 index 00000000..4f6ae802 --- /dev/null +++ b/tests/requests/valid/019.http @@ -0,0 +1,4 @@ +GET /first HTTP/1.0\r\n +\r\n +GET /second HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/019.py b/tests/requests/valid/019.py new file mode 100644 index 00000000..76d941d5 --- /dev/null +++ b/tests/requests/valid/019.py @@ -0,0 +1,7 @@ +request = { + "method": "GET", + "uri": uri("/first"), + "version": (1, 0), + "headers": [], + "body": "" +} \ No newline at end of file diff --git a/tests/requests/valid/020.http b/tests/requests/valid/020.http new file mode 100644 index 00000000..0dde0e3f --- /dev/null +++ b/tests/requests/valid/020.http @@ -0,0 +1,5 @@ +GET /first HTTP/1.0\r\n +Content-Length: 24\r\n +\r\n +GET /second HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/020.py b/tests/requests/valid/020.py new file mode 100644 index 00000000..56702278 --- /dev/null +++ b/tests/requests/valid/020.py @@ -0,0 +1,7 @@ +request = { + "method": "GET", + "uri": uri("/first"), + "version": (1, 0), + "headers": [('CONTENT-LENGTH', '24')], + "body": "GET /second HTTP/1.1\r\n\r\n" +} \ No newline at end of file diff --git a/tests/requests/valid/021.http b/tests/requests/valid/021.http new file mode 100644 index 00000000..4e2dd2be --- /dev/null +++ b/tests/requests/valid/021.http @@ -0,0 +1,5 @@ +GET /first HTTP/1.1\r\n +Connection: Close\r\n +\r\n +GET /second HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/021.py b/tests/requests/valid/021.py new file mode 100644 index 00000000..4815a1c5 --- /dev/null +++ b/tests/requests/valid/021.py @@ -0,0 +1,7 @@ +request = { + "method": "GET", + "uri": uri("/first"), + "version": (1, 1), + "headers": [("CONNECTION", "Close")], + "body": "" +} \ No newline at end of file diff --git a/tests/requests/valid/022.http b/tests/requests/valid/022.http new file mode 100644 index 00000000..b65aae17 --- /dev/null +++ b/tests/requests/valid/022.http @@ -0,0 +1,5 @@ +GET /first HTTP/1.0\r\n +Connection: Keep-Alive\r\n +\r\n +GET /second HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/022.py b/tests/requests/valid/022.py new file mode 100644 index 00000000..a51a6f74 --- /dev/null +++ b/tests/requests/valid/022.py @@ -0,0 +1,17 @@ +req1 = { + "method": "GET", + "uri": uri("/first"), + "version": (1, 0), + "headers": [("CONNECTION", "Keep-Alive")], + "body": "" +} + +req2 = { + "method": "GET", + "uri": uri("/second"), + "version": (1, 1), + "headers": [], + "body": "" +} + +request = [req1, req2] \ No newline at end of file diff --git a/tests/requests/valid/023.http b/tests/requests/valid/023.http new file mode 100644 index 00000000..89fd727a --- /dev/null +++ b/tests/requests/valid/023.http @@ -0,0 +1,11 @@ +POST /two_chunks_mult_zero_end HTTP/1.1\r\n +Transfer-Encoding: chunked\r\n +\r\n +5\r\n +hello\r\n +6\r\n + world\r\n +000\r\n +\r\n +GET /second HTTP/1.1\r\n +\r\n \ No newline at end of file diff --git a/tests/requests/valid/023.py b/tests/requests/valid/023.py new file mode 100644 index 00000000..0fa05f1a --- /dev/null +++ b/tests/requests/valid/023.py @@ -0,0 +1,19 @@ +req1 = { + "method": "POST", + "uri": uri("/two_chunks_mult_zero_end"), + "version": (1, 1), + "headers": [ + ("TRANSFER-ENCODING", "chunked") + ], + "body": "hello world" +} + +req2 = { + "method": "GET", + "uri": uri("/second"), + "version": (1, 1), + "headers": [], + "body": "" +} + +request = [req1, req2] \ No newline at end of file diff --git a/tests/treq.py b/tests/treq.py index e9aa246a..3aa325ad 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -89,7 +89,7 @@ class request(object): return 1 def size_small_random(self): - return random.randint(0, 2) + return random.randint(0, 4) def size_random(self): return random.randint(1, 4096) @@ -117,6 +117,7 @@ class request(object): if not data: count -= 1 if count <= 0: + print "BOD: %r" % body raise AssertionError("Unexpected apparent EOF") if len(body): @@ -201,14 +202,7 @@ class request(object): for sz in sizers for sn in senders ] - # Strip out match_readlines, match_iter for all but one sizer - cfgs = [ - (mt, sz, sn) - for (mt, sz, sn) in cfgs - if mt in [self.match_readlines, self.match_iter] - and sz != self.size_all - ] - + ret = [] for (mt, sz, sn) in cfgs: mtn = mt.func_name[6:] @@ -223,43 +217,12 @@ class request(object): def check(self, sender, sizer, matcher): cases = self.expect[:] - ended = False - try: - p = RequestParser(sender()) - except Exception, e: - if not isinstance(cases[0], Exception): - raise - self.same_error(e, cases[0]) - t.eq(len(casese), 1) - while True: - try: - req = p.next() - except StopIteration, e: - t.eq(len(cases), 0) - ended = True - break - except ParseException, e: - if not issubclass(cases[0], Exception): - raise - self.same_error(e, cases.pop(0)) - t.eq(len(cases), 0) - return - else: - self.same(req, sizer, matcher, cases.pop(0)) + p = RequestParser(sender()) + for req in p: + self.same(req, sizer, matcher, cases.pop(0)) t.eq(len(cases), 0) - t.eq(ended, True) def same(self, req, sizer, matcher, exp): - if isinstance(req, Exception): - self.same_error(req, exp) - else: - self.same_obj(req, sizer, matcher, exp) - - def same_error(self, req, exp): - t.istype(req, Exception) - t.istype(req, exp) - - def same_obj(self, req, sizer, matcher, exp): t.eq(req.method, exp["method"]) t.eq(req.uri, exp["uri"]["raw"]) t.eq(req.scheme, exp["uri"]["scheme"])