Added more valid request tests.

Found and fixed a couple read and readline related bugs.

gunicorn/http/body.py

@@ -3,6 +3,8 @@
 # This file is part of gunicorn released under the MIT license. 
 # See the NOTICE for more information.
 
+import sys
+
 try:
     from cStringIO import StringIO
 except ImportError:
@@ -178,49 +180,68 @@ class Body(object):
             raise StopIteration()
         return ret
 
-    def read(self, size=None):
+    def getsize(self, size):
-        if size is not None and not isinstance(size, (int, long)):
+        if size is None:
+            return sys.maxint
+        elif not isinstance(size, (int, long)):
             raise TypeError("size must be an integral type")
+        elif size < 0:
+            return sys.maxint
+        return size
     
-        if size is not None and size < self.buf.tell():
+    def read(self, size=None):
+        size = self.getsize(size)
+        if size == 0:
+            return ""
+
+        if size < self.buf.tell():
             data = self.buf.getvalue()
             ret, rest = data[:size], data[size:]
             self.buf.truncate(0)
             self.buf.write(rest)
             return ret
 
-        if size > 0:
+        while size > self.buf.tell():
-            size -= self.buf.tell()
+            data = self.reader.read(1024)
-        else:
+            if not len(data):
-            size = None
+                break
+            self.buf.write(data)
 
-        ret = self.buf.getvalue() + self.reader.read(size=size)
+        data = self.buf.getvalue()
+        ret, rest = data[:size], data[size:]
         self.buf.truncate(0)
+        self.buf.write(rest)
         return ret
     
     def readline(self, size=None):
+        size = self.getsize(size)
         if size == 0:
             return ""
-        if size < 0:
-            size = None
         
-        idx = -1
+        idx = self.buf.getvalue().find("\n")
         while idx < 0:
             data = self.reader.read(1024)
             if not len(data):
                 break
             self.buf.write(data)
-            if size is not None and self.buf.tell() > size:
+            idx = self.buf.getvalue().find("\n")
+            if size < self.buf.tell():
                 break
-            idx = self.buf.getvalue().find("\r\n")
         
-        if idx < 0 and size is not None:
+        # If we didn't find it, and we got here, we've
-            idx = size
+        # exceeded size or run out of data.
-        elif idx < 0:
+        if idx < 0:
-            idx = self.buf.tell()
+            rlen = min(size, self.buf.tell())
+        else:
+            rlen = idx + 1
+
+            # If rlen is beyond our size threshold, trim back
+            if rlen > size:
+                rlen = size
         
         data = self.buf.getvalue()
-        ret, rest = data[:idx], data[idx:]
+        ret, rest = data[:rlen], data[rlen:]
+        
         self.buf.truncate(0)
         self.buf.write(rest)
         return ret

tests/001-test-parser.py

@@ -1,25 +0,0 @@
-# -*- coding: utf-8 -
-#
-# This file is part of gunicorn released under the MIT license. 
-# See the NOTICE for more information.
-
-import t
-import treq
-
-import glob
-import os
-dirname = os.path.dirname(__file__)
-reqdir = os.path.join(dirname, "requests")
-
-def load_py(fname):
-    config = globals().copy()
-    config["uri"] = treq.uri
-    execfile(fname, config)
-    return config["request"]
-
-def test_http_parser():
-    for fname in glob.glob(os.path.join(reqdir, "*.http")):
-        expect = load_py(os.path.splitext(fname)[0] + ".py")
-        req = treq.request(fname, expect)
-        for case in req.gen_cases():
-            yield case

tests/001-test-valid-requests.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -
+#
+# This file is part of gunicorn released under the MIT license. 
+# See the NOTICE for more information.
+
+import t
+import treq
+
+import glob
+import os
+dirname = os.path.dirname(__file__)
+reqdir = os.path.join(dirname, "requests", "valid")
+
+def load_py(fname):
+    config = globals().copy()
+    config["uri"] = treq.uri
+    execfile(fname, config)
+    return config["request"]
+
+def a_case(fname):
+    expect = load_py(os.path.splitext(fname)[0] + ".py")
+    req = treq.request(fname, expect)
+    for case in req.gen_cases():
+        case[0](*case[1:])
+
+def test_http_parser():
+    for fname in glob.glob(os.path.join(reqdir, "*.http")):
+        if os.getenv("GUNS_BLAZING"):
+            expect = load_py(os.path.splitext(fname)[0] + ".py")
+            req = treq.request(fname, expect)
+            for case in req.gen_cases():
+                yield case
+        else:
+            yield (a_case, fname)

tests/requests/valid/018.http

@@ -0,0 +1,4 @@
+GET /first HTTP/1.1\r\n
+\r\n
+GET /second HTTP/1.1\r\n
+\r\n

tests/requests/valid/018.py

@@ -0,0 +1,17 @@
+req1 = {
+    "method": "GET",
+    "uri": uri("/first"),
+    "version": (1, 1),
+    "headers": [],
+    "body": ""
+}
+
+req2 = {
+    "method": "GET",
+    "uri": uri("/second"),
+    "version": (1, 1),
+    "headers": [],
+    "body": ""
+}
+
+request = [req1, req2]

tests/requests/valid/019.http

@@ -0,0 +1,4 @@
+GET /first HTTP/1.0\r\n
+\r\n
+GET /second HTTP/1.1\r\n
+\r\n

tests/requests/valid/019.py

@@ -0,0 +1,7 @@
+request = {
+    "method": "GET",
+    "uri": uri("/first"),
+    "version": (1, 0),
+    "headers": [],
+    "body": ""
+}

tests/requests/valid/020.http

@@ -0,0 +1,5 @@
+GET /first HTTP/1.0\r\n
+Content-Length: 24\r\n
+\r\n
+GET /second HTTP/1.1\r\n
+\r\n

tests/requests/valid/020.py

@@ -0,0 +1,7 @@
+request = {
+    "method": "GET",
+    "uri": uri("/first"),
+    "version": (1, 0),
+    "headers": [('CONTENT-LENGTH', '24')],
+    "body": "GET /second HTTP/1.1\r\n\r\n"
+}

tests/requests/valid/021.http

@@ -0,0 +1,5 @@
+GET /first HTTP/1.1\r\n
+Connection: Close\r\n
+\r\n
+GET /second HTTP/1.1\r\n
+\r\n

tests/requests/valid/021.py

@@ -0,0 +1,7 @@
+request = {
+    "method": "GET",
+    "uri": uri("/first"),
+    "version": (1, 1),
+    "headers": [("CONNECTION", "Close")],
+    "body": ""
+}

tests/requests/valid/022.http

@@ -0,0 +1,5 @@
+GET /first HTTP/1.0\r\n
+Connection: Keep-Alive\r\n
+\r\n
+GET /second HTTP/1.1\r\n
+\r\n

tests/requests/valid/022.py

@@ -0,0 +1,17 @@
+req1 = {
+    "method": "GET",
+    "uri": uri("/first"),
+    "version": (1, 0),
+    "headers": [("CONNECTION", "Keep-Alive")],
+    "body": ""
+}
+
+req2 = {
+    "method": "GET",
+    "uri": uri("/second"),
+    "version": (1, 1),
+    "headers": [],
+    "body": ""
+}
+
+request = [req1, req2]

tests/requests/valid/023.http

@@ -0,0 +1,11 @@
+POST /two_chunks_mult_zero_end HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+000\r\n
+\r\n
+GET /second HTTP/1.1\r\n
+\r\n

tests/requests/valid/023.py

@@ -0,0 +1,19 @@
+req1 = {
+    "method": "POST",
+    "uri": uri("/two_chunks_mult_zero_end"),
+    "version": (1, 1),
+    "headers": [
+        ("TRANSFER-ENCODING", "chunked")
+    ],
+    "body": "hello world"
+}
+
+req2 = {
+    "method": "GET",
+    "uri": uri("/second"),
+    "version": (1, 1),
+    "headers": [],
+    "body": ""
+}
+
+request = [req1, req2]

tests/treq.py

@@ -89,7 +89,7 @@ class request(object):
         return 1
     
     def size_small_random(self):
-        return random.randint(0, 2)
+        return random.randint(0, 4)
     
     def size_random(self):
         return random.randint(1, 4096)
@@ -117,6 +117,7 @@ class request(object):
             if not data:
                 count -= 1
             if count <= 0:
+                print "BOD: %r" % body
                 raise AssertionError("Unexpected apparent EOF")
 
         if len(body):
@@ -201,13 +202,6 @@ class request(object):
             for sz in sizers
             for sn in senders
         ]
-        # Strip out match_readlines, match_iter for all but one sizer
-        cfgs = [
-            (mt, sz, sn)
-            for (mt, sz, sn) in cfgs
-            if mt in [self.match_readlines, self.match_iter]
-            and sz != self.size_all
-        ]
 
         ret = []
         for (mt, sz, sn) in cfgs:
@@ -223,43 +217,12 @@ class request(object):
 
     def check(self, sender, sizer, matcher):
         cases = self.expect[:]
-        ended = False
+        p = RequestParser(sender())
-        try:
+        for req in p:
-            p = RequestParser(sender())
+            self.same(req, sizer, matcher, cases.pop(0))
-        except Exception, e:
-            if not isinstance(cases[0], Exception):
-                raise
-            self.same_error(e, cases[0])
-            t.eq(len(casese), 1)
-        while True:
-            try:
-                req = p.next()
-            except StopIteration, e:
-                t.eq(len(cases), 0)
-                ended = True
-                break
-            except ParseException, e:
-                if not issubclass(cases[0], Exception):
-                    raise
-                self.same_error(e, cases.pop(0))
-                t.eq(len(cases), 0)
-                return
-            else:
-                self.same(req, sizer, matcher, cases.pop(0))
         t.eq(len(cases), 0)
-        t.eq(ended, True)
 
     def same(self, req, sizer, matcher, exp):
-        if isinstance(req, Exception):
-            self.same_error(req, exp)
-        else:
-            self.same_obj(req, sizer, matcher, exp)
-    
-    def same_error(self, req, exp):
-        t.istype(req, Exception)
-        t.istype(req, exp)
-    
-    def same_obj(self, req, sizer, matcher, exp):
         t.eq(req.method, exp["method"])
         t.eq(req.uri, exp["uri"]["raw"])
         t.eq(req.scheme, exp["uri"]["scheme"])