fix chunked encoding and add tests

gunicorn/http/http_parser.py

@@ -131,33 +131,34 @@ class HttpParser(object):
             
     def body_eof(self):
         """do we have all the body ?"""
-        if self.is_chunked and self._chunk_eof:
+        import sys
-            return True
+        if self.is_chunked:
-        if self._content_len == 0:
+            if self._chunk_eof:
-            return True
+                return True
+        elif self._content_len == 0:
+                return True
         return False
         
     def read_chunk(self, data):
         dlen = len(data)
         if not self.start_offset:
-            i = data.find("\n")
+            i = data.find("\r\n")
             if i != -1:
                 chunk = data[:i].strip().split(";", 1)
-                chunk_size = int(line.pop(0), 16)
+                chunk_size = int(chunk.pop(0), 16)
-                self.start_offset = i+1
+                self.start_offset = i+2
                 self.chunk_size = chunk_size
-        else:
+                if self.chunk_size == 0:
-            buf = self.data[self.start_offset:]
-            
-            end_offset = chunk_size + 2
-            # we wait CRLF else return None
-            if len(buf) == end_offset:
-                if chunk_size <= 0:
                     self._chunk_eof = True
-                    # we put data 
+                    return '', data[:self.start_offset]
-                    return '', data[:end_offset]
+        else:
+            buf = data[self.start_offset:self.start_offset+self.chunk_size]
+            end_offset = self.start_offset + self.chunk_size + 2
+            # we wait CRLF else return None
+            if len(data) >= end_offset:
+                ret = buf, data[end_offset:]
                 self.chunk_size = 0
-                return buf[chunk_size:], data[:end_offset]
+                return ret
         return '', data
         
     def trailing_header(self, data):
@@ -173,7 +174,9 @@ class HttpParser(object):
         dlen = len(data)
         chunk = ''
         if self.is_chunked:
+
             chunk, data = self.read_chunk(data)
+            
             if not chunk:
                 return '', data
         else:

tests/001-test-parser.py

@@ -156,10 +156,33 @@ def test_010(buf, p):
     t.eq(p.path, "/post_chunked_all_your_base")
     t.eq(p.headers, [('Transfer-Encoding', 'chunked')])
     t.eq(p.is_chunked, True)
+    t.eq(p._chunk_eof, False)
+    t.ne(p.body_eof(), True)
     body = ""
     buf = buf[i:]
-    print buf
     while not p.body_eof():
         chunk, buf = p.filter_body(buf)
-        body += chunk
+        print chunk
+        if chunk:
+            body += chunk
     t.eq(body, "all your base are belong to us")
+    
+@t.request("011.http")
+def test_011(buf, p):
+    headers = []
+    i = p.filter_headers(headers, buf)
+    t.ne(i, -1)
+    t.eq(p.method, "POST")
+    t.eq(p.version, (1, 1))
+    t.eq(p.path, "/two_chunks_mult_zero_end")
+    t.eq(p.headers, [('Transfer-Encoding', 'chunked')])
+    t.eq(p.is_chunked, True)
+    t.eq(p._chunk_eof, False)
+    t.ne(p.body_eof(), True)
+    body = ""
+    buf = buf[i:]
+    while not p.body_eof():
+        chunk, buf = p.filter_body(buf)
+        if chunk:
+            body += chunk
+    t.eq(body, "hello world")

tests/requests/001.http

@@ -1,6 +1,6 @@
-PUT /stuff/here?foo=bar HTTP/1.0
+PUT /stuff/here?foo=bar HTTP/1.0\r\n
-Server: http://127.0.0.1:5984
+Server: http://127.0.0.1:5984\r\n
-Content-Type: application/json
+Content-Type: application/json\r\n
-Content-Length: 14
+Content-Length: 14\r\n
-
+\r\n
 {"nom": "nom"}

tests/requests/002.http

@@ -1,5 +1,5 @@
-GET /test HTTP/1.1
+GET /test HTTP/1.1\r\n
-User-Agent: curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1
+User-Agent: curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1\r\n
-Host: 0.0.0.0=5000
+Host: 0.0.0.0=5000\r\n
-Accept: */*
+Accept: */*\r\n
-
+\r\n

tests/requests/003.http

@@ -1,10 +1,10 @@
-GET /favicon.ico HTTP/1.1
+GET /favicon.ico HTTP/1.1\r\n
-Host: 0.0.0.0=5000
+Host: 0.0.0.0=5000\r\n
-User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0
+User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0\r\n
-Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n
-Accept-Language: en-us,en;q=0.5
+Accept-Language: en-us,en;q=0.5\r\n
-Accept-Encoding: gzip,deflate
+Accept-Encoding: gzip,deflate\r\n
-Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n
-Keep-Alive: 300
+Keep-Alive: 300\r\n
-Connection: keep-alive
+Connection: keep-alive\r\n
-
+\r\n

tests/requests/004.http

@@ -1,3 +1,3 @@
-GET /dumbfuck HTTP/1.1
+GET /dumbfuck HTTP/1.1\r\n
-aaaaaaaaaaaaa:++++++++++
+aaaaaaaaaaaaa:++++++++++\r\n
-
+\r\n

tests/requests/005.http

@@ -1,2 +1,2 @@
-GET /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1
+GET /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1\r\n
-
+\r\n

tests/requests/006.http

@@ -1,2 +1,2 @@
-GET /get_no_headers_no_body/world HTTP/1.1
+GET /get_no_headers_no_body/world HTTP/1.1\r\n
-
+\r\n

tests/requests/007.http

@@ -1,3 +1,3 @@
-GET /get_one_header_no_body HTTP/1.1
+GET /get_one_header_no_body HTTP/1.1\r\n
-Accept: */*
+Accept: */*\r\n
-
+\r\n

tests/requests/008.http

@@ -1,4 +1,4 @@
-GET /get_funky_content_length_body_hello HTTP/1.0
+GET /get_funky_content_length_body_hello HTTP/1.0\r\n
-conTENT-Length: 5
+conTENT-Length: 5\r\n
-
+\r\n
 HELLO

tests/requests/009.http

@@ -1,6 +1,6 @@
-POST /post_identity_body_world?q=search#hey HTTP/1.1
+POST /post_identity_body_world?q=search#hey HTTP/1.1\r\n
-Accept: */*
+Accept: */*\r\n
-Transfer-Encoding: identity
+Transfer-Encoding: identity\r\n
-Content-Length: 5
+Content-Length: 5\r\n
-
+\r\n
 World

tests/requests/010.http

@@ -1,6 +1,7 @@
-POST /post_chunked_all_your_base HTTP/1.1
+POST /post_chunked_all_your_base HTTP/1.1\r\n
-Transfer-Encoding: chunked
+Transfer-Encoding: chunked\r\n
+\r\n
+1e\r\n
+all your base are belong to us\r\n
+0\r\n
 
-1e
-all your base are belong to us
-0

tests/requests/011.http

@@ -1,8 +1,9 @@
-POST /two_chunks_mult_zero_end HTTP/1.1
+POST /two_chunks_mult_zero_end HTTP/1.1\r\n
-Transfer-Encoding: chunked
+Transfer-Encoding: chunked\r\n
-
+\r\n
-5
+5\r\n
-hello
+hello\r\n
-6
+6\r\n
- world
+ world\r\n
-000
+000\r\n
+\r\n

tests/requests/012.http

@@ -1,10 +1,11 @@
-POST /chunked_w_trailing_headers HTTP/1.1
+POST /chunked_w_trailing_headers HTTP/1.1\r\n
-Transfer-Encoding: chunked
+Transfer-Encoding: chunked\r\n
-
+\r\n
-5
+5\r\n
-hello
+hello\r\n
-6
+6\r\n
- world
+ world\r\n
-0
+0\r\n
-Vary: *
+Vary: *\r\n
-Content-Type: text/plain
+Content-Type: text/plain\r\n
+\r\n

tests/t.py

@@ -16,10 +16,8 @@ def data_source(fname, eol):
     with open(fname) as handle:
         lines = []
         for line in handle:
-            next = line.rstrip("\r\n") + eol
+            line = line.rstrip("\n").replace("\\r\\n", "\r\n")
-            if next == "\r\n":
+            lines.append(line)
-                eol = ""
-            lines.append(next)
         return "".join(lines)
 
 class request(object):