optimize readline()

Use less memory when processing file uploading, to be fast and safe

gunicorn/http/body.py

@@ -225,21 +225,25 @@ class Body(object):
         if size == 0:
             return b""
 
-        line = self.buf.getvalue()
+        data = self.buf.getvalue()
         self.buf = six.BytesIO()
-        if len(line) < size:
-            line += self.reader.read(size - len(line))
-        extra_buf_data = line[size:]
-        line = line[:size]
 
-        idx = line.find(b"\n")
+        ret = []
-        if idx >= 0:
+        while 1:
-            ret = line[:idx + 1]
+            idx = data.find(b"\n", 0, size)
-            self.buf.write(line[idx + 1:])
+            idx = idx + 1 if idx >= 0 else size if len(data) >= size else 0
-            self.buf.write(extra_buf_data)
+            if idx:
-            return ret
+                ret.append(data[:idx])
-        self.buf.write(extra_buf_data)
+                self.buf.write(data[idx:])
-        return line
+                break
+
+            ret.append(data)
+            size -= len(data)
+            data = self.reader.read(min(1024, size))
+            if not data:
+                break
+
+        return b"".join(ret)
 
     def readlines(self, size=None):
         ret = []

gunicorn/six.py

@@ -274,8 +274,14 @@ else:
     def u(s):
         return unicode(s, "unicode_escape")
     int2byte = chr
-    import StringIO
+    import cStringIO
-    StringIO = BytesIO = StringIO.StringIO
+    def StringIO(buf=''):
+        sio = cStringIO.StringIO()
+        if buf:
+            sio.write(buf)
+            sio.seek(0)
+        return sio
+    BytesIO = StringIO
 _add_doc(b, """Byte literal""")
 _add_doc(u, """Text literal""")