From 18d2b92146592b3f5370fb1d964ce91a416c693e Mon Sep 17 00:00:00 2001 From: Randall Leeds Date: Wed, 5 Aug 2015 16:49:43 -0700 Subject: [PATCH] Simplify sendfile logic A safe and reliable check for whether a file descriptor supports mmap is to directly check if it is seekable. However, some seekable file descriptors may also report a zero size when calling fstat. If there is no content length specified for the response and it cannot be determined from the file descriptor then it is not possible to know what chunk size to send to the client. In this case, is it necessary to fall back to unwinding the body by iteration. The above conditions together reveal a straightforward and reliable way to check for sendfile support. This patch modifies the Response class to assert these conditions using a try/catch block as part of a new, simplified sendfile method. This method returns False if it is not possible to serve the response using sendfile. Otherwise, it serves the response and returns True. By returning False when SSL is in use, the code is made even simpler by removing the special support for SSL, which is served well enough by the iteration protocol. Fix #1038 --- gunicorn/http/wsgi.py | 99 +++++++++++++++++-------------------------- gunicorn/util.py | 14 ------ 2 files changed, 38 insertions(+), 75 deletions(-) diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index b8b56934..6804eb80 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -12,7 +12,6 @@ import sys from gunicorn._compat import unquote_to_wsgi_str from gunicorn.six import string_types, binary_type, reraise from gunicorn import SERVER_SOFTWARE -import gunicorn.six as six import gunicorn.util as util try: @@ -24,6 +23,10 @@ except ImportError: except ImportError: sendfile = None +# Send files in at most 1GB blocks as some operating systems can have problems +# with sending files in blocks over 2GB. +BLKSIZE = 0x3FFFFFFF + NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') log = logging.getLogger(__name__) @@ -344,77 +347,51 @@ class Response(object): util.write(self.sock, arg, self.chunked) def can_sendfile(self): - return (self.cfg.sendfile and (sendfile is not None)) + return self.cfg.sendfile and sendfile is not None - def sendfile_all(self, fileno, sockno, offset, nbytes): - # Send file in at most 1GB blocks as some operating - # systems can have problems with sending files in blocks - # over 2GB. + def sendfile(self, respiter): + if self.cfg.is_ssl or not self.can_sendfile(): + return False - BLKSIZE = 0x3FFFFFFF + try: + fileno = respiter.filelike.fileno() + offset = os.lseek(fileno, 0, os.SEEK_CUR) + if self.response_length is None: + filesize = os.fstat(fileno).st_size - if nbytes > BLKSIZE: - for m in range(0, nbytes, BLKSIZE): - self.sendfile_all(fileno, sockno, offset, min(nbytes, BLKSIZE)) - offset += BLKSIZE - nbytes -= BLKSIZE - else: - sent = 0 - sent += sendfile(sockno, fileno, offset + sent, nbytes - sent) - while sent != nbytes: - sent += sendfile(sockno, fileno, offset + sent, nbytes - sent) + # The file may be special and sendfile will fail. + # It may also be zero-length, but that is okay. + if filesize == 0: + return False - def sendfile_use_send(self, fileno, fo_offset, nbytes): + nbytes = filesize - offset + else: + nbytes = self.response_length + except (OSError, io.UnsupportedOperation): + return False - # send file in blocks of 8182 bytes - BLKSIZE = 8192 + self.send_headers() + if self.is_chunked(): + chunk_size = "%X\r\n" % nbytes + self.sock.sendall(chunk_size.encode('utf-8')) + + sockno = self.sock.fileno() sent = 0 - while sent != nbytes: - data = os.read(fileno, BLKSIZE) - if not data: - break - sent += len(data) - if sent > nbytes: - data = data[:nbytes - sent] + for m in range(0, nbytes, BLKSIZE): + count = min(nbytes - sent, BLKSIZE) + sent += sendfile(sockno, fileno, offset + sent, count) - util.write(self.sock, data, self.chunked) + if self.is_chunked(): + self.sock.sendall(b"\r\n") + + os.lseek(fileno, offset, os.SEEK_SET) + + return True def write_file(self, respiter): - if self.can_sendfile() and util.is_fileobject(respiter.filelike): - # sometimes the fileno isn't a callable - if six.callable(respiter.filelike.fileno): - fileno = respiter.filelike.fileno() - else: - fileno = respiter.filelike.fileno - - fd_offset = os.lseek(fileno, 0, os.SEEK_CUR) - fo_offset = respiter.filelike.tell() - nbytes = max(os.fstat(fileno).st_size - fo_offset, 0) - - if self.response_length: - nbytes = min(nbytes, self.response_length) - - if nbytes == 0: - return - - self.send_headers() - - if self.cfg.is_ssl: - self.sendfile_use_send(fileno, fo_offset, nbytes) - else: - if self.is_chunked(): - chunk_size = "%X\r\n" % nbytes - self.sock.sendall(chunk_size.encode('utf-8')) - - self.sendfile_all(fileno, self.sock.fileno(), fo_offset, nbytes) - - if self.is_chunked(): - self.sock.sendall(b"\r\n") - - os.lseek(fileno, fd_offset, os.SEEK_SET) - else: + if not self.sendfile(respiter): for item in respiter: self.write(item) diff --git a/gunicorn/util.py b/gunicorn/util.py index ad5dd343..ad99fb9d 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -7,7 +7,6 @@ from __future__ import print_function import email.utils import fcntl -import io import os import pkg_resources import random @@ -517,19 +516,6 @@ def to_latin1(value): return value.encode("latin-1") -def is_fileobject(obj): - if not hasattr(obj, "tell") or not hasattr(obj, "fileno"): - return False - - # check BytesIO case and maybe others - try: - obj.fileno() - except (IOError, io.UnsupportedOperation): - return False - - return True - - def warn(msg): print("!!!", file=sys.stderr)