Merge pull request #3113 from pajod/patch-security

Fix numerous message parsing issues (v2)

SECURITY.md

@@ -0,0 +1,22 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+**Please note that public Github issues are open for everyone to see!**
+
+If you believe you are found a problem in Gunicorn software, examples or documentation, we encourage you to send your report privately via email, or via Github using the *Report a vulnerability* button in the [Security](https://github.com/benoitc/gunicorn/security) section.
+
+## Supported Releases
+
+At this time, **only the latest release** receives any security attention whatsoever.
+
+| Version | Status          |
+| ------- | ------------------ |
+| latest release  | :white_check_mark: |
+| 21.2.0  | :x: |
+| 20.0.0  | :x: |
+| < 20.0  | :x: |
+
+## Python Versions
+
+Gunicorn runs on Python 3.7+, we *highly recommend* the latest release of a [supported series](https://devguide.python.org/version/) and will not prioritize issues exclusively affecting in EoL environments.

docs/source/2023-news.rst

@@ -2,6 +2,28 @@
 Changelog - 2023
 ================
 
+22.0.0 - TBDTBDTBD
+==================
+
+- fix numerous security vulnerabilites in HTTP parser (closing some request smuggling vectors)
+- parsing additional requests is no longer attempted past unsupported request framing
+- on HTTP versions < 1.1 support for chunked transfer is refused (only used in exploits)
+- requests conflicting configured or passed SCRIPT_NAME now produce a verbose error
+- Trailer fields are no longer inspected for headers indicating secure scheme
+
+** Breaking changes **
+
+- the limitations on valid characters in the HTTP method have been bounded to Internet Standards
+- requests specifying unsupported transfer coding (order) are refused by default (rare)
+- HTTP methods are no longer casefolded by default (IANA method registry contains none affacted)
+- HTTP methods containing the number sign (#) are no longer accepted by default (rare)
+- HTTP versions < 1.0 or >= 2.0 are no longer accepted by default (rare, only HTTP/1.1 is supported)
+- HTTP versions consisting of multiple digits or containing a prefix/suffix are no longer accepted
+- HTTP header field names Gunicorn cannot safely map to variables are silently dropped, as in other software
+- HTTP headers with empty field name are refused by default (no legitimate use cases, used in exploits)
+- requests with both Transfer-Encoding and Content-Length are refused by default (such a message might indicate an attempt to perform request smuggling)
+- empty transfer codings are no longer permitted (reportedly seen with really old & broken proxies)
+
 21.2.0 - 2023-07-19
 ===================
 

gunicorn/config.py

@@ -2254,5 +2254,131 @@ class StripHeaderSpaces(Setting):
         This is known to induce vulnerabilities and is not compliant with the HTTP/1.1 standard.
         See https://portswigger.net/research/http-desync-attacks-request-smuggling-reborn.
 
-        Use with care and only if necessary.
+        Use with care and only if necessary. May be removed in a future version.
+
+        .. versionadded:: 20.0.1
+        """
+
+
+class PermitUnconventionalHTTPMethod(Setting):
+    name = "permit_unconventional_http_method"
+    section = "Server Mechanics"
+    cli = ["--permit-unconventional-http-method"]
+    validator = validate_bool
+    action = "store_true"
+    default = False
+    desc = """\
+        Permit HTTP methods not matching conventions, such as IANA registration guidelines
+
+        This permits request methods of length less than 3 or more than 20,
+        methods with lowercase characters or methods containing the # character.
+        HTTP methods are case sensitive by definition, and merely uppercase by convention.
+
+        This option is provided to diagnose backwards-incompatible changes.
+
+        Use with care and only if necessary. May be removed in a future version.
+
+        .. versionadded:: 22.0.0
+        """
+
+
+class PermitUnconventionalHTTPVersion(Setting):
+    name = "permit_unconventional_http_version"
+    section = "Server Mechanics"
+    cli = ["--permit-unconventional-http-version"]
+    validator = validate_bool
+    action = "store_true"
+    default = False
+    desc = """\
+        Permit HTTP version not matching conventions of 2023
+
+        This disables the refusal of likely malformed request lines.
+        It is unusual to specify HTTP 1 versions other than 1.0 and 1.1.
+
+        This option is provided to diagnose backwards-incompatible changes.
+        Use with care and only if necessary. May be removed in a future version.
+
+        .. versionadded:: 22.0.0
+        """
+
+
+class CasefoldHTTPMethod(Setting):
+    name = "casefold_http_method"
+    section = "Server Mechanics"
+    cli = ["--casefold-http-method"]
+    validator = validate_bool
+    action = "store_true"
+    default = False
+    desc = """\
+         Transform received HTTP methods to uppercase
+
+         HTTP methods are case sensitive by definition, and merely uppercase by convention.
+
+         This option is provided because previous versions of gunicorn defaulted to this behaviour.
+
+         Use with care and only if necessary. May be removed in a future version.
+
+         .. versionadded:: 22.0.0
+         """
+
+
+def validate_header_map_behaviour(val):
+    # FIXME: refactor all of this subclassing stdlib argparse
+
+    if val is None:
+        return
+
+    if not isinstance(val, str):
+        raise TypeError("Invalid type for casting: %s" % val)
+    if val.lower().strip() == "drop":
+        return "drop"
+    elif val.lower().strip() == "refuse":
+        return "refuse"
+    elif val.lower().strip() == "dangerous":
+        return "dangerous"
+    else:
+        raise ValueError("Invalid header map behaviour: %s" % val)
+
+
+class HeaderMap(Setting):
+    name = "header_map"
+    section = "Server Mechanics"
+    cli = ["--header-map"]
+    validator = validate_header_map_behaviour
+    default = "drop"
+    desc = """\
+        Configure how header field names are mapped into environ
+
+        Headers containing underscores are permitted by RFC9110,
+        but gunicorn joining headers of different names into
+        the same environment variable will dangerously confuse applications as to which is which.
+
+        The safe default ``drop`` is to silently drop headers that cannot be unambiguously mapped.
+        The value ``refuse`` will return an error if a request contains *any* such header.
+        The value ``dangerous`` matches the previous, not advisabble, behaviour of mapping different
+        header field names into the same environ name.
+
+        Use with care and only if necessary and after considering if your problem could
+        instead be solved by specifically renaming or rewriting only the intended headers
+        on a proxy in front of Gunicorn.
+
+        .. versionadded:: 22.0.0
+        """
+
+
+class TolerateDangerousFraming(Setting):
+    name = "tolerate_dangerous_framing"
+    section = "Server Mechanics"
+    cli = ["--tolerate-dangerous-framing"]
+    validator = validate_bool
+    action = "store_true"
+    default = False
+    desc = """\
+        Process requests with both Transfer-Encoding and Content-Length
+
+        This is known to induce vulnerabilities, but not strictly forbidden by RFC9112.
+
+        Use with care and only if necessary. May be removed in a future version.
+
+        .. versionadded:: 22.0.0
         """

gunicorn/http/body.py

@@ -51,7 +51,7 @@ class ChunkedReader(object):
         if done:
             unreader.unread(buf.getvalue()[2:])
             return b""
-        self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx])
+        self.req.trailers = self.req.parse_headers(buf.getvalue()[:idx], from_trailer=True)
         unreader.unread(buf.getvalue()[idx + 4:])
 
     def parse_chunked(self, unreader):
@@ -85,11 +85,13 @@ class ChunkedReader(object):
         data = buf.getvalue()
         line, rest_chunk = data[:idx], data[idx + 2:]
 
-        chunk_size = line.split(b";", 1)[0].strip()
+        # RFC9112 7.1.1: BWS before chunk-ext - but ONLY then
-        try:
+        chunk_size, *chunk_ext = line.split(b";", 1)
-            chunk_size = int(chunk_size, 16)
+        if chunk_ext:
-        except ValueError:
+            chunk_size = chunk_size.rstrip(b" \t")
+        if any(n not in b"0123456789abcdefABCDEF" for n in chunk_size):
             raise InvalidChunkSize(chunk_size)
+        chunk_size = int(chunk_size, 16)
 
         if chunk_size == 0:
             try:

gunicorn/http/errors.py

@@ -22,6 +22,15 @@ class NoMoreData(IOError):
         return "No more data after: %r" % self.buf
 
 
+class ConfigurationProblem(ParseException):
+    def __init__(self, info):
+        self.info = info
+        self.code = 500
+
+    def __str__(self):
+        return "Configuration problem: %s" % self.info
+
+
 class InvalidRequestLine(ParseException):
     def __init__(self, req):
         self.req = req
@@ -64,6 +73,15 @@ class InvalidHeaderName(ParseException):
         return "Invalid HTTP header name: %r" % self.hdr
 
 
+class UnsupportedTransferCoding(ParseException):
+    def __init__(self, hdr):
+        self.hdr = hdr
+        self.code = 501
+
+    def __str__(self):
+        return "Unsupported transfer coding: %r" % self.hdr
+
+
 class InvalidChunkSize(IOError):
     def __init__(self, data):
         self.data = data

gunicorn/http/message.py

@@ -12,6 +12,7 @@ from gunicorn.http.errors import (
     InvalidHeader, InvalidHeaderName, NoMoreData,
     InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion,
     LimitRequestLine, LimitRequestHeaders,
+    UnsupportedTransferCoding,
 )
 from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest
 from gunicorn.http.errors import InvalidSchemeHeaders
@@ -21,9 +22,12 @@ MAX_REQUEST_LINE = 8190
 MAX_HEADERS = 32768
 DEFAULT_MAX_HEADERFIELD_SIZE = 8190
 
-HEADER_RE = re.compile(r"[\x00-\x1F\x7F()<>@,;:\[\]={} \t\\\"]")
+# verbosely on purpose, avoid backslash ambiguity
-METH_RE = re.compile(r"[A-Z0-9$-_.]{3,20}")
+RFC9110_5_6_2_TOKEN_SPECIALS = r"!#$%&'*+-.^_`|~"
-VERSION_RE = re.compile(r"HTTP/(\d+)\.(\d+)")
+TOKEN_RE = re.compile(r"[%s0-9a-zA-Z]+" % (re.escape(RFC9110_5_6_2_TOKEN_SPECIALS)))
+METHOD_BADCHAR_RE = re.compile("[a-z#]")
+# usually 1.0 or 1.1 - RFC9112 permits restricting to single-digit versions
+VERSION_RE = re.compile(r"HTTP/(\d)\.(\d)")
 
 
 class Message(object):
@@ -37,6 +41,7 @@ class Message(object):
         self.trailers = []
         self.body = None
         self.scheme = "https" if cfg.is_ssl else "http"
+        self.must_close = False
 
         # set headers limits
         self.limit_request_fields = cfg.limit_request_fields
@@ -56,22 +61,29 @@ class Message(object):
         self.unreader.unread(unused)
         self.set_body_reader()
 
+    def force_close(self):
+        self.must_close = True
+
     def parse(self, unreader):
         raise NotImplementedError()
 
-    def parse_headers(self, data):
+    def parse_headers(self, data, from_trailer=False):
         cfg = self.cfg
         headers = []
 
-        # Split lines on \r\n keeping the \r\n on each line
+        # Split lines on \r\n
-        lines = [bytes_to_str(line) + "\r\n" for line in data.split(b"\r\n")]
+        lines = [bytes_to_str(line) for line in data.split(b"\r\n")]
 
         # handle scheme headers
         scheme_header = False
         secure_scheme_headers = {}
-        if ('*' in cfg.forwarded_allow_ips or
+        if from_trailer:
-            not isinstance(self.peer_addr, tuple)
+            # nonsense. either a request is https from the beginning
-                or self.peer_addr[0] in cfg.forwarded_allow_ips):
+            #  .. or we are just behind a proxy who does not remove conflicting trailers
+            pass
+        elif ('*' in cfg.forwarded_allow_ips or
+              not isinstance(self.peer_addr, tuple)
+              or self.peer_addr[0] in cfg.forwarded_allow_ips):
             secure_scheme_headers = cfg.secure_scheme_headers
 
         # Parse headers into key/value pairs paying attention
@@ -80,30 +92,34 @@ class Message(object):
             if len(headers) >= self.limit_request_fields:
                 raise LimitRequestHeaders("limit request headers fields")
 
-            # Parse initial header name : value pair.
+            # Parse initial header name: value pair.
             curr = lines.pop(0)
-            header_length = len(curr)
+            header_length = len(curr) + len("\r\n")
-            if curr.find(":") < 0:
+            if curr.find(":") <= 0:
-                raise InvalidHeader(curr.strip())
+                raise InvalidHeader(curr)
             name, value = curr.split(":", 1)
             if self.cfg.strip_header_spaces:
-                name = name.rstrip(" \t").upper()
+                name = name.rstrip(" \t")
-            else:
+            if not TOKEN_RE.fullmatch(name):
-                name = name.upper()
-            if HEADER_RE.search(name):
                 raise InvalidHeaderName(name)
 
-            name, value = name.strip(), [value.lstrip()]
+            # this is still a dangerous place to do this
+            #  but it is more correct than doing it before the pattern match:
+            # after we entered Unicode wonderland, 8bits could case-shift into ASCII:
+            # b"\xDF".decode("latin-1").upper().encode("ascii") == b"SS"
+            name = name.upper()
+
+            value = [value.lstrip(" \t")]
 
             # Consume value continuation lines
             while lines and lines[0].startswith((" ", "\t")):
                 curr = lines.pop(0)
-                header_length += len(curr)
+                header_length += len(curr) + len("\r\n")
                 if header_length > self.limit_request_field_size > 0:
                     raise LimitRequestHeaders("limit request headers "
                                               "fields size")
-                value.append(curr)
+                value.append(curr.strip("\t "))
-            value = ''.join(value).rstrip()
+            value = " ".join(value)
 
             if header_length > self.limit_request_field_size > 0:
                 raise LimitRequestHeaders("limit request headers fields size")
@@ -118,6 +134,23 @@ class Message(object):
                     scheme_header = True
                     self.scheme = scheme
 
+            # ambiguous mapping allows fooling downstream, e.g. merging non-identical headers:
+            # X-Forwarded-For: 2001:db8::ha:cc:ed
+            # X_Forwarded_For: 127.0.0.1,::1
+            # HTTP_X_FORWARDED_FOR = 2001:db8::ha:cc:ed,127.0.0.1,::1
+            # Only modify after fixing *ALL* header transformations; network to wsgi env
+            if "_" in name:
+                if self.cfg.header_map == "dangerous":
+                    # as if we did not know we cannot safely map this
+                    pass
+                elif self.cfg.header_map == "drop":
+                    # almost as if it never had been there
+                    # but still counts against resource limits
+                    continue
+                else:
+                    # fail-safe fallthrough: refuse
+                    raise InvalidHeaderName(name)
+
             headers.append((name, value))
 
         return headers
@@ -133,9 +166,47 @@ class Message(object):
                 content_length = value
             elif name == "TRANSFER-ENCODING":
                 if value.lower() == "chunked":
+                    # DANGER: transer codings stack, and stacked chunking is never intended
+                    if chunked:
+                        raise InvalidHeader("TRANSFER-ENCODING", req=self)
                     chunked = True
+                elif value.lower() == "identity":
+                    # does not do much, could still plausibly desync from what the proxy does
+                    # safe option: nuke it, its never needed
+                    if chunked:
+                        raise InvalidHeader("TRANSFER-ENCODING", req=self)
+                elif value.lower() == "":
+                    # lacking security review on this case
+                    # offer the option to restore previous behaviour, but refuse by default, for now
+                    self.force_close()
+                    if not self.cfg.tolerate_dangerous_framing:
+                        raise UnsupportedTransferCoding(value)
+                # DANGER: do not change lightly; ref: request smuggling
+                # T-E is a list and we *could* support correctly parsing its elements
+                #  .. but that is only safe after getting all the edge cases right
+                #  .. for which no real-world need exists, so best to NOT open that can of worms
+                else:
+                    self.force_close()
+                    # even if parser is extended, retain this branch:
+                    #  the "chunked not last" case remains to be rejected!
+                    raise UnsupportedTransferCoding(value)
 
         if chunked:
+            # two potentially dangerous cases:
+            #  a) CL + TE (TE overrides CL.. only safe if the recipient sees it that way too)
+            #  b) chunked HTTP/1.0 (always faulty)
+            if self.version < (1, 1):
+                # framing wonky, see RFC 9112 Section 6.1
+                self.force_close()
+                if not self.cfg.tolerate_dangerous_framing:
+                    raise InvalidHeader("TRANSFER-ENCODING", req=self)
+            if content_length is not None:
+                # we cannot be certain the message framing we understood matches proxy intent
+                #  -> whatever happens next, remaining input must not be trusted
+                self.force_close()
+                # either processing or rejecting is permitted in RFC 9112 Section 6.1
+                if not self.cfg.tolerate_dangerous_framing:
+                    raise InvalidHeader("CONTENT-LENGTH", req=self)
             self.body = Body(ChunkedReader(self, self.unreader))
         elif content_length is not None:
             try:
@@ -154,9 +225,11 @@ class Message(object):
             self.body = Body(EOFReader(self.unreader))
 
     def should_close(self):
+        if self.must_close:
+            return True
         for (h, v) in self.headers:
             if h == "CONNECTION":
-                v = v.lower().strip()
+                v = v.lower().strip(" \t")
                 if v == "close":
                     return True
                 elif v == "keep-alive":
@@ -230,7 +303,7 @@ class Request(Message):
             self.unreader.unread(data[2:])
             return b""
 
-        self.headers = self.parse_headers(data[:idx])
+        self.headers = self.parse_headers(data[:idx], from_trailer=False)
 
         ret = data[idx + 4:]
         buf = None
@@ -283,7 +356,7 @@ class Request(Message):
             raise ForbiddenProxyRequest(self.peer_addr[0])
 
     def parse_proxy_protocol(self, line):
-        bits = line.split()
+        bits = line.split(" ")
 
         if len(bits) != 6:
             raise InvalidProxyLine(line)
@@ -328,14 +401,27 @@ class Request(Message):
         }
 
     def parse_request_line(self, line_bytes):
-        bits = [bytes_to_str(bit) for bit in line_bytes.split(None, 2)]
+        bits = [bytes_to_str(bit) for bit in line_bytes.split(b" ", 2)]
         if len(bits) != 3:
             raise InvalidRequestLine(bytes_to_str(line_bytes))
 
-        # Method
+        # Method: RFC9110 Section 9
-        if not METH_RE.match(bits[0]):
+        self.method = bits[0]
-            raise InvalidRequestMethod(bits[0])
+
-        self.method = bits[0].upper()
+        # nonstandard restriction, suitable for all IANA registered methods
+        # partially enforced in previous gunicorn versions
+        if not self.cfg.permit_unconventional_http_method:
+            if METHOD_BADCHAR_RE.search(self.method):
+                raise InvalidRequestMethod(self.method)
+            if not 3 <= len(bits[0]) <= 20:
+                raise InvalidRequestMethod(self.method)
+        # standard restriction: RFC9110 token
+        if not TOKEN_RE.fullmatch(self.method):
+            raise InvalidRequestMethod(self.method)
+        # nonstandard and dangerous
+        # methods are merely uppercase by convention, no case-insensitive treatment is intended
+        if self.cfg.casefold_http_method:
+            self.method = self.method.upper()
 
         # URI
         self.uri = bits[1]
@@ -349,10 +435,14 @@ class Request(Message):
         self.fragment = parts.fragment or ""
 
         # Version
-        match = VERSION_RE.match(bits[2])
+        match = VERSION_RE.fullmatch(bits[2])
         if match is None:
             raise InvalidHTTPVersion(bits[2])
         self.version = (int(match.group(1)), int(match.group(2)))
+        if not (1, 0) <= self.version < (2, 0):
+            # if ever relaxing this, carefully review Content-Encoding processing
+            if not self.cfg.permit_unconventional_http_version:
+                raise InvalidHTTPVersion(self.version)
 
     def set_body_reader(self):
         super().set_body_reader()

gunicorn/http/wsgi.py

@@ -9,8 +9,8 @@ import os
 import re
 import sys
 
-from gunicorn.http.message import HEADER_RE
+from gunicorn.http.message import TOKEN_RE
-from gunicorn.http.errors import InvalidHeader, InvalidHeaderName
+from gunicorn.http.errors import ConfigurationProblem, InvalidHeader, InvalidHeaderName
 from gunicorn import SERVER_SOFTWARE, SERVER
 from gunicorn import util
 
@@ -18,7 +18,9 @@ from gunicorn import util
 # with sending files in blocks over 2GB.
 BLKSIZE = 0x3FFFFFFF
 
-HEADER_VALUE_RE = re.compile(r'[\x00-\x1F\x7F]')
+# RFC9110 5.5: field-vchar = VCHAR / obs-text
+# RFC4234 B.1: VCHAR = 0x21-x07E = printable ASCII
+HEADER_VALUE_RE = re.compile(r'[ \t\x21-\x7e\x80-\xff]*')
 
 log = logging.getLogger(__name__)
 
@@ -133,6 +135,8 @@ def create(req, sock, client, server, cfg):
             environ['CONTENT_LENGTH'] = hdr_value
             continue
 
+        # do not change lightly, this is a common source of security problems
+        # RFC9110 Section 17.10 discourages ambiguous or incomplete mappings
         key = 'HTTP_' + hdr_name.replace('-', '_')
         if key in environ:
             hdr_value = "%s,%s" % (environ[key], hdr_value)
@@ -180,7 +184,11 @@ def create(req, sock, client, server, cfg):
     # set the path and script name
     path_info = req.path
     if script_name:
-        path_info = path_info.split(script_name, 1)[1]
+        if not path_info.startswith(script_name):
+            raise ConfigurationProblem(
+                "Request path %r does not start with SCRIPT_NAME %r" %
+                (path_info, script_name))
+        path_info = path_info[len(script_name):]
     environ['PATH_INFO'] = util.unquote_to_wsgi_str(path_info)
     environ['SCRIPT_NAME'] = script_name
 
@@ -249,31 +257,32 @@ class Response(object):
             if not isinstance(name, str):
                 raise TypeError('%r is not a string' % name)
 
-            if HEADER_RE.search(name):
+            if not TOKEN_RE.fullmatch(name):
                 raise InvalidHeaderName('%r' % name)
 
             if not isinstance(value, str):
                 raise TypeError('%r is not a string' % value)
 
-            if HEADER_VALUE_RE.search(value):
+            if not HEADER_VALUE_RE.fullmatch(value):
                 raise InvalidHeader('%r' % value)
 
-            value = value.strip()
+            # RFC9110 5.5
-            lname = name.lower().strip()
+            value = value.strip(" \t")
+            lname = name.lower()
             if lname == "content-length":
                 self.response_length = int(value)
             elif util.is_hoppish(name):
                 if lname == "connection":
                     # handle websocket
-                    if value.lower().strip() == "upgrade":
+                    if value.lower() == "upgrade":
                         self.upgrade = True
                 elif lname == "upgrade":
-                    if value.lower().strip() == "websocket":
+                    if value.lower() == "websocket":
-                        self.headers.append((name.strip(), value))
+                        self.headers.append((name, value))
 
                 # ignore hopbyhop headers
                 continue
-            self.headers.append((name.strip(), value))
+            self.headers.append((name, value))
 
     def is_chunked(self):
         # Only use chunked responses when the client is

gunicorn/workers/base.py

@@ -251,6 +251,8 @@ class Worker(object):
         else:
             if hasattr(req, "uri"):
                 self.log.exception("Error handling request %s", req.uri)
+            else:
+                self.log.exception("Error handling request (no URI read)")
             status_int = 500
             reason = "Internal Server Error"
             mesg = ""

tests/requests/invalid/003.http

@@ -1,2 +1,2 @@
--blargh /foo HTTP/1.1\r\n
+GET\n/\nHTTP/1.1\r\n
-\r\n
+\r\n

tests/requests/invalid/003.py

@@ -1,2 +1,2 @@
-from gunicorn.http.errors import InvalidRequestMethod
+from gunicorn.http.errors import InvalidRequestLine
-request = InvalidRequestMethod
+request = InvalidRequestLine

tests/requests/invalid/003b.http

@@ -0,0 +1,2 @@
+bla:rgh /foo HTTP/1.1\r\n
+\r\n

tests/requests/invalid/003b.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidRequestMethod
+request = InvalidRequestMethod

tests/requests/invalid/003c.http

@@ -0,0 +1,2 @@
+-bl /foo HTTP/1.1\r\n
+\r\n

tests/requests/invalid/003c.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidRequestMethod
+request = InvalidRequestMethod

tests/requests/invalid/040.http

@@ -0,0 +1,6 @@
+GET /keep/same/as?invalid/040 HTTP/1.0\r\n
+Transfer_Encoding: tricked\r\n
+Content-Length: 7\r\n
+Content_Length: -1E23\r\n
+\r\n
+tricked\r\n

tests/requests/invalid/040.py

@@ -0,0 +1,7 @@
+from gunicorn.http.errors import InvalidHeaderName
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("header_map", "refuse")
+
+request = InvalidHeaderName

tests/requests/invalid/chunked_01.http

@@ -0,0 +1,12 @@
+POST /chunked_w_underscore_chunk_size HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6_0\r\n
+ world\r\n
+0\r\n
+\r\n
+POST /after HTTP/1.1\r\n
+Transfer-Encoding: identity\r\n
+\r\n

tests/requests/invalid/chunked_01.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidChunkSize
+request = InvalidChunkSize

tests/requests/invalid/chunked_02.http

@@ -0,0 +1,9 @@
+POST /chunked_with_prefixed_value HTTP/1.1\r\n
+Content-Length: 12\r\n
+Transfer-Encoding: \tchunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+\r\n

tests/requests/invalid/chunked_02.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidHeader
+request = InvalidHeader

tests/requests/invalid/chunked_03.http

@@ -0,0 +1,8 @@
+POST /double_chunked HTTP/1.1\r\n
+Transfer-Encoding: identity, chunked, identity, chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+\r\n

tests/requests/invalid/chunked_03.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import UnsupportedTransferCoding
+request = UnsupportedTransferCoding

tests/requests/invalid/chunked_04.http

@@ -0,0 +1,11 @@
+POST /chunked_twice HTTP/1.1\r\n
+Transfer-Encoding: identity\r\n
+Transfer-Encoding: chunked\r\n
+Transfer-Encoding: identity\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+\r\n

tests/requests/invalid/chunked_04.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidHeader
+request = InvalidHeader

tests/requests/invalid/chunked_05.http

@@ -0,0 +1,11 @@
+POST /chunked_HTTP_1.0 HTTP/1.0\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+0\r\n
+Vary: *\r\n
+Content-Type: text/plain\r\n
+\r\n

tests/requests/invalid/chunked_05.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidHeader
+request = InvalidHeader

tests/requests/invalid/chunked_06.http

@@ -0,0 +1,9 @@
+POST /chunked_not_last HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+Transfer-Encoding: gzip\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+\r\n

tests/requests/invalid/chunked_06.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import UnsupportedTransferCoding
+request = UnsupportedTransferCoding

tests/requests/invalid/chunked_07.http

@@ -0,0 +1,10 @@
+POST /chunked_ambiguous_header_mapping HTTP/1.1\r\n
+Transfer_Encoding: gzip\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+0\r\n
+\r\n

tests/requests/invalid/chunked_07.py

@@ -0,0 +1,7 @@
+from gunicorn.http.errors import InvalidHeaderName
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("header_map", "refuse")
+
+request = InvalidHeaderName

tests/requests/invalid/chunked_08.http

@@ -0,0 +1,9 @@
+POST /chunked_not_last HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+Transfer-Encoding: identity\r\n
+\r\n
+5\r\n
+hello\r\n
+6\r\n
+ world\r\n
+\r\n

tests/requests/invalid/chunked_08.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidHeader
+request = InvalidHeader

tests/requests/invalid/chunked_09.http

@@ -0,0 +1,7 @@
+POST /chunked_ows_without_ext HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+0 \r\n
+\r\n

tests/requests/invalid/chunked_09.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidChunkSize
+request = InvalidChunkSize

tests/requests/invalid/chunked_10.http

@@ -0,0 +1,7 @@
+POST /chunked_ows_before HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\r\n
+hello\r\n
+ 0\r\n
+\r\n

tests/requests/invalid/chunked_10.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidChunkSize
+request = InvalidChunkSize

tests/requests/invalid/chunked_11.http

@@ -0,0 +1,7 @@
+POST /chunked_ows_before HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5\n;\r\n
+hello\r\n
+0\r\n
+\r\n

tests/requests/invalid/chunked_11.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidChunkSize
+request = InvalidChunkSize

tests/requests/invalid/nonascii_01.http

@@ -0,0 +1,4 @@
+GETß /germans.. HTTP/1.1\r\n
+Content-Length: 3\r\n
+\r\n
+ÄÄÄ

tests/requests/invalid/nonascii_01.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidRequestMethod
+
+cfg = Config()
+request = InvalidRequestMethod

tests/requests/invalid/nonascii_02.http

@@ -0,0 +1,4 @@
+GETÿ /french.. HTTP/1.1\r\n
+Content-Length: 3\r\n
+\r\n
+ÄÄÄ

tests/requests/invalid/nonascii_02.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidRequestMethod
+
+cfg = Config()
+request = InvalidRequestMethod

tests/requests/invalid/nonascii_03.http

@@ -0,0 +1,5 @@
+GET /germans.. HTTP/1.1\r\n
+Content-Lengthß: 3\r\n
+Content-Length: 3\r\n
+\r\n
+ÄÄÄ

tests/requests/invalid/nonascii_03.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidHeaderName
+
+cfg = Config()
+request = InvalidHeaderName

tests/requests/invalid/nonascii_04.http

@@ -0,0 +1,5 @@
+GET /french.. HTTP/1.1\r\n
+Content-Lengthÿ: 3\r\n
+Content-Length: 3\r\n
+\r\n
+ÄÄÄ

tests/requests/invalid/nonascii_04.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidHeaderName
+
+cfg = Config()
+request = InvalidHeaderName

tests/requests/invalid/prefix_01.http

@@ -0,0 +1,2 @@
+GET\0PROXY /foo HTTP/1.1\r\n
+\r\n

tests/requests/invalid/prefix_01.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidRequestMethod
+request = InvalidRequestMethod

tests/requests/invalid/prefix_02.http

@@ -0,0 +1,2 @@
+GET\0 /foo HTTP/1.1\r\n
+\r\n

tests/requests/invalid/prefix_02.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidRequestMethod
+request = InvalidRequestMethod

tests/requests/invalid/prefix_03.http

@@ -0,0 +1,4 @@
+GET /stuff/here?foo=bar HTTP/1.1\r\n
+Content-Length: 0 1\r\n
+\r\n
+x

tests/requests/invalid/prefix_03.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidHeader
+
+cfg = Config()
+request = InvalidHeader

tests/requests/invalid/prefix_04.http

@@ -0,0 +1,5 @@
+GET /stuff/here?foo=bar HTTP/1.1\r\n
+Content-Length: 3 1\r\n
+\r\n
+xyz
+abc123

tests/requests/invalid/prefix_04.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidHeader
+
+cfg = Config()
+request = InvalidHeader

tests/requests/invalid/prefix_05.http

@@ -0,0 +1,4 @@
+GET: /stuff/here?foo=bar HTTP/1.1\r\n
+Content-Length: 3\r\n
+\r\n
+xyz

tests/requests/invalid/prefix_05.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidRequestMethod
+
+cfg = Config()
+request = InvalidRequestMethod

tests/requests/invalid/prefix_06.http

@@ -0,0 +1,4 @@
+GET /the/future HTTP/1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111.1\r\n
+Content-Length: 7\r\n
+\r\n
+Old Man

tests/requests/invalid/prefix_06.py

@@ -0,0 +1,5 @@
+from gunicorn.config import Config
+from gunicorn.http.errors import InvalidHTTPVersion
+
+cfg = Config()
+request = InvalidHTTPVersion

tests/requests/invalid/version_01.http

@@ -0,0 +1,2 @@
+GET /foo HTTP/0.99\r\n
+\r\n

tests/requests/invalid/version_01.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidHTTPVersion
+request = InvalidHTTPVersion

tests/requests/invalid/version_02.http

@@ -0,0 +1,2 @@
+GET /foo HTTP/2.0\r\n
+\r\n

tests/requests/invalid/version_02.py

@@ -0,0 +1,2 @@
+from gunicorn.http.errors import InvalidHTTPVersion
+request = InvalidHTTPVersion

tests/requests/valid/016.py

@@ -1,35 +1,35 @@
-certificate = """-----BEGIN CERTIFICATE-----\r\n
+certificate = """-----BEGIN CERTIFICATE-----
-    MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx\r\n
+ MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx
-    ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT\r\n
+ ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT
-    AkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu\r\n
+ AkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu
-    dWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV\r\n
+ dWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV
-    SzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV\r\n
+ SzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV
-    BAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB\r\n
+ BAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB
-    BQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF\r\n
+ BQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF
-    W51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR\r\n
+ W51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR
-    gW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL\r\n
+ gW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL
-    0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP\r\n
+ 0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP
-    u2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR\r\n
+ u2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR
-    wgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG\r\n
+ wgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG
-    1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs\r\n
+ 1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs
-    BglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD\r\n
+ BglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD
-    VR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj\r\n
+ VR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj
-    loCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj\r\n
+ loCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj
-    aWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG\r\n
+ aWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG
-    9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE\r\n
+ 9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE
-    IjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO\r\n
+ IjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO
-    BgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1\r\n
+ BgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1
-    cHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4Qg\r\n
+ cHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4Qg
-    EDBDAWLmh0dHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC\r\n
+ EDBDAWLmh0dHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC
-    5jcmwwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv\r\n
+ 5jcmwwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv
-    Y3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3\r\n
+ Y3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3
-    XCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8\r\n
+ XCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8
-    UO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk\r\n
+ UO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk
-    hTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK\r\n
+ hTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK
-    wTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu\r\n
+ wTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu
-    Yhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3\r\n
+ Yhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3
-    RA==\r\n
+ RA==
-    -----END CERTIFICATE-----""".replace("\n\n", "\n")
+ -----END CERTIFICATE-----""".replace("\n", "")
 
 request = {
     "method": "GET",

tests/requests/valid/025.http

@@ -1,10 +1,9 @@
 POST /chunked_cont_h_at_first HTTP/1.1\r\n
-Content-Length: -1\r\n
 Transfer-Encoding: chunked\r\n
 \r\n
 5; some; parameters=stuff\r\n
 hello\r\n
-6; blahblah; blah\r\n
+6 \t;\tblahblah; blah\r\n
  world\r\n
 0\r\n
 \r\n
@@ -16,4 +15,10 @@ Content-Length: -1\r\n
 hello\r\n
 6; blahblah; blah\r\n
  world\r\n
-0\r\n
+0\r\n
+\r\n
+PUT /ignored_after_dangerous_framing HTTP/1.1\r\n
+Content-Length: 3\r\n
+\r\n
+foo\r\n
+\r\n

tests/requests/valid/025.py

@@ -1,9 +1,13 @@
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("tolerate_dangerous_framing", True)
+
 req1 = {
     "method": "POST",
     "uri": uri("/chunked_cont_h_at_first"),
     "version": (1, 1),
     "headers": [
-        ("CONTENT-LENGTH", "-1"),
         ("TRANSFER-ENCODING", "chunked")
     ],
     "body": b"hello world"

tests/requests/valid/025compat.http

@@ -0,0 +1,18 @@
+POST /chunked_cont_h_at_first HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+\r\n
+5; some; parameters=stuff\r\n
+hello\r\n
+6; blahblah; blah\r\n
+ world\r\n
+0\r\n
+\r\n
+PUT /chunked_cont_h_at_last HTTP/1.1\r\n
+Transfer-Encoding: chunked\r\n
+Content-Length: -1\r\n
+\r\n
+5; some; parameters=stuff\r\n
+hello\r\n
+6; blahblah; blah\r\n
+ world\r\n
+0\r\n

tests/requests/valid/025compat.py

@@ -0,0 +1,27 @@
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("tolerate_dangerous_framing", True)
+
+req1 = {
+    "method": "POST",
+    "uri": uri("/chunked_cont_h_at_first"),
+    "version": (1, 1),
+    "headers": [
+        ("TRANSFER-ENCODING", "chunked")
+    ],
+    "body": b"hello world"
+}
+
+req2 = {
+    "method": "PUT",
+    "uri": uri("/chunked_cont_h_at_last"),
+    "version": (1, 1),
+    "headers": [
+        ("TRANSFER-ENCODING", "chunked"),
+        ("CONTENT-LENGTH", "-1"),
+    ],
+    "body": b"hello world"
+}
+
+request = [req1, req2]

tests/requests/valid/029.http

@@ -1,6 +1,6 @@
 GET /stuff/here?foo=bar HTTP/1.1\r\n
-Transfer-Encoding: chunked\r\n
 Transfer-Encoding: identity\r\n
+Transfer-Encoding: chunked\r\n
 \r\n
 5\r\n
 hello\r\n

tests/requests/valid/029.py

@@ -7,8 +7,8 @@ request = {
     "uri": uri("/stuff/here?foo=bar"),
     "version": (1, 1),
     "headers": [
+        ('TRANSFER-ENCODING', 'identity'),
         ('TRANSFER-ENCODING', 'chunked'),
-        ('TRANSFER-ENCODING', 'identity')
     ],
     "body": b"hello"
 }

tests/requests/valid/031.http

@@ -0,0 +1,2 @@
+-BLARGH /foo HTTP/1.1\r\n
+\r\n

tests/requests/valid/031.py

@@ -0,0 +1,7 @@
+request = {
+    "method": "-BLARGH",
+    "uri": uri("/foo"),
+    "version": (1, 1),
+    "headers": [],
+    "body": b""
+}

tests/requests/valid/031compat.http

@@ -0,0 +1,2 @@
+-blargh /foo HTTP/1.1\r\n
+\r\n

tests/requests/valid/031compat.py

@@ -0,0 +1,13 @@
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("permit_unconventional_http_method", True)
+cfg.set("casefold_http_method", True)
+
+request = {
+    "method": "-BLARGH",
+    "uri": uri("/foo"),
+    "version": (1, 1),
+    "headers": [],
+    "body": b""
+}

tests/requests/valid/031compat2.http

@@ -0,0 +1,2 @@
+-blargh /foo HTTP/1.1\r\n
+\r\n

tests/requests/valid/031compat2.py

@@ -0,0 +1,12 @@
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("permit_unconventional_http_method", True)
+
+request = {
+    "method": "-blargh",
+    "uri": uri("/foo"),
+    "version": (1, 1),
+    "headers": [],
+    "body": b""
+}

tests/requests/valid/040.http

@@ -0,0 +1,6 @@
+GET /keep/same/as?invalid/040 HTTP/1.0\r\n
+Transfer_Encoding: tricked\r\n
+Content-Length: 7\r\n
+Content_Length: -1E23\r\n
+\r\n
+tricked\r\n

tests/requests/valid/040.py

@@ -0,0 +1,9 @@
+request = {
+    "method": "GET",
+    "uri": uri("/keep/same/as?invalid/040"),
+    "version": (1, 0),
+    "headers": [
+        ("CONTENT-LENGTH", "7")
+    ],
+    "body": b'tricked'
+}

tests/requests/valid/040_compat.http

@@ -0,0 +1,6 @@
+GET /keep/same/as?invalid/040 HTTP/1.0\r\n
+Transfer_Encoding: tricked\r\n
+Content-Length: 7\r\n
+Content_Length: -1E23\r\n
+\r\n
+tricked\r\n

tests/requests/valid/040_compat.py

@@ -0,0 +1,16 @@
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set("header_map", "dangerous")
+
+request = {
+    "method": "GET",
+    "uri": uri("/keep/same/as?invalid/040"),
+    "version": (1, 0),
+    "headers": [
+        ("TRANSFER_ENCODING", "tricked"),
+        ("CONTENT-LENGTH", "7"),
+        ("CONTENT_LENGTH", "-1E23"),
+    ],
+    "body": b'tricked'
+}

tests/test_http.py

@@ -10,6 +10,17 @@ from gunicorn.http.body import Body, LengthReader, EOFReader
 from gunicorn.http.wsgi import Response
 from gunicorn.http.unreader import Unreader, IterUnreader, SocketUnreader
 from gunicorn.http.errors import InvalidHeader, InvalidHeaderName
+from gunicorn.http.message import TOKEN_RE
+
+
+def test_method_pattern():
+    assert TOKEN_RE.fullmatch("GET")
+    assert TOKEN_RE.fullmatch("MKCALENDAR")
+    assert not TOKEN_RE.fullmatch("GET:")
+    assert not TOKEN_RE.fullmatch("GET;")
+    RFC9110_5_6_2_TOKEN_DELIM = r'"(),/:;<=>?@[\]{}'
+    for bad_char in RFC9110_5_6_2_TOKEN_DELIM:
+        assert not TOKEN_RE.match(bad_char)
 
 
 def assert_readline(payload, size, expected):

tests/treq.py

@@ -51,7 +51,9 @@ class request(object):
         with open(self.fname, 'rb') as handle:
             self.data = handle.read()
         self.data = self.data.replace(b"\n", b"").replace(b"\\r\\n", b"\r\n")
-        self.data = self.data.replace(b"\\0", b"\000")
+        self.data = self.data.replace(b"\\0", b"\000").replace(b"\\n", b"\n").replace(b"\\t", b"\t")
+        if b"\\" in self.data:
+            raise AssertionError("Unexpected backslash in test data - only handling HTAB, NUL and CRLF")
 
     # Functions for sending data to the parser.
     # These functions mock out reading from a
@@ -246,8 +248,10 @@ class request(object):
     def check(self, cfg, sender, sizer, matcher):
         cases = self.expect[:]
         p = RequestParser(cfg, sender(), None)
-        for req in p:
+        parsed_request_idx = -1
+        for parsed_request_idx, req in enumerate(p):
             self.same(req, sizer, matcher, cases.pop(0))
+        assert len(self.expect) == parsed_request_idx + 1
         assert not cases
 
     def same(self, req, sizer, matcher, exp):
@@ -262,7 +266,8 @@ class request(object):
         assert req.trailers == exp.get("trailers", [])
 
 
-class badrequest(object):
+class badrequest:
+    # FIXME: no good reason why this cannot match what the more extensive mechanism above
     def __init__(self, fname):
         self.fname = fname
         self.name = os.path.basename(fname)
@@ -270,7 +275,9 @@ class badrequest(object):
         with open(self.fname) as handle:
             self.data = handle.read()
         self.data = self.data.replace("\n", "").replace("\\r\\n", "\r\n")
-        self.data = self.data.replace("\\0", "\000")
+        self.data = self.data.replace("\\0", "\000").replace("\\n", "\n").replace("\\t", "\t")
+        if "\\" in self.data:
+            raise AssertionError("Unexpected backslash in test data - only handling HTAB, NUL and CRLF")
         self.data = self.data.encode('latin1')
 
     def send(self):
@@ -283,4 +290,6 @@ class badrequest(object):
 
     def check(self, cfg):
         p = RequestParser(cfg, self.send(), None)
-        next(p)
+        # must fully consume iterator, otherwise EOF errors could go unnoticed
+        for _ in p:
+            pass