whitespace handling in header field values

Strip whitespace also *after* header field value.
Simply refuse obsolete header folding (a default-off
option to revert is temporarily provided).
While we are at it, explicitly handle recently
introduced http error classes with intended status code.

docs/source/settings.rst

@@ -1396,6 +1396,26 @@ The variables are passed to the PasteDeploy entrypoint. Example::
 
 .. versionadded:: 19.7
 
+.. _permit-obsolete-folding:
+
+``permit_obsolete_folding``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Command line:** ``--permit-obsolete-folding``
+
+**Default:** ``False``
+
+Permit requests employing obsolete HTTP line folding mechanism
+
+The folding mechanism was deprecated by rfc7230 Section 3.2.4 and will not be
+ employed in HTTP request headers from standards-compliant HTTP clients.
+
+This option is provided to diagnose backwards-incompatible changes.
+Use with care and only if necessary. Temporary; the precise effect of this option may
+change in a future version, or it may be removed altogether.
+
+.. versionadded:: 23.0.0
+
 .. _strip-header-spaces:
 
 ``strip_header_spaces``

gunicorn/config.py

@@ -2243,6 +2243,27 @@ class PasteGlobalConf(Setting):
         """
 
 
+class PermitObsoleteFolding(Setting):
+    name = "permit_obsolete_folding"
+    section = "Server Mechanics"
+    cli = ["--permit-obsolete-folding"]
+    validator = validate_bool
+    action = "store_true"
+    default = False
+    desc = """\
+        Permit requests employing obsolete HTTP line folding mechanism
+
+        The folding mechanism was deprecated by rfc7230 Section 3.2.4 and will not be
+         employed in HTTP request headers from standards-compliant HTTP clients.
+
+        This option is provided to diagnose backwards-incompatible changes.
+        Use with care and only if necessary. Temporary; the precise effect of this option may
+        change in a future version, or it may be removed altogether.
+
+        .. versionadded:: 23.0.0
+        """
+
+
 class StripHeaderSpaces(Setting):
     name = "strip_header_spaces"
     section = "Server Mechanics"

gunicorn/http/errors.py

@@ -65,6 +65,14 @@ class InvalidHeader(ParseException):
         return "Invalid HTTP Header: %r" % self.hdr
 
 
+class ObsoleteFolding(ParseException):
+    def __init__(self, hdr):
+        self.hdr = hdr
+
+    def __str__(self):
+        return "Obsolete line folding is unacceptable: %r" % (self.hdr, )
+
+
 class InvalidHeaderName(ParseException):
     def __init__(self, hdr):
         self.hdr = hdr

gunicorn/http/message.py

@@ -12,7 +12,7 @@ from gunicorn.http.errors import (
     InvalidHeader, InvalidHeaderName, NoMoreData,
     InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion,
     LimitRequestLine, LimitRequestHeaders,
-    UnsupportedTransferCoding,
+    UnsupportedTransferCoding, ObsoleteFolding,
 )
 from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest
 from gunicorn.http.errors import InvalidSchemeHeaders
@@ -110,10 +110,13 @@ class Message(object):
             # b"\xDF".decode("latin-1").upper().encode("ascii") == b"SS"
             name = name.upper()
 
-            value = [value.lstrip(" \t")]
+            value = [value.strip(" \t")]
 
-            # Consume value continuation lines
+            # Consume value continuation lines..
             while lines and lines[0].startswith((" ", "\t")):
+                # .. which is obsolete here, and no longer done by default
+                if not self.cfg.permit_obsolete_folding:
+                    raise ObsoleteFolding(name)
                 curr = lines.pop(0)
                 header_length += len(curr) + len("\r\n")
                 if header_length > self.limit_request_field_size > 0:

gunicorn/workers/base.py

@@ -20,6 +20,8 @@ from gunicorn.http.errors import (
     InvalidProxyLine, InvalidRequestLine,
     InvalidRequestMethod, InvalidSchemeHeaders,
     LimitRequestHeaders, LimitRequestLine,
+    UnsupportedTransferCoding,
+    ConfigurationProblem, ObsoleteFolding,
 )
 from gunicorn.http.wsgi import Response, default_environ
 from gunicorn.reloader import reloader_engines
@@ -210,7 +212,8 @@ class Worker(object):
             InvalidHTTPVersion, InvalidHeader, InvalidHeaderName,
             LimitRequestLine, LimitRequestHeaders,
             InvalidProxyLine, ForbiddenProxyRequest,
-            InvalidSchemeHeaders,
+            InvalidSchemeHeaders, UnsupportedTransferCoding,
+            ConfigurationProblem, ObsoleteFolding,
             SSLError,
         )):
 
@@ -223,6 +226,14 @@ class Worker(object):
                 mesg = "Invalid Method '%s'" % str(exc)
             elif isinstance(exc, InvalidHTTPVersion):
                 mesg = "Invalid HTTP Version '%s'" % str(exc)
+            elif isinstance(exc, UnsupportedTransferCoding):
+                mesg = "%s" % str(exc)
+                status_int = 501
+            elif isinstance(exc, ConfigurationProblem):
+                mesg = "%s" % str(exc)
+                status_int = 500
+            elif isinstance(exc, ObsoleteFolding):
+                mesg = "%s" % str(exc)
             elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)):
                 mesg = "%s" % str(exc)
                 if not req and hasattr(exc, "req"):

tests/requests/invalid/013.py

@@ -4,3 +4,7 @@ from gunicorn.http.errors import LimitRequestHeaders
 request = LimitRequestHeaders
 cfg = Config()
 cfg.set('limit_request_field_size', 14)
+
+# once this option is removed, this test should not be dropped;
+#  rather, add something involving unnessessary padding
+cfg.set('permit_obsolete_folding', True)

tests/requests/invalid/obs_fold_01.http

@@ -0,0 +1,5 @@
+GET / HTTP/1.1\r\n
+Long: one\r\n
+ two\r\n
+Host: localhost\r\n
+\r\n

tests/requests/invalid/obs_fold_01.py

@@ -0,0 +1,3 @@
+from gunicorn.http.errors import ObsoleteFolding
+
+request = ObsoleteFolding

tests/requests/valid/compat_obs_fold.http

@@ -0,0 +1,5 @@
+GET / HTTP/1.1\r\n
+Long: one\r\n
+ two\r\n
+Host: localhost\r\n
+\r\n

tests/requests/valid/compat_obs_fold.py

@@ -0,0 +1,16 @@
+from gunicorn.http.errors import ObsoleteFolding
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set('permit_obsolete_folding', True)
+
+request = {
+    "method": "GET",
+    "uri": uri("/"),
+    "version": (1, 1),
+    "headers": [
+        ("LONG", "one two"),
+        ("HOST", "localhost"),
+    ],
+    "body": b""
+}

tests/requests/valid/compat_obs_fold_huge.py

@@ -1,3 +1,8 @@
+from gunicorn.config import Config
+
+cfg = Config()
+cfg.set('permit_obsolete_folding', True)
+
 certificate = """-----BEGIN CERTIFICATE-----
  MIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx
  ETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT

tests/requests/valid/padding_01.http

@@ -0,0 +1,4 @@
+GET / HTTP/1.1\r\n
+Host: localhost\r\n
+Name: \t value \t \r\n
+\r\n

tests/requests/valid/padding_01.py

@@ -0,0 +1,11 @@
+
+request = {
+    "method": "GET",
+    "uri": uri("/"),
+    "version": (1, 1),
+    "headers": [
+        ("HOST", "localhost"),
+        ("NAME", "value")
+    ],
+    "body": b"",
+}