Added ability to use LF, not only CRLF delimiter for response Headers and Body

h11/_connection.py

@@ -425,7 +425,6 @@ def next_event(self):
  event = self._extract_next_receive_event()
  if event not in [NEED_DATA, PAUSED]:
  self._process_event(self.their_role, event)
- self._receive_buffer.compress()
  if event is NEED_DATA:
  if len(self._receive_buffer) > self._max_incomplete_event_size:
  # 431 is "Request header fields too large" which is pretty

h11/_readers.py

@@ -153,7 +153,7 @@ def __call__(self, buf):
  assert self._bytes_to_discard == 0
  if self._bytes_in_chunk == 0:
  # We need to refill our chunk count
- chunk_header = buf.maybe_extract_until_next(b"\r\n")
+ chunk_header = buf.maybe_extract_next_line()
  if chunk_header is None:
  return None
  matches = validate(

h11/_receivebuffer.py

@@ -1,10 +1,12 @@
+import re
 import sys
 
 __all__ = ["ReceiveBuffer"]
 
 
 # Operations we want to support:
-# - find next \r\n or \r\n\r\n, or wait until there is one
+# - find next \r\n or \r\n\r\n (\n or \n\n are also acceptable),
+# or wait until there is one
 # - read at-most-N bytes
 # Goals:
 # - on average, do this fast
@@ -38,75 +40,101 @@
 # slightly clever thing where we delay calling compress() until we've
 # processed a whole event, which could in theory be slightly more efficient
 # than the internal bytearray support.)
+
+blank_line_regex = re.compile(b"\n\r?\n", re.MULTILINE)
+
+
 class ReceiveBuffer(object):
  def __init__(self):
  self._data = bytearray()
- # These are both absolute offsets into self._data:
- self._start = 0
- self._looked_at = 0
- self._looked_for = b""
+ self._next_line_search = 0
+ self._multiple_lines_search = 0
+
+ def __iadd__(self, byteslike):
+ self._data += byteslike
+ return self
 
  def __bool__(self):
  return bool(len(self))
 
+ def __len__(self):
+ return len(self._data)
+
  # for @property unprocessed_data
  def __bytes__(self):
- return bytes(self._data[self._start :])
+ return bytes(self._data)
 
  if sys.version_info[0] < 3: # version specific: Python 2
  __str__ = __bytes__
  __nonzero__ = __bool__
 
- def __len__(self):
- return len(self._data) - self._start
+ def _extract(self, count):
+ # extracting an initial slice of the data buffer and return it
+ out = self._data[:count]
+ del self._data[:count]
 
- def compress(self):
- # Heuristic: only compress if it lets us reduce size by a factor
- # of 2
- if self._start > len(self._data) // 2:
- del self._data[: self._start]
- self._looked_at -= self._start
- self._start -= self._start
+ self._next_line_search = 0
+ self._multiple_lines_search = 0
 
- def __iadd__(self, byteslike):
- self._data += byteslike
- return self
+ return out
 
  def maybe_extract_at_most(self, count):
- out = self._data[self._start : self._start + count]
+ """
+ Extract a fixed number of bytes from the buffer.
+ """
+ out = self._data[:count]
  if not out:
  return None
- self._start += len(out)
- return out
 
- def maybe_extract_until_next(self, needle):
- # Returns extracted bytes on success (advancing offset), or None on
- # failure
- if self._looked_for == needle:
- search_start = max(self._start, self._looked_at - len(needle) + 1)
- else:
- search_start = self._start
- offset = self._data.find(needle, search_start)
- if offset == -1:
- self._looked_at = len(self._data)
- self._looked_for = needle
+ return self._extract(count)
+
+ def maybe_extract_next_line(self):
+ """
+ Extract the first line, if it is completed in the buffer.
+ """
+ # Only search in buffer space that we've not already looked at.
+ search_start_index = max(0, self._next_line_search - 1)
+ partial_idx = self._data.find(b"\r\n", search_start_index)
+
+ if partial_idx == -1:
+ self._next_line_search = len(self._data)
  return None
- new_start = offset + len(needle)
- out = self._data[self._start : new_start]
- self._start = new_start
- return out
 
- # HTTP/1.1 has a number of constructs where you keep reading lines until
- # you see a blank one. This does that, and then returns the lines.
+ # + 2 is to compensate len(b"\r\n")
+ idx = partial_idx + 2
+
+ return self._extract(idx)
+
  def maybe_extract_lines(self):
- if self._data[self._start : self._start + 2] == b"\r\n":
- self._start += 2
+ """
+ Extract everything up to the first blank line, and return a list of lines.
+ """
+ # Handle the case where we have an immediate empty line.
+ if self._data[:1] == b"\n":
+ self._extract(1)
+ return []
+
+ if self._data[:2] == b"\r\n":
+ self._extract(2)
  return []
- else:
- data = self.maybe_extract_until_next(b"\r\n\r\n")
- if data is None:
- return None
- lines = data.split(b"\r\n")
- assert lines[-2] == lines[-1] == b""
- del lines[-2:]
- return lines
+
+ # Only search in buffer space that we've not already looked at.
+ match = blank_line_regex.search(self._data, self._multiple_lines_search)
+ if match is None:
+ self._multiple_lines_search = max(0, len(self._data) - 2)
+ return None
+
+ # Truncate the buffer and return it.
+ idx = match.span(0)[-1]
+ out = self._extract(idx)
+ lines = out.split(b"\n")
+
+ for line in lines:
+ if line.endswith(b"\r"):
+ del line[-1]
+
+ assert lines[-2] == lines[-1] == b""
+
+ del lines[-2:]
+
+ return lines

h11/tests/test_io.py

@@ -215,6 +215,43 @@ def test_readers_unusual():
  ),
  )
 
+ # Tolerate headers line endings (\r\n and \n)
+ # \n\r\b between headers and body
+ tr(
+ READERS[SERVER, SEND_RESPONSE],
+ b"HTTP/1.1 200 OK\r\nSomeHeader: val\n\r\n",
+ Response(
+ status_code=200,
+ headers=[("SomeHeader", "val")],
+ http_version="1.1",
+ reason="OK",
+ ),
+ )
+
+ # delimited only with \n
+ tr(
+ READERS[SERVER, SEND_RESPONSE],
+ b"HTTP/1.1 200 OK\nSomeHeader1: val1\nSomeHeader2: val2\n\n",
+ Response(
+ status_code=200,
+ headers=[("SomeHeader1", "val1"), ("SomeHeader2", "val2")],
+ http_version="1.1",
+ reason="OK",
+ ),
+ )
+
+ # mixed \r\n and \n
+ tr(
+ READERS[SERVER, SEND_RESPONSE],
+ b"HTTP/1.1 200 OK\r\nSomeHeader1: val1\nSomeHeader2: val2\n\r\n",
+ Response(
+ status_code=200,
+ headers=[("SomeHeader1", "val1"), ("SomeHeader2", "val2")],
+ http_version="1.1",
+ reason="OK",
+ ),
+ )
+
  # obsolete line folding
  tr(
  READERS[CLIENT, IDLE],

h11/tests/test_receivebuffer.py

@@ -1,3 +1,7 @@
+import re
+
+import pytest
+
 from .._receivebuffer import ReceiveBuffer
 
 
@@ -12,15 +16,13 @@ def test_receivebuffer():
  assert len(b) == 3
  assert bytes(b) == b"123"
 
- b.compress()
  assert bytes(b) == b"123"
 
  assert b.maybe_extract_at_most(2) == b"12"
  assert b
  assert len(b) == 1
  assert bytes(b) == b"3"
 
- b.compress()
  assert bytes(b) == b"3"
 
  assert b.maybe_extract_at_most(10) == b"3"
@@ -33,32 +35,35 @@ def test_receivebuffer():
  # maybe_extract_until_next
  ################################################################
 
- b += b"12345a6789aa"
+ b += b"123\n456\r\n789\r\n"
 
- assert b.maybe_extract_until_next(b"a") == b"12345a"
- assert bytes(b) == b"6789aa"
+ assert b.maybe_extract_next_line() == b"123\n456\r\n"
+ assert bytes(b) == b"789\r\n"
 
- assert b.maybe_extract_until_next(b"aaa") is None
- assert bytes(b) == b"6789aa"
+ assert b.maybe_extract_next_line() == b"789\r\n"
+ assert bytes(b) == b""
 
- b += b"a12"
- assert b.maybe_extract_until_next(b"aaa") == b"6789aaa"
- assert bytes(b) == b"12"
+ b += b"12\r"
+ assert b.maybe_extract_next_line() is None
+ assert bytes(b) == b"12\r"
 
- # check repeated searches for the same needle, triggering the
- # pickup-where-we-left-off logic
- b += b"345"
- assert b.maybe_extract_until_next(b"aaa") is None
+ b += b"345\n\r"
+ assert b.maybe_extract_next_line() is None
+ assert bytes(b) == b"12\r345\n\r"
 
- b += b"6789aaa123"
- assert b.maybe_extract_until_next(b"aaa") == b"123456789aaa"
- assert bytes(b) == b"123"
+ # here we stopped at the middle of b"\r\n" delimiter
+
+ b += b"\n6789aaa123\r\n"
+ assert b.maybe_extract_next_line() == b"12\r345\n\r\n"
+ assert b.maybe_extract_next_line() == b"6789aaa123\r\n"
+ assert b.maybe_extract_next_line() is None
+ assert bytes(b) == b""
 
  ################################################################
  # maybe_extract_lines
  ################################################################
 
- b += b"\r\na: b\r\nfoo:bar\r\n\r\ntrailing"
+ b += b"123\r\na: b\r\nfoo:bar\r\n\r\ntrailing"
  lines = b.maybe_extract_lines()
  assert lines == [b"123", b"a: b", b"foo:bar"]
  assert bytes(b) == b"trailing"
@@ -76,3 +81,54 @@ def test_receivebuffer():
  b += b"\r\ntrailing"
  assert b.maybe_extract_lines() == []
  assert bytes(b) == b"trailing"
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ pytest.param(
+ (
+ b"HTTP/1.1 200 OK\r\n",
+ b"Content-type: text/plain\r\n",
+ b"Connection: close\r\n",
+ b"\r\n",
+ b"Some body",
+ ),
+ id="with_crlf_delimiter",
+ ),
+ pytest.param(
+ (
+ b"HTTP/1.1 200 OK\n",
+ b"Content-type: text/plain\n",
+ b"Connection: close\n",
+ b"\n",
+ b"Some body",
+ ),
+ id="with_lf_only_delimiter",
+ ),
+ pytest.param(
+ (
+ b"HTTP/1.1 200 OK\n",
+ b"Content-type: text/plain\r\n",
+ b"Connection: close\n",
+ b"\n",
+ b"Some body",
+ ),
+ id="with_mixed_crlf_and_lf",
+ ),
+ ],
+)
+def test_receivebuffer_for_invalid_delimiter(data):
+ b = ReceiveBuffer()
+
+ for line in data:
+ b += line
+
+ lines = b.maybe_extract_lines()
+
+ assert lines == [
+ b"HTTP/1.1 200 OK",
+ b"Content-type: text/plain",
+ b"Connection: close",
+ ]
+ assert bytes(b) == b"Some body"

newsfragments/7.feature.rst

@@ -0,0 +1,3 @@
+Added support for servers with broken line endings.
+
+After this change h11 accepts both ``\r\n`` and ``\n`` as a headers delimiter.