Preserve header casing. Take two. 🎬

docs/source/api.rst

@@ -128,6 +128,21 @@ have a newline inside a header value, and ``Content-Length: hello`` is
 an error because `Content-Length` should always be an integer. We may
 add additional checks in the future.
 
+While we make sure to expose header names as lowercased bytes, we also
+preserve the original header casing that is used. Compliant HTTP
+agents should always treat headers in a case insensitive manner, but
+this may not always be the case. When sending bytes over the wire we
+send headers preserving whatever original header casing was used.
+
+It is possible to access the headers in their raw original casing,
+which may be useful for some user output or debugging purposes.
+
+.. ipython:: python
+
+ original_headers = [("Host", "example.com")]
+ req = h11.Request(method="GET", target="/", headers=original_headers)
+ req.headers.raw_items()
+
 .. _http_version-format:
 
 It's not just headers we normalize to being byte-strings: the same

h11/_connection.py

@@ -534,7 +534,7 @@ def send_failed(self):
  def _clean_up_response_headers_for_sending(self, response):
  assert type(response) is Response
 
- headers = list(response.headers)
+ headers = response.headers
  need_close = False
 
  # HEAD requests need some special handling: they always act like they
@@ -560,27 +560,27 @@ def _clean_up_response_headers_for_sending(self, response):
  # but the HTTP spec says that if our peer does this then we have
  # to fix it instead of erroring out, so we'll accord the user the
  # same respect).
- set_comma_header(headers, b"content-length", [])
+headers = set_comma_header(headers, b"content-length", [])
  if self.their_http_version is None or self.their_http_version < b"1.1":
  # Either we never got a valid request and are sending back an
  # error (their_http_version is None), so we assume the worst;
  # or else we did get a valid HTTP/1.0 request, so we know that
  # they don't understand chunked encoding.
- set_comma_header(headers, b"transfer-encoding", [])
+headers = set_comma_header(headers, b"transfer-encoding", [])
  # This is actually redundant ATM, since currently we
  # unconditionally disable keep-alive when talking to HTTP/1.0
  # peers. But let's be defensive just in case we add
  # Connection: keep-alive support later:
  if self._request_method != b"HEAD":
  need_close = True
  else:
- set_comma_header(headers, b"transfer-encoding", ["chunked"])
+headers = set_comma_header(headers, b"transfer-encoding", ["chunked"])
 
  if not self._cstate.keep_alive or need_close:
  # Make sure Connection: close is set
  connection = set(get_comma_header(headers, b"connection"))
  connection.discard(b"keep-alive")
  connection.add(b"close")
- set_comma_header(headers, b"connection", sorted(connection))
+headers = set_comma_header(headers, b"connection", sorted(connection))
 
  response.headers = headers

h11/_headers.py

@@ -62,6 +62,59 @@
 _field_value_re = re.compile(field_value.encode("ascii"))
 
 
+class Headers:
+ """
+ A list-like interface that allows iterating over headers as byte-pairs
+ of (lowercased-name, value).
+
+ Internally we actually store the representation as three-tuples,
+ including both the raw original casing, in order to preserve casing
+ over-the-wire, and the lowercased name, for case-insensitive comparisions.
+
+ r = Request(
+ method="GET",
+ target="/",
+ headers=[("Host", "example.org"), ("Connection", "keep-alive")],
+ http_version="1.1",
+ )
+ assert r.headers == [
+ (b"host", b"example.org"),
+ (b"connection", b"keep-alive")
+ ]
+ assert r.headers.raw_items() == [
+ (b"Host", b"example.org"),
+ (b"Connection", b"keep-alive")
+ ]
+ """
+ __slots__ = '_full_items'
+
+ def __init__(self, full_items):
+ self._full_items = full_items
+
+ def __iter__(self):
+ for _, name, value in self._full_items:
+ yield name, value
+
+ def __bool__(self):
+ return bool(self._full_items)
+
+ def __eq__(self, other):
+ return list(self) == list(other)
+
+ def __len__(self):
+ return len(self._full_items)
+
+ def __repr__(self):
+ return "<Headers(%s)>" % repr(list(self))
+
+ def __getitem__(self, idx):
+ _, name, value = self._full_items[idx]
+ return (name, value)
+
+ def raw_items(self):
+ return [(raw_name, value) for raw_name, _, value in self._full_items]
+
+
 def normalize_and_validate(headers, _parsed=False):
  new_headers = []
  saw_content_length = False
@@ -75,6 +128,7 @@ def normalize_and_validate(headers, _parsed=False):
  value = bytesify(value)
  validate(_field_name_re, name, "Illegal header name {!r}", name)
  validate(_field_value_re, value, "Illegal header value {!r}", value)
+ raw_name = name
  name = name.lower()
  if name == b"content-length":
  if saw_content_length:
@@ -99,8 +153,8 @@ def normalize_and_validate(headers, _parsed=False):
  error_status_hint=501,
  )
  saw_transfer_encoding = True
- new_headers.append((name, value))
- return new_headers
+ new_headers.append((raw_name, name, value))
+ return Headers(new_headers)
 
 
 def get_comma_header(headers, name):
@@ -140,7 +194,7 @@ def get_comma_header(headers, name):
  # "100-continue". Splitting on commas is harmless. Case insensitive.
  #
  out = []
- for found_name, found_raw_value in headers:
+ for _, found_name, found_raw_value in headers._full_items:
  if found_name == name:
  found_raw_value = found_raw_value.lower()
  for found_split_value in found_raw_value.split(b","):
@@ -152,13 +206,21 @@ def get_comma_header(headers, name):
 
 def set_comma_header(headers, name, new_values):
  # The header name `name` is expected to be lower-case bytes.
+ #
+ # Note that when we store the header we use title casing for the header
+ # names, in order to match the conventional HTTP header style.
+ #
+ # Simply calling `.title()` is a blunt approach, but it's correct
+ # here given the cases where we're using `set_comma_header`...
+ #
+ # Connection, Content-Length, Transfer-Encoding.
  new_headers = []
- for found_name, found_raw_value in headers:
+ for found_raw_name, found_name, found_raw_value in headers._full_items:
  if found_name != name:
- new_headers.append((found_name, found_raw_value))
+ new_headers.append((found_raw_name, found_raw_value))
  for new_value in new_values:
- new_headers.append((name, new_value))
-headers[:] = normalize_and_validate(new_headers)
+ new_headers.append((name.title(), new_value))
+return normalize_and_validate(new_headers)
 
 
 def has_expect_100_continue(request):

h11/_readers.py

@@ -58,7 +58,9 @@ def _decode_header_lines(lines):
  # Python 3, validate() takes either and returns matches as bytes. But
  # on Python 2, validate can return matches as bytearrays, so we have
  # to explicitly cast back.
- matches = validate(header_field_re, bytes(line), "illegal header line: {!r}", bytes(line))
+ matches = validate(
+ header_field_re, bytes(line), "illegal header line: {!r}", bytes(line)
+ )
  yield (matches["field_name"], matches["field_value"])
 
 
@@ -71,7 +73,9 @@ def maybe_read_from_IDLE_client(buf):
  return None
  if not lines:
  raise LocalProtocolError("no request line received")
- matches = validate(request_line_re, lines[0], "illegal request line: {!r}", lines[0])
+ matches = validate(
+ request_line_re, lines[0], "illegal request line: {!r}", lines[0]
+ )
  return Request(
  headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches
  )
@@ -152,7 +156,12 @@ def __call__(self, buf):
  chunk_header = buf.maybe_extract_until_next(b"\r\n")
  if chunk_header is None:
  return None
- matches = validate(chunk_header_re, chunk_header, "illegal chunk header: {!r}", chunk_header)
+ matches = validate(
+ chunk_header_re,
+ chunk_header,
+ "illegal chunk header: {!r}",
+ chunk_header,
+ )
  # XX FIXME: we discard chunk extensions. Does anyone care?
  # We convert to bytes because Python 2's `int()` function doesn't
  # work properly on bytearray objects.

h11/_writers.py

@@ -38,12 +38,13 @@ def write_headers(headers, write):
  # "Since the Host field-value is critical information for handling a
  # request, a user agent SHOULD generate Host as the first header field
  # following the request-line." - RFC 7230
- for name, value in headers:
+ raw_items = headers._full_items
+ for raw_name, name, value in raw_items:
  if name == b"host":
- write(bytesmod(b"%s: %s\r\n", (name, value)))
- for name, value in headers:
+ write(bytesmod(b"%s: %s\r\n", (raw_name, value)))
+ for raw_name, name, value in raw_items:
  if name != b"host":
- write(bytesmod(b"%s: %s\r\n", (name, value)))
+ write(bytesmod(b"%s: %s\r\n", (raw_name, value)))
  write(b"\r\n")
 
 

h11/tests/test_connection.py

@@ -96,7 +96,7 @@ def test_Connection_basics_and_content_length():
  ),
  )
  assert data == (
- b"GET / HTTP/1.1\r\n" b"host: example.com\r\n" b"content-length: 10\r\n\r\n"
+ b"GET / HTTP/1.1\r\n" b"Host: example.com\r\n" b"Content-Length: 10\r\n\r\n"
  )
 
  for conn in p.conns:
@@ -113,7 +113,7 @@ def test_Connection_basics_and_content_length():
  assert data == b"HTTP/1.1 100 \r\n\r\n"
 
  data = p.send(SERVER, Response(status_code=200, headers=[("Content-Length", "11")]))
- assert data == b"HTTP/1.1 200 \r\ncontent-length: 11\r\n\r\n"
+ assert data == b"HTTP/1.1 200 \r\nContent-Length: 11\r\n\r\n"
 
  for conn in p.conns:
  assert conn.states == {CLIENT: SEND_BODY, SERVER: SEND_BODY}
@@ -243,7 +243,7 @@ def test_server_talking_to_http10_client():
  # We automatically Connection: close back at them
  assert (
  c.send(Response(status_code=200, headers=[]))
- == b"HTTP/1.1 200 \r\nconnection: close\r\n\r\n"
+ == b"HTTP/1.1 200 \r\nConnection: close\r\n\r\n"
  )
 
  assert c.send(Data(data=b"12345")) == b"12345"
@@ -303,7 +303,7 @@ def test_automatic_transfer_encoding_in_response():
  receive_and_get(c, b"GET / HTTP/1.0\r\n\r\n")
  assert (
  c.send(Response(status_code=200, headers=user_headers))
- == b"HTTP/1.1 200 \r\nconnection: close\r\n\r\n"
+ == b"HTTP/1.1 200 \r\nConnection: close\r\n\r\n"
  )
  assert c.send(Data(data=b"12345")) == b"12345"
 
@@ -876,7 +876,7 @@ def test_errors():
  if role is SERVER:
  assert (
  c.send(Response(status_code=400, headers=[]))
- == b"HTTP/1.1 400 \r\nconnection: close\r\n\r\n"
+ == b"HTTP/1.1 400 \r\nConnection: close\r\n\r\n"
  )
 
  # After an error sending, you can no longer send
@@ -988,14 +988,14 @@ def setup(method, http_version):
  c = setup(method, b"1.1")
  assert (
  c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\n"
- b"transfer-encoding: chunked\r\n\r\n"
+ b"Transfer-Encoding: chunked\r\n\r\n"
  )
 
  # No Content-Length, HTTP/1.0 peer, frame with connection: close
  c = setup(method, b"1.0")
  assert (
  c.send(Response(status_code=200, headers=[])) == b"HTTP/1.1 200 \r\n"
- b"connection: close\r\n\r\n"
+ b"Connection: close\r\n\r\n"
  )
 
  # Content-Length + Transfer-Encoding, TE wins
@@ -1011,7 +1011,7 @@ def setup(method, http_version):
  )
  )
  == b"HTTP/1.1 200 \r\n"
- b"transfer-encoding: chunked\r\n\r\n"
+ b"Transfer-Encoding: chunked\r\n\r\n"
  )
 
 

h11/tests/test_events.py

@@ -163,3 +163,19 @@ def test_intenum_status_code():
  assert r.status_code == HTTPStatus.OK
  assert type(r.status_code) is not type(HTTPStatus.OK)
  assert type(r.status_code) is int
+
+
+def test_header_casing():
+ r = Request(
+ method="GET",
+ target="/",
+ headers=[("Host", "example.org"), ("Connection", "keep-alive")],
+ http_version="1.1",
+ )
+ assert len(r.headers) == 2
+ assert r.headers[0] == (b"host", b"example.org")
+ assert r.headers == [(b"host", b"example.org"), (b"connection", b"keep-alive")]
+ assert r.headers.raw_items() == [
+ (b"Host", b"example.org"),
+ (b"Connection", b"keep-alive"),
+ ]

h11/tests/test_headers.py

@@ -83,7 +83,7 @@ def test_get_set_comma_header():
 
  assert get_comma_header(headers, b"connection") == [b"close", b"foo", b"bar"]
 
- set_comma_header(headers, b"newthing", ["a", "b"])
+headers = set_comma_header(headers, b"newthing", ["a", "b"])
 
  with pytest.raises(LocalProtocolError):
  set_comma_header(headers, b"newthing", [" a", "b"])
@@ -96,7 +96,7 @@ def test_get_set_comma_header():
  (b"newthing", b"b"),
  ]
 
- set_comma_header(headers, b"whatever", ["different thing"])
+headers = set_comma_header(headers, b"whatever", ["different thing"])
 
  assert headers == [
  (b"connection", b"close"),

h11/tests/test_io.py

@@ -1,7 +1,7 @@
 import pytest
 
 from .._events import *
-from .._headers import normalize_and_validate
+from .._headers import normalize_and_validate, Headers
 from .._readers import (
  _obsolete_line_fold,
  ChunkedReader,
@@ -31,12 +31,12 @@
  target="/a",
  headers=[("Host", "foo"), ("Connection", "close")],
  ),
- b"GET /a HTTP/1.1\r\nhost: foo\r\nconnection: close\r\n\r\n",
+ b"GET /a HTTP/1.1\r\nHost: foo\r\nConnection: close\r\n\r\n",
  ),
  (
  (SERVER, SEND_RESPONSE),
  Response(status_code=200, headers=[("Connection", "close")], reason=b"OK"),
- b"HTTP/1.1 200 OK\r\nconnection: close\r\n\r\n",
+ b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n",
  ),
  (
  (SERVER, SEND_RESPONSE),
@@ -48,7 +48,7 @@
  InformationalResponse(
  status_code=101, headers=[("Upgrade", "websocket")], reason=b"Upgrade"
  ),
- b"HTTP/1.1 101 Upgrade\r\nupgrade: websocket\r\n\r\n",
+ b"HTTP/1.1 101 Upgrade\r\nUpgrade: websocket\r\n\r\n",
  ),
  (
  (SERVER, SEND_RESPONSE),
@@ -121,7 +121,7 @@ def test_writers_unusual():
  normalize_and_validate([("foo", "bar"), ("baz", "quux")]),
  b"foo: bar\r\nbaz: quux\r\n\r\n",
  )
- tw(write_headers, [], b"\r\n")
+ tw(write_headers, Headers([]), b"\r\n")
 
  # We understand HTTP/1.0, but we don't speak it
  with pytest.raises(LocalProtocolError):
@@ -435,7 +435,7 @@ def test_ChunkedWriter():
 
  assert (
  dowrite(w, EndOfMessage(headers=[("Etag", "asdf"), ("a", "b")]))
- == b"0\r\netag: asdf\r\na: b\r\n\r\n"
+ == b"0\r\nEtag: asdf\r\na: b\r\n\r\n"
  )
 
 
@@ -503,5 +503,5 @@ def test_host_comes_first():
  tw(
  write_headers,
  normalize_and_validate([("foo", "bar"), ("Host", "example.com")]),
- b"host: example.com\r\nfoo: bar\r\n\r\n",
+ b"Host: example.com\r\nfoo: bar\r\n\r\n",
  )