Fix Content-Lenght calculation of unicode string

httpx/_multipart.py

@@ -3,7 +3,7 @@
 import typing
 from pathlib import Path
 
-from ._types import FileContent, FileTypes, RequestFiles
+from ._types import FileTypes, RequestFiles
 from ._utils import (
  format_form_param,
  guess_content_type,
@@ -40,11 +40,7 @@ def render_headers(self) -> bytes:
 
  def render_data(self) -> bytes:
  if not hasattr(self, "_data"):
- self._data = (
- self.value
- if isinstance(self.value, bytes)
- else self.value.encode("utf-8")
- )
+ self._data = to_bytes(self.value)
 
  return self._data
 
@@ -66,8 +62,6 @@ class FileField:
  def __init__(self, name: str, value: FileTypes) -> None:
  self.name = name
 
- fileobj: FileContent
-
  if isinstance(value, tuple):
  try:
  filename, fileobj, content_type = value # type: ignore
@@ -79,6 +73,12 @@ def __init__(self, name: str, value: FileTypes) -> None:
  fileobj = value
  content_type = guess_content_type(filename)
 
+ if isinstance(fileobj, str):
+ # Ensure we only deal with bytes to prevent any content-length
+ # mismatch due to str -> bytes encoding.
+ # See: https://github.com/encode/httpx/issues/1482
+ fileobj = to_bytes(fileobj)
+
  self.filename = filename
  self.file = fileobj
  self.content_type = content_type
@@ -87,7 +87,7 @@ def __init__(self, name: str, value: FileTypes) -> None:
  def get_length(self) -> int:
  headers = self.render_headers()
 
- if isinstance(self.file, (str, bytes)):
+ if isinstance(self.file, bytes):
  return len(headers) + len(self.file)
 
  # Let's do our best not to read `file` into memory.
@@ -119,8 +119,8 @@ def render_headers(self) -> bytes:
  return self._headers
 
  def render_data(self) -> typing.Iterator[bytes]:
- if isinstance(self.file, (str, bytes)):
- yield to_bytes(self.file)
+ if isinstance(self.file, bytes):
+ yield self.file
  return
 
  if hasattr(self, "_data"):

httpx/_types.py

@@ -80,13 +80,15 @@
 
 RequestData = dict
 
-FileContent = Union[IO[str], IO[bytes], str, bytes]
+FileContent = Union[IO[str], IO[bytes], bytes]
 FileTypes = Union[
- # file (or text)
+ # text
+ str,
+ # file
  FileContent,
  # (filename, file (or text))
- Tuple[Optional[str], FileContent],
+ Tuple[Optional[str], Union[str, FileContent]],
  # (filename, file (or text), content_type)
- Tuple[Optional[str], FileContent, Optional[str]],
+ Tuple[Optional[str], Union[str, FileContent], Optional[str]],
 ]
 RequestFiles = Union[Mapping[str, FileTypes], Sequence[Tuple[str, FileTypes]]]

tests/test_multipart.py

@@ -8,6 +8,7 @@
 
 import httpx
 from httpx._content import encode_request
+from httpx._types import RequestFiles
 from httpx._utils import format_form_param
 
 
@@ -105,7 +106,10 @@ def test_multipart_encode(tmp_path: typing.Any) -> None:
  "c": ["11", "22", "33"],
  "d": "",
  }
- files = {"file": ("name.txt", open(path, "rb"))}
+ files: RequestFiles = {
+ "file": ("name.txt", open(path, "rb")),
+ "file2": ("file2.txt", "<únicode string>"),
+ }
 
  with mock.patch("os.urandom", return_value=os.urandom(16)):
  boundary = os.urandom(16).hex()
@@ -123,8 +127,11 @@ def test_multipart_encode(tmp_path: typing.Any) -> None:
  '--{0}\r\nContent-Disposition: form-data; name="file";'
  ' filename="name.txt"\r\n'
  "Content-Type: text/plain\r\n\r\n<file content>\r\n"
+ '--{0}\r\nContent-Disposition: form-data; name="file2";'
+ ' filename="file2.txt"\r\n'
+ "Content-Type: text/plain\r\n\r\n<únicode string>\r\n"
  "--{0}--\r\n"
- "".format(boundary).encode("ascii")
+ "".format(boundary).encode("utf-8")
  )
  assert headers == {
  "Content-Type": f"multipart/form-data; boundary={boundary}",