gh-119609, PEP 756: Add PyUnicode_Export() function

Doc/c-api/unicode.rst

@@ -341,6 +341,82 @@ APIs:
  .. versionadded:: 3.3
 
 
+.. c:function:: int32_t PyUnicode_Export(PyObject *unicode, int32_t requested_formats, uint32_t flags, Py_buffer *view)
+
+ Export the contents of the *unicode* string in one of the *requested_formats*.
+
+ * On success, fill *view*, and return a format (greater than ``0``).
+ * On error, set an exception, and return ``-1``.
+ *view* is left unchanged.
+
+ After a successful call to :c:func:`PyUnicode_Export`,
+ the *view* buffer must be released by :c:func:`PyBuffer_Release`.
+ The contents of the buffer are valid until they are released.
+
+ The buffer is read-only and must not be modified.
+
+ *unicode* and *view* must not be NULL.
+
+ Available formats:
+
+ .. c:namespace:: NULL
+
+ =================================== ======== ===========================
+ Constant Identifier Value Description
+ =================================== ======== ===========================
+ .. c:macro:: PyUnicode_FORMAT_UCS1 ``0x01`` UCS-1 string (``Py_UCS1*``)
+ .. c:macro:: PyUnicode_FORMAT_UCS2 ``0x02`` UCS-2 string (``Py_UCS2*``)
+ .. c:macro:: PyUnicode_FORMAT_UCS4 ``0x04`` UCS-4 string (``Py_UCS4*``)
+ .. c:macro:: PyUnicode_FORMAT_UTF8 ``0x08`` UTF-8 string (``char*``)
+ .. c:macro:: PyUnicode_FORMAT_ASCII ``0x10`` ASCII string (``Py_UCS1*``)
+ =================================== ======== ===========================
+
+ UCS-2 and UCS-4 use the native byte order.
+
+ *requested_formats* can be a single format or a bitwise combination of the
+ formats in the table above.
+ On success, the returned format will be set to a single one of the requested
+ flags.
+
+ Note that future versions of Python may introduce additional formats.
+
+ By default, if the :c:macro:`PyUnicode_EXPORT_COPY` flag is not set in
+ *flags*, no memory is copied and no conversion is done.
+
+ If the :c:macro:`PyUnicode_EXPORT_COPY` flag is set in *flags*, the function
+ can copy memory to provide the requested format and convert from a format
+ to another.
+
+ The :c:macro:`PyUnicode_EXPORT_COPY` flag is needed to export to
+ :c:macro:`PyUnicode_FORMAT_UTF8` a string containing surrogate characters.
+
+ Available flags:
+
+ .. c:namespace:: NULL
+
+ ================================== ======== ===================================
+ Flag Value Description
+ ================================== ======== ===================================
+ .. c:macro:: PyUnicode_EXPORT_COPY ``0x01`` Allow memory copies and conversions
+ ================================== ======== ===================================
+
+ .. versionadded:: 3.14
+
+
+.. c:function:: PyObject* PyUnicode_Import(const void *data, Py_ssize_t nbytes, int32_t format)
+
+ Create a Unicode string object from a buffer in a supported format.
+
+ * Return a reference to a new string object on success.
+ * Set an exception and return ``NULL`` on error.
+
+ *data* must not be NULL. *nbytes* must be positive or zero.
+
+ See :c:func:`PyUnicode_Export` for the available formats.
+
+ .. versionadded:: 3.14
+
+
 .. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
  Py_ssize_t size)
 

Doc/whatsnew/3.14.rst

@@ -554,6 +554,10 @@ New Features
 
  (Contributed by Victor Stinner in :gh:`107954`.)
 
+* Add :c:func:`PyUnicode_Export` and :c:func:`PyUnicode_Import` functions
+ to export and import strings.
+ (Contributed by Victor Stinner in :gh:`119609`.)
+
 
 Porting to Python 3.14
 ----------------------

Include/unicodeobject.h

@@ -248,6 +248,27 @@ PyAPI_FUNC(PyObject *) PyUnicode_InternFromString(
  const char *u /* UTF-8 encoded string */
  );
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030e0000
+#define PyUnicode_FORMAT_UCS1 0x01 // Py_UCS1*
+#define PyUnicode_FORMAT_UCS2 0x02 // Py_UCS2*
+#define PyUnicode_FORMAT_UCS4 0x04 // Py_UCS4*
+#define PyUnicode_FORMAT_UTF8 0x08 // char*
+#define PyUnicode_FORMAT_ASCII 0x10 // char* (ASCII string)
+
+#define PyUnicode_EXPORT_COPY 0x01
+
+
+PyAPI_FUNC(int32_t) PyUnicode_Export(
+ PyObject *unicode,
+ int32_t requested_formats,
+ uint32_t flags,
+ Py_buffer *view);
+PyAPI_FUNC(PyObject*) PyUnicode_Import(
+ const void *data,
+ Py_ssize_t nbytes,
+ int32_t format);
+#endif
+
 /* --- wchar_t support for platforms which support it --------------------- */
 
 #ifdef HAVE_WCHAR_H

Lib/test/test_capi/test_unicode.py

@@ -1,5 +1,6 @@
-import unittest
+import struct
 import sys
+import unittest
 from test import support
 from test.support import import_helper
 
@@ -28,6 +29,17 @@ class Str(str):
  pass
 
 
+PyUnicode_FORMAT_UCS1 = 0x01
+PyUnicode_FORMAT_UCS2 = 0x02
+PyUnicode_FORMAT_UCS4 = 0x04
+PyUnicode_FORMAT_UTF8 = 0x08
+PyUnicode_FORMAT_ASCII = 0x10
+# Invalid native format
+PyUnicode_FORMAT_INVALID = 0x20
+
+PyUnicode_EXPORT_COPY = 0x01
+
+
 class CAPITest(unittest.TestCase):
 
  @support.cpython_only
@@ -1721,6 +1733,183 @@ def test_pep393_utf8_caching_bug(self):
  # Check that the second call returns the same result
  self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
 
+ def test_unicode_export(self):
+ # Test PyUnicode_Export() and PyUnicode_FreeExport()
+ unicode_export = _testlimitedcapi.unicode_export
+ if sys.byteorder == 'little':
+ ucs2_enc = 'utf-16le'
+ ucs4_enc = 'utf-32le'
+ else:
+ ucs2_enc = 'utf-16be'
+ ucs4_enc = 'utf-32be'
+
+ # export to the native format
+ formats = (PyUnicode_FORMAT_ASCII
+ | PyUnicode_FORMAT_UCS1
+ | PyUnicode_FORMAT_UCS2
+ | PyUnicode_FORMAT_UCS4)
+ BUFFER_UCS1 = 'B'
+ BUFFER_UCS2 = '=H'
+ BUFFER_UCS4 = '=I'
+
+ def check_ucs1(text, formats, flags=0):
+ if formats == PyUnicode_FORMAT_UCS1:
+ export_format = PyUnicode_FORMAT_UCS1
+ elif text.isascii():
+ export_format = PyUnicode_FORMAT_ASCII
+ else:
+ export_format = PyUnicode_FORMAT_UCS1
+ self.assertEqual(unicode_export(text, formats, flags),
+ (text.encode('latin1'), export_format, 1, BUFFER_UCS1))
+
+ def check_ucs2(text, formats, flags=0):
+ self.assertEqual(unicode_export(text, formats, flags),
+ (text.encode(ucs2_enc, 'surrogatepass'),
+ PyUnicode_FORMAT_UCS2, 2, BUFFER_UCS2))
+
+ def check_ucs4(text, formats, flags=0):
+ self.assertEqual(unicode_export(text, formats, flags),
+ (text.encode(ucs4_enc, 'surrogatepass'),
+ PyUnicode_FORMAT_UCS4, 4, BUFFER_UCS4))
+
+ def check_utf8(text, flags=0):
+ self.assertEqual(unicode_export(text, PyUnicode_FORMAT_UTF8, flags),
+ (text.encode('utf8', 'surrogatepass'),
+ PyUnicode_FORMAT_UTF8, 1, 'B'))
+
+ def check_no_matching_format(text, formats, flags=0):
+ err_msg = "unable to find a matching export format"
+ with self.assertRaisesRegex(ValueError, err_msg):
+ unicode_export('abc', formats, flags)
+
+ # export as native format
+ check_ucs1("abc", formats)
+ check_ucs1("latin1:\xe9", formats)
+ check_ucs2('ucs2:\u20ac', formats)
+ check_ucs4('ucs4:\U0010ffff', formats)
+
+ # convert ASCII to UCS1
+ check_ucs1("abc", PyUnicode_FORMAT_UCS1)
+
+ # convert to UCS2 (need PyUnicode_EXPORT_COPY)
+ check_no_matching_format("abc", PyUnicode_FORMAT_UCS2)
+ check_no_matching_format("latin1:\xe9", PyUnicode_FORMAT_UCS2)
+ check_ucs2("abc", PyUnicode_FORMAT_UCS2, PyUnicode_EXPORT_COPY)
+ check_ucs2("latin1:\xe9", PyUnicode_FORMAT_UCS2, PyUnicode_EXPORT_COPY)
+
+ # convert to UCS4 (need PyUnicode_EXPORT_COPY)
+ check_no_matching_format("abc", PyUnicode_FORMAT_UCS4)
+ check_no_matching_format("latin1:\xe9", PyUnicode_FORMAT_UCS4)
+ check_no_matching_format('ucs2:\u20ac', PyUnicode_FORMAT_UCS4)
+ check_ucs4("abc", PyUnicode_FORMAT_UCS4, PyUnicode_EXPORT_COPY)
+ check_ucs4("latin1:\xe9", PyUnicode_FORMAT_UCS4, PyUnicode_EXPORT_COPY)
+ check_ucs4('ucs2:\u20ac', PyUnicode_FORMAT_UCS4, PyUnicode_EXPORT_COPY)
+
+ # always encode to UTF8
+ check_utf8("abc")
+ check_utf8("latin1:\xe9")
+ check_utf8('ucs2:\u20ac')
+ check_utf8('ucs4:\U0010ffff')
+
+ # surrogates
+ check_ucs2('\udc80', PyUnicode_FORMAT_UCS2)
+ check_ucs4('\udc80', PyUnicode_FORMAT_UCS4, PyUnicode_EXPORT_COPY)
+ check_utf8('\udc80', PyUnicode_EXPORT_COPY)
+
+ # No supported format or invalid format
+ for formats in (0, PyUnicode_FORMAT_INVALID):
+ with self.subTest(formats=formats):
+ check_no_matching_format('abc', formats)
+
+ def test_unicode_import(self):
+ # Test PyUnicode_Import()
+ unicode_import = _testlimitedcapi.unicode_import
+ if sys.byteorder == 'little':
+ ucs2_enc = 'utf-16le'
+ ucs4_enc = 'utf-32le'
+ else:
+ ucs2_enc = 'utf-16be'
+ ucs4_enc = 'utf-32be'
+
+ self.assertEqual(unicode_import(b'abc', PyUnicode_FORMAT_ASCII),
+ "abc")
+ self.assertEqual(unicode_import(b'latin1:\xe9', PyUnicode_FORMAT_UCS1),
+ "latin1:\xe9")
+
+ self.assertEqual(unicode_import('ucs2:\u20ac'.encode(ucs2_enc),
+ PyUnicode_FORMAT_UCS2),
+ 'ucs2:\u20ac')
+
+ self.assertEqual(unicode_import('ucs4:\U0010ffff'.encode(ucs4_enc),
+ PyUnicode_FORMAT_UCS4),
+ 'ucs4:\U0010ffff')
+
+ text = "abc\xe9\U0010ffff"
+ self.assertEqual(unicode_import(text.encode('utf8'),
+ PyUnicode_FORMAT_UTF8),
+ text)
+
+ # Empty string
+ for native_format in (
+ PyUnicode_FORMAT_ASCII,
+ PyUnicode_FORMAT_UCS1,
+ PyUnicode_FORMAT_UCS2,
+ PyUnicode_FORMAT_UCS4,
+ PyUnicode_FORMAT_UTF8,
+ ):
+ with self.subTest(native_format=native_format):
+ self.assertEqual(unicode_import(b'', native_format),
+ '')
+
+ # Invalid format
+ with self.assertRaises(ValueError):
+ unicode_import(b'', PyUnicode_FORMAT_INVALID)
+
+ # Invalid size
+ ucs2 = 'ucs2:\u20ac'.encode(ucs2_enc)
+ with self.assertRaises(ValueError):
+ unicode_import(ucs2[:-1], PyUnicode_FORMAT_UCS2)
+ ucs4 = 'ucs4:\U0010ffff'.encode(ucs4_enc)
+ with self.assertRaises(ValueError):
+ unicode_import(ucs4[:-1], PyUnicode_FORMAT_UCS4)
+ with self.assertRaises(ValueError):
+ unicode_import(ucs4[:-2], PyUnicode_FORMAT_UCS4)
+ with self.assertRaises(ValueError):
+ unicode_import(ucs4[:-3], PyUnicode_FORMAT_UCS4)
+
+ def test_unicode_export_import_roundtrip(self):
+ unicode_export = _testlimitedcapi.unicode_export
+ unicode_import = _testlimitedcapi.unicode_import
+
+ ASCII = PyUnicode_FORMAT_ASCII
+ UCS1 = PyUnicode_FORMAT_UCS1
+ UCS2 = PyUnicode_FORMAT_UCS2
+ UCS4 = PyUnicode_FORMAT_UCS4
+ UTF8 = PyUnicode_FORMAT_UTF8
+ ALL = (ASCII | UCS1 | UCS2 | UCS4 | UTF8)
+
+ def roundtrip(string, formats):
+ export = unicode_export(string, formats, PyUnicode_EXPORT_COPY)
+ buf, buf_fmt, item_size, view_fmt = export
+ self.assertEqual(unicode_import(buf, buf_fmt), string)
+
+ for string, allowed_formats in (
+ ('', {ASCII, UCS1, UCS2, UCS4, UTF8}),
+ ('ascii', {ASCII, UCS1, UCS2, UCS4, UTF8}),
+ ('latin1:\xe9', {UCS1, UCS2, UCS4, UTF8}),
+ ('ucs2:\u20ac', {UCS2, UCS4, UTF8}),
+ ('ucs4:\U0001f638', {UCS4, UTF8}),
+ ):
+ for formats in ASCII, UCS1, UCS2, UCS4, UTF8:
+ with self.subTest(string=string, formats=formats):
+ if formats not in allowed_formats:
+ with self.assertRaises(ValueError):
+ unicode_export(string, formats, PyUnicode_EXPORT_COPY)
+ else:
+ roundtrip(string, formats)
+
+ roundtrip(string, ALL)
+
 
 class PyUnicodeWriterTest(unittest.TestCase):
  def create_writer(self, size):
@@ -1904,5 +2093,5 @@ def test_recover_error(self):
  self.assertEqual(writer.finish(), 'Hello World.')
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
  unittest.main()

Misc/NEWS.d/next/C_API/2024-09-05-16-57-51.gh-issue-119609.5EZ-kg.rst

@@ -0,0 +1,2 @@
+Add :c:func:`PyUnicode_Export` and :c:func:`PyUnicode_Import` functions to
+export and import strings. Patch by Victor Stinner.

Misc/stable_abi.toml

@@ -2527,4 +2527,18 @@
 [function.PyLong_AsUInt64]
  added = '3.14'
 [const.Py_tp_vectorcall]
- added = '3.14'
+ added = '3.14'
+[const.PyUnicode_FORMAT_UCS1]
+ added = '3.14'
+[const.PyUnicode_FORMAT_UCS2]
+ added = '3.14'
+[const.PyUnicode_FORMAT_UCS4]
+ added = '3.14'
+[const.PyUnicode_FORMAT_UTF8]
+ added = '3.14'
+[const.PyUnicode_FORMAT_ASCII]
+ added = '3.14'
+[function.PyUnicode_Export]
+ added = '3.14'
+[function.PyUnicode_Import]
+ added = '3.14'