gh-119609: Add PyUnicode_AsNativeFormat() function

Add PyUnicode_AsNativeFormat() and PyUnicode_FromNativeFormat()
functions to the C API.

Author: vstinner

Doc/c-api/unicode.rst

@@ -341,6 +341,53 @@ APIs:
  .. versionadded:: 3.3
 
 
+.. c:function:: const void* PyUnicode_AsNativeFormat(PyObject *unicode, Py_ssize_t *size, int *native_format)
+
+ Get the contents of a string in its native format.
+
+ * Return the contents, set *\*size* and *\*native_format* on success.
+ * Set an exception and return ``NULL`` on error.
+
+ The contents is valid as long as *unicode* is valid.
+
+ *unicode*, *size* and *native_format* must not be NULL.
+
+ *\*native_format* is set to one of these native formats:
+
+ .. c:namespace::NULL
+
+ ======================================== ===== ============================
+ Constant Identifier Value Description
+ ======================================== ===== ============================
+.. c:macro:: PyUnicode_NATIVE_ASCII ``1`` ASCII string (``Py_UCS1*``)
+.. c:macro:: PyUnicode_NATIVE_UCS1 ``2`` UCS-1 string (``Py_UCS1*``)
+.. c:macro:: PyUnicode_NATIVE_UCS2 ``3`` UCS-2 string (``Py_UCS2*``)
+.. c:macro:: PyUnicode_NATIVE_UCS4 ``4`` UCS-4 string (``Py_UCS4*``)
+.. c:macro:: PyUnicode_NATIVE_UTF8 ``5`` UTF-8 string (``char*``)
+ ======================================== ===== ============================
+
+ .. impl-detail::
+ In CPython, the :c:macro:`PyUnicode_NATIVE_UTF8` format is not used by
+:c:func:`PyUnicode_AsNativeFormat`, but it's accepted by
+:c:func:`PyUnicode_FromNativeFormat`.
+
+ .. versionadded:: 3.14
+
+
+.. c:function:: PyObject* PyUnicode_FromNativeFormat(const void *data, Py_ssize_t size, int native_format)
+
+ Create a string object from a native format string.
+
+ * Return a reference to a new string object on success.
+ * Set an exception and return ``NULL`` on error.
+
+ *data* must not be NULL. *size* must be positive or zero.
+
+ See :c:func:`PyUnicode_AsNativeFormat` for the available native formats.
+
+ .. versionadded:: 3.14
+
+
 .. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
  Py_ssize_t size)
 

Doc/data/stable_abi.dat

@@ -217,6 +217,12 @@ C API Changes
 New Features
 ------------
 
+* Add :c:func:`PyUnicode_AsNativeFormat` and
+:c:func:`PyUnicode_FromNativeFormat` functions to import and export strings
+ in their native format.
+ (Contributed by Victor Stinner in :gh:`119609`.)
+
+
 Porting to Python 3.14
 ----------------------
 

Doc/whatsnew/3.14.rst

@@ -248,6 +248,28 @@ PyAPI_FUNC(PyObject *) PyUnicode_InternFromString(
 constchar*u/* UTF-8 encoded string */
  );
 
+#definePyUnicode_NATIVE_ASCII 1
+#definePyUnicode_NATIVE_UCS1 2
+#definePyUnicode_NATIVE_UCS2 3
+#definePyUnicode_NATIVE_UCS4 4
+#definePyUnicode_NATIVE_UTF8 5
+
+// Get the content of a string in its native format.
+// - Return the content, set '*size' and '*native_format' on success.
+// - Set an exception and return NULL on error.
+PyAPI_FUNC(constvoid*) PyUnicode_AsNativeFormat(
+PyObject*unicode,
+Py_ssize_t*size,
+int*native_format);
+
+// Create a string object from a native format string.
+// - Return a reference to a new string object on success.
+// - Set an exception and return NULL on error.
+PyAPI_FUNC(PyObject*) PyUnicode_FromNativeFormat(
+constvoid*data,
+Py_ssize_tsize,
+intnative_format);
+
 /* --- wchar_t support for platforms which support it --------------------- */
 
 #ifdefHAVE_WCHAR_H

Include/unicodeobject.h

@@ -24,6 +24,14 @@ class Str(str):
 pass
 
 
+PyUnicode_NATIVE_ASCII=1
+PyUnicode_NATIVE_UCS1=2
+PyUnicode_NATIVE_UCS2=3
+PyUnicode_NATIVE_UCS4=4
+PyUnicode_NATIVE_UTF8=5
+# Invalid native format
+PyUnicode_NATIVE_INVALID=0
+
 classCAPITest(unittest.TestCase):
 
 @support.cpython_only
@@ -1675,6 +1683,75 @@ def test_pep393_utf8_caching_bug(self):
 # Check that the second call returns the same result
 self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i+1))
 
-
-if__name__=="__main__":
+deftest_unicode_asnativeformat(self):
+# Test PyUnicode_AsNativeFormat()
+asnativeformat=_testlimitedcapi.unicode_asnativeformat
+self.assertEqual(asnativeformat("abc"),
+ (b'abc', PyUnicode_NATIVE_ASCII))
+self.assertEqual(asnativeformat("latin1:\xe9"),
+ (b'latin1:\xe9', PyUnicode_NATIVE_UCS1))
+
+ucs2_enc='utf-16le'ifsys.byteorder=='little'else'utf-16be'
+self.assertEqual(asnativeformat('ucs2:\u20ac'),
+ ('ucs2:\u20ac'.encode(ucs2_enc),
+PyUnicode_NATIVE_UCS2))
+
+ucs4_enc='utf-32le'ifsys.byteorder=='little'else'utf-32be'
+self.assertEqual(asnativeformat('ucs4:\U0010ffff'),
+ ('ucs4:\U0010ffff'.encode(ucs4_enc),
+PyUnicode_NATIVE_UCS4))
+
+deftest_unicode_fromnativeformat(self):
+# Test PyUnicode_FromNativeFormat()
+fromnativeformat=_testlimitedcapi.unicode_fromnativeformat
+self.assertEqual(fromnativeformat(b'abc', PyUnicode_NATIVE_ASCII),
+"abc")
+self.assertEqual(fromnativeformat(b'latin1:\xe9', PyUnicode_NATIVE_UCS1),
+"latin1:\xe9")
+
+ucs2_enc='utf-16le'ifsys.byteorder=='little'else'utf-16be'
+self.assertEqual(fromnativeformat('ucs2:\u20ac'.encode(ucs2_enc),
+PyUnicode_NATIVE_UCS2),
+'ucs2:\u20ac')
+
+ucs4_enc='utf-32le'ifsys.byteorder=='little'else'utf-32be'
+self.assertEqual(fromnativeformat('ucs4:\U0010ffff'.encode(ucs4_enc),
+PyUnicode_NATIVE_UCS4),
+'ucs4:\U0010ffff')
+
+text="abc\xe9\U0010ffff"
+self.assertEqual(fromnativeformat(text.encode('utf8'),
+PyUnicode_NATIVE_UTF8),
+text)
+
+# Empty string
+fornative_formatin (
+PyUnicode_NATIVE_ASCII,
+PyUnicode_NATIVE_UCS1,
+PyUnicode_NATIVE_UCS2,
+PyUnicode_NATIVE_UCS4,
+PyUnicode_NATIVE_UTF8,
+ ):
+withself.subTest(native_format=native_format):
+self.assertEqual(fromnativeformat(b'', native_format),
+'')
+
+# Invalid format
+withself.assertRaises(ValueError):
+fromnativeformat(b'', PyUnicode_NATIVE_INVALID)
+
+# Invalid size
+ucs2='ucs2:\u20ac'.encode(ucs2_enc)
+withself.assertRaises(ValueError):
+fromnativeformat(ucs2[:-1], PyUnicode_NATIVE_UCS2)
+ucs4='ucs4:\U0010ffff'.encode(ucs4_enc)
+withself.assertRaises(ValueError):
+fromnativeformat(ucs4[:-1], PyUnicode_NATIVE_UCS4)
+withself.assertRaises(ValueError):
+fromnativeformat(ucs4[:-2], PyUnicode_NATIVE_UCS4)
+withself.assertRaises(ValueError):
+fromnativeformat(ucs4[:-3], PyUnicode_NATIVE_UCS4)
+
+
+if__name__=='__main__':
 unittest.main()

Lib/test/test_capi/test_unicode.py

@@ -0,0 +1,3 @@
+Add :c:func:`PyUnicode_AsNativeFormat` and
+:c:func:`PyUnicode_FromNativeFormat` functions to import and export strings
+in their native format. Patch by Victor Stinner.

Lib/test/test_stable_abi_ctypes.py

@@ -2507,3 +2507,7 @@
 added = '3.13'
 [function.PyEval_GetFrameLocals]
 added = '3.13'
+[function.PyUnicode_AsNativeFormat]
+added = '3.14'
+[function.PyUnicode_FromNativeFormat]
+added = '3.14'

Misc/NEWS.d/next/C API/2024-05-27-17-46-17.gh-issue-119609.kPIx6S.rst

@@ -1837,6 +1837,35 @@ test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored))
 #undef CHECK_FORMAT_0
 }
 
+
+// Test PyUnicode_AsNativeFormat()
+staticPyObject*
+unicode_asnativeformat(PyObject*self, PyObject*obj)
+{
+Py_ssize_tsize;
+intnative_format;
+constvoid*data=PyUnicode_AsNativeFormat(obj, &size, &native_format);
+if (data==NULL) {
+returnNULL;
+ }
+returnPy_BuildValue("y#i", data, size, native_format);
+}
+
+
+// Test PyUnicode_FromNativeFormat()
+staticPyObject*
+unicode_fromnativeformat(PyObject*self, PyObject*args)
+{
+constvoid*data;
+Py_ssize_tsize;
+intnative_format;
+if (!PyArg_ParseTuple(args, "y#i", &data, &size, &native_format)) {
+returnNULL;
+ }
+returnPyUnicode_FromNativeFormat(data, size, native_format);
+}
+
+
 staticPyMethodDefTestMethods[] = {
  {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS},
  {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS},
@@ -1924,6 +1953,8 @@ static PyMethodDef TestMethods[] = {
  {"unicode_format", unicode_format, METH_VARARGS},
  {"unicode_contains", unicode_contains, METH_VARARGS},
  {"unicode_isidentifier", unicode_isidentifier, METH_O},
+ {"unicode_asnativeformat", unicode_asnativeformat, METH_O},
+ {"unicode_fromnativeformat", unicode_fromnativeformat, METH_VARARGS},
  {NULL},
 };
 

Misc/stable_abi.toml

@@ -2094,6 +2094,89 @@ _PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size)
 returnres;
 }
 
+constvoid*
+PyUnicode_AsNativeFormat(PyObject*unicode,
+Py_ssize_t*size, int*native_format)
+{
+if (!PyUnicode_Check(unicode)) {
+*size=0;
+*native_format=0;
+PyErr_Format(PyExc_TypeError, "must be str, not %T", unicode);
+returnNULL;
+ }
+
+Py_ssize_tlen=PyUnicode_GET_LENGTH(unicode);
+
+if (PyUnicode_IS_ASCII(unicode)) {
+*native_format=PyUnicode_NATIVE_ASCII;
+*size=len;
+returnPyUnicode_1BYTE_DATA(unicode);
+ }
+intkind=PyUnicode_KIND(unicode);
+
+switch (kind)
+ {
+casePyUnicode_1BYTE_KIND:
+*native_format=PyUnicode_NATIVE_UCS1;
+*size=len;
+returnPyUnicode_1BYTE_DATA(unicode);
+
+casePyUnicode_2BYTE_KIND:
+*native_format=PyUnicode_NATIVE_UCS2;
+*size=len*2;
+returnPyUnicode_2BYTE_DATA(unicode);
+
+default:
+assert(kind==PyUnicode_4BYTE_KIND);
+*native_format=PyUnicode_NATIVE_UCS4;
+*size=len*4;
+returnPyUnicode_4BYTE_DATA(unicode);
+ }
+}
+
+PyObject*
+PyUnicode_FromNativeFormat(constvoid*data, Py_ssize_tsize,
+intnative_format)
+{
+if (size<0) {
+PyErr_SetString(PyExc_ValueError, "Negative size");
+returnNULL;
+ }
+
+switch (native_format)
+ {
+casePyUnicode_NATIVE_ASCII:
+returnPyUnicode_DecodeASCII((constchar*)data, size, NULL);
+
+casePyUnicode_NATIVE_UCS1:
+return_PyUnicode_FromUCS1(data, size);
+
+casePyUnicode_NATIVE_UCS2:
+if (size % 2) {
+PyErr_Format(PyExc_ValueError, "size must be a multiple of 2: %zd",
+size);
+returnNULL;
+ }
+return_PyUnicode_FromUCS2(data, size / 2);
+
+casePyUnicode_NATIVE_UCS4:
+if (size % 4) {
+PyErr_Format(PyExc_ValueError, "size must be a multiple of 4: %zd",
+size);
+returnNULL;
+ }
+return_PyUnicode_FromUCS4(data, size / 4);
+
+casePyUnicode_NATIVE_UTF8:
+returnPyUnicode_DecodeUTF8((constchar*)data, size, NULL);
+
+default:
+PyErr_Format(PyExc_ValueError, "unknown native format %i",
+native_format);
+returnNULL;
+ }
+}
+
 PyObject*
 PyUnicode_FromKindAndData(intkind, constvoid*buffer, Py_ssize_tsize)
 {