Visitar URL original
gh-119609, PEP 756: Add PyUnicode_Export() function by vstinner 路 Pull Request #123738 路 python/cpython 路 GitHub
Skip to content

gh-119609, PEP 756: Add PyUnicode_Export() function #123738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
vstinner wants to merge 28 commits into from
Closed

gh-119609, PEP 756: Add PyUnicode_Export() function #123738

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
c84f314
gh-119609: Add PyUnicode_Export() function
vstinner May 27, 2024
d0cdbd1
Address reviews
vstinner Sep 5, 2024
9b33dca
Exclude from limited C API 3.13 and older
vstinner Sep 5, 2024
cf1f74a
Replace PyErr_Format() with PyErr_SetString()
vstinner Sep 5, 2024
93d4470
Fix test_collections: implement UserString.__release_buffer__()
vstinner Sep 5, 2024
17ad7b9
Add format parameter to PyUnicode_Export()
vstinner Sep 9, 2024
d683d0a
format must not be NULL
vstinner Sep 9, 2024
78a70fa
Fix memory leak in unicode_releasebuffer()
vstinner Sep 10, 2024
79207f5
Remove PyUnicode_GetBufferFormat() documentation
vstinner Sep 10, 2024
bc0fb69
Apply suggestions from code review
vstinner Sep 10, 2024
2cdbc27
Set format to 0 on error
vstinner Sep 10, 2024
b5be22d
Remove trailing space
vstinner Sep 10, 2024
2960b25
Change constant values
vstinner Sep 10, 2024
bcb41f3
Update constants value in the doc
vstinner Sep 11, 2024
44cb702
Remove unicode_releasebuffer(); use bytes instead
vstinner Sep 12, 2024
1809d8d
PyUnicode_Export() returns the format
vstinner Sep 12, 2024
6707ef4
Fix PyUnicode_Export() signature in doc
vstinner Sep 12, 2024
abf5c58
Use _PyUnicode_EncodeUTF16() and _PyUnicode_EncodeUTF32()
vstinner Sep 12, 2024
033fc07
Use signed int in C tests
vstinner Sep 12, 2024
078dfcf
Update stable_abi: remove PyUnicode_GetBufferFormat()
vstinner Sep 12, 2024
79c6d01
Revert "Use _PyUnicode_EncodeUTF16() and _PyUnicode_EncodeUTF32()"
vstinner Sep 12, 2024
5479ab2
Allow surrogate characters in UTF-8
vstinner Sep 12, 2024
ab2f9b0
Merge branch 'main' into unicode_view
vstinner Sep 13, 2024
f71f230
Avoid a second copy in the UTF-8 export
vstinner Sep 13, 2024
492f10a
UCS-4 export: remove one memory copy
vstinner Sep 13, 2024
b031163
Update Py_buffer format
vstinner Sep 16, 2024
21e6012
Add PyUnicode_EXPORT_COPY flag
vstinner Sep 23, 2024
3267ce6
doc
vstinner Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add format parameter to PyUnicode_Export()
  • Loading branch information
@vstinner
vstinner committed Sep 9, 2024
commit 17ad7b9c88156adfa413fb9922991e1fb85aaa77
20 changes: 3 additions & 17 deletions Doc/c-api/unicode.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,12 +341,12 @@ APIs:
.. versionadded:: 3.3


.. c:function:: int PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
.. c:function:: int PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view, uint32_t *format)

Export the contents of the *unicode* string in one of the requested format
*requested_formats*.

* On success, fill *view*, and return ``0``.
* On success, fill *view* and set *\*format*, and return ``0``.
* On error, set an exception and return ``-1``.

The *view* buffer must be released by :c:func:`PyBuffer_Release`.
Comment thread
vstinner marked this conversation as resolved.
Outdated
Expand All @@ -372,27 +372,13 @@ APIs:

*requested_formats* can be a single format or a bitwise combination of the
formats in the table above.
To determine the format that was selected for output, call
:c:func:`PyUnicode_GetBufferFormat`.
On success, *\*format* will be set to a single one of the requested flags.
Comment thread
vstinner marked this conversation as resolved.
Outdated

Note that future versions of Python may introduce additional formats.

.. versionadded:: 3.14


.. c:function:: int PyUnicode_GetBufferFormat(const Py_buffer *view, uint32_t *format)

Get the format of the buffer *view*.

* On success, set *\*format* to the corresponding ``PyUnicode_FORMAT_*`` value
and return ``0``.
* On error, set an exception and return ``-1``.

*view* must be a buffer filled by :c:func:`PyUnicode_Export`.

.. versionadded:: 3.14


.. c:function:: PyObject* PyUnicode_Import(const void *data, Py_ssize_t nbytes, uint32_t format)

Create a Unicode string object from a buffer in a supported format.
Expand Down
4 changes: 1 addition & 3 deletions Include/unicodeobject.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,9 +258,7 @@ PyAPI_FUNC(PyObject *) PyUnicode_InternFromString(
PyAPI_FUNC(int) PyUnicode_Export(
PyObject *unicode,
uint32_t requested_formats,
Py_buffer *view);
PyAPI_FUNC(int) PyUnicode_GetBufferFormat(
const Py_buffer *view,
Py_buffer *view,
uint32_t *format);
PyAPI_FUNC(PyObject*) PyUnicode_Import(
const void *data,
Expand Down
5 changes: 1 addition & 4 deletions Modules/_testlimitedcapi/unicode.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1849,11 +1849,8 @@ unicode_export(PyObject *self, PyObject *args)
}

Py_buffer view;
if (PyUnicode_Export(obj, requested_formats, &view) < 0) {
return NULL;
}
uint32_t format;
if (PyUnicode_GetBufferFormat(&view, &format) < 0) {
if (PyUnicode_Export(obj, requested_formats, &view, &format) < 0) {
return NULL;
}

Expand Down
47 changes: 11 additions & 36 deletions Objects/unicodeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2333,7 +2333,7 @@ PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer,


static int
unicode_export(PyObject *unicode, Py_buffer *view,
unicode_export(PyObject *unicode, Py_buffer *view, uint32_t *pformat,
Py_ssize_t len, const void *buf,
int itemsize, const char *format, uint32_t internal_format)
{
Expand All @@ -2344,12 +2344,14 @@ unicode_export(PyObject *unicode, Py_buffer *view,
view->itemsize = itemsize;
view->format = (char*)format;
view->internal = (void*)(uintptr_t)internal_format;
Comment thread
vstinner marked this conversation as resolved.
Outdated
*pformat = internal_format;
return 0;
}


int
PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
PyUnicode_Export(PyObject *unicode, uint32_t requested_formats,
Py_buffer *view, uint32_t *format)
{
#if SIZEOF_INT == 4
# define BUFFER_UCS4 "I"
Expand All @@ -2369,7 +2371,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
if (PyUnicode_IS_ASCII(unicode)
&& (requested_formats & PyUnicode_FORMAT_ASCII))
{
return unicode_export(unicode, view,
return unicode_export(unicode, view, format,
len, PyUnicode_1BYTE_DATA(unicode),
1, "B", PyUnicode_FORMAT_ASCII);
}
Expand All @@ -2379,7 +2381,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
if (kind == PyUnicode_1BYTE_KIND
&& (requested_formats & PyUnicode_FORMAT_UCS1))
{
return unicode_export(unicode, view,
return unicode_export(unicode, view, format,
len, PyUnicode_1BYTE_DATA(unicode),
1, "B", PyUnicode_FORMAT_UCS1);
}
Expand All @@ -2388,7 +2390,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
if (kind == PyUnicode_2BYTE_KIND
&& (requested_formats & PyUnicode_FORMAT_UCS2))
{
return unicode_export(unicode, view,
return unicode_export(unicode, view, format,
len, PyUnicode_2BYTE_DATA(unicode),
2, "H", PyUnicode_FORMAT_UCS2);
}
Expand All @@ -2409,7 +2411,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
ucs2);
ucs2[len] = 0;

return unicode_export(unicode, view,
return unicode_export(unicode, view, format,

@mdboom mdboom Sep 9, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the cases where there is a conversion, does the buffer need to hold a reference to the unicode object? Shouldn't we instead be passing NULL here?

@vstinner vstinner Sep 10, 2024

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer to store unicode as view.obj in all cases. So PyBuffer_Release() is able to find the unicode_releasebuffer() function. Technically, you're right that in UCS2 and UCS4 cases, when we allocate a buffer, we don't need to hold a reference to unicode to make sure that data remains valid.

len, ucs2,
2, "H", PyUnicode_FORMAT_UCS2);
}
Expand All @@ -2418,7 +2420,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
if (kind == PyUnicode_4BYTE_KIND
&& (requested_formats & PyUnicode_FORMAT_UCS4))
{
return unicode_export(unicode, view,
return unicode_export(unicode, view, format,
len, PyUnicode_4BYTE_DATA(unicode),
4, BUFFER_UCS4, PyUnicode_FORMAT_UCS4);
}
Expand All @@ -2429,7 +2431,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
if (ucs4 == NULL) {
return -1;
}
return unicode_export(unicode, view,
return unicode_export(unicode, view, format,
len, ucs4,
4, BUFFER_UCS4, PyUnicode_FORMAT_UCS4);
}
Expand All @@ -2441,7 +2443,7 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
if (utf8 == NULL) {
return -1;
}
return unicode_export(unicode, view,
return unicode_export(unicode, view, format,
nbytes, utf8,
1, "B", PyUnicode_FORMAT_UTF8);
}
Expand All @@ -2454,33 +2456,6 @@ PyUnicode_Export(PyObject *unicode, uint32_t requested_formats, Py_buffer *view)
}


int
PyUnicode_GetBufferFormat(const Py_buffer *view, uint32_t *format)
{
if (view->obj == NULL || !PyUnicode_Check(view->obj)) {
PyErr_SetString(PyExc_ValueError, "not a str export");
return -1;
}

uintptr_t internal_format = (uintptr_t)view->internal;
switch (internal_format)
{
case PyUnicode_FORMAT_ASCII:
case PyUnicode_FORMAT_UCS1:
case PyUnicode_FORMAT_UCS2:
case PyUnicode_FORMAT_UCS4:
case PyUnicode_FORMAT_UTF8:
break;
default:
PyErr_SetString(PyExc_ValueError, "invalid format");
return -1;
}

*format = (uint32_t)internal_format;
return 0;
}


static void
unicode_releasebuffer(PyObject *unicode, Py_buffer *view)
{
Expand Down