Visitar URL original
gh-119609, PEP 756: Add PyUnicode_Export() function by vstinner · Pull Request #123738 · python/cpython · GitHub
Skip to content

gh-119609, PEP 756: Add PyUnicode_Export() function #123738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
vstinner wants to merge 28 commits into from
+593 −15
Closed

gh-119609, PEP 756: Add PyUnicode_Export() function #123738

Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
c84f314
gh-119609: Add PyUnicode_Export() function
vstinner May 27, 2024
d0cdbd1
Address reviews
vstinner Sep 5, 2024
9b33dca
Exclude from limited C API 3.13 and older
vstinner Sep 5, 2024
cf1f74a
Replace PyErr_Format() with PyErr_SetString()
vstinner Sep 5, 2024
93d4470
Fix test_collections: implement UserString.__release_buffer__()
vstinner Sep 5, 2024
17ad7b9
Add format parameter to PyUnicode_Export()
vstinner Sep 9, 2024
d683d0a
format must not be NULL
vstinner Sep 9, 2024
78a70fa
Fix memory leak in unicode_releasebuffer()
vstinner Sep 10, 2024
79207f5
Remove PyUnicode_GetBufferFormat() documentation
vstinner Sep 10, 2024
bc0fb69
Apply suggestions from code review
vstinner Sep 10, 2024
2cdbc27
Set format to 0 on error
vstinner Sep 10, 2024
b5be22d
Remove trailing space
vstinner Sep 10, 2024
2960b25
Change constant values
vstinner Sep 10, 2024
bcb41f3
Update constants value in the doc
vstinner Sep 11, 2024
44cb702
Remove unicode_releasebuffer(); use bytes instead
vstinner Sep 12, 2024
1809d8d
PyUnicode_Export() returns the format
vstinner Sep 12, 2024
6707ef4
Fix PyUnicode_Export() signature in doc
vstinner Sep 12, 2024
abf5c58
Use _PyUnicode_EncodeUTF16() and _PyUnicode_EncodeUTF32()
vstinner Sep 12, 2024
033fc07
Use signed int in C tests
vstinner Sep 12, 2024
078dfcf
Update stable_abi: remove PyUnicode_GetBufferFormat()
vstinner Sep 12, 2024
79c6d01
Revert "Use _PyUnicode_EncodeUTF16() and _PyUnicode_EncodeUTF32()"
vstinner Sep 12, 2024
5479ab2
Allow surrogate characters in UTF-8
vstinner Sep 12, 2024
ab2f9b0
Merge branch 'main' into unicode_view
vstinner Sep 13, 2024
f71f230
Avoid a second copy in the UTF-8 export
vstinner Sep 13, 2024
492f10a
UCS-4 export: remove one memory copy
vstinner Sep 13, 2024
b031163
Update Py_buffer format
vstinner Sep 16, 2024
21e6012
Add PyUnicode_EXPORT_COPY flag
vstinner Sep 23, 2024
3267ce6
doc
vstinner Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
UCS-4 export: remove one memory copy
  • Loading branch information
@vstinner
vstinner committed Sep 13, 2024
commit 492f10a1c120008fa9cea81d7c1c8c5e67ac46bf
37 changes: 20 additions & 17 deletions Objects/unicodeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ unicode_decode_utf8_writer(_PyUnicodeWriter *writer,
static inline int unicode_is_finalizing(void);
static int unicode_is_singleton(PyObject *unicode);
#endif
static Py_UCS4*
as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize,
int copy_null);


// Return a reference to the immortal empty string singleton.
Expand Down Expand Up @@ -2444,16 +2447,13 @@ PyUnicode_Export(PyObject *unicode, int32_t requested_formats,

// Convert ASCII, UCS1 or UCS2 to UCS4
if (requested_formats & PyUnicode_FORMAT_UCS4) {
Py_UCS4 *ucs4 = PyUnicode_AsUCS4Copy(unicode);
if (ucs4 == NULL) {
return -1;
}

PyObject *bytes = PyBytes_FromStringAndSize((char*)ucs4, (len + 1) * 4);
PyMem_Free(ucs4);
PyObject *bytes = PyBytes_FromStringAndSize(NULL, (len + 1) * 4);
if (bytes == NULL) {
return -1;
}
Py_UCS4 *ucs4 = (Py_UCS4*)PyBytes_AS_STRING(bytes);

(void)as_ucs4(unicode, ucs4, len + 1, 1);

return unicode_export_bytes(bytes, view, len,
4, BUFFER_UCS4, PyUnicode_FORMAT_UCS4);
Expand Down Expand Up @@ -2709,15 +2709,14 @@ static Py_UCS4*
as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize,
int copy_null)
{
int kind;
const void *data;
Py_ssize_t len, targetlen;
kind = PyUnicode_KIND(string);
data = PyUnicode_DATA(string);
len = PyUnicode_GET_LENGTH(string);
targetlen = len;
if (copy_null)
int kind = PyUnicode_KIND(string);
const void *data = PyUnicode_DATA(string);
Py_ssize_t len = PyUnicode_GET_LENGTH(string);
Py_ssize_t targetlen = len;
if (copy_null) {
targetlen++;
}

if (!target) {
target = PyMem_New(Py_UCS4, targetlen);
if (!target) {
Expand All @@ -2729,11 +2728,13 @@ as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize,
if (targetsize < targetlen) {
PyErr_Format(PyExc_SystemError,
"string is longer than the buffer");
if (copy_null && 0 < targetsize)
if (copy_null && 0 < targetsize) {
target[0] = 0;
}
return NULL;
}
}

if (kind == PyUnicode_1BYTE_KIND) {
const Py_UCS1 *start = (const Py_UCS1 *) data;
_PyUnicode_CONVERT_BYTES(Py_UCS1, Py_UCS4, start, start + len, target);
Expand All @@ -2748,8 +2749,10 @@ as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize,
else {
Py_UNREACHABLE();
}
if (copy_null)
if (copy_null) {
target[len] = 0;
}

return target;
}

Expand Down