Visitar URL original
Normalize hex codes in unicode escape sequences in f-strings (#4523) … · psf/black@158aa3c · GitHub
Skip to content

Commit 158aa3c

Browse files
Normalize hex codes in unicode escape sequences in f-strings (#4523) (#5526)
Co-authored-by: cobalt <61329810+cobaltt7@users.noreply.github.com>
1 parent ac9a6ed commit 158aa3c

5 files changed

Lines changed: 88 additions & 9 deletions

File tree

‎CHANGES.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
- Fix crash when `# fmt: skip` follows the closing bracket of a multi-line call or
1717
subscript that isn't the last element of a collection or argument list (#5522)
18+
- Normalize hex codes in unicode escape sequences in f-strings (#5526)
1819
- Fix `--stdin` output ending in `\r\r\n` instead of `\r\n` when the input uses CRLF
1920
line endings and does not end with a newline, e.g. a notebook piped to `black -`
2021
(#5559)

‎src/black/linegen.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
from black.strings import (
8383
fix_multiline_docstring,
8484
get_string_prefix,
85+
normalize_fstring_unicode_escape_sequences,
8586
normalize_string_prefix,
8687
normalize_string_quotes,
8788
normalize_unicode_escape_sequences,
@@ -648,6 +649,7 @@ def visit_fstring(self, node: Node) -> Iterator[Line]:
648649
if any(child.type == STANDALONE_COMMENT for child in node.children):
649650
yield from self.visit_default(node)
650651
return
652+
normalize_fstring_unicode_escape_sequences(node)
651653
# currently we don't want to format and split f-strings at all.
652654
string_leaf = fstring_tstring_to_string(node)
653655
node.replace(string_leaf)
@@ -668,6 +670,7 @@ def visit_tstring(self, node: Node) -> Iterator[Line]:
668670
if any(child.type == STANDALONE_COMMENT for child in node.children):
669671
yield from self.visit_default(node)
670672
return
673+
normalize_fstring_unicode_escape_sequences(node)
671674
# currently we don't want to format and split t-strings at all.
672675
string_leaf = fstring_tstring_to_string(node)
673676
node.replace(string_leaf)

‎src/black/strings.py‎

Lines changed: 64 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010

1111
from black._width_table import WIDTH_TABLE
1212
from black.mode import Mode, Preview
13-
from blib2to3.pytree import Leaf
13+
from blib2to3.pgen2 import token
14+
from blib2to3.pytree import Leaf, Node
1415

1516
STRING_PREFIX_CHARS: Final = "fturbFTURB" # All possible string prefix characters.
1617
STRING_PREFIX_RE: Final = re.compile(
@@ -330,13 +331,7 @@ def normalize_fstring_quotes(
330331
return middles, new_quote
331332

332333

333-
def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
334-
"""Replace hex codes in Unicode escape sequences with lowercase representation."""
335-
text = leaf.value
336-
prefix = get_string_prefix(text)
337-
if "r" in prefix.lower():
338-
return
339-
334+
def _normalize_unicode_escape_sequences_in_text(text: str) -> str:
340335
def replace(m: Match[str]) -> str:
341336
groups = m.groupdict()
342337
back_slashes = groups["backslashes"]
@@ -358,7 +353,67 @@ def replace(m: Match[str]) -> str:
358353
# \N{}
359354
return back_slashes + "N{" + groups["N"].upper() + "}"
360355

361-
leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text)
356+
return re.sub(UNICODE_ESCAPE_RE, replace, text)
357+
358+
359+
def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
360+
"""Replace hex codes in Unicode escape sequences with lowercase representation."""
361+
text = leaf.value
362+
prefix = get_string_prefix(text)
363+
if "r" in prefix.lower():
364+
return
365+
366+
leaf.value = _normalize_unicode_escape_sequences_in_text(text)
367+
368+
369+
def _normalize_fstring_field_node(node: Leaf | Node) -> None:
370+
from black.nodes import syms
371+
372+
if isinstance(node, Leaf):
373+
if node.type == token.STRING:
374+
normalize_unicode_escape_sequences(node)
375+
return
376+
377+
if node.type in (syms.fstring, syms.tstring):
378+
normalize_fstring_unicode_escape_sequences(node)
379+
return
380+
381+
for child in node.children:
382+
_normalize_fstring_field_node(child)
383+
384+
385+
def normalize_fstring_unicode_escape_sequences(node: Node) -> None:
386+
"""Replace hex codes in Unicode escape sequences with lowercase representation
387+
within an f-string or t-string node."""
388+
from black.nodes import syms
389+
390+
if not node.children:
391+
return
392+
393+
first_child = node.children[0]
394+
is_raw_fstring = (
395+
isinstance(first_child, Leaf) and "r" in first_child.value.rstrip("'\"").lower()
396+
)
397+
398+
for child in node.children:
399+
if child.type in (token.FSTRING_MIDDLE, token.TSTRING_MIDDLE):
400+
if not is_raw_fstring:
401+
assert isinstance(child, Leaf)
402+
child.value = _normalize_unicode_escape_sequences_in_text(child.value)
403+
elif child.type in (
404+
syms.fstring_replacement_field,
405+
syms.tstring_replacement_field,
406+
):
407+
# If this is a debug f-string (e.g. f"{foo=}"), do not touch the
408+
# expression before '=', as Python preserves the exact source expression.
409+
if any(
410+
isinstance(c, Leaf) and c.type == token.EQUAL and c.value == "="
411+
for c in child.children
412+
):
413+
continue
414+
415+
for subchild in child.children:
416+
_normalize_fstring_field_node(subchild)
362417

363418

364419
@lru_cache(maxsize=4096)

‎tests/data/cases/format_unicode_escape_seq.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
x = "\N{CYRILLIC small LETTER BYELORUSSIAN-UKRAINIAN I}"
1414
x = b"\x1Fdon't byte"
1515
x = rb"\x1Fdon't format"
16+
x = f"foo \xFF {1}"
1617

1718
# output
1819

@@ -31,3 +32,4 @@
3132
x = "\N{CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I}"
3233
x = b"\x1fdon't byte"
3334
x = rb"\x1Fdon't format"
35+
x = f"foo \xff {1}"

‎tests/data/cases/pep_701.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,15 @@
130130

131131
f"{'\''}"
132132
f"{f'\''}"
133+
f"{'\xFF'}"
134+
f"{r'\xFF'}"
135+
f"{'\xFF'=}"
136+
f"outside \xAA {'\xBB'}"
137+
rf"outside \xAA {'\xBB'}"
138+
f"{f'hello \xEE'}"
139+
f"{'\N{ox}'}"
140+
f"{'\uFaCe'}"
141+
f"{'\U0001F60E'}"
133142

134143
f'{1}\{{'
135144
f'{2} foo \{{[\}}'
@@ -269,6 +278,15 @@
269278

270279
f"{'\''}"
271280
f"{f'\''}"
281+
f"{'\xff'}"
282+
f"{r'\xFF'}"
283+
f"{'\xFF'=}"
284+
f"outside \xaa {'\xbb'}"
285+
rf"outside \xAA {'\xbb'}"
286+
f"{f'hello \xee'}"
287+
f"{'\N{OX}'}"
288+
f"{'\uface'}"
289+
f"{'\U0001f60e'}"
272290

273291
f"{1}\{{"
274292
f"{2} foo \{{[\}}"

0 commit comments

Comments
 (0)