Skip to content

Commit af8d1f5

Browse files
lysnikolaoudavepecksobolevn
authored
[3.14] gh-132661: Disallow Template/str concatenation after PEP 750 spec update (#135996) (#136901)
Co-authored-by: Dave Peck <davepeck@gmail.com> Co-authored-by: sobolevn <mail@sobolevn.me>
1 parent 0d87bb6 commit af8d1f5

File tree

13 files changed

+1406
-1442
lines changed

13 files changed

+1406
-1442
lines changed

‎Grammar/python.gram

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -985,7 +985,10 @@ tstring[expr_ty] (memo):
985985
_PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }
986986

987987
string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
988-
strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
988+
strings[expr_ty] (memo):
989+
| invalid_string_tstring_concat
990+
| a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
991+
| a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }
989992

990993
list[expr_ty]:
991994
| '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
@@ -1546,6 +1549,12 @@ invalid_tstring_conversion_character:
15461549
| '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
15471550
| '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
15481551

1552+
invalid_string_tstring_concat:
1553+
| a=(fstring|string)+ b[expr_ty]=tstring {
1554+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
1555+
| a=tstring+ b[expr_ty]=(fstring|string) {
1556+
RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
1557+
15491558
invalid_arithmetic:
15501559
| sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
15511560
invalid_factor:

‎Lib/_ast_unparse.py

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix):
626626
)
627627
self._ftstring_helper(fstring_parts)
628628

629-
def _tstring_helper(self, node):
630-
if not node.values:
631-
self._write_ftstring([], "t")
632-
return
633-
last_idx = 0
634-
for i, value in enumerate(node.values):
635-
# This can happen if we have an implicit concat of a t-string
636-
# with an f-string
637-
if isinstance(value, FormattedValue):
638-
if i > last_idx:
639-
# Write t-string until here
640-
self._write_ftstring(node.values[last_idx:i], "t")
641-
self.write(" ")
642-
# Write f-string with the current formatted value
643-
self._write_ftstring([node.values[i]], "f")
644-
if i + 1 < len(node.values):
645-
# Only add a space if there are more values after this
646-
self.write(" ")
647-
last_idx = i + 1
648-
649-
if last_idx < len(node.values):
650-
# Write t-string from last_idx to end
651-
self._write_ftstring(node.values[last_idx:], "t")
652-
653629
def visit_JoinedStr(self, node):
654630
self._write_ftstring(node.values, "f")
655631

656632
def visit_TemplateStr(self, node):
657-
self._tstring_helper(node)
633+
self._write_ftstring(node.values, "t")
658634

659635
def _write_ftstring_inner(self, node, is_format_spec=False):
660636
if isinstance(node, JoinedStr):

‎Lib/test/test_ast/test_ast.py

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -999,13 +999,6 @@ def test_tstring(self):
999999
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
10001000
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
10011001

1002-
# Test AST for implicit concat of t-string with f-string
1003-
tree = ast.parse('t"Hello {name}" f"{name}"')
1004-
self.assertIsInstance(tree.body[0].value, ast.TemplateStr)
1005-
self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
1006-
self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
1007-
self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue)
1008-
10091002

10101003
class CopyTests(unittest.TestCase):
10111004
"""Test copying and pickling AST nodes."""

‎Lib/test/test_tstring.py

Lines changed: 25 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,6 @@ def test_raw_tstrings(self):
150150
t = tr"{path}\Documents"
151151
self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
152152

153-
154153
def test_template_concatenation(self):
155154
# Test template + template
156155
t1 = t"Hello, "
@@ -161,9 +160,10 @@ def test_template_concatenation(self):
161160

162161
# Test template + string
163162
t1 = t"Hello"
164-
combined = t1 + ", world"
165-
self.assertTStringEqual(combined, ("Hello, world",), ())
166-
self.assertEqual(fstring(combined), "Hello, world")
163+
expected_msg = 'can only concatenate string.templatelib.Template ' \
164+
'\\(not "str"\\) to string.templatelib.Template'
165+
with self.assertRaisesRegex(TypeError, expected_msg):
166+
t1 + ", world"
167167

168168
# Test template + template with interpolation
169169
name = "Python"
@@ -174,9 +174,10 @@ def test_template_concatenation(self):
174174
self.assertEqual(fstring(combined), "Hello, Python")
175175

176176
# Test string + template
177-
t = "Hello, " + t"{name}"
178-
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
179-
self.assertEqual(fstring(t), "Hello, Python")
177+
expected_msg = 'can only concatenate str ' \
178+
'\\(not "string.templatelib.Template"\\) to str'
179+
with self.assertRaisesRegex(TypeError, expected_msg):
180+
"Hello, " + t"{name}"
180181

181182
def test_nested_templates(self):
182183
# Test a template inside another template expression
@@ -241,52 +242,28 @@ def test_literal_concatenation(self):
241242
self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
242243
self.assertEqual(fstring(t), "Hello, Python")
243244

244-
# Test concatenation with string literal
245-
name = "Python"
246-
t = t"Hello, {name}" "and welcome!"
247-
self.assertTStringEqual(
248-
t, ("Hello, ", "and welcome!"), [(name, "name")]
249-
)
250-
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
251-
252-
# Test concatenation with Unicode literal
253-
name = "Python"
254-
t = t"Hello, {name}" u"and welcome!"
255-
self.assertTStringEqual(
256-
t, ("Hello, ", "and welcome!"), [(name, "name")]
257-
)
258-
self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
259-
260-
# Test concatenation with f-string literal
261-
tab = '\t'
262-
t = t"Tab: {tab}. " f"f-tab: {tab}."
263-
self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")])
264-
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.")
265-
266-
# Test concatenation with raw string literal
267-
tab = '\t'
268-
t = t"Tab: {tab}. " r"Raw tab: \t."
269-
self.assertTStringEqual(
270-
t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")]
271-
)
272-
self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.")
273-
274-
# Test concatenation with raw f-string literal
275-
tab = '\t'
276-
t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t."
277-
self.assertTStringEqual(
278-
t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")]
279-
)
280-
self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.")
281-
245+
# Test disallowed mix of t-string and string/f-string (incl. bytes)
282246
what = 't'
283-
expected_msg = 'cannot mix bytes and nonbytes literals'
247+
expected_msg = 'cannot mix t-string literals with string or bytes literals'
284248
for case in (
249+
"t'{what}-string literal' 'str literal'",
250+
"t'{what}-string literal' u'unicode literal'",
251+
"t'{what}-string literal' f'f-string literal'",
252+
"t'{what}-string literal' r'raw string literal'",
253+
"t'{what}-string literal' rf'raw f-string literal'",
285254
"t'{what}-string literal' b'bytes literal'",
286255
"t'{what}-string literal' br'raw bytes literal'",
256+
"'str literal' t'{what}-string literal'",
257+
"u'unicode literal' t'{what}-string literal'",
258+
"f'f-string literal' t'{what}-string literal'",
259+
"r'raw string literal' t'{what}-string literal'",
260+
"rf'raw f-string literal' t'{what}-string literal'",
261+
"b'bytes literal' t'{what}-string literal'",
262+
"br'raw bytes literal' t'{what}-string literal'",
287263
):
288-
with self.assertRaisesRegex(SyntaxError, expected_msg):
289-
eval(case)
264+
with self.subTest(case):
265+
with self.assertRaisesRegex(SyntaxError, expected_msg):
266+
eval(case)
290267

291268
def test_triple_quoted(self):
292269
# Test triple-quoted t-strings

‎Lib/test/test_unparse.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -206,10 +206,6 @@ def test_tstrings(self):
206206
self.check_ast_roundtrip("t'foo'")
207207
self.check_ast_roundtrip("t'foo {bar}'")
208208
self.check_ast_roundtrip("t'foo {bar!s:.2f}'")
209-
self.check_ast_roundtrip("t'foo {bar}' f'{bar}'")
210-
self.check_ast_roundtrip("f'{bar}' t'foo {bar}'")
211-
self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'")
212-
self.check_ast_roundtrip("t'foo {bar}' u'bar'")
213209

214210
def test_strings(self):
215211
self.check_ast_roundtrip("u'foo'")

‎Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Reflect recent :pep:`750` change.
2+
3+
Disallow concatenation of ``string.templatelib.Template`` and :class:`str`.
4+
Also, disallow implicit concatenation of t-string literals with string or
5+
f-string literals.

‎Objects/templateobject.c

Lines changed: 7 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ templateiter_next(PyObject *op)
3030
Py_SETREF(item, PyIter_Next(self->interpolationsiter));
3131
self->from_strings = 1;
3232
}
33-
} else {
33+
}
34+
else {
3435
item = PyIter_Next(self->interpolationsiter);
3536
self->from_strings = 1;
3637
}
@@ -245,54 +246,6 @@ template_iter(PyObject *op)
245246
return (PyObject *)iter;
246247
}
247248

248-
static PyObject *
249-
template_strings_append_str(PyObject *strings, PyObject *str)
250-
{
251-
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
252-
PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1);
253-
PyObject *concat = PyUnicode_Concat(string, str);
254-
if (concat == NULL) {
255-
return NULL;
256-
}
257-
258-
PyObject *newstrings = PyTuple_New(stringslen);
259-
if (newstrings == NULL) {
260-
Py_DECREF(concat);
261-
return NULL;
262-
}
263-
264-
for (Py_ssize_t i = 0; i < stringslen - 1; i++) {
265-
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
266-
}
267-
PyTuple_SET_ITEM(newstrings, stringslen - 1, concat);
268-
269-
return newstrings;
270-
}
271-
272-
static PyObject *
273-
template_strings_prepend_str(PyObject *strings, PyObject *str)
274-
{
275-
Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
276-
PyObject *string = PyTuple_GET_ITEM(strings, 0);
277-
PyObject *concat = PyUnicode_Concat(str, string);
278-
if (concat == NULL) {
279-
return NULL;
280-
}
281-
282-
PyObject *newstrings = PyTuple_New(stringslen);
283-
if (newstrings == NULL) {
284-
Py_DECREF(concat);
285-
return NULL;
286-
}
287-
288-
PyTuple_SET_ITEM(newstrings, 0, concat);
289-
for (Py_ssize_t i = 1; i < stringslen; i++) {
290-
PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
291-
}
292-
293-
return newstrings;
294-
}
295-
296249
static PyObject *
297250
template_strings_concat(PyObject *left, PyObject *right)
298251
{
@@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other)
344297
return newtemplate;
345298
}
346299

347-
static PyObject *
348-
template_concat_template_str(templateobject *self, PyObject *other)
349-
{
350-
PyObject *newstrings = template_strings_append_str(self->strings, other);
351-
if (newstrings == NULL) {
352-
return NULL;
353-
}
354-
355-
PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
356-
Py_DECREF(newstrings);
357-
return newtemplate;
358-
}
359-
360-
static PyObject *
361-
template_concat_str_template(templateobject *self, PyObject *other)
362-
{
363-
PyObject *newstrings = template_strings_prepend_str(self->strings, other);
364-
if (newstrings == NULL) {
365-
return NULL;
366-
}
367-
368-
PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
369-
Py_DECREF(newstrings);
370-
return newtemplate;
371-
}
372-
373300
PyObject *
374301
_PyTemplate_Concat(PyObject *self, PyObject *other)
375302
{
376303
if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) {
377304
return template_concat_templates((templateobject *) self, (templateobject *) other);
378305
}
379-
else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) {
380-
return template_concat_template_str((templateobject *) self, other);
381-
}
382-
else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) {
383-
return template_concat_str_template((templateobject *) other, self);
384-
}
385-
else {
386-
Py_RETURN_NOTIMPLEMENTED;
387-
}
306+
307+
PyErr_Format(PyExc_TypeError,
308+
"can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template",
309+
other);
310+
return NULL;
388311
}
389312

390313
static PyObject *

‎Objects/unicodeobject.c

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
5656
#include "pycore_pyhash.h" // _Py_HashSecret_t
5757
#include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding()
5858
#include "pycore_pystate.h" // _PyInterpreterState_GET()
59-
#include "pycore_template.h" // _PyTemplate_Concat()
6059
#include "pycore_tuple.h" // _PyTuple_FromArray()
6160
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
6261
#include "pycore_unicodeobject.h" // struct _Py_unicode_state
@@ -11639,16 +11638,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
1163911638
return NULL;
1164011639

1164111640
if (!PyUnicode_Check(right)) {
11642-
if (_PyTemplate_CheckExact(right)) {
11643-
// str + tstring is implemented in the tstring type
11644-
return _PyTemplate_Concat(left, right);
11645-
}
11646-
else {
11647-
PyErr_Format(PyExc_TypeError,
11648-
"can only concatenate str (not \"%.200s\") to str",
11649-
Py_TYPE(right)->tp_name);
11650-
return NULL;
11651-
}
11641+
PyErr_Format(PyExc_TypeError,
11642+
"can only concatenate str (not \"%.200s\") to str",
11643+
Py_TYPE(right)->tp_name);
11644+
return NULL;
1165211645
}
1165311646

1165411647
/* Shortcuts */

0 commit comments

Comments
 (0)