From 40a9ddad4e5597b0cd68adb168b57f4ffe7b7030 Mon Sep 17 00:00:00 2001 From: Myeongseon Choi Date: Thu, 5 Dec 2024 15:04:56 +0900 Subject: [PATCH 001/317] update test_fstring.py from cpython 3.12.7 add expectedFailure to tag what should rustpython do add comment for some syntaxerror which make test run broken --- Lib/test/test_fstring.py | 1800 +++++++++++++++++++++++++------------- 1 file changed, 1181 insertions(+), 619 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index b1cd9b1604..b3a38dd652 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -8,15 +8,18 @@ # Unicode identifiers in tests is allowed by PEP 3131. import ast +import datetime import os import re import types import decimal import unittest +import warnings +from test import support from test.support.os_helper import temp_cwd -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok -a_global = 'global variable' +a_global = "global variable" # You could argue that I'm too strict in looking for specific error # values with assertRaisesRegex, but without it it's way too easy to @@ -25,6 +28,7 @@ # worthwhile tradeoff. When I switched to this method, I found many # examples where I wasn't testing what I thought I was. + class TestCase(unittest.TestCase): def assertAllRaise(self, exception_type, regex, error_strings): for str in error_strings: @@ -36,43 +40,45 @@ def test__format__lookup(self): # Make sure __format__ is looked up on the type, not the instance. class X: def __format__(self, spec): - return 'class' + return "class" x = X() # Add a bound __format__ method to the 'y' instance, but not # the 'x' instance. y = X() - y.__format__ = types.MethodType(lambda self, spec: 'instance', y) + y.__format__ = types.MethodType(lambda self, spec: "instance", y) - self.assertEqual(f'{y}', format(y)) - self.assertEqual(f'{y}', 'class') + self.assertEqual(f"{y}", format(y)) + self.assertEqual(f"{y}", "class") self.assertEqual(format(x), format(y)) # __format__ is not called this way, but still make sure it # returns what we expect (so we can make sure we're bypassing # it). - self.assertEqual(x.__format__(''), 'class') - self.assertEqual(y.__format__(''), 'instance') + self.assertEqual(x.__format__(""), "class") + self.assertEqual(y.__format__(""), "instance") # This is how __format__ is actually called. - self.assertEqual(type(x).__format__(x, ''), 'class') - self.assertEqual(type(y).__format__(y, ''), 'class') + self.assertEqual(type(x).__format__(x, ""), "class") + self.assertEqual(type(y).__format__(y, ""), "class") def test_ast(self): # Inspired by http://bugs.python.org/issue24975 class X: def __init__(self): self.called = False + def __call__(self): self.called = True return 4 + x = X() expr = """ a = 10 f'{a * x()}'""" t = ast.parse(expr) - c = compile(t, '', 'exec') + c = compile(t, "", "exec") # Make sure x was not called. self.assertFalse(x.called) @@ -278,7 +284,6 @@ def test_ast_line_numbers_duplicate_expression(self): self.assertEqual(binop.right.col_offset, 27) def test_ast_numbers_fstring_with_formatting(self): - t = ast.parse('f"Here is that pesky {xxx:.3f} again"') self.assertEqual(len(t.body), 1) self.assertEqual(t.body[0].lineno, 1) @@ -300,6 +305,8 @@ def test_ast_numbers_fstring_with_formatting(self): self.assertEqual(name.col_offset, 22) self.assertEqual(name.end_col_offset, 25) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_ast_line_numbers_multiline_fstring(self): # See bpo-30465 for details. expr = """ @@ -329,13 +336,13 @@ def test_ast_line_numbers_multiline_fstring(self): self.assertEqual(t.body[1].lineno, 3) self.assertEqual(t.body[1].value.lineno, 3) self.assertEqual(t.body[1].value.values[0].lineno, 3) - self.assertEqual(t.body[1].value.values[1].lineno, 3) - self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 4) + self.assertEqual(t.body[1].value.values[2].lineno, 6) self.assertEqual(t.body[1].col_offset, 0) self.assertEqual(t.body[1].value.col_offset, 0) - self.assertEqual(t.body[1].value.values[0].col_offset, 0) - self.assertEqual(t.body[1].value.values[1].col_offset, 0) - self.assertEqual(t.body[1].value.values[2].col_offset, 0) + self.assertEqual(t.body[1].value.values[0].col_offset, 4) + self.assertEqual(t.body[1].value.values[1].col_offset, 2) + self.assertEqual(t.body[1].value.values[2].col_offset, 11) # NOTE: the following lineno information and col_offset is correct for # expressions within FormattedValues. binop = t.body[1].value.values[1].value @@ -366,19 +373,21 @@ def test_ast_line_numbers_multiline_fstring(self): self.assertEqual(t.body[0].lineno, 2) self.assertEqual(t.body[0].value.lineno, 2) self.assertEqual(t.body[0].value.values[0].lineno, 2) - self.assertEqual(t.body[0].value.values[1].lineno, 2) - self.assertEqual(t.body[0].value.values[2].lineno, 2) + self.assertEqual(t.body[0].value.values[1].lineno, 3) + self.assertEqual(t.body[0].value.values[2].lineno, 3) self.assertEqual(t.body[0].col_offset, 0) self.assertEqual(t.body[0].value.col_offset, 4) - self.assertEqual(t.body[0].value.values[0].col_offset, 4) - self.assertEqual(t.body[0].value.values[1].col_offset, 4) - self.assertEqual(t.body[0].value.values[2].col_offset, 4) + self.assertEqual(t.body[0].value.values[0].col_offset, 8) + self.assertEqual(t.body[0].value.values[1].col_offset, 10) + self.assertEqual(t.body[0].value.values[2].col_offset, 17) # Check {blech} self.assertEqual(t.body[0].value.values[1].value.lineno, 3) self.assertEqual(t.body[0].value.values[1].value.end_lineno, 3) self.assertEqual(t.body[0].value.values[1].value.col_offset, 11) self.assertEqual(t.body[0].value.values[1].value.end_col_offset, 16) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_ast_line_numbers_with_parentheses(self): expr = """ x = ( @@ -387,6 +396,20 @@ def test_ast_line_numbers_with_parentheses(self): t = ast.parse(expr) self.assertEqual(type(t), ast.Module) self.assertEqual(len(t.body), 1) + # check the joinedstr location + joinedstr = t.body[0].value + self.assertEqual(type(joinedstr), ast.JoinedStr) + self.assertEqual(joinedstr.lineno, 3) + self.assertEqual(joinedstr.end_lineno, 3) + self.assertEqual(joinedstr.col_offset, 4) + self.assertEqual(joinedstr.end_col_offset, 17) + # check the formatted value location + fv = t.body[0].value.values[1] + self.assertEqual(type(fv), ast.FormattedValue) + self.assertEqual(fv.lineno, 3) + self.assertEqual(fv.end_lineno, 3) + self.assertEqual(fv.col_offset, 7) + self.assertEqual(fv.end_col_offset, 16) # check the test(t) location call = t.body[0].value.values[1].value self.assertEqual(type(call), ast.Call) @@ -397,6 +420,38 @@ def test_ast_line_numbers_with_parentheses(self): expr = """ x = ( + u'wat', + u"wat", + b'wat', + b"wat", + f'wat', + f"wat", +) + +y = ( + u'''wat''', + u\"\"\"wat\"\"\", + b'''wat''', + b\"\"\"wat\"\"\", + f'''wat''', + f\"\"\"wat\"\"\", +) + """ + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + x, y = t.body + + # Check the single quoted string offsets first. + offsets = [(elt.col_offset, elt.end_col_offset) for elt in x.value.elts] + self.assertTrue(all(offset == (4, 10) for offset in offsets)) + + # Check the triple quoted string offsets. + offsets = [(elt.col_offset, elt.end_col_offset) for elt in y.value.elts] + self.assertTrue(all(offset == (4, 14) for offset in offsets)) + + expr = """ +x = ( 'PERL_MM_OPT', ( f'wat' f'some_string={f(x)} ' @@ -415,9 +470,9 @@ def test_ast_line_numbers_with_parentheses(self): # check the first wat self.assertEqual(type(wat1), ast.Constant) self.assertEqual(wat1.lineno, 4) - self.assertEqual(wat1.end_lineno, 6) - self.assertEqual(wat1.col_offset, 12) - self.assertEqual(wat1.end_col_offset, 18) + self.assertEqual(wat1.end_lineno, 5) + self.assertEqual(wat1.col_offset, 14) + self.assertEqual(wat1.end_col_offset, 26) # check the call call = middle.value self.assertEqual(type(call), ast.Call) @@ -427,427 +482,673 @@ def test_ast_line_numbers_with_parentheses(self): self.assertEqual(call.end_col_offset, 31) # check the second wat self.assertEqual(type(wat2), ast.Constant) - self.assertEqual(wat2.lineno, 4) + self.assertEqual(wat2.lineno, 5) self.assertEqual(wat2.end_lineno, 6) - self.assertEqual(wat2.col_offset, 12) - self.assertEqual(wat2.end_col_offset, 18) + self.assertEqual(wat2.col_offset, 32) + # wat ends at the offset 17, but the whole f-string + # ends at the offset 18 (since the quote is part of the + # f-string but not the wat string) + self.assertEqual(wat2.end_col_offset, 17) + self.assertEqual(fstring.end_col_offset, 18) + + def test_ast_fstring_empty_format_spec(self): + expr = "f'{expr:}'" + + mod = ast.parse(expr) + self.assertEqual(type(mod), ast.Module) + self.assertEqual(len(mod.body), 1) + + fstring = mod.body[0].value + self.assertEqual(type(fstring), ast.JoinedStr) + self.assertEqual(len(fstring.values), 1) + + fv = fstring.values[0] + self.assertEqual(type(fv), ast.FormattedValue) + + format_spec = fv.format_spec + self.assertEqual(type(format_spec), ast.JoinedStr) + self.assertEqual(len(format_spec.values), 0) # TODO: RUSTPYTHON @unittest.expectedFailure def test_docstring(self): def f(): - f'''Not a docstring''' + f"""Not a docstring""" + self.assertIsNone(f.__doc__) + def g(): - '''Not a docstring''' \ - f'' + """Not a docstring""" f"" + self.assertIsNone(g.__doc__) def test_literal_eval(self): - with self.assertRaisesRegex(ValueError, 'malformed node or string'): + with self.assertRaisesRegex(ValueError, "malformed node or string"): ast.literal_eval("f'x'") def test_ast_compile_time_concat(self): - x = [''] + x = [""] expr = """x[0] = 'foo' f'{3}'""" t = ast.parse(expr) - c = compile(t, '', 'exec') + c = compile(t, "", "exec") exec(c) - self.assertEqual(x[0], 'foo3') + self.assertEqual(x[0], "foo3") def test_compile_time_concat_errors(self): - self.assertAllRaise(SyntaxError, - 'cannot mix bytes and nonbytes literals', - [r"""f'' b''""", - r"""b'' f''""", - ]) + self.assertAllRaise( + SyntaxError, + "cannot mix bytes and nonbytes literals", + [ + r"""f'' b''""", + r"""b'' f''""", + ], + ) def test_literal(self): - self.assertEqual(f'', '') - self.assertEqual(f'a', 'a') - self.assertEqual(f' ', ' ') + self.assertEqual(f"", "") + self.assertEqual(f"a", "a") + self.assertEqual(f" ", " ") def test_unterminated_string(self): - self.assertAllRaise(SyntaxError, 'f-string: unterminated string', - [r"""f'{"x'""", - r"""f'{"x}'""", - r"""f'{("x'""", - r"""f'{("x}'""", - ]) - + self.assertAllRaise( + SyntaxError, + "unterminated string", + [ + r"""f'{"x'""", + r"""f'{"x}'""", + r"""f'{("x'""", + r"""f'{("x}'""", + ], + ) + + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): - self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " - r"does not match opening parenthesis '\('", - ["f'{((}'", - ]) - self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\)' " - r"does not match opening parenthesis '\['", - ["f'{a[4)}'", - ]) - self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\]' " - r"does not match opening parenthesis '\('", - ["f'{a(4]}'", - ]) - self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " - r"does not match opening parenthesis '\['", - ["f'{a[4}'", - ]) - self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " - r"does not match opening parenthesis '\('", - ["f'{a(4}'", - ]) - self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") + self.assertAllRaise( + SyntaxError, + r"closing parenthesis '\}' " r"does not match opening parenthesis '\('", + [ + "f'{((}'", + ], + ) + self.assertAllRaise( + SyntaxError, + r"closing parenthesis '\)' " r"does not match opening parenthesis '\['", + [ + "f'{a[4)}'", + ], + ) + self.assertAllRaise( + SyntaxError, + r"closing parenthesis '\]' " r"does not match opening parenthesis '\('", + [ + "f'{a(4]}'", + ], + ) + self.assertAllRaise( + SyntaxError, + r"closing parenthesis '\}' " r"does not match opening parenthesis '\['", + [ + "f'{a[4}'", + ], + ) + self.assertAllRaise( + SyntaxError, + r"closing parenthesis '\}' " r"does not match opening parenthesis '\('", + [ + "f'{a(4}'", + ], + ) + self.assertRaises(SyntaxError, eval, "f'{" + "(" * 500 + "}'") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + def test_fstring_nested_too_deeply(self): + self.assertAllRaise( + SyntaxError, + "f-string: expressions nested too deeply", + ['f"{1+2:{1+2:{1+1:{1}}}}"'], + ) + + def create_nested_fstring(n): + if n == 0: + return "1+1" + prev = create_nested_fstring(n - 1) + return f'f"{{{prev}}}"' + + self.assertAllRaise( + SyntaxError, "too many nested f-strings", [create_nested_fstring(160)] + ) + + def test_syntax_error_in_nested_fstring(self): + # See gh-104016 for more information on this crash + self.assertAllRaise( + SyntaxError, "invalid syntax", ['f"{1 1:' + ('{f"1:' * 199)] + ) def test_double_braces(self): - self.assertEqual(f'{{', '{') - self.assertEqual(f'a{{', 'a{') - self.assertEqual(f'{{b', '{b') - self.assertEqual(f'a{{b', 'a{b') - self.assertEqual(f'}}', '}') - self.assertEqual(f'a}}', 'a}') - self.assertEqual(f'}}b', '}b') - self.assertEqual(f'a}}b', 'a}b') - self.assertEqual(f'{{}}', '{}') - self.assertEqual(f'a{{}}', 'a{}') - self.assertEqual(f'{{b}}', '{b}') - self.assertEqual(f'{{}}c', '{}c') - self.assertEqual(f'a{{b}}', 'a{b}') - self.assertEqual(f'a{{}}c', 'a{}c') - self.assertEqual(f'{{b}}c', '{b}c') - self.assertEqual(f'a{{b}}c', 'a{b}c') - - self.assertEqual(f'{{{10}', '{10') - self.assertEqual(f'}}{10}', '}10') - self.assertEqual(f'}}{{{10}', '}{10') - self.assertEqual(f'}}a{{{10}', '}a{10') - - self.assertEqual(f'{10}{{', '10{') - self.assertEqual(f'{10}}}', '10}') - self.assertEqual(f'{10}}}{{', '10}{') - self.assertEqual(f'{10}}}a{{' '}', '10}a{}') + self.assertEqual(f"{{", "{") + self.assertEqual(f"a{{", "a{") + self.assertEqual(f"{{b", "{b") + self.assertEqual(f"a{{b", "a{b") + self.assertEqual(f"}}", "}") + self.assertEqual(f"a}}", "a}") + self.assertEqual(f"}}b", "}b") + self.assertEqual(f"a}}b", "a}b") + self.assertEqual(f"{{}}", "{}") + self.assertEqual(f"a{{}}", "a{}") + self.assertEqual(f"{{b}}", "{b}") + self.assertEqual(f"{{}}c", "{}c") + self.assertEqual(f"a{{b}}", "a{b}") + self.assertEqual(f"a{{}}c", "a{}c") + self.assertEqual(f"{{b}}c", "{b}c") + self.assertEqual(f"a{{b}}c", "a{b}c") + + self.assertEqual(f"{{{10}", "{10") + self.assertEqual(f"}}{10}", "}10") + self.assertEqual(f"}}{{{10}", "}{10") + self.assertEqual(f"}}a{{{10}", "}a{10") + + self.assertEqual(f"{10}{{", "10{") + self.assertEqual(f"{10}}}", "10}") + self.assertEqual(f"{10}}}{{", "10}{") + self.assertEqual(f"{10}}}a{{" "}", "10}a{}") # Inside of strings, don't interpret doubled brackets. - self.assertEqual(f'{"{{}}"}', '{{}}') + self.assertEqual(f'{"{{}}"}', "{{}}") - self.assertAllRaise(TypeError, 'unhashable type', - ["f'{ {{}} }'", # dict in a set - ]) + self.assertAllRaise( + TypeError, + "unhashable type", + [ + "f'{ {{}} }'", # dict in a set + ], + ) def test_compile_time_concat(self): - x = 'def' - self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') - self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi') - self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ') - self.assertEqual('{x}' f'{x}', '{x}def') - self.assertEqual('{x' f'{x}', '{xdef') - self.assertEqual('{x}' f'{x}', '{x}def') - self.assertEqual('{{x}}' f'{x}', '{{x}}def') - self.assertEqual('{{x' f'{x}', '{{xdef') - self.assertEqual('x}}' f'{x}', 'x}}def') - self.assertEqual(f'{x}' 'x}}', 'defx}}') - self.assertEqual(f'{x}' '', 'def') - self.assertEqual('' f'{x}' '', 'def') - self.assertEqual('' f'{x}', 'def') - self.assertEqual(f'{x}' '2', 'def2') - self.assertEqual('1' f'{x}' '2', '1def2') - self.assertEqual('1' f'{x}', '1def') - self.assertEqual(f'{x}' f'-{x}', 'def-def') - self.assertEqual('' f'', '') - self.assertEqual('' f'' '', '') - self.assertEqual('' f'' '' f'', '') - self.assertEqual(f'', '') - self.assertEqual(f'' '', '') - self.assertEqual(f'' '' f'', '') - self.assertEqual(f'' '' f'' '', '') - - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", - ["f'{3' f'}'", # can't concat to get a valid f-string - ]) + x = "def" + self.assertEqual("abc" f"## {x}ghi", "abc## defghi") + self.assertEqual("abc" f"{x}" "ghi", "abcdefghi") + self.assertEqual("abc" f"{x}" "gh" f"i{x:4}", "abcdefghidef ") + self.assertEqual("{x}" f"{x}", "{x}def") + self.assertEqual("{x" f"{x}", "{xdef") + self.assertEqual("{x}" f"{x}", "{x}def") + self.assertEqual("{{x}}" f"{x}", "{{x}}def") + self.assertEqual("{{x" f"{x}", "{{xdef") + self.assertEqual("x}}" f"{x}", "x}}def") + self.assertEqual(f"{x}" "x}}", "defx}}") + self.assertEqual(f"{x}" "", "def") + self.assertEqual("" f"{x}" "", "def") + self.assertEqual("" f"{x}", "def") + self.assertEqual(f"{x}" "2", "def2") + self.assertEqual("1" f"{x}" "2", "1def2") + self.assertEqual("1" f"{x}", "1def") + self.assertEqual(f"{x}" f"-{x}", "def-def") + self.assertEqual("" f"", "") + self.assertEqual("" f"" "", "") + self.assertEqual("" f"" "" f"", "") + self.assertEqual(f"", "") + self.assertEqual(f"" "", "") + self.assertEqual(f"" "" f"", "") + self.assertEqual(f"" "" f"" "", "") + + # This is not really [f'{'] + [f'}'] since we treat the inside + # of braces as a purely new context, so it is actually f'{ and + # then eval(' f') (a valid expression) and then }' which would + # constitute a valid f-string. + # TODO: RUSTPYTHON SyntaxError + # self.assertEqual(f'{' f'}', " f") + + self.assertAllRaise( + SyntaxError, + "expecting '}'", + [ + '''f'{3' f"}"''', # can't concat to get a valid f-string + ], + ) # TODO: RUSTPYTHON @unittest.expectedFailure def test_comments(self): # These aren't comments, since they're in strings. - d = {'#': 'hash'} - self.assertEqual(f'{"#"}', '#') - self.assertEqual(f'{d["#"]}', 'hash') - - self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'", - ["f'{1#}'", # error because the expression becomes "(1#)" - "f'{3(#)}'", - "f'{#}'", - ]) - self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", - ["f'{)#}'", # When wrapped in parens, this becomes - # '()#)'. Make sure that doesn't compile. - ]) + d = {"#": "hash"} + self.assertEqual(f'{"#"}', "#") + self.assertEqual(f'{d["#"]}', "hash") + + self.assertAllRaise( + SyntaxError, + "'{' was never closed", + [ + "f'{1#}'", # error because everything after '#' is a comment + "f'{#}'", + "f'one: {1#}'", + "f'{1# one} {2 this is a comment still#}'", + ], + ) + self.assertAllRaise( + SyntaxError, + r"f-string: unmatched '\)'", + [ + "f'{)#}'", # When wrapped in parens, this becomes + # '()#)'. Make sure that doesn't compile. + ], + ) + self.assertEqual( + f"""A complex trick: { +2 # two +}""", + "A complex trick: 2", + ) + self.assertEqual( + f""" +{ +40 # forty ++ # plus +2 # two +}""", + "\n42", + ) + self.assertEqual( + f""" +{ +40 # forty ++ # plus +2 # two +}""", + "\n42", + ) +# TODO: RUSTPYTHON SyntaxError +# self.assertEqual( +# f""" +# # this is not a comment +# { # the following operation it's +# 3 # this is a number +# * 2}""", +# "\n# this is not a comment\n6", +# ) + self.assertEqual( + f""" +{# f'a {comment}' +86 # constant +# nothing more +}""", + "\n86", + ) + + self.assertAllRaise( + SyntaxError, + r"f-string: valid expression required before '}'", + [ + """f''' +{ +# only a comment +}''' +""", # this is equivalent to f'{}' + ], + ) def test_many_expressions(self): # Create a string with many expressions in it. Note that # because we have a space in here as a literal, we're actually # going to use twice as many ast nodes: one for each literal # plus one for each expression. - def build_fstr(n, extra=''): - return "f'" + ('{x} ' * n) + extra + "'" + def build_fstr(n, extra=""): + return "f'" + ("{x} " * n) + extra + "'" - x = 'X' + x = "X" width = 1 # Test around 256. for i in range(250, 260): - self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + self.assertEqual(eval(build_fstr(i)), (x + " ") * i) # Test concatenating 2 largs fstrings. - self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) + self.assertEqual(eval(build_fstr(255) * 256), (x + " ") * (255 * 256)) - s = build_fstr(253, '{x:{width}} ') - self.assertEqual(eval(s), (x+' ')*254) + s = build_fstr(253, "{x:{width}} ") + self.assertEqual(eval(s), (x + " ") * 254) # Test lots of expressions and constants, concatenated. s = "f'{1}' 'x' 'y'" * 1024 - self.assertEqual(eval(s), '1xy' * 1024) + self.assertEqual(eval(s), "1xy" * 1024) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_format_specifier_expressions(self): width = 10 precision = 4 - value = decimal.Decimal('12.34567') - self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') - self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') - self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') - self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') - self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') - self.assertEqual(f'{10:#{1}0x}', ' 0xa') - self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') - self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') - self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') - self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') - - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", - ["""f'{"s"!r{":10"}}'""", - - # This looks like a nested format spec. - ]) - - self.assertAllRaise(SyntaxError, "f-string: invalid syntax", - [# Invalid syntax inside a nested spec. - "f'{4:{/5}}'", - ]) - - self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", - [# Can't nest format specifiers. - "f'result: {value:{width:{0}}.{precision:1}}'", - ]) - - self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', - [# No expansion inside conversion or for - # the : or ! itself. - """f'{"s"!{"r"}}'""", - ]) + value = decimal.Decimal("12.34567") + self.assertEqual(f"result: {value:{width}.{precision}}", "result: 12.35") + self.assertEqual(f"result: {value:{width!r}.{precision}}", "result: 12.35") + self.assertEqual( + f"result: {value:{width:0}.{precision:1}}", "result: 12.35" + ) + self.assertEqual( + f"result: {value:{1}{0:0}.{precision:1}}", "result: 12.35" + ) + self.assertEqual( + f"result: {value:{ 1}{ 0:0}.{ precision:1}}", "result: 12.35" + ) + self.assertEqual(f"{10:#{1}0x}", " 0xa") + self.assertEqual(f'{10:{"#"}1{0}{"x"}}', " 0xa") + self.assertEqual(f'{-10:-{"#"}1{0}x}', " -0xa") + self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', " -0xa") + self.assertEqual(f"{10:#{3 != {4:5} and width}x}", " 0xa") + + # TODO: RUSTPYTHON SyntaxError + # self.assertEqual( + # f"result: {value:{width:{0}}.{precision:1}}", "result: 12.35" + # ) + + + self.assertAllRaise( + SyntaxError, + "f-string: expecting ':' or '}'", + [ + """f'{"s"!r{":10"}}'""", + # This looks like a nested format spec. + ], + ) + + + self.assertAllRaise( + SyntaxError, + "f-string: expecting a valid expression after '{'", + [ # Invalid syntax inside a nested spec. + "f'{4:{/5}}'", + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: invalid conversion character", + [ # No expansion inside conversion or for + # the : or ! itself. + """f'{"s"!{"r"}}'""", + ], + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_custom_format_specifier(self): + class CustomFormat: + def __format__(self, format_spec): + return format_spec + + self.assertEqual(f"{CustomFormat():\n}", "\n") + self.assertEqual(f"{CustomFormat():\u2603}", "☃") + with self.assertWarns(SyntaxWarning): + exec(r'f"{F():¯\_(ツ)_/¯}"', {"F": CustomFormat}) def test_side_effect_order(self): class X: def __init__(self): self.i = 0 + def __format__(self, spec): self.i += 1 return str(self.i) x = X() - self.assertEqual(f'{x} {x}', '1 2') + self.assertEqual(f"{x} {x}", "1 2") # TODO: RUSTPYTHON @unittest.expectedFailure def test_missing_expression(self): - self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', - ["f'{}'", - "f'{ }'" - "f' {} '", - "f'{10:{ }}'", - "f' { } '", - - # The Python parser ignores also the following - # whitespace characters in additional to a space. - "f'''{\t\f\r\n}'''", - ]) - - # Different error messeges are raised when a specfier ('!', ':' or '=') is used after an empty expression - self.assertAllRaise(SyntaxError, "f-string: expression required before '!'", - ["f'{!r}'", - "f'{ !r}'", - "f'{!}'", - "f'''{\t\f\r\n!a}'''", - - # Catch empty expression before the - # missing closing brace. - "f'{!'", - "f'{!s:'", - - # Catch empty expression before the - # invalid conversion. - "f'{!x}'", - "f'{ !xr}'", - "f'{!x:}'", - "f'{!x:a}'", - "f'{ !xr:}'", - "f'{ !xr:a}'", - ]) - - self.assertAllRaise(SyntaxError, "f-string: expression required before ':'", - ["f'{:}'", - "f'{ :!}'", - "f'{:2}'", - "f'''{\t\f\r\n:a}'''", - "f'{:'", - ]) - - self.assertAllRaise(SyntaxError, "f-string: expression required before '='", - ["f'{=}'", - "f'{ =}'", - "f'{ =:}'", - "f'{ =!}'", - "f'''{\t\f\r\n=}'''", - "f'{='", - ]) + self.assertAllRaise( + SyntaxError, + "f-string: valid expression required before '}'", + [ + "f'{}'", + "f'{ }'" "f' {} '", + "f'{10:{ }}'", + "f' { } '", + # The Python parser ignores also the following + # whitespace characters in additional to a space. + "f'''{\t\f\r\n}'''", + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: valid expression required before '!'", + [ + "f'{!r}'", + "f'{ !r}'", + "f'{!}'", + "f'''{\t\f\r\n!a}'''", + # Catch empty expression before the + # missing closing brace. + "f'{!'", + "f'{!s:'", + # Catch empty expression before the + # invalid conversion. + "f'{!x}'", + "f'{ !xr}'", + "f'{!x:}'", + "f'{!x:a}'", + "f'{ !xr:}'", + "f'{ !xr:a}'", + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: valid expression required before ':'", + [ + "f'{:}'", + "f'{ :!}'", + "f'{:2}'", + "f'''{\t\f\r\n:a}'''", + "f'{:'", + "F'{[F'{:'}[F'{:'}]]]", + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: valid expression required before '='", + [ + "f'{=}'", + "f'{ =}'", + "f'{ =:}'", + "f'{ =!}'", + "f'''{\t\f\r\n=}'''", + "f'{='", + ], + ) # Different error message is raised for other whitespace characters. - self.assertAllRaise(SyntaxError, r"invalid non-printable character U\+00A0", - ["f'''{\xa0}'''", - "\xa0", - ]) + self.assertAllRaise( + SyntaxError, + r"invalid non-printable character U\+00A0", + [ + "f'''{\xa0}'''", + "\xa0", + ], + ) # TODO: RUSTPYTHON @unittest.expectedFailure def test_parens_in_expressions(self): - self.assertEqual(f'{3,}', '(3,)') - - # Add these because when an expression is evaluated, parens - # are added around it. But we shouldn't go from an invalid - # expression to a valid one. The added parens are just - # supposed to allow whitespace (including newlines). - self.assertAllRaise(SyntaxError, 'f-string: invalid syntax', - ["f'{,}'", - "f'{,}'", # this is (,), which is an error - ]) - - self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", - ["f'{3)+(4}'", - ]) - - self.assertAllRaise(SyntaxError, 'unterminated string literal', - ["f'{\n}'", - ]) + self.assertEqual(f"{3,}", "(3,)") + + self.assertAllRaise( + SyntaxError, + "f-string: expecting a valid expression after '{'", + [ + "f'{,}'", + ], + ) + + self.assertAllRaise( + SyntaxError, + r"f-string: unmatched '\)'", + [ + "f'{3)+(4}'", + ], + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_newlines_before_syntax_error(self): - self.assertAllRaise(SyntaxError, "invalid syntax", - ["f'{.}'", "\nf'{.}'", "\n\nf'{.}'"]) + self.assertAllRaise( + SyntaxError, + "f-string: expecting a valid expression after '{'", + ["f'{.}'", "\nf'{.}'", "\n\nf'{.}'"], + ) # TODO: RUSTPYTHON @unittest.expectedFailure def test_backslashes_in_string_part(self): - self.assertEqual(f'\t', '\t') - self.assertEqual(r'\t', '\\t') - self.assertEqual(rf'\t', '\\t') - self.assertEqual(f'{2}\t', '2\t') - self.assertEqual(f'{2}\t{3}', '2\t3') - self.assertEqual(f'\t{3}', '\t3') - - self.assertEqual(f'\u0394', '\u0394') - self.assertEqual(r'\u0394', '\\u0394') - self.assertEqual(rf'\u0394', '\\u0394') - self.assertEqual(f'{2}\u0394', '2\u0394') - self.assertEqual(f'{2}\u0394{3}', '2\u03943') - self.assertEqual(f'\u0394{3}', '\u03943') - - self.assertEqual(f'\U00000394', '\u0394') - self.assertEqual(r'\U00000394', '\\U00000394') - self.assertEqual(rf'\U00000394', '\\U00000394') - self.assertEqual(f'{2}\U00000394', '2\u0394') - self.assertEqual(f'{2}\U00000394{3}', '2\u03943') - self.assertEqual(f'\U00000394{3}', '\u03943') - - self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', '\u0394') - self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}', '2\u0394') - self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}{3}', '2\u03943') - self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}{3}', '\u03943') - self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}', '2\u0394') - self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}3', '2\u03943') - self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}3', '\u03943') - - self.assertEqual(f'\x20', ' ') - self.assertEqual(r'\x20', '\\x20') - self.assertEqual(rf'\x20', '\\x20') - self.assertEqual(f'{2}\x20', '2 ') - self.assertEqual(f'{2}\x20{3}', '2 3') - self.assertEqual(f'\x20{3}', ' 3') - - self.assertEqual(f'2\x20', '2 ') - self.assertEqual(f'2\x203', '2 3') - self.assertEqual(f'\x203', ' 3') - - with self.assertWarns(DeprecationWarning): # invalid escape sequence + self.assertEqual(f"\t", "\t") + self.assertEqual(r"\t", "\\t") + self.assertEqual(rf"\t", "\\t") + self.assertEqual(f"{2}\t", "2\t") + self.assertEqual(f"{2}\t{3}", "2\t3") + self.assertEqual(f"\t{3}", "\t3") + + self.assertEqual(f"\u0394", "\u0394") + self.assertEqual(r"\u0394", "\\u0394") + self.assertEqual(rf"\u0394", "\\u0394") + self.assertEqual(f"{2}\u0394", "2\u0394") + self.assertEqual(f"{2}\u0394{3}", "2\u03943") + self.assertEqual(f"\u0394{3}", "\u03943") + + self.assertEqual(f"\U00000394", "\u0394") + self.assertEqual(r"\U00000394", "\\U00000394") + self.assertEqual(rf"\U00000394", "\\U00000394") + self.assertEqual(f"{2}\U00000394", "2\u0394") + self.assertEqual(f"{2}\U00000394{3}", "2\u03943") + self.assertEqual(f"\U00000394{3}", "\u03943") + + self.assertEqual(f"\N{GREEK CAPITAL LETTER DELTA}", "\u0394") + self.assertEqual(f"{2}\N{GREEK CAPITAL LETTER DELTA}", "2\u0394") + self.assertEqual(f"{2}\N{GREEK CAPITAL LETTER DELTA}{3}", "2\u03943") + self.assertEqual(f"\N{GREEK CAPITAL LETTER DELTA}{3}", "\u03943") + self.assertEqual(f"2\N{GREEK CAPITAL LETTER DELTA}", "2\u0394") + self.assertEqual(f"2\N{GREEK CAPITAL LETTER DELTA}3", "2\u03943") + self.assertEqual(f"\N{GREEK CAPITAL LETTER DELTA}3", "\u03943") + + self.assertEqual(f"\x20", " ") + self.assertEqual(r"\x20", "\\x20") + self.assertEqual(rf"\x20", "\\x20") + self.assertEqual(f"{2}\x20", "2 ") + self.assertEqual(f"{2}\x20{3}", "2 3") + self.assertEqual(f"\x20{3}", " 3") + + self.assertEqual(f"2\x20", "2 ") + self.assertEqual(f"2\x203", "2 3") + self.assertEqual(f"\x203", " 3") + + with self.assertWarns(SyntaxWarning): # invalid escape sequence value = eval(r"f'\{6*7}'") - self.assertEqual(value, '\\42') - self.assertEqual(f'\\{6*7}', '\\42') - self.assertEqual(fr'\{6*7}', '\\42') - - AMPERSAND = 'spam' + self.assertEqual(value, "\\42") + with self.assertWarns(SyntaxWarning): # invalid escape sequence + value = eval(r"f'\g'") + self.assertEqual(value, "\\g") + self.assertEqual(f"\\{6*7}", "\\42") + self.assertEqual(rf"\{6*7}", "\\42") + + AMPERSAND = "spam" # Get the right unicode character (&), or pick up local variable # depending on the number of backslashes. - self.assertEqual(f'\N{AMPERSAND}', '&') - self.assertEqual(f'\\N{AMPERSAND}', '\\Nspam') - self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam') - self.assertEqual(f'\\\N{AMPERSAND}', '\\&') + self.assertEqual(f"\N{AMPERSAND}", "&") + self.assertEqual(f"\\N{AMPERSAND}", "\\Nspam") + self.assertEqual(rf"\N{AMPERSAND}", "\\Nspam") + self.assertEqual(f"\\\N{AMPERSAND}", "\\&") # TODO: RUSTPYTHON @unittest.expectedFailure def test_misformed_unicode_character_name(self): # These test are needed because unicode names are parsed # differently inside f-strings. - self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape", - [r"f'\N'", - r"f'\N '", - r"f'\N '", # See bpo-46503. - r"f'\N{'", - r"f'\N{GREEK CAPITAL LETTER DELTA'", - - # Here are the non-f-string versions, - # which should give the same errors. - r"'\N'", - r"'\N '", - r"'\N '", - r"'\N{'", - r"'\N{GREEK CAPITAL LETTER DELTA'", - ]) + self.assertAllRaise( + SyntaxError, + r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape", + [ + r"f'\N'", + r"f'\N '", + r"f'\N '", # See bpo-46503. + r"f'\N{'", + r"f'\N{GREEK CAPITAL LETTER DELTA'", + # Here are the non-f-string versions, + # which should give the same errors. + r"'\N'", + r"'\N '", + r"'\N '", + r"'\N{'", + r"'\N{GREEK CAPITAL LETTER DELTA'", + ], + ) # TODO: RUSTPYTHON @unittest.expectedFailure - def test_no_backslashes_in_expression_part(self): - self.assertAllRaise(SyntaxError, 'f-string expression part cannot include a backslash', - [r"f'{\'a\'}'", - r"f'{\t3}'", - r"f'{\}'", - r"rf'{\'a\'}'", - r"rf'{\t3}'", - r"rf'{\}'", - r"""rf'{"\N{LEFT CURLY BRACKET}"}'""", - r"f'{\n}'", - ]) + def test_backslashes_in_expression_part(self): + # TODO: RUSTPYTHON SyntaxError + # self.assertEqual( + # f"{( + # 1 + + # 2 + # )}", + # "3", + # ) + + self.assertEqual("\N{LEFT CURLY BRACKET}", "{") + self.assertEqual(f'{"\N{LEFT CURLY BRACKET}"}', "{") + self.assertEqual(rf'{"\N{LEFT CURLY BRACKET}"}', "{") + + self.assertAllRaise( + SyntaxError, + "f-string: valid expression required before '}'", + [ + "f'{\n}'", + ], + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_invalid_backslashes_inside_fstring_context(self): + # All of these variations are invalid python syntax, + # so they are also invalid in f-strings as well. + cases = [ + formatting.format(expr=expr) + for formatting in [ + "{expr}", + "f'{{{expr}}}'", + "rf'{{{expr}}}'", + ] + for expr in [ + r"\'a\'", + r"\t3", + r"\\"[0], + ] + ] + self.assertAllRaise( + SyntaxError, "unexpected character after line continuation", cases + ) def test_no_escapes_for_braces(self): """ Only literal curly braces begin an expression. """ # \x7b is '{'. - self.assertEqual(f'\x7b1+1}}', '{1+1}') - self.assertEqual(f'\x7b1+1', '{1+1') - self.assertEqual(f'\u007b1+1', '{1+1') - self.assertEqual(f'\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}', '{1+1}') + self.assertEqual(f"\x7b1+1}}", "{1+1}") + self.assertEqual(f"\x7b1+1", "{1+1") + self.assertEqual(f"\u007b1+1", "{1+1") + self.assertEqual(f"\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}", "{1+1}") def test_newlines_in_expressions(self): - self.assertEqual(f'{0}', '0') - self.assertEqual(rf'''{3+ -4}''', '7') + self.assertEqual(f"{0}", "0") + self.assertEqual( + rf"""{3+ +4}""", + "7", + ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_lambda(self): x = 5 self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") @@ -855,17 +1156,99 @@ def test_lambda(self): self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888 ") # lambda doesn't work without parens, because the colon - # makes the parser think it's a format_spec - self.assertAllRaise(SyntaxError, 'f-string: invalid syntax', - ["f'{lambda x:x}'", - ]) + # makes the parser think it's a format_spec + # emit warning if we can match a format_spec + self.assertAllRaise( + SyntaxError, + "f-string: lambda expressions are not allowed " "without parentheses", + [ + "f'{lambda x:x}'", + "f'{lambda :x}'", + "f'{lambda *arg, :x}'", + "f'{1, lambda:x}'", + "f'{lambda x:}'", + "f'{lambda :}'", + ], + ) + # Ensure the detection of invalid lambdas doesn't trigger detection + # for valid lambdas in the second error pass + with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + compile("lambda name_3=f'{name_4}': {name_3}\n1 $ 1", "", "exec") + + # but don't emit the paren warning in general cases + with self.assertRaisesRegex( + SyntaxError, "f-string: expecting a valid expression after '{'" + ): + eval("f'{+ lambda:None}'") + + def test_valid_prefixes(self): + self.assertEqual(f"{1}", "1") + self.assertEqual(Rf"{2}", "2") + self.assertEqual(Rf"{3}", "3") + + def test_roundtrip_raw_quotes(self): + self.assertEqual(rf"\'", "\\'") + self.assertEqual(rf"\"", '\\"') + self.assertEqual(rf"\"\'", "\\\"\\'") + self.assertEqual(rf"\'\"", "\\'\\\"") + self.assertEqual(rf"\"\'\"", '\\"\\\'\\"') + self.assertEqual(rf"\'\"\'", "\\'\\\"\\'") + self.assertEqual(rf"\"\'\"\'", "\\\"\\'\\\"\\'") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_fstring_backslash_before_double_bracket(self): + deprecated_cases = [ + (r"f'\{{\}}'", "\\{\\}"), + (r"f'\{{'", "\\{"), + (r"f'\{{{1+1}'", "\\{2"), + (r"f'\}}{1+1}'", "\\}2"), + (r"f'{1+1}\}}'", "2\\}"), + ] + + for case, expected_result in deprecated_cases: + with self.subTest(case=case, expected_result=expected_result): + with self.assertWarns(SyntaxWarning): + result = eval(case) + self.assertEqual(result, expected_result) + self.assertEqual(rf"\{{\}}", "\\{\\}") + self.assertEqual(rf"\{{", "\\{") + self.assertEqual(rf"\{{{1+1}", "\\{2") + self.assertEqual(rf"\}}{1+1}", "\\}2") + self.assertEqual(rf"{1+1}\}}", "2\\}") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_fstring_backslash_before_double_bracket_warns_once(self): + with self.assertWarns(SyntaxWarning) as w: + eval(r"f'\{{'") + self.assertEqual(len(w.warnings), 1) + self.assertEqual(w.warnings[0].category, SyntaxWarning) + + def test_fstring_backslash_prefix_raw(self): + self.assertEqual(f"\\", "\\") + self.assertEqual(f"\\\\", "\\\\") + self.assertEqual(rf"\\", r"\\") + self.assertEqual(rf"\\\\", r"\\\\") + self.assertEqual(rf"\\", r"\\") + self.assertEqual(rf"\\\\", r"\\\\") + self.assertEqual(Rf"\\", R"\\") + self.assertEqual(Rf"\\\\", R"\\\\") + self.assertEqual(Rf"\\", R"\\") + self.assertEqual(Rf"\\\\", R"\\\\") + self.assertEqual(Rf"\\", R"\\") + self.assertEqual(Rf"\\\\", R"\\\\") + + def test_fstring_format_spec_greedy_matching(self): + self.assertEqual(f"{1:}}}", "1}") + self.assertEqual(f"{1:>3{5}}}}", " 1}") def test_yield(self): # Not terribly useful, but make sure the yield turns # a function into a generator def fn(y): - f'y:{yield y*2}' - f'{yield}' + f"y:{yield y*2}" + f"{yield}" g = fn(4) self.assertEqual(next(g), 8) @@ -873,257 +1256,329 @@ def fn(y): def test_yield_send(self): def fn(x): - yield f'x:{yield (lambda i: x * i)}' + yield f"x:{yield (lambda i: x * i)}" g = fn(10) the_lambda = next(g) self.assertEqual(the_lambda(4), 40) - self.assertEqual(g.send('string'), 'x:string') + self.assertEqual(g.send("string"), "x:string") - def test_expressions_with_triple_quoted_strings(self): - self.assertEqual(f"{'''x'''}", 'x') - # TODO: RUSTPYTHON self.assertEqual(f"{'''eric's'''}", "eric's") + # TODO: RUSTPYTHON SyntaxError + # def test_expressions_with_triple_quoted_strings(self): + # self.assertEqual(f"{'''x'''}", 'x') + # self.assertEqual(f"{'''eric's'''}", "eric's") - # Test concatenation within an expression - # TODO: RUSTPYTHON self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') - # TODO: RUSTPYTHON self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') - # TODO: RUSTPYTHON self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') - # TODO: RUSTPYTHON self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') - # TODO: RUSTPYTHON self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') - # TODO: RUSTPYTHON self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') + # # Test concatenation within an expression + # self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') + # self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') + # self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') + # self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') + # self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') + # self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') def test_multiple_vars(self): x = 98 - y = 'abc' - self.assertEqual(f'{x}{y}', '98abc') + y = "abc" + self.assertEqual(f"{x}{y}", "98abc") - self.assertEqual(f'X{x}{y}', 'X98abc') - self.assertEqual(f'{x}X{y}', '98Xabc') - self.assertEqual(f'{x}{y}X', '98abcX') + self.assertEqual(f"X{x}{y}", "X98abc") + self.assertEqual(f"{x}X{y}", "98Xabc") + self.assertEqual(f"{x}{y}X", "98abcX") - self.assertEqual(f'X{x}Y{y}', 'X98Yabc') - self.assertEqual(f'X{x}{y}Y', 'X98abcY') - self.assertEqual(f'{x}X{y}Y', '98XabcY') + self.assertEqual(f"X{x}Y{y}", "X98Yabc") + self.assertEqual(f"X{x}{y}Y", "X98abcY") + self.assertEqual(f"{x}X{y}Y", "98XabcY") - self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ') + self.assertEqual(f"X{x}Y{y}Z", "X98YabcZ") def test_closure(self): def outer(x): def inner(): - return f'x:{x}' + return f"x:{x}" + return inner - self.assertEqual(outer('987')(), 'x:987') - self.assertEqual(outer(7)(), 'x:7') + self.assertEqual(outer("987")(), "x:987") + self.assertEqual(outer(7)(), "x:7") def test_arguments(self): y = 2 + def f(x, width): - return f'x={x*y:{width}}' + return f"x={x*y:{width}}" - self.assertEqual(f('foo', 10), 'x=foofoo ') - x = 'bar' - self.assertEqual(f(10, 10), 'x= 20') + self.assertEqual(f("foo", 10), "x=foofoo ") + x = "bar" + self.assertEqual(f(10, 10), "x= 20") def test_locals(self): value = 123 - self.assertEqual(f'v:{value}', 'v:123') + self.assertEqual(f"v:{value}", "v:123") def test_missing_variable(self): with self.assertRaises(NameError): - f'v:{value}' + f"v:{value}" def test_missing_format_spec(self): class O: def __format__(self, spec): if not spec: - return '*' + return "*" return spec - self.assertEqual(f'{O():x}', 'x') - self.assertEqual(f'{O()}', '*') - self.assertEqual(f'{O():}', '*') + self.assertEqual(f"{O():x}", "x") + self.assertEqual(f"{O()}", "*") + self.assertEqual(f"{O():}", "*") - self.assertEqual(f'{3:}', '3') - self.assertEqual(f'{3!s:}', '3') + self.assertEqual(f"{3:}", "3") + self.assertEqual(f"{3!s:}", "3") def test_global(self): - self.assertEqual(f'g:{a_global}', 'g:global variable') - self.assertEqual(f'g:{a_global!r}', "g:'global variable'") + self.assertEqual(f"g:{a_global}", "g:global variable") + self.assertEqual(f"g:{a_global!r}", "g:'global variable'") - a_local = 'local variable' - self.assertEqual(f'g:{a_global} l:{a_local}', - 'g:global variable l:local variable') - self.assertEqual(f'g:{a_global!r}', - "g:'global variable'") - self.assertEqual(f'g:{a_global} l:{a_local!r}', - "g:global variable l:'local variable'") + a_local = "local variable" + self.assertEqual( + f"g:{a_global} l:{a_local}", "g:global variable l:local variable" + ) + self.assertEqual(f"g:{a_global!r}", "g:'global variable'") + self.assertEqual( + f"g:{a_global} l:{a_local!r}", "g:global variable l:'local variable'" + ) - self.assertIn("module 'unittest' from", f'{unittest}') + self.assertIn("module 'unittest' from", f"{unittest}") def test_shadowed_global(self): - a_global = 'really a local' - self.assertEqual(f'g:{a_global}', 'g:really a local') - self.assertEqual(f'g:{a_global!r}', "g:'really a local'") - - a_local = 'local variable' - self.assertEqual(f'g:{a_global} l:{a_local}', - 'g:really a local l:local variable') - self.assertEqual(f'g:{a_global!r}', - "g:'really a local'") - self.assertEqual(f'g:{a_global} l:{a_local!r}', - "g:really a local l:'local variable'") + a_global = "really a local" + self.assertEqual(f"g:{a_global}", "g:really a local") + self.assertEqual(f"g:{a_global!r}", "g:'really a local'") + + a_local = "local variable" + self.assertEqual( + f"g:{a_global} l:{a_local}", "g:really a local l:local variable" + ) + self.assertEqual(f"g:{a_global!r}", "g:'really a local'") + self.assertEqual( + f"g:{a_global} l:{a_local!r}", "g:really a local l:'local variable'" + ) def test_call(self): def foo(x): - return 'x=' + str(x) + return "x=" + str(x) - self.assertEqual(f'{foo(10)}', 'x=10') + self.assertEqual(f"{foo(10)}", "x=10") def test_nested_fstrings(self): y = 5 - self.assertEqual(f'{f"{0}"*3}', '000') - self.assertEqual(f'{f"{y}"*3}', '555') + self.assertEqual(f'{f"{0}"*3}', "000") + self.assertEqual(f'{f"{y}"*3}', "555") def test_invalid_string_prefixes(self): - single_quote_cases = ["fu''", - "uf''", - "Fu''", - "fU''", - "Uf''", - "uF''", - "ufr''", - "urf''", - "fur''", - "fru''", - "rfu''", - "ruf''", - "FUR''", - "Fur''", - "fb''", - "fB''", - "Fb''", - "FB''", - "bf''", - "bF''", - "Bf''", - "BF''",] + single_quote_cases = [ + "fu''", + "uf''", + "Fu''", + "fU''", + "Uf''", + "uF''", + "ufr''", + "urf''", + "fur''", + "fru''", + "rfu''", + "ruf''", + "FUR''", + "Fur''", + "fb''", + "fB''", + "Fb''", + "FB''", + "bf''", + "bF''", + "Bf''", + "BF''", + ] double_quote_cases = [case.replace("'", '"') for case in single_quote_cases] - self.assertAllRaise(SyntaxError, 'invalid syntax', - single_quote_cases + double_quote_cases) + self.assertAllRaise( + SyntaxError, "invalid syntax", single_quote_cases + double_quote_cases + ) def test_leading_trailing_spaces(self): - self.assertEqual(f'{ 3}', '3') - self.assertEqual(f'{ 3}', '3') - self.assertEqual(f'{3 }', '3') - self.assertEqual(f'{3 }', '3') + self.assertEqual(f"{ 3}", "3") + self.assertEqual(f"{ 3}", "3") + self.assertEqual(f"{3 }", "3") + self.assertEqual(f"{3 }", "3") - self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}', - 'expr={1: 2}') - self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }', - 'expr={1: 2}') + self.assertEqual(f"expr={ {x: y for x, y in [(1, 2), ]}}", "expr={1: 2}") + self.assertEqual(f"expr={ {x: y for x, y in [(1, 2), ]} }", "expr={1: 2}") def test_not_equal(self): # There's a special test for this because there's a special # case in the f-string parser to look for != as not ending an # expression. Normally it would, while looking for !s or !r. - self.assertEqual(f'{3!=4}', 'True') - self.assertEqual(f'{3!=4:}', 'True') - self.assertEqual(f'{3!=4!s}', 'True') - self.assertEqual(f'{3!=4!s:.3}', 'Tru') + self.assertEqual(f"{3!=4}", "True") + self.assertEqual(f"{3!=4:}", "True") + self.assertEqual(f"{3!=4!s}", "True") + self.assertEqual(f"{3!=4!s:.3}", "Tru") def test_equal_equal(self): # Because an expression ending in = has special meaning, # there's a special test for ==. Make sure it works. - self.assertEqual(f'{0==1}', 'False') + self.assertEqual(f"{0==1}", "False") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_conversions(self): - self.assertEqual(f'{3.14:10.10}', ' 3.14') - self.assertEqual(f'{3.14!s:10.10}', '3.14 ') - self.assertEqual(f'{3.14!r:10.10}', '3.14 ') - self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + self.assertEqual(f"{3.14:10.10}", " 3.14") + self.assertEqual(f"{3.14!s:10.10}", "3.14 ") + self.assertEqual(f"{3.14!r:10.10}", "3.14 ") + self.assertEqual(f"{3.14!a:10.10}", "3.14 ") - self.assertEqual(f'{"a"}', 'a') + self.assertEqual(f'{"a"}', "a") self.assertEqual(f'{"a"!r}', "'a'") self.assertEqual(f'{"a"!a}', "'a'") + # Conversions can have trailing whitespace after them since it + # does not provide any significance + # TODO: RUSTPYTHON SyntaxError + # self.assertEqual(f"{3!s }", "3") + # self.assertEqual(f"{3.14!s :10.10}", "3.14 ") + # Not a conversion. self.assertEqual(f'{"a!r"}', "a!r") # Not a conversion, but show that ! is allowed in a format spec. - self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') - - self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', - ["f'{3!g}'", - "f'{3!A}'", - "f'{3!3}'", - "f'{3!G}'", - "f'{3!!}'", - "f'{3!:}'", - "f'{3! s}'", # no space before conversion char - ]) - - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", - ["f'{x!s{y}}'", - "f'{3!ss}'", - "f'{3!ss:}'", - "f'{3!ss:s}'", - ]) + self.assertEqual(f"{3.14:!<10.10}", "3.14!!!!!!") + + self.assertAllRaise( + SyntaxError, + "f-string: expecting '}'", + [ + "f'{3!'", + "f'{3!s'", + "f'{3!g'", + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: missing conversion character", + [ + "f'{3!}'", + "f'{3!:'", + "f'{3!:}'", + ], + ) + + for conv_identifier in "g", "A", "G", "ä", "ɐ": + self.assertAllRaise( + SyntaxError, + "f-string: invalid conversion character %r: " + "expected 's', 'r', or 'a'" % conv_identifier, + ["f'{3!" + conv_identifier + "}'"], + ) + + for conv_non_identifier in "3", "!": + self.assertAllRaise( + SyntaxError, + "f-string: invalid conversion character", + ["f'{3!" + conv_non_identifier + "}'"], + ) + + for conv in " s", " s ": + self.assertAllRaise( + SyntaxError, + "f-string: conversion type must come right after the" + " exclamanation mark", + ["f'{3!" + conv + "}'"], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: invalid conversion character 'ss': " "expected 's', 'r', or 'a'", + [ + "f'{3!ss}'", + "f'{3!ss:}'", + "f'{3!ss:s}'", + ], + ) def test_assignment(self): - self.assertAllRaise(SyntaxError, r'invalid syntax', - ["f'' = 3", - "f'{0}' = x", - "f'{x}' = x", - ]) + self.assertAllRaise( + SyntaxError, + r"invalid syntax", + [ + "f'' = 3", + "f'{0}' = x", + "f'{x}' = x", + ], + ) def test_del(self): - self.assertAllRaise(SyntaxError, 'invalid syntax', - ["del f''", - "del '' f''", - ]) + self.assertAllRaise( + SyntaxError, + "invalid syntax", + [ + "del f''", + "del '' f''", + ], + ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_mismatched_braces(self): - self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", - ["f'{{}'", - "f'{{}}}'", - "f'}'", - "f'x}'", - "f'x}x'", - r"f'\u007b}'", - - # Can't have { or } in a format spec. - "f'{3:}>10}'", - "f'{3:}}>10}'", - ]) - - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", - ["f'{3:{{>10}'", - "f'{3'", - "f'{3!'", - "f'{3:'", - "f'{3!s'", - "f'{3!s:'", - "f'{3!s:3'", - "f'x{'", - "f'x{x'", - "f'{x'", - "f'{3:s'", - "f'{{{'", - "f'{{}}{'", - "f'{'", - "f'x{<'", # See bpo-46762. - "f'x{>'", - "f'{i='", # See gh-93418. - ]) + self.assertAllRaise( + SyntaxError, + "f-string: single '}' is not allowed", + [ + "f'{{}'", + "f'{{}}}'", + "f'}'", + "f'x}'", + "f'x}x'", + r"f'\u007b}'", + # Can't have { or } in a format spec. + "f'{3:}>10}'", + "f'{3:}}>10}'", + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: expecting '}'", + [ + "f'{3'", + "f'{3!'", + "f'{3:'", + "f'{3!s'", + "f'{3!s:'", + "f'{3!s:3'", + "f'x{'", + "f'x{x'", + "f'{x'", + "f'{3:s'", + "f'{{{'", + "f'{{}}{'", + "f'{'", + "f'{i='", # See gh-93418. + ], + ) + + self.assertAllRaise( + SyntaxError, + "f-string: expecting a valid expression after '{'", + [ + "f'{3:{{>10}'", + ], + ) # But these are just normal strings. - self.assertEqual(f'{"{"}', '{') - self.assertEqual(f'{"}"}', '}') - self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3') - self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2') + self.assertEqual(f'{"{"}', "{") + self.assertEqual(f'{"}"}', "}") + self.assertEqual(f'{3:{"}"}>10}', "}}}}}}}}}3") + self.assertEqual(f'{2:{"{"}>10}', "{{{{{{{{{2") def test_if_conditional(self): # There's special logic in compile.c to test if the @@ -1132,7 +1587,7 @@ def test_if_conditional(self): def test_fstring(x, expected): flag = 0 - if f'{x}': + if f"{x}": flag = 1 else: flag = 2 @@ -1140,7 +1595,7 @@ def test_fstring(x, expected): def test_concat_empty(x, expected): flag = 0 - if '' f'{x}': + if "" f"{x}": flag = 1 else: flag = 2 @@ -1148,141 +1603,153 @@ def test_concat_empty(x, expected): def test_concat_non_empty(x, expected): flag = 0 - if ' ' f'{x}': + if " " f"{x}": flag = 1 else: flag = 2 self.assertEqual(flag, expected) - test_fstring('', 2) - test_fstring(' ', 1) + test_fstring("", 2) + test_fstring(" ", 1) - test_concat_empty('', 2) - test_concat_empty(' ', 1) + test_concat_empty("", 2) + test_concat_empty(" ", 1) - test_concat_non_empty('', 1) - test_concat_non_empty(' ', 1) + test_concat_non_empty("", 1) + test_concat_non_empty(" ", 1) def test_empty_format_specifier(self): - x = 'test' - self.assertEqual(f'{x}', 'test') - self.assertEqual(f'{x:}', 'test') - self.assertEqual(f'{x!s:}', 'test') - self.assertEqual(f'{x!r:}', "'test'") + x = "test" + self.assertEqual(f"{x}", "test") + self.assertEqual(f"{x:}", "test") + self.assertEqual(f"{x!s:}", "test") + self.assertEqual(f"{x!r:}", "'test'") # TODO: RUSTPYTHON d[0] error @unittest.expectedFailure def test_str_format_differences(self): - d = {'a': 'string', - 0: 'integer', - } + d = { + "a": "string", + 0: "integer", + } a = 0 - self.assertEqual(f'{d[0]}', 'integer') - self.assertEqual(f'{d["a"]}', 'string') - self.assertEqual(f'{d[a]}', 'integer') - self.assertEqual('{d[a]}'.format(d=d), 'string') - self.assertEqual('{d[0]}'.format(d=d), 'integer') + self.assertEqual(f"{d[0]}", "integer") + self.assertEqual(f'{d["a"]}', "string") + self.assertEqual(f"{d[a]}", "integer") + self.assertEqual("{d[a]}".format(d=d), "string") + self.assertEqual("{d[0]}".format(d=d), "integer") # TODO: RUSTPYTHON @unittest.expectedFailure def test_errors(self): # see issue 26287 - self.assertAllRaise(TypeError, 'unsupported', - [r"f'{(lambda: 0):x}'", - r"f'{(0,):x}'", - ]) - self.assertAllRaise(ValueError, 'Unknown format code', - [r"f'{1000:j}'", - r"f'{1000:j}'", - ]) + self.assertAllRaise( + TypeError, + "unsupported", + [ + r"f'{(lambda: 0):x}'", + r"f'{(0,):x}'", + ], + ) + self.assertAllRaise( + ValueError, + "Unknown format code", + [ + r"f'{1000:j}'", + r"f'{1000:j}'", + ], + ) def test_filename_in_syntaxerror(self): # see issue 38964 with temp_cwd() as cwd: - file_path = os.path.join(cwd, 't.py') - with open(file_path, 'w', encoding="utf-8") as f: - f.write('f"{a b}"') # This generates a SyntaxError - _, _, stderr = assert_python_failure(file_path, - PYTHONIOENCODING='ascii') - self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr) + file_path = os.path.join(cwd, "t.py") + with open(file_path, "w", encoding="utf-8") as f: + f.write('f"{a b}"') # This generates a SyntaxError + _, _, stderr = assert_python_failure(file_path, PYTHONIOENCODING="ascii") + self.assertIn(file_path.encode("ascii", "backslashreplace"), stderr) def test_loop(self): for i in range(1000): - self.assertEqual(f'i:{i}', 'i:' + str(i)) + self.assertEqual(f"i:{i}", "i:" + str(i)) def test_dict(self): - d = {'"': 'dquote', - "'": 'squote', - 'foo': 'bar', - } - self.assertEqual(f'''{d["'"]}''', 'squote') - self.assertEqual(f"""{d['"']}""", 'dquote') + d = { + '"': "dquote", + "'": "squote", + "foo": "bar", + } + self.assertEqual(f"""{d["'"]}""", "squote") + self.assertEqual(f"""{d['"']}""", "dquote") - self.assertEqual(f'{d["foo"]}', 'bar') - self.assertEqual(f"{d['foo']}", 'bar') + self.assertEqual(f'{d["foo"]}', "bar") + self.assertEqual(f"{d['foo']}", "bar") def test_backslash_char(self): # Check eval of a backslash followed by a control char. # See bpo-30682: this used to raise an assert in pydebug mode. - self.assertEqual(eval('f"\\\n"'), '') - self.assertEqual(eval('f"\\\r"'), '') + self.assertEqual(eval('f"\\\n"'), "") + self.assertEqual(eval('f"\\\r"'), "") def test_debug_conversion(self): - x = 'A string' - self.assertEqual(f'{x=}', 'x=' + repr(x)) - self.assertEqual(f'{x =}', 'x =' + repr(x)) - self.assertEqual(f'{x=!s}', 'x=' + str(x)) - self.assertEqual(f'{x=!r}', 'x=' + repr(x)) - self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) + x = "A string" + self.assertEqual(f"{x=}", "x=" + repr(x)) + self.assertEqual(f"{x =}", "x =" + repr(x)) + self.assertEqual(f"{x=!s}", "x=" + str(x)) + self.assertEqual(f"{x=!r}", "x=" + repr(x)) + self.assertEqual(f"{x=!a}", "x=" + ascii(x)) x = 2.71828 - self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) - self.assertEqual(f'{x=:}', 'x=' + format(x, '')) - self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) - self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) - self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) + self.assertEqual(f"{x=:.2f}", "x=" + format(x, ".2f")) + self.assertEqual(f"{x=:}", "x=" + format(x, "")) + self.assertEqual(f"{x=!r:^20}", "x=" + format(repr(x), "^20")) + self.assertEqual(f"{x=!s:^20}", "x=" + format(str(x), "^20")) + self.assertEqual(f"{x=!a:^20}", "x=" + format(ascii(x), "^20")) x = 9 - self.assertEqual(f'{3*x+15=}', '3*x+15=42') + self.assertEqual(f"{3*x+15=}", "3*x+15=42") # There is code in ast.c that deals with non-ascii expression values. So, # use a unicode identifier to trigger that. tenπ = 31.4 - self.assertEqual(f'{tenπ=:.2f}', 'tenπ=31.40') + self.assertEqual(f"{tenπ=:.2f}", "tenπ=31.40") # Also test with Unicode in non-identifiers. - self.assertEqual(f'{"Σ"=}', '"Σ"=\'Σ\'') + self.assertEqual(f'{"Σ"=}', "\"Σ\"='Σ'") # Make sure nested fstrings still work. - self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', '*****3.1415=3.1*****') + self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', "*****3.1415=3.1*****") # Make sure text before and after an expression with = works # correctly. - pi = 'π' - self.assertEqual(f'alpha α {pi=} ω omega', "alpha α pi='π' ω omega") + pi = "π" + self.assertEqual(f"alpha α {pi=} ω omega", "alpha α pi='π' ω omega") # Check multi-line expressions. - self.assertEqual(f'''{ + self.assertEqual( + f"""{ 3 -=}''', '\n3\n=3') +=}""", + "\n3\n=3", + ) # Since = is handled specially, make sure all existing uses of # it still work. - self.assertEqual(f'{0==1}', 'False') - self.assertEqual(f'{0!=1}', 'True') - self.assertEqual(f'{0<=1}', 'True') - self.assertEqual(f'{0>=1}', 'False') - self.assertEqual(f'{(x:="5")}', '5') - self.assertEqual(x, '5') - self.assertEqual(f'{(x:=5)}', '5') + self.assertEqual(f"{0==1}", "False") + self.assertEqual(f"{0!=1}", "True") + self.assertEqual(f"{0<=1}", "True") + self.assertEqual(f"{0>=1}", "False") + self.assertEqual(f'{(x:="5")}', "5") + self.assertEqual(x, "5") + self.assertEqual(f"{(x:=5)}", "5") self.assertEqual(x, 5) - self.assertEqual(f'{"="}', '=') + self.assertEqual(f'{"="}', "=") x = 20 # This isn't an assignment expression, it's 'x', with a format # spec of '=10'. See test_walrus: you need to use parens. - self.assertEqual(f'{x:=10}', ' 20') + self.assertEqual(f"{x:=10}", " 20") # Test named function parameters, to make sure '=' parsing works # there. @@ -1291,36 +1758,54 @@ def f(a): oldx = x x = a return oldx + x = 0 - self.assertEqual(f'{f(a="3=")}', '0') - self.assertEqual(x, '3=') - self.assertEqual(f'{f(a=4)}', '3=') + self.assertEqual(f'{f(a="3=")}', "0") + self.assertEqual(x, "3=") + self.assertEqual(f"{f(a=4)}", "3=") self.assertEqual(x, 4) + # Check debug expressions in format spec + y = 20 + self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") + self.assertEqual( + f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", "h1y=20h2y=20h3y=20" + ) + # Make sure __format__ is being called. class C: def __format__(self, s): - return f'FORMAT-{s}' + return f"FORMAT-{s}" + def __repr__(self): - return 'REPR' + return "REPR" - self.assertEqual(f'{C()=}', 'C()=REPR') - self.assertEqual(f'{C()=!r}', 'C()=REPR') - self.assertEqual(f'{C()=:}', 'C()=FORMAT-') - self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') - self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') - self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + self.assertEqual(f"{C()=}", "C()=REPR") + self.assertEqual(f"{C()=!r}", "C()=REPR") + self.assertEqual(f"{C()=:}", "C()=FORMAT-") + self.assertEqual(f"{C()=: }", "C()=FORMAT- ") + self.assertEqual(f"{C()=:x}", "C()=FORMAT-x") + self.assertEqual(f"{C()=!r:*^20}", "C()=********REPR********") + self.assertEqual(f"{C():{20=}}", "FORMAT-20=20") self.assertRaises(SyntaxError, eval, "f'{C=]'") # Make sure leading and following text works. - x = 'foo' - self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') + x = "foo" + self.assertEqual(f"X{x=}Y", "Xx=" + repr(x) + "Y") # Make sure whitespace around the = works. - self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') - self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') - self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') + self.assertEqual(f"X{x =}Y", "Xx =" + repr(x) + "Y") + self.assertEqual(f"X{x= }Y", "Xx= " + repr(x) + "Y") + self.assertEqual(f"X{x = }Y", "Xx = " + repr(x) + "Y") + self.assertEqual(f"sadsd {1 + 1 = :{1 + 1:1d}f}", "sadsd 1 + 1 = 2.000000") + +# TODO: RUSTPYTHON SyntaxError +# self.assertEqual( +# f"{1+2 = # my comment +# }", +# "1+2 = \n 3", +# ) # These next lines contains tabs. Backslash escapes don't # work in f-strings. @@ -1328,23 +1813,25 @@ def __repr__(self): # this will be to dynamically created and exec the f-strings. But # that's such a hassle I'll save it for another day. For now, convert # the tabs to spaces just to shut up patchcheck. - #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') - #self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + # self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') + # self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') def test_walrus(self): x = 20 # This isn't an assignment expression, it's 'x', with a format # spec of '=10'. - self.assertEqual(f'{x:=10}', ' 20') + self.assertEqual(f"{x:=10}", " 20") # This is an assignment expression, which requires parens. - self.assertEqual(f'{(x:=10)}', '10') + self.assertEqual(f"{(x:=10)}", "10") self.assertEqual(x, 10) # TODO: RUSTPYTHON @unittest.expectedFailure def test_invalid_syntax_error_message(self): - with self.assertRaisesRegex(SyntaxError, "f-string: invalid syntax"): + with self.assertRaisesRegex( + SyntaxError, "f-string: expecting '=', or '!', or ':', or '}'" + ): compile("f'{a $ b}'", "?", "exec") # TODO: RUSTPYTHON @@ -1352,37 +1839,112 @@ def test_invalid_syntax_error_message(self): def test_with_two_commas_in_format_specifier(self): error_msg = re.escape("Cannot specify ',' with ','.") with self.assertRaisesRegex(ValueError, error_msg): - f'{1:,,}' + f"{1:,,}" # TODO: RUSTPYTHON @unittest.expectedFailure def test_with_two_underscore_in_format_specifier(self): error_msg = re.escape("Cannot specify '_' with '_'.") with self.assertRaisesRegex(ValueError, error_msg): - f'{1:__}' + f"{1:__}" # TODO: RUSTPYTHON @unittest.expectedFailure def test_with_a_commas_and_an_underscore_in_format_specifier(self): error_msg = re.escape("Cannot specify both ',' and '_'.") with self.assertRaisesRegex(ValueError, error_msg): - f'{1:,_}' + f"{1:,_}" # TODO: RUSTPYTHON @unittest.expectedFailure def test_with_an_underscore_and_a_comma_in_format_specifier(self): error_msg = re.escape("Cannot specify both ',' and '_'.") with self.assertRaisesRegex(ValueError, error_msg): - f'{1:_,}' + f"{1:_,}" + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_syntax_error_for_starred_expressions(self): - error_msg = re.escape("cannot use starred expression here") - with self.assertRaisesRegex(SyntaxError, error_msg): + with self.assertRaisesRegex(SyntaxError, "can't use starred expression here"): compile("f'{*a}'", "?", "exec") - error_msg = re.escape("cannot use double starred expression here") - with self.assertRaisesRegex(SyntaxError, error_msg): + with self.assertRaisesRegex( + SyntaxError, "f-string: expecting a valid expression after '{'" + ): compile("f'{**a}'", "?", "exec") -if __name__ == '__main__': + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_not_closing_quotes(self): + self.assertAllRaise(SyntaxError, "unterminated f-string literal", ['f"', "f'"]) + self.assertAllRaise( + SyntaxError, "unterminated triple-quoted f-string literal", ['f"""', "f'''"] + ) + # Ensure that the errors are reported at the correct line number. + data = '''\ +x = 1 + 1 +y = 2 + 2 +z = f""" +sdfjnsdfjsdf +sdfsdfs{1+ +2} dfigdf {3+ +4}sdufsd"" +''' + try: + compile(data, "?", "exec") + except SyntaxError as e: + self.assertEqual(e.text, 'z = f"""') + self.assertEqual(e.lineno, 3) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_syntax_error_after_debug(self): + self.assertAllRaise( + SyntaxError, + "f-string: expecting a valid expression after '{'", + [ + "f'{1=}{;'", + "f'{1=}{+;'", + "f'{1=}{2}{;'", + "f'{1=}{3}{;'", + ], + ) + self.assertAllRaise( + SyntaxError, + "f-string: expecting '=', or '!', or ':', or '}'", + [ + "f'{1=}{1;'", + "f'{1=}{1;}'", + ], + ) + + def test_debug_in_file(self): + with temp_cwd(): + script = "script.py" + with open("script.py", "w") as f: + f.write(f"""\ +print(f'''{{ +3 +=}}''')""") + + _, stdout, _ = assert_python_ok(script) + self.assertEqual( + stdout.decode("utf-8").strip().replace("\r\n", "\n").replace("\r", "\n"), + "3\n=3", + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_syntax_warning_infinite_recursion_in_file(self): + with temp_cwd(): + script = "script.py" + with open(script, "w") as f: + f.write(r"print(f'\{1}')") + + _, stdout, stderr = assert_python_ok(script) + self.assertIn(rb"\1", stdout) + self.assertEqual(len(stderr.strip().splitlines()), 2) + + +if __name__ == "__main__": unittest.main() From e2a55cbf34dc999c29f5e0984a24e727246f9fcd Mon Sep 17 00:00:00 2001 From: Shubham Patil Date: Mon, 6 Jan 2025 19:35:14 +0530 Subject: [PATCH 002/317] Handle pre-release flag being empty for schedule triggers in release workflow GitHub workflow_dispatch input variables are always empty for other triggers --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8933d2107d..70255d8082 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -127,12 +127,12 @@ jobs: tag: ${{ github.ref_name }} run: ${{ github.run_number }} run: | - if [[ "${{ github.event.inputs.pre-release }}" == "true" ]]; then - RELEASE_TYPE_NAME=Pre-Release - PRERELEASE_ARG=--prerelease - else + if [[ "${{ github.event.inputs.pre-release }}" == "false" ]]; then RELEASE_TYPE_NAME=Release PRERELEASE_ARG= + else + RELEASE_TYPE_NAME=Pre-Release + PRERELEASE_ARG=--prerelease fi today=$(date '+%Y-%m-%d') From d06459fa49b12cd0164844d1164d0ba97ec7db0a Mon Sep 17 00:00:00 2001 From: Bob McWhirter Date: Wed, 8 Jan 2025 14:03:15 -0500 Subject: [PATCH 003/317] guard signal-handling init more broadly If `install_signal_handlers` is false (due to embedding), the VM still inits the signal stdlib and installs a lot of signal-handling, including touching *ever* signal, including SIGINT. When running multiple concurrent interpreters with varying inits at varying times, this can break the hosting application's signal-handling so lovingly set up before starting anything with RustPython. --- vm/src/stdlib/signal.rs | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 83e9d861b2..73c9d978e6 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -110,28 +110,28 @@ pub(crate) mod _signal { module: &Py, vm: &VirtualMachine, ) { - let sig_dfl = vm.new_pyobj(SIG_DFL as u8); - let sig_ign = vm.new_pyobj(SIG_IGN as u8); + if vm.state.settings.install_signal_handlers { + let sig_dfl = vm.new_pyobj(SIG_DFL as u8); + let sig_ign = vm.new_pyobj(SIG_IGN as u8); - for signum in 1..NSIG { - let handler = unsafe { libc::signal(signum as i32, SIG_IGN) }; - if handler != SIG_ERR { - unsafe { libc::signal(signum as i32, handler) }; + for signum in 1..NSIG { + let handler = unsafe { libc::signal(signum as i32, SIG_IGN) }; + if handler != SIG_ERR { + unsafe { libc::signal(signum as i32, handler) }; + } + let py_handler = if handler == SIG_DFL { + Some(sig_dfl.clone()) + } else if handler == SIG_IGN { + Some(sig_ign.clone()) + } else { + None + }; + vm.signal_handlers.as_deref().unwrap().borrow_mut()[signum] = py_handler; } - let py_handler = if handler == SIG_DFL { - Some(sig_dfl.clone()) - } else if handler == SIG_IGN { - Some(sig_ign.clone()) - } else { - None - }; - vm.signal_handlers.as_deref().unwrap().borrow_mut()[signum] = py_handler; - } - let int_handler = module - .get_attr("default_int_handler", vm) - .expect("_signal does not have this attr?"); - if vm.state.settings.install_signal_handlers { + let int_handler = module + .get_attr("default_int_handler", vm) + .expect("_signal does not have this attr?"); signal(libc::SIGINT, int_handler, vm).expect("Failed to set sigint handler"); } } From b7db23bbae1198cc0c97e35e0db02fca84371509 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 9 Jan 2025 15:59:44 -0600 Subject: [PATCH 004/317] Fix warnings for Rust 1.84 --- Cargo.lock | 21 ++++++++++----------- common/src/int.rs | 6 +++--- compiler/codegen/src/compile.rs | 15 ++++++--------- derive-impl/src/pyclass.rs | 2 +- derive-impl/src/util.rs | 2 +- src/shell/helper.rs | 2 +- stdlib/src/array.rs | 2 +- stdlib/src/contextvars.rs | 14 ++++++-------- stdlib/src/unicodedata.rs | 2 +- vm/src/anystr.rs | 7 ++----- vm/src/builtins/asyncgenerator.rs | 2 +- vm/src/builtins/descriptor.rs | 2 +- vm/src/builtins/mappingproxy.rs | 2 +- vm/src/builtins/str.rs | 9 +++------ vm/src/cformat.rs | 4 ++-- vm/src/protocol/mapping.rs | 2 +- vm/src/stdlib/builtins.rs | 3 +-- vm/src/stdlib/itertools.rs | 2 +- vm/src/stdlib/nt.rs | 2 +- vm/src/stdlib/os.rs | 4 ++-- vm/src/types/slot.rs | 4 ++-- vm/src/vm/vm_ops.rs | 4 ++-- vm/src/warn.rs | 2 +- wasm/lib/src/convert.rs | 2 +- 24 files changed, 52 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a9e1c9508..5d5db7f5ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2946,9 +2946,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -2957,13 +2957,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.77", @@ -2984,9 +2983,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2994,9 +2993,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -3007,9 +3006,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "web-sys" diff --git a/common/src/int.rs b/common/src/int.rs index 0c19f55141..ca449ac708 100644 --- a/common/src/int.rs +++ b/common/src/int.rs @@ -60,9 +60,9 @@ pub fn bytes_to_int(lit: &[u8], mut base: u32) -> Option { return Some(BigInt::zero()); } } - 16 => lit.get(1).map_or(false, |&b| matches!(b, b'x' | b'X')), - 2 => lit.get(1).map_or(false, |&b| matches!(b, b'b' | b'B')), - 8 => lit.get(1).map_or(false, |&b| matches!(b, b'o' | b'O')), + 16 => lit.get(1).is_some_and(|&b| matches!(b, b'x' | b'X')), + 2 => lit.get(1).is_some_and(|&b| matches!(b, b'b' | b'B')), + 8 => lit.get(1).is_some_and(|&b| matches!(b, b'o' | b'O')), _ => false, } } else { diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index dc9f8a976e..eeb4b7dec6 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -3102,19 +3102,18 @@ impl Compiler { Expr::Tuple(ExprTuple { elts, .. }) => elts.iter().any(Self::contains_await), Expr::Set(ExprSet { elts, .. }) => elts.iter().any(Self::contains_await), Expr::Dict(ExprDict { keys, values, .. }) => { - keys.iter() - .any(|key| key.as_ref().map_or(false, Self::contains_await)) + keys.iter().flatten().any(Self::contains_await) || values.iter().any(Self::contains_await) } Expr::Slice(ExprSlice { lower, upper, step, .. }) => { - lower.as_ref().map_or(false, |l| Self::contains_await(l)) - || upper.as_ref().map_or(false, |u| Self::contains_await(u)) - || step.as_ref().map_or(false, |s| Self::contains_await(s)) + lower.as_deref().is_some_and(Self::contains_await) + || upper.as_deref().is_some_and(Self::contains_await) + || step.as_deref().is_some_and(Self::contains_await) } Expr::Yield(ExprYield { value, .. }) => { - value.as_ref().map_or(false, |v| Self::contains_await(v)) + value.as_deref().is_some_and(Self::contains_await) } Expr::Await(ExprAwait { .. }) => true, Expr::YieldFrom(ExprYieldFrom { value, .. }) => Self::contains_await(value), @@ -3128,9 +3127,7 @@ impl Compiler { .. }) => { Self::contains_await(value) - || format_spec - .as_ref() - .map_or(false, |fs| Self::contains_await(fs)) + || format_spec.as_deref().is_some_and(Self::contains_await) } Expr::Name(located_ast::ExprName { .. }) => false, Expr::Lambda(located_ast::ExprLambda { body, .. }) => Self::contains_await(body), diff --git a/derive-impl/src/pyclass.rs b/derive-impl/src/pyclass.rs index a2896ffeed..dedb3eb77a 100644 --- a/derive-impl/src/pyclass.rs +++ b/derive-impl/src/pyclass.rs @@ -348,7 +348,7 @@ fn generate_class_def( && if let Ok(Meta::List(l)) = attr.parse_meta() { l.nested .into_iter() - .any(|n| n.get_ident().map_or(false, |p| p == "PyStructSequence")) + .any(|n| n.get_ident().is_some_and(|p| p == "PyStructSequence")) } else { false } diff --git a/derive-impl/src/util.rs b/derive-impl/src/util.rs index 916b19db06..f016b0d1e9 100644 --- a/derive-impl/src/util.rs +++ b/derive-impl/src/util.rs @@ -558,7 +558,7 @@ impl AttributeExt for Attribute { let has_name = list .nested .iter() - .any(|nested_meta| nested_meta.get_path().map_or(false, |p| p.is_ident(name))); + .any(|nested_meta| nested_meta.get_path().is_some_and(|p| p.is_ident(name))); if !has_name { list.nested.push(new_item()) } diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 34691e7995..c228dbdf65 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -22,7 +22,7 @@ fn split_idents_on_dot(line: &str) -> Option<(usize, Vec)> { match c { '.' => { // check for a double dot - if i != 0 && words.last().map_or(false, |s| s.is_empty()) { + if i != 0 && words.last().is_some_and(|s| s.is_empty()) { return None; } reverse_string(words.last_mut().unwrap()); diff --git a/stdlib/src/array.rs b/stdlib/src/array.rs index 9f4540aa24..9bd58f8043 100644 --- a/stdlib/src/array.rs +++ b/stdlib/src/array.rs @@ -1238,7 +1238,7 @@ mod array { let res = match array_a.cmp(&array_b) { // fast path for same ArrayContentType type - Ok(partial_ord) => partial_ord.map_or(false, |ord| op.eval_ord(ord)), + Ok(partial_ord) => partial_ord.is_some_and(|ord| op.eval_ord(ord)), Err(()) => { let iter = Iterator::zip(array_a.iter(vm), array_b.iter(vm)); diff --git a/stdlib/src/contextvars.rs b/stdlib/src/contextvars.rs index 8e1bfca005..40a59050b3 100644 --- a/stdlib/src/contextvars.rs +++ b/stdlib/src/contextvars.rs @@ -129,18 +129,16 @@ mod _contextvars { super::CONTEXTS.with(|ctxs| { let mut ctxs = ctxs.borrow_mut(); - if !ctxs - .last() - .map_or(false, |ctx| ctx.get_id() == zelf.get_id()) - { + // TODO: use Vec::pop_if once stabilized + if ctxs.last().is_some_and(|ctx| ctx.get_id() == zelf.get_id()) { + let _ = ctxs.pop(); + Ok(()) + } else { let msg = "cannot exit context: thread state references a different context object" .to_owned(); - return Err(vm.new_runtime_error(msg)); + Err(vm.new_runtime_error(msg)) } - - let _ = ctxs.pop(); - Ok(()) })?; zelf.inner.entered.set(false); diff --git a/stdlib/src/unicodedata.rs b/stdlib/src/unicodedata.rs index 56105ceca6..70483073a7 100644 --- a/stdlib/src/unicodedata.rs +++ b/stdlib/src/unicodedata.rs @@ -85,7 +85,7 @@ mod unicodedata { } fn check_age(&self, c: char) -> bool { - Age::of(c).map_or(false, |age| age.actual() <= self.unic_version) + Age::of(c).is_some_and(|age| age.actual() <= self.unic_version) } fn extract_char(&self, character: PyStrRef, vm: &VirtualMachine) -> PyResult> { diff --git a/vm/src/anystr.rs b/vm/src/anystr.rs index be04cba641..f5bfab1c57 100644 --- a/vm/src/anystr.rs +++ b/vm/src/anystr.rs @@ -385,17 +385,14 @@ pub trait AnyStr { let (end_len, i_diff) = match *ch { b'\n' => (keep, 1), b'\r' => { - let is_rn = enumerated.peek().map_or(false, |(_, ch)| **ch == b'\n'); + let is_rn = enumerated.next_if(|(_, ch)| **ch == b'\n').is_some(); if is_rn { - let _ = enumerated.next(); (keep + keep, 2) } else { (keep, 1) } } - _ => { - continue; - } + _ => continue, }; let range = last_i..i + end_len; last_i = i + i_diff; diff --git a/vm/src/builtins/asyncgenerator.rs b/vm/src/builtins/asyncgenerator.rs index 4efb41b450..f9ed4719d8 100644 --- a/vm/src/builtins/asyncgenerator.rs +++ b/vm/src/builtins/asyncgenerator.rs @@ -390,7 +390,7 @@ impl PyAsyncGenAThrow { } fn ignored_close(&self, res: &PyResult) -> bool { - res.as_ref().map_or(false, |v| match v { + res.as_ref().is_ok_and(|v| match v { PyIterReturn::Return(obj) => obj.payload_is::(), PyIterReturn::StopIteration(_) => false, }) diff --git a/vm/src/builtins/descriptor.rs b/vm/src/builtins/descriptor.rs index 408b0efc84..4b10f63f21 100644 --- a/vm/src/builtins/descriptor.rs +++ b/vm/src/builtins/descriptor.rs @@ -67,7 +67,7 @@ impl GetDescriptor for PyMethodDescriptor { let bound = match obj { Some(obj) => { if descr.method.flags.contains(PyMethodFlags::METHOD) { - if cls.map_or(false, |c| c.fast_isinstance(vm.ctx.types.type_type)) { + if cls.is_some_and(|c| c.fast_isinstance(vm.ctx.types.type_type)) { obj } else { return Err(vm.new_type_error(format!( diff --git a/vm/src/builtins/mappingproxy.rs b/vm/src/builtins/mappingproxy.rs index 5e7608c778..7b2386a39b 100644 --- a/vm/src/builtins/mappingproxy.rs +++ b/vm/src/builtins/mappingproxy.rs @@ -126,7 +126,7 @@ impl PyMappingProxy { match &self.mapping { MappingProxyInner::Class(class) => Ok(key .as_interned_str(vm) - .map_or(false, |key| class.attributes.read().contains_key(key))), + .is_some_and(|key| class.attributes.read().contains_key(key))), MappingProxyInner::Mapping(mapping) => mapping.to_sequence().contains(key, vm), } } diff --git a/vm/src/builtins/str.rs b/vm/src/builtins/str.rs index ddf2493ded..c0a096b211 100644 --- a/vm/src/builtins/str.rs +++ b/vm/src/builtins/str.rs @@ -903,9 +903,8 @@ impl PyStr { let end_len = match ch { '\n' => 1, '\r' => { - let is_rn = enumerated.peek().map_or(false, |(_, ch)| *ch == '\n'); + let is_rn = enumerated.next_if(|(_, ch)| *ch == '\n').is_some(); if is_rn { - let _ = enumerated.next(); 2 } else { 1 @@ -913,9 +912,7 @@ impl PyStr { } '\x0b' | '\x0c' | '\x1c' | '\x1d' | '\x1e' | '\u{0085}' | '\u{2028}' | '\u{2029}' => ch.len_utf8(), - _ => { - continue; - } + _ => continue, }; let range = if args.keepends { last_i..i + end_len @@ -1160,7 +1157,7 @@ impl PyStr { #[pymethod] fn isidentifier(&self) -> bool { let mut chars = self.as_str().chars(); - let is_identifier_start = chars.next().map_or(false, |c| c == '_' || is_xid_start(c)); + let is_identifier_start = chars.next().is_some_and(|c| c == '_' || is_xid_start(c)); // a string is not an identifier if it has whitespace or starts with a number is_identifier_start && chars.all(is_xid_continue) } diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 2ba7f11238..79f4d68833 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -296,7 +296,7 @@ pub(crate) fn cformat_bytes( return if is_mapping || values_obj .payload::() - .map_or(false, |e| e.is_empty()) + .is_some_and(|e| e.is_empty()) { for (_, part) in format.iter_mut() { match part { @@ -397,7 +397,7 @@ pub(crate) fn cformat_string( return if is_mapping || values_obj .payload::() - .map_or(false, |e| e.is_empty()) + .is_some_and(|e| e.is_empty()) { for (_, part) in format.iter() { match part { diff --git a/vm/src/protocol/mapping.rs b/vm/src/protocol/mapping.rs index 11cd03d445..986b806f6c 100644 --- a/vm/src/protocol/mapping.rs +++ b/vm/src/protocol/mapping.rs @@ -95,7 +95,7 @@ impl PyMapping<'_> { // PyMapping::Check #[inline] pub fn check(obj: &PyObject) -> bool { - Self::find_methods(obj).map_or(false, |x| x.as_ref().check()) + Self::find_methods(obj).is_some_and(|x| x.as_ref().check()) } pub fn find_methods(obj: &PyObject) -> Option> { diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index b151f58323..cc9b0a8be7 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -429,8 +429,7 @@ mod builtins { let fd_matches = |obj, expected| { vm.call_method(obj, "fileno", ()) .and_then(|o| i64::try_from_object(vm, o)) - .ok() - .map_or(false, |fd| fd == expected) + .is_ok_and(|fd| fd == expected) }; // everything is normalish, we can just rely on rustyline to use stdin/stdout diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs index f3d476497d..eb800a0b1f 100644 --- a/vm/src/stdlib/itertools.rs +++ b/vm/src/stdlib/itertools.rs @@ -684,7 +684,7 @@ mod decl { self.grouper .as_ref() .and_then(|g| g.upgrade()) - .map_or(false, |ref current_grouper| grouper.is(current_grouper)) + .is_some_and(|current_grouper| grouper.is(¤t_grouper)) } } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 4cfcef5afc..803cade630 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -62,7 +62,7 @@ pub(crate) mod module { .as_path() .parent() .and_then(|dst_parent| dst_parent.join(&args.src).symlink_metadata().ok()) - .map_or(false, |meta| meta.is_dir()); + .is_some_and(|meta| meta.is_dir()); let res = if dir { win_fs::symlink_dir(args.src.path, args.dst.path) } else { diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 24637d5a90..0b3f617552 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -274,8 +274,8 @@ pub(super) mod _os { fn remove(path: OsPath, dir_fd: DirFd<0>, vm: &VirtualMachine) -> PyResult<()> { let [] = dir_fd.0; let is_junction = cfg!(windows) - && fs::metadata(&path).map_or(false, |meta| meta.file_type().is_dir()) - && fs::symlink_metadata(&path).map_or(false, |meta| meta.file_type().is_symlink()); + && fs::metadata(&path).is_ok_and(|meta| meta.file_type().is_dir()) + && fs::symlink_metadata(&path).is_ok_and(|meta| meta.file_type().is_symlink()); let res = if is_junction { fs::remove_dir(&path) } else { diff --git a/vm/src/types/slot.rs b/vm/src/types/slot.rs index 3bbf3fc813..75f113977d 100644 --- a/vm/src/types/slot.rs +++ b/vm/src/types/slot.rs @@ -470,7 +470,7 @@ impl PyType { .attributes .read() .get(identifier!(ctx, __hash__)) - .map_or(false, |a| a.is(&ctx.none)); + .is_some_and(|a| a.is(&ctx.none)); let wrapper = if is_unhashable { hash_not_implemented } else { @@ -945,7 +945,7 @@ pub trait GetDescriptor: PyPayload { #[inline] fn _cls_is(cls: &Option, other: &impl Borrow) -> bool { - cls.as_ref().map_or(false, |cls| other.borrow().is(cls)) + cls.as_ref().is_some_and(|cls| other.borrow().is(cls)) } } diff --git a/vm/src/vm/vm_ops.rs b/vm/src/vm/vm_ops.rs index 3069f6ebe2..56c32d2927 100644 --- a/vm/src/vm/vm_ops.rs +++ b/vm/src/vm/vm_ops.rs @@ -298,8 +298,8 @@ impl VirtualMachine { } if let Some(slot_c) = class_c.slots.as_number.left_ternary_op(op_slot) { - if slot_a.map_or(false, |slot_a| (slot_a as usize) != (slot_c as usize)) - && slot_b.map_or(false, |slot_b| (slot_b as usize) != (slot_c as usize)) + if slot_a.is_some_and(|slot_a| slot_a != slot_c) + && slot_b.is_some_and(|slot_b| slot_b != slot_c) { let ret = slot_c(a, b, c, self)?; if !ret.is(&self.ctx.not_implemented) { diff --git a/vm/src/warn.rs b/vm/src/warn.rs index c1e4dc3473..ab316c8559 100644 --- a/vm/src/warn.rs +++ b/vm/src/warn.rs @@ -367,7 +367,7 @@ fn setup_context( // Stack level comparisons to Python code is off by one as there is no // warnings-related stack level to avoid. - if stack_level <= 0 || f.as_ref().map_or(false, |frame| frame.is_internal_frame()) { + if stack_level <= 0 || f.as_ref().is_some_and(|frame| frame.is_internal_frame()) { loop { stack_level -= 1; if stack_level <= 0 { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 7ebc4e257b..0618d9a37b 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -34,7 +34,7 @@ extern "C" { pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyBaseExceptionRef) -> JsValue { let jserr = vm.try_class("_js", "JSError").ok(); - let js_arg = if jserr.map_or(false, |jserr| py_err.fast_isinstance(&jserr)) { + let js_arg = if jserr.is_some_and(|jserr| py_err.fast_isinstance(&jserr)) { py_err.get_arg(0) } else { None From c901bc07a48f1709b6fe87f2bedac1b29a5c8e7f Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 9 Jan 2025 22:09:33 -0600 Subject: [PATCH 005/317] Upgrade wasm deps + fix demo --- Lib/_dummy_os.py | 22 ++++--- Lib/io.py | 12 +++- wasm/demo/package.json | 29 ++++----- wasm/demo/snippets/import_pypi.py | 11 ++-- wasm/demo/src/browser_module.rs | 0 wasm/demo/src/index.ejs | 3 +- wasm/demo/src/index.js | 101 +++++++++++++++--------------- wasm/demo/src/style.css | 2 +- wasm/demo/webpack.config.js | 14 ++++- wasm/lib/src/convert.rs | 12 +++- 10 files changed, 121 insertions(+), 85 deletions(-) delete mode 100644 wasm/demo/src/browser_module.rs diff --git a/Lib/_dummy_os.py b/Lib/_dummy_os.py index 5bd5ec0a13..38e287af69 100644 --- a/Lib/_dummy_os.py +++ b/Lib/_dummy_os.py @@ -5,22 +5,30 @@ try: from os import * except ImportError: - import abc + import abc, sys def __getattr__(name): - raise OSError("no os specific module found") + if name in {"_path_normpath", "__path__"}: + raise AttributeError(name) + if name.isupper(): + return 0 + def dummy(*args, **kwargs): + import io + return io.UnsupportedOperation(f"{name}: no os specific module found") + dummy.__name__ = f"dummy_{name}" + return dummy - def _shim(): - import _dummy_os, sys - sys.modules['os'] = _dummy_os - sys.modules['os.path'] = _dummy_os.path + sys.modules['os'] = sys.modules['posix'] = sys.modules[__name__] import posixpath as path - import sys sys.modules['os.path'] = path del sys sep = path.sep + supports_dir_fd = set() + supports_effective_ids = set() + supports_fd = set() + supports_follow_symlinks = set() def fspath(path): diff --git a/Lib/io.py b/Lib/io.py index 7362ae305d..c2812876d3 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -55,10 +55,15 @@ import abc from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, - open, open_code, FileIO, BytesIO, StringIO, BufferedReader, + open, open_code, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, IncrementalNewlineDecoder, text_encoding, TextIOWrapper) +try: + from _io import FileIO +except ImportError: + pass + # Pretend this exception was created here. UnsupportedOperation.__module__ = "io" @@ -82,7 +87,10 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase): class TextIOBase(_io._TextIOBase, IOBase): __doc__ = _io._TextIOBase.__doc__ -RawIOBase.register(FileIO) +try: + RawIOBase.register(FileIO) +except NameError: + pass for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom, BufferedRWPair): diff --git a/wasm/demo/package.json b/wasm/demo/package.json index c875132508..53ed264ea7 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -4,24 +4,25 @@ "description": "Bindings to the RustPython library for WebAssembly", "main": "index.js", "dependencies": { - "codemirror": "^5.42.0", - "local-echo": "^0.2.0", - "xterm": "^3.8.0" + "@codemirror/lang-python": "^6.1.6", + "@xterm/addon-fit": "^0.10.0", + "@xterm/xterm": "^5.3.0", + "codemirror": "^6.0.1", + "upgrade": "^1.1.0", + "xterm-readline": "^1.1.2" }, "devDependencies": { - "@wasm-tool/wasm-pack-plugin": "^1.1.0", - "clean-webpack-plugin": "^3.0.0", - "css-loader": "^3.4.1", - "html-webpack-plugin": "^3.2.0", - "mini-css-extract-plugin": "^0.9.0", - "raw-loader": "^4.0.0", - "serve": "^11.0.2", - "webpack": "^4.16.3", - "webpack-cli": "^3.1.0", - "webpack-dev-server": "^3.1.5" + "@wasm-tool/wasm-pack-plugin": "^1.7.0", + "css-loader": "^7.1.2", + "html-webpack-plugin": "^5.6.3", + "mini-css-extract-plugin": "^2.9.2", + "serve": "^14.2.4", + "webpack": "^5.97.1", + "webpack-cli": "^6.0.1", + "webpack-dev-server": "^5.2.0" }, "scripts": { - "dev": "webpack-dev-server -d", + "dev": "webpack serve", "build": "webpack", "dist": "webpack --mode production", "test": "webpack --mode production && cd ../tests && pytest" diff --git a/wasm/demo/snippets/import_pypi.py b/wasm/demo/snippets/import_pypi.py index cbe837cd05..a7c8a8e7fd 100644 --- a/wasm/demo/snippets/import_pypi.py +++ b/wasm/demo/snippets/import_pypi.py @@ -1,13 +1,12 @@ -import asyncweb -import whlimport - -whlimport.setup() - # make sys.modules['os'] a dumb version of the os module, which has posixpath # available as os.path as well as a few other utilities, but will raise an # OSError for anything that actually requires an OS import _dummy_os -_dummy_os._shim() + +import asyncweb +import whlimport + +whlimport.setup() @asyncweb.main async def main(): diff --git a/wasm/demo/src/browser_module.rs b/wasm/demo/src/browser_module.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/wasm/demo/src/index.ejs b/wasm/demo/src/index.ejs index 02f2f52f5c..d261b110e5 100644 --- a/wasm/demo/src/index.ejs +++ b/wasm/demo/src/index.ejs @@ -14,7 +14,6 @@ browser's devtools and play with rp.pyEval('1 + 1')

- ", symbol="single"): + try: + tree = self.compile.compiler( + source, + filename, + "exec", + ast.PyCF_ONLY_AST, + incomplete_input=False, + ) + except (SyntaxError, OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + if tree.body: + *_, last_stmt = tree.body + for stmt in tree.body: + wrapper = ast.Interactive if stmt is last_stmt else ast.Module + the_symbol = symbol if stmt is last_stmt else "exec" + item = wrapper([stmt]) + try: + code = self.compile.compiler(item, filename, the_symbol) + except SyntaxError as e: + if e.args[0] == "'await' outside function": + python = os.path.basename(sys.executable) + e.add_note( + f"Try the asyncio REPL ({python} -m asyncio) to use" + f" top-level 'await' and run background asyncio tasks." + ) + self.showsyntaxerror(filename, source=source) + return False + except (OverflowError, ValueError): + self.showsyntaxerror(filename, source=source) + return False + + if code is None: + return True + + self.runcode(code) + return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 0000000000..3a624d9f68 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 0000000000..0468b9a267 --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,76 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + # Always return a copy of the control characters list to ensure + # there are not any additional references to self.cc + self.cc[:], + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 0000000000..c4b95fa2e8 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,419 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class history_search_backward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=False) + + +class history_search_forward(commands.Command): + def do(self) -> None: + r = self.reader + r.search_next(forwards=True) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + history_search_backward, + history_search_forward, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "history-search-forward"), + (r"\x1b[6~", "history-search-forward"), + (r"\", "history-search-backward"), + (r"\x1b[5~", "history-search-backward"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i].rstrip() + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + self.last_refresh_cache.invalidated = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(), self.suspend_history(): + yield + + @contextmanager + def suspend_history(self) -> SimpleContextManager: + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def search_next(self, *, forwards: bool) -> None: + """Search history for the current line contents up to the cursor. + + Selects the first item found. If nothing is under the cursor, any next + item in history is selected. + """ + pos = self.pos + s = self.get_unicode() + history_index = self.historyi + + # In multiline contexts, we're only interested in the current line. + nl_index = s.rfind('\n', 0, pos) + prefix = s[nl_index + 1:pos] + pos = len(prefix) + + match_prefix = len(prefix) + len_item = 0 + if history_index < len(self.history): + len_item = len(self.get_item(history_index)) + if len_item and pos == len_item: + match_prefix = False + elif not pos: + match_prefix = False + + while 1: + if forwards: + out_of_bounds = history_index >= len(self.history) - 1 + else: + out_of_bounds = history_index == 0 + if out_of_bounds: + if forwards and not match_prefix: + self.pos = 0 + self.buffer = [] + self.dirty = True + else: + self.error("not found") + return + + history_index += 1 if forwards else -1 + s = self.get_item(history_index) + + if not match_prefix: + self.select_item(history_index) + return + + len_acc = 0 + for i, line in enumerate(s.splitlines(keepends=True)): + if line.startswith(prefix): + self.select_item(history_index) + self.pos = pos + len_acc + return + len_acc += len(line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 0000000000..21c24eb5cd --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self) -> bool: + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 0000000000..2fb03d1952 --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,213 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +Keymap contains functions for parsing keyspecs and turning keyspecs into +appropriate sequences. + +A keyspec is a string representing a sequence of key presses that can +be bound to a command. All characters other than the backslash represent +themselves. In the traditional manner, a backslash introduces an escape +sequence. + +pyrepl uses its own keyspec format that is meant to be a strict superset of +readline's KEYSEQ format. This means that if a spec is found that readline +accepts that this doesn't, it should be logged as a bug. Note that this means +we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort. + +The extension to readline is that the sequence \\ denotes the +sequence of characters produced by hitting KEY. + +Examples: +`a' - what you get when you hit the `a' key +`\\EOA' - Escape - O - A (up, on my terminal) +`\\' - the up arrow key +`\\' - ditto (keynames are case-insensitive) +`\\C-o', `\\c-o' - control-o +`\\M-.' - meta-period +`\\E.' - ditto (that's how meta works for pyrepl) +`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def parse_keys(keys: str) -> list[str]: + """Parse keys in keyspec format to a sequence of keys.""" + s = 0 + r: list[str] = [] + while s < len(keys): + k, s = _parse_single_key_sequence(keys, s) + r.extend(k) + return r + + +def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]: + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) == 1: + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + elif ret in {"left", "right"}: + ret = f"ctrl {ret}" + else: + raise KeySpecError("\\C- followed by invalid key") + + result = [ret], s + if meta: + result[0].insert(0, "\033") + return result + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py new file mode 100644 index 0000000000..a6f824dcc4 --- /dev/null +++ b/Lib/_pyrepl/main.py @@ -0,0 +1,59 @@ +import errno +import os +import sys + + +CAN_USE_PYREPL: bool +FAIL_REASON: str +try: + if sys.platform == "win32" and sys.getwindowsversion().build < 10586: + raise RuntimeError("Windows 10 TH2 or later required") + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) +except Exception as e: + CAN_USE_PYREPL = False + FAIL_REASON = f"warning: can't use pyrepl: {e}" +else: + CAN_USE_PYREPL = True + FAIL_REASON = "" + + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + if not CAN_USE_PYREPL: + if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON: + from .trace import trace + trace(FAIL_REASON) + print(FAIL_REASON, file=sys.stderr) + return sys._baserepl() + + if mainmodule: + namespace = mainmodule.__dict__ + else: + import __main__ + namespace = __main__.__dict__ + namespace.pop("__pyrepl_interactive_console", None) + + # sys._baserepl() above does this internally, we do it here + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + sys.audit("cpython.run_startup", startup_path) + + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code, namespace) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + + from .console import InteractiveColoredConsole + from .simple_interact import run_multiline_interactive_console + console = InteractiveColoredConsole(namespace, filename="") + run_multiline_interactive_console(console) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 0000000000..395f5945ab --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,24 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code,redundant-expr +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 0000000000..1fddc63e3e --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plain_pager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfile_pager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'pager', title) + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + has_tty = True + + def getchar() -> str: + return sys.stdin.read(1) + + except (ImportError, AttributeError, io.UnsupportedOperation): + def getchar() -> str: + return sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 0000000000..dc26bfd3a3 --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,816 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import sys + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .trace import trace + + +# types +Command = commands.Command +from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed representation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if c == '\x1a': + s.append(c) + b.append(2) + elif ord(c) < 128: + s.append(c) + b.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.extend([0] * (len(c) - 1)) + else: + s.append(c) + b.append(str_width(c)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + (r"\M-\n", "accept"), + ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x03", "ctrl-c"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\C-\", "backward-word"), + (r"\", "right"), + (r"\C-\", "forward-word"), + (r"\", "delete"), + (r"\x1b[3~", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into `buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to `syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + in_bracketed_paste: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) + can_colorize: bool = False + threading_hook: Callback | None = None + + ## cached metadata to speed up screen refreshes + @dataclass + class RefreshCache: + in_bracketed_paste: bool = False + screen: list[str] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + line_end_offsets: list[int] = field(default_factory=list) + pos: int = field(init=False) + cxy: tuple[int, int] = field(init=False) + dimensions: tuple[int, int] = field(init=False) + invalidated: bool = False + + def update_cache(self, + reader: Reader, + screen: list[str], + screeninfo: list[tuple[int, list[int]]], + ) -> None: + self.in_bracketed_paste = reader.in_bracketed_paste + self.screen = screen.copy() + self.screeninfo = screeninfo.copy() + self.pos = reader.pos + self.cxy = reader.cxy + self.dimensions = reader.console.width, reader.console.height + self.invalidated = False + + def valid(self, reader: Reader) -> bool: + if self.invalidated: + return False + dimensions = reader.console.width, reader.console.height + dimensions_changed = dimensions != self.dimensions + paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste + return not (dimensions_changed or paste_changed) + + def get_cached_location(self, reader: Reader) -> tuple[int, int]: + if self.invalidated: + raise ValueError("Cache is invalidated") + offset = 0 + earliest_common_pos = min(reader.pos, self.pos) + num_common_lines = len(self.line_end_offsets) + while num_common_lines > 0: + offset = self.line_end_offsets[num_common_lines - 1] + if earliest_common_pos > offset: + break + num_common_lines -= 1 + else: + offset = 0 + return offset, num_common_lines + + last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.can_colorize = can_colorize() + + self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.pos = self.pos + self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """Translate changes in self.buffer into changes in self.console.screen.""" + # Since the last call to calc_screen: + # screen and screeninfo may differ due to a completion menu being shown + # pos and cxy may differ due to edits, cursor movements, or completion menus + + # Lines that are above both the old and new cursor position can't have changed, + # unless the terminal has been resized (which might cause reflowing) or we've + # entered or left paste mode (which changes prompts, causing reflowing). + num_common_lines = 0 + offset = 0 + if self.last_refresh_cache.valid(self): + offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) + + screen = self.last_refresh_cache.screen + del screen[num_common_lines:] + + screeninfo = self.last_refresh_cache.screeninfo + del screeninfo[num_common_lines:] + + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets + del last_refresh_line_end_offsets[num_common_lines:] + + pos = self.pos + pos -= offset + + prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + + lines = "".join(self.buffer[offset:]).split("\n") + + cursor_found = False + lines_beyond_cursor = 0 + for ln, line in enumerate(lines, num_common_lines): + ll = len(line) + if 0 <= pos <= ll: + self.lxy = pos, ln + cursor_found = True + elif cursor_found: + lines_beyond_cursor += 1 + if lines_beyond_cursor > self.console.height: + # No need to keep formatting lines. + # The console can't show them. + break + if prompt_from_cache: + # Only the first line's prompt can come from the cache + prompt_from_cache = False + prompt = "" + else: + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + last_refresh_line_end_offsets.append(offset) + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + offset += ll + 1 # Takes all of the line plus the newline + last_refresh_line_end_offsets.append(offset) + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + i = 0 + while l: + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + if len(l) > index_to_wrap_before: + offset += index_to_wrap_before + post = "\\" + after = [1] + else: + offset += index_to_wrap_before + 1 # Takes the newline + post = "" + after = [] + last_refresh_line_end_offsets.append(offset) + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + i += 1 + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + + self.last_refresh_cache.update_cache(self, screen, screeninfo) + return screen + + @staticmethod + def process_prompt(prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, + # which breaks the logic below so we redefine it here. + def wlen(s: str) -> int: + return sum(str_width(i) for i in s) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning `default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + `lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = f"(arg: {self.arg}) " + elif self.paste_mode and not self.in_bracketed_paste: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif self.ps4 and lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if self.can_colorize: + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + p, y = 0, 0 + l2: list[int] = [] + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer) and len(self.screeninfo) > 0: + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": + return + + # this call sets up self.cxy, so call it first. + self.screen = self.calc_screen() + self.console.refresh(self.screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + + trace("received command {cmd}", cmd=cmd) + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + + command = command_type(self, *cmd) # type: ignore[arg-type] + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def run_hooks(self) -> None: + threading_hook = self.threading_hook + if threading_hook is None and 'threading' in sys.modules: + from ._threading_handler import install_threading_hook + install_threading_hook(self) + if threading_hook is not None: + try: + threading_hook() + except Exception: + pass + + input_hook = self.console.input_hook + if input_hook: + try: + input_hook() + except Exception: + pass + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + # We use the same timeout as in readline.c: 100ms + self.run_hooks() + self.console.wait(100) + event = self.console.get_event(block=False) + if not event: + if block: + continue + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 0000000000..888185eb03 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,598 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field + +import os +from site import gethistoryfile # type: ignore[attr-defined] +import sys +from rlcompleter import Completer as RLCompleter + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .console import Console as ConsoleType + +Console: type[ConsoleType] +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import UnixConsole as Console, _error +except ImportError: + from .windows_console import WindowsConsole as Console, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Mapping + + +MoreLinesCallable = Callable[[str], bool] + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = None + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: MoreLinesCallable | None = None + last_used_indentation: str | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + def update_last_used_indentation(self) -> None: + indentation = _get_first_indentation(self.buffer) + if indentation is not None: + self.last_used_indentation = indentation + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +def _get_first_indentation(buffer: list[str]) -> str | None: + indented_line_start = None + for i in range(len(buffer)): + if (i < len(buffer) - 1 + and buffer[i] == "\n" + and buffer[i + 1] in " \t" + ): + indented_line_start = i + 1 + elif indented_line_start is not None and buffer[i] not in " \t\n": + return ''.join(buffer[indented_line_start : i]) + return None + + +def _should_auto_indent(buffer: list[str], pos: int) -> bool: + # check if last character before "pos" is a colon, ignoring + # whitespaces and comments. + last_char = None + while pos > 0: + pos -= 1 + if last_char is None: + if buffer[pos] not in " \t\n#": # ignore whitespaces and comments + last_char = buffer[pos] + else: + # even if we found a non-whitespace character before + # original pos, we keep going back until newline is reached + # to make sure we ignore comments + if buffer[pos] == "\n": + break + if buffer[pos] == "#": + last_char = None + return last_char == ":" + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + + if self.reader.in_bracketed_paste: + r.insert("\n") + return + + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + def _newline_before_pos(): + before_idx = r.pos - 1 + while before_idx > 0 and text[before_idx].isspace(): + before_idx -= 1 + return text[before_idx : r.pos].count("\n") > 0 + + # if there's already a new line before the cursor then + # even if the cursor is followed by whitespace, we assume + # the user is trying to terminate the block + if _newline_before_pos() and text[r.pos:].isspace(): + self.finish = True + return + + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode: + if indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + r.update_last_used_indentation() + if _should_auto_indent(r.buffer, r.pos): + if r.last_used_indentation is not None: + indentation = r.last_used_indentation + else: + # default + indentation = " " * 4 + r.insert(indentation) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = field(default=None, repr=False) + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = Console(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + prompt_str = str(prompt) + reader.ps1 = prompt_str + sys.audit("builtins.input", prompt_str) + result = reader.readline(startup_hook=self.startup_hook) + sys.audit("builtins.input/result", result) + return result + + def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str: + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = ps1 + reader.ps2 = ps1 + reader.ps3 = ps2 + reader.ps4 = "" + with warnings.catch_warnings(action="ignore"): + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + is_editline = f.readline().startswith(b"_HiStOrY_V2_") + if is_editline: + encoding = "unicode-escape" + else: + f.seek(0) + encoding = "utf-8" + + lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = self._histline("".join(buffer).replace("\r", "") + line) + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + f = open(os.path.expanduser(filename), "w", + encoding="utf-8", newline="\n") + with f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> str: + return self.get_reader().get_unicode() + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if buf[i] in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup(namespace: Mapping[str, Any]) -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # set up namespace in rlcompleter, which requires it to be a bona fide dict + if not isinstance(namespace, dict): + namespace = dict(namespace) + _wrapper.config.readline_completer = RLCompleter(namespace).complete + + # this is not really what readline.c does. Better than nothing I guess + import builtins + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 0000000000..66e66eae7e --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,167 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _sitebuiltins +import linecache +import functools +import os +import sys +import code + +from .readline import _get_reader, multiline_input + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + + +_error: tuple[type[Exception], ...] | type[Exception] +try: + from .unix_console import _error +except ModuleNotFoundError: + from .windows_console import _error + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + if term := os.environ.get("TERM", ""): + term = f"; TERM={term}" + return str(str(e) or repr(e) or "unknown error") + term + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": _sitebuiltins._Helper(), + "clear": _clear_screen, + "\x1a": _sitebuiltins.Quitter('\x1a', ''), +} + + +def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + lines = src.splitlines(keepends=True) + if len(lines) == 1: + return False + + last_line = lines[-1] + was_indented = last_line.startswith((" ", "\t")) + not_empty = last_line.strip() != "" + incomplete = not last_line.endswith("\n") + return (was_indented or not_empty) and incomplete + else: + return code is None + + +def run_multiline_interactive_console( + console: code.InteractiveConsole, + *, + future_flags: int = 0, +) -> None: + from .readline import _setup + _setup(console.locals) + if future_flags: + console.compile.compiler.flags |= future_flags + + more_lines = functools.partial(_more_lines, console) + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + # Make sure that history does not change because of commands + with reader.suspend_history(): + command() + return True + return False + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"" + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + r = _get_reader() + if r.input_trans is r.isearch_trans: + r.do_cmd(("isearch-end", [""])) + r.pos = len(r.get_unicode()) + r.dirty = True + r.refresh() + r.in_bracketed_paste = False + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 0000000000..a8eb2433cd --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 0000000000..f9d48b828c --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 0000000000..e69c96b115 --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,810 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import termios +import time +import platform +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +TYPE_CHECKING = False + +# types +if TYPE_CHECKING: + from typing import IO, Literal, overload +else: + overload = lambda func: None + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minumum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* + def poll(self, timeout: float | None = None) -> list[int]: + if timeout is None: + r, w, e = select.select([self.fd], [], []) + else: + r, w, e = select.select([self.fd], [], [], timeout/1000) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + super().__init__(f_in, f_out, term, encoding) + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + self.input_buffer = b"" + self.input_buffer_pos = 0 + curses.setupterm(term or None, self.output_fd) + self.term = term + + @overload + def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + + @overload + def _my_getstr(cap: str, optional: bool) -> bytes | None: ... + + def _my_getstr(cap: str, optional: bool = False) -> bytes | None: + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def more_in_buffer(self) -> bool: + return bool( + self.input_buffer + and self.input_buffer_pos < len(self.input_buffer) + ) + + def __read(self, n: int) -> bytes: + if not self.more_in_buffer(): + self.input_buffer = os.read(self.input_fd, 10000) + + ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] + self.input_buffer_pos += len(ret) + if self.input_buffer_pos >= len(self.input_buffer): + self.input_buffer = b"" + self.input_buffer_pos = 0 + return ret + + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen.copy() + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.iflag |= termios.BRKINT + raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) + raw.lflag |= termios.ISIG + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + # In macOS terminal we need to deactivate line wrap via ANSI escape code + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7l") + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + self.__enable_bracketed_paste() + + def restore(self): + """ + Restore the console to the default state + """ + self.__disable_bracketed_paste() + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + os.write(self.output_fd, b"\033[?7h") + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + if not block and not self.wait(timeout=0): + return None + + while self.event_queue.empty(): + while True: + try: + self.push_char(self.__read(1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + return self.event_queue.get() + + def wait(self, timeout: float | None = None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.more_in_buffer() + or bool(self.pollob.poll(timeout)) + ) + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except (KeyError, TypeError, ValueError): + try: + size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + except OSError: + return 25, 80 + height, width = struct.unpack("hhhh", size)[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except (KeyError, TypeError, ValueError): + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = self.__read(amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.posxy = 0, 0 + self.screen = [] + + @property + def input_hook(self): + try: + import posix + except ImportError: + return None + if posix._is_inputhook_installed(): + return posix._inputhook + + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.posxy[1] + and x_coord > self.posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif ( + x_coord < minlen + and oldline[x_pos + 1 :] == newline[x_pos + 1 :] + and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + ): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + assert self._cud1 is not None + assert self._cuu1 is not None + dy = y - self.posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x: int) -> None: + if x != self.posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x: int) -> None: + assert self._cuf1 is not None + assert self._cub1 is not None + dx = x - self.posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x: int) -> None: + dx = x - self.posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.posxy = 0, self.posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 0000000000..70cfade26e --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 0000000000..4651717bd7 --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,25 @@ +import re +import unicodedata +import functools + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +@functools.cache +def str_width(c: str) -> int: + if ord(c) < 128: + return 1 + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + if len(s) == 1 and s != '\x1a': + return str_width(s) + length = sum(str_width(i) for i in s) + # remove lengths of any escape sequences + sequence = ANSI_ESCAPE_SEQUENCE.findall(s) + ctrl_z_cnt = s.count('\x1a') + return length - sum(len(i) for i in sequence) + ctrl_z_cnt diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py new file mode 100644 index 0000000000..fffadd5e2e --- /dev/null +++ b/Lib/_pyrepl/windows_console.py @@ -0,0 +1,618 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import io +import os +import sys +import time +import msvcrt + +from collections import deque +import ctypes +from ctypes.wintypes import ( + _COORD, + WORD, + SMALL_RECT, + BOOL, + HANDLE, + CHAR, + DWORD, + WCHAR, + SHORT, +) +from ctypes import Structure, POINTER, Union +from .console import Event, Console +from .trace import trace +from .utils import wlen + +try: + from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] +except: + # Keep MyPy happy off Windows + from ctypes import CDLL as WinDLL, cdll as windll + + def GetLastError() -> int: + return 42 + + class WinError(OSError): # type: ignore[no-redef] + def __init__(self, err: int | None, descr: str | None = None) -> None: + self.err = err + self.descr = descr + + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import IO + +# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes +VK_MAP: dict[int, str] = { + 0x23: "end", # VK_END + 0x24: "home", # VK_HOME + 0x25: "left", # VK_LEFT + 0x26: "up", # VK_UP + 0x27: "right", # VK_RIGHT + 0x28: "down", # VK_DOWN + 0x2E: "delete", # VK_DELETE + 0x70: "f1", # VK_F1 + 0x71: "f2", # VK_F2 + 0x72: "f3", # VK_F3 + 0x73: "f4", # VK_F4 + 0x74: "f5", # VK_F5 + 0x75: "f6", # VK_F6 + 0x76: "f7", # VK_F7 + 0x77: "f8", # VK_F8 + 0x78: "f9", # VK_F9 + 0x79: "f10", # VK_F10 + 0x7A: "f11", # VK_F11 + 0x7B: "f12", # VK_F12 + 0x7C: "f13", # VK_F13 + 0x7D: "f14", # VK_F14 + 0x7E: "f15", # VK_F15 + 0x7F: "f16", # VK_F16 + 0x80: "f17", # VK_F17 + 0x81: "f18", # VK_F18 + 0x82: "f19", # VK_F19 + 0x83: "f20", # VK_F20 +} + +# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +ERASE_IN_LINE = "\x1b[K" +MOVE_LEFT = "\x1b[{}D" +MOVE_RIGHT = "\x1b[{}C" +MOVE_UP = "\x1b[{}A" +MOVE_DOWN = "\x1b[{}B" +CLEAR = "\x1b[H\x1b[J" + + +class _error(Exception): + pass + + +class WindowsConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + super().__init__(f_in, f_out, term, encoding) + + SetConsoleMode( + OutHandle, + ENABLE_WRAP_AT_EOL_OUTPUT + | ENABLE_PROCESSED_OUTPUT + | ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + self.screen: list[str] = [] + self.width = 80 + self.height = 25 + self.__offset = 0 + self.event_queue: deque[Event] = deque() + try: + self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] + except ValueError: + # Console I/O is redirected, fallback... + self.out = None + + def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + + while len(self.screen) < min(len(screen), self.height): + self._hide_cursor() + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") + self.posxy = 0, len(self.screen) + self.screen.append("") + + px, py = self.posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + scroll_lines = offset - old_offset + + # Scrolling the buffer as the current input is greater than the visible + # portion of the window. We need to scroll the visible portion and the + # entire history + self._scroll(scroll_lines, self._getscrollbacksize()) + self.posxy = self.posxy[0], self.posxy[1] + scroll_lines + self.__offset += scroll_lines + + for i in range(scroll_lines): + self.screen.append("") + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + self.__offset = offset + + self._hide_cursor() + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self._move_relative(0, y) + self.posxy = 0, y + self._erase_to_end() + y += 1 + + self._show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + + @property + def input_hook(self): + try: + import nt + except ImportError: + return None + if nt._is_inputhook_installed(): + return nt._inputhook + + def __write_changed_line( + self, y: int, oldline: str, newline: str, px_coord: int + ) -> None: + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: + break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while ( + x_coord < minlen + and oldline[x_pos] == newline[x_pos] + and newline[x_pos] != "\x1b" + ): + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + self._hide_cursor() + self._move_relative(x_coord, y) + if wlen(oldline) > wlen(newline): + self._erase_to_end() + + self.__write(newline[x_pos:]) + if wlen(newline) == self.width: + # If we wrapped we want to start at the next line + self._move_relative(0, y + 1) + self.posxy = 0, y + 1 + else: + self.posxy = wlen(newline), y + + if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def _scroll( + self, top: int, bottom: int, left: int | None = None, right: int | None = None + ) -> None: + scroll_rect = SMALL_RECT() + scroll_rect.Top = SHORT(top) + scroll_rect.Bottom = SHORT(bottom) + scroll_rect.Left = SHORT(0 if left is None else left) + scroll_rect.Right = SHORT( + self.getheightwidth()[1] - 1 if right is None else right + ) + destination_origin = _COORD() + fill_info = CHAR_INFO() + fill_info.UnicodeChar = " " + + if not ScrollConsoleScreenBuffer( + OutHandle, scroll_rect, None, destination_origin, fill_info + ): + raise WinError(GetLastError()) + + def _hide_cursor(self): + self.__write("\x1b[?25l") + + def _show_cursor(self): + self.__write("\x1b[?25h") + + def _enable_blinking(self): + self.__write("\x1b[?12h") + + def _disable_blinking(self): + self.__write("\x1b[?12l") + + def __write(self, text: str) -> None: + if "\x1a" in text: + text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + + if self.out is not None: + self.out.write(text.encode(self.encoding, "replace")) + self.out.flush() + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + + @property + def screen_xy(self) -> tuple[int, int]: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return info.dwCursorPosition.X, info.dwCursorPosition.Y + + def _erase_to_end(self) -> None: + self.__write(ERASE_IN_LINE) + + def prepare(self) -> None: + trace("prepare") + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.posxy = 0, 0 + self.__gone_tall = 0 + self.__offset = 0 + + def restore(self) -> None: + pass + + def _move_relative(self, x: int, y: int) -> None: + """Moves relative to the current posxy""" + dx = x - self.posxy[0] + dy = y - self.posxy[1] + if dx < 0: + self.__write(MOVE_LEFT.format(-dx)) + elif dx > 0: + self.__write(MOVE_RIGHT.format(dx)) + + if dy < 0: + self.__write(MOVE_UP.format(-dy)) + elif dy > 0: + self.__write(MOVE_DOWN.format(dy)) + + def move_cursor(self, x: int, y: int) -> None: + if x < 0 or y < 0: + raise ValueError(f"Bad cursor position {x}, {y}") + + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(0, Event("scroll", "")) + else: + self._move_relative(x, y) + self.posxy = x, y + + def set_cursor_vis(self, visible: bool) -> None: + if visible: + self._show_cursor() + else: + self._hide_cursor() + + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + return ( + info.srWindow.Bottom - info.srWindow.Top + 1, + info.srWindow.Right - info.srWindow.Left + 1, + ) + + def _getscrollbacksize(self) -> int: + info = CONSOLE_SCREEN_BUFFER_INFO() + if not GetConsoleScreenBufferInfo(OutHandle, info): + raise WinError(GetLastError()) + + return info.srWindow.Bottom # type: ignore[no-any-return] + + def _read_input(self, block: bool = True) -> INPUT_RECORD | None: + if not block: + events = DWORD() + if not GetNumberOfConsoleInputEvents(InHandle, events): + raise WinError(GetLastError()) + if not events.value: + return None + + rec = INPUT_RECORD() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, 1, read): + raise WinError(GetLastError()) + + return rec + + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + if self.event_queue: + return self.event_queue.pop() + + while True: + rec = self._read_input(block) + if rec is None: + return None + + if rec.EventType == WINDOW_BUFFER_SIZE_EVENT: + return Event("resize", "") + + if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown: + # Only process keys and keydown events + if block: + continue + return None + + key = rec.Event.KeyEvent.uChar.UnicodeChar + + if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": + # Make enter make unix-like + return Event(evt="key", data="\n", raw=b"\n") + elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + # Turn backspace directly into the command + return Event( + evt="key", + data="backspace", + raw=rec.Event.KeyEvent.uChar.UnicodeChar, + ) + elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + # Handle special keys like arrow keys and translate them into the appropriate command + code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) + if code: + return Event( + evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar + ) + if block: + continue + + return None + + return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + raise NotImplementedError("push_char not supported on Windows") + + def beep(self) -> None: + self.__write("\x07") + + def clear(self) -> None: + """Wipe the screen""" + self.__write(CLEAR) + self.posxy = 0, 0 + self.screen = [""] + + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write("\r\n") + + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere). + + All output on Windows is unbuffered so this is a nop""" + pass + + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + if not FlushConsoleInputBuffer(InHandle): + raise WinError(GetLastError()) + + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + return Event("key", "", b"") + + def wait(self, timeout: float | None) -> bool: + """Wait for an event.""" + # Poor man's Windows select loop + start_time = time.time() + while True: + if msvcrt.kbhit(): # type: ignore[attr-defined] + return True + if timeout and time.time() - start_time > timeout / 1000: + return False + time.sleep(0.01) + + def repaint(self) -> None: + raise NotImplementedError("No repaint support") + + +# Windows interop +class CONSOLE_SCREEN_BUFFER_INFO(Structure): + _fields_ = [ + ("dwSize", _COORD), + ("dwCursorPosition", _COORD), + ("wAttributes", WORD), + ("srWindow", SMALL_RECT), + ("dwMaximumWindowSize", _COORD), + ] + + +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [ + ("dwSize", DWORD), + ("bVisible", BOOL), + ] + + +class CHAR_INFO(Structure): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Attributes", WORD), + ] + + +class Char(Union): + _fields_ = [ + ("UnicodeChar", WCHAR), + ("Char", CHAR), + ] + + +class KeyEvent(ctypes.Structure): + _fields_ = [ + ("bKeyDown", BOOL), + ("wRepeatCount", WORD), + ("wVirtualKeyCode", WORD), + ("wVirtualScanCode", WORD), + ("uChar", Char), + ("dwControlKeyState", DWORD), + ] + + +class WindowsBufferSizeEvent(ctypes.Structure): + _fields_ = [("dwSize", _COORD)] + + +class ConsoleEvent(ctypes.Union): + _fields_ = [ + ("KeyEvent", KeyEvent), + ("WindowsBufferSizeEvent", WindowsBufferSizeEvent), + ] + + +class INPUT_RECORD(Structure): + _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)] + + +KEY_EVENT = 0x01 +FOCUS_EVENT = 0x10 +MENU_EVENT = 0x08 +MOUSE_EVENT = 0x02 +WINDOW_BUFFER_SIZE_EVENT = 0x04 + +ENABLE_PROCESSED_OUTPUT = 0x01 +ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 +ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 + +STD_INPUT_HANDLE = -10 +STD_OUTPUT_HANDLE = -11 + +if sys.platform == "win32": + _KERNEL32 = WinDLL("kernel32", use_last_error=True) + + GetStdHandle = windll.kernel32.GetStdHandle + GetStdHandle.argtypes = [DWORD] + GetStdHandle.restype = HANDLE + + GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo + GetConsoleScreenBufferInfo.argtypes = [ + HANDLE, + ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + GetConsoleScreenBufferInfo.restype = BOOL + + ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW + ScrollConsoleScreenBuffer.argtypes = [ + HANDLE, + POINTER(SMALL_RECT), + POINTER(SMALL_RECT), + _COORD, + POINTER(CHAR_INFO), + ] + ScrollConsoleScreenBuffer.restype = BOOL + + SetConsoleMode = _KERNEL32.SetConsoleMode + SetConsoleMode.argtypes = [HANDLE, DWORD] + SetConsoleMode.restype = BOOL + + ReadConsoleInput = _KERNEL32.ReadConsoleInputW + ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] + ReadConsoleInput.restype = BOOL + + GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents + GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] + GetNumberOfConsoleInputEvents.restype = BOOL + + FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer + FlushConsoleInputBuffer.argtypes = [HANDLE] + FlushConsoleInputBuffer.restype = BOOL + + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) + InHandle = GetStdHandle(STD_INPUT_HANDLE) +else: + + def _win_only(*args, **kwargs): + raise NotImplementedError("Windows only") + + GetStdHandle = _win_only + GetConsoleScreenBufferInfo = _win_only + ScrollConsoleScreenBuffer = _win_only + SetConsoleMode = _win_only + ReadConsoleInput = _win_only + GetNumberOfConsoleInputEvents = _win_only + FlushConsoleInputBuffer = _win_only + OutHandle = 0 + InHandle = 0 diff --git a/Lib/site.py b/Lib/site.py index 271524c0cf..acc8481b13 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -679,5 +679,17 @@ def exists(path): print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) sys.exit(10) +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + if __name__ == '__main__': _script() diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index befa0f8dff..3a66f7f80d 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -26,6 +26,7 @@ mod sys { use std::os::windows::ffi::OsStrExt; use std::{ env::{self, VarError}, + io::Read, path, sync::atomic::Ordering, }; @@ -307,6 +308,21 @@ mod sys { .collect() } + #[cfg(feature = "rustpython-compiler")] + #[pyfunction] + fn _baserepl(vm: &VirtualMachine) -> PyResult<()> { + // read stdin to end + let stdin = std::io::stdin(); + let mut handle = stdin.lock(); + let mut source = String::new(); + handle + .read_to_string(&mut source) + .map_err(|e| vm.new_os_error(format!("Error reading from stdin: {}", e)))?; + vm.compile(&source, crate::compiler::Mode::Single, "".to_owned()) + .map_err(|e| vm.new_os_error(format!("Error running stdin: {}", e)))?; + Ok(()) + } + #[pyfunction] fn audit(_args: FuncArgs) { // TODO: sys.audit implementation From b81ae9b954079d07b8112422572e08f99939f311 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 10 Apr 2025 17:37:20 -0700 Subject: [PATCH 212/317] More cspell fixes (#5670) --- .cspell.dict/rust-more.txt | 1 + .cspell.json | 9 +++++++-- Lib/test/test_bytes.py | 4 ++-- Lib/test/test_sqlite3/test_dbapi.py | 2 +- benches/execution.rs | 6 +++--- benches/microbenchmarks.rs | 4 ++-- common/src/float_ops.rs | 2 +- common/src/hash.rs | 2 +- stdlib/src/math.rs | 2 +- wasm/demo/src/index.js | 6 +++--- wasm/lib/src/js_module.rs | 2 +- wasm/notebook/src/index.js | 6 +++--- wtf8/src/lib.rs | 2 +- 13 files changed, 27 insertions(+), 21 deletions(-) diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index 99e87e532c..e92f1ff8c1 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -43,6 +43,7 @@ modpow msvc muldiv nanos +nonoverlapping objclass peekable powc diff --git a/.cspell.json b/.cspell.json index caa53a2879..69ace83483 100644 --- a/.cspell.json +++ b/.cspell.json @@ -47,7 +47,7 @@ // words - list of words to be always considered correct "words": [ "RUSTPYTHONPATH", - // RustPython + // RustPython terms "aiterable", "alnum", "baseclass", @@ -67,6 +67,7 @@ "GetSet", "groupref", "internable", + "lossily", "makeunicodedata", "miri", "notrace", @@ -96,6 +97,7 @@ "PyResult", "pyslot", "PyStaticMethod", + "pystone", "pystr", "pystruct", "pystructseq", @@ -104,13 +106,16 @@ "richcompare", "RustPython", "struc", + // plural of summand + "summands", "sysmodule", "tracebacks", "typealiases", - "Unconstructible", + "unconstructible", "unhashable", "uninit", "unraisable", + "unresizable", "wasi", "zelf", // unix diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index baf84642ee..cc1affc669 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1992,7 +1992,7 @@ def test_join(self): s3 = s1.join([b"abcd"]) self.assertIs(type(s3), self.basetype) - @unittest.skip("TODO: RUSTPYHON, Fails on ByteArraySubclassWithSlotsTest") + @unittest.skip("TODO: RUSTPYTHON, Fails on ByteArraySubclassWithSlotsTest") def test_pickle(self): a = self.type2test(b"abcd") a.x = 10 @@ -2007,7 +2007,7 @@ def test_pickle(self): self.assertEqual(type(a.z), type(b.z)) self.assertFalse(hasattr(b, 'y')) - @unittest.skip("TODO: RUSTPYHON, Fails on ByteArraySubclassWithSlotsTest") + @unittest.skip("TODO: RUSTPYTHON, Fails on ByteArraySubclassWithSlotsTest") def test_copy(self): a = self.type2test(b"abcd") a.x = 10 diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index bbc151fcb7..56cd8f8a68 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -591,7 +591,7 @@ def test_connection_bad_reinit(self): ((v,) for v in range(3))) -@unittest.skip("TODO: RUSTPYHON") +@unittest.skip("TODO: RUSTPYTHON") class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): self.cx = sqlite.Connection.__new__(sqlite.Connection) diff --git a/benches/execution.rs b/benches/execution.rs index d38dab0890..88e5e3650b 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -18,7 +18,7 @@ fn bench_cpython_code(b: &mut Bencher, source: &str) { }) } -fn bench_rustpy_code(b: &mut Bencher, name: &str, source: &str) { +fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) { // NOTE: Take long time. let mut settings = Settings::default(); settings.path_list.push("Lib/".to_string()); @@ -41,7 +41,7 @@ pub fn benchmark_file_execution(group: &mut BenchmarkGroup, name: &str bench_cpython_code(b, contents) }); group.bench_function(BenchmarkId::new(name, "rustpython"), |b| { - bench_rustpy_code(b, name, contents) + bench_rustpython_code(b, name, contents) }); } @@ -77,7 +77,7 @@ pub fn benchmark_pystone(group: &mut BenchmarkGroup, contents: String) bench_cpython_code(b, code_str) }); group.bench_function(BenchmarkId::new("rustpython", idx), |b| { - bench_rustpy_code(b, "pystone", code_str) + bench_rustpython_code(b, "pystone", code_str) }); } } diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 6f41f00d6c..c62ecf239a 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -104,7 +104,7 @@ fn cpy_compile_code<'a>( compile.call1((code, name, "exec"))?.extract() } -fn bench_rustpy_code(group: &mut BenchmarkGroup, bench: &MicroBenchmark) { +fn bench_rustpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchmark) { let mut settings = Settings::default(); settings.path_list.push("Lib/".to_string()); settings.write_bytecode = false; @@ -169,7 +169,7 @@ pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) { let mut group = c.benchmark_group("microbenchmarks"); bench_cpython_code(&mut group, &benchmark); - bench_rustpy_code(&mut group, &benchmark); + bench_rustpython_code(&mut group, &benchmark); group.finish(); } diff --git a/common/src/float_ops.rs b/common/src/float_ops.rs index 46e2d57067..b3c90d0ac6 100644 --- a/common/src/float_ops.rs +++ b/common/src/float_ops.rs @@ -2,7 +2,7 @@ use malachite_bigint::{BigInt, ToBigInt}; use num_traits::{Float, Signed, ToPrimitive, Zero}; use std::f64; -pub fn ufrexp(value: f64) -> (f64, i32) { +pub fn decompose_float(value: f64) -> (f64, i32) { if 0.0 == value { (0.0, 0i32) } else { diff --git a/common/src/hash.rs b/common/src/hash.rs index 1ae561650c..88af9b6f56 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -97,7 +97,7 @@ pub fn hash_float(value: f64) -> Option { }; } - let frexp = super::float_ops::ufrexp(value); + let frexp = super::float_ops::decompose_float(value); // process 28 bits at a time; this should work well both for binary // and hexadecimal floating point. diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index a7da60949c..4854bdd75e 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -557,7 +557,7 @@ mod math { fn frexp(x: ArgIntoFloat) -> (f64, i32) { let value = *x; if value.is_finite() { - let (m, exp) = float_ops::ufrexp(value); + let (m, exp) = float_ops::decompose_float(value); (m * value.signum(), exp) } else { (value, 0) diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index 1af847d59d..0b568fa1d9 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -13,10 +13,10 @@ let rp; // A dependency graph that contains any wasm must be imported asynchronously. import('rustpython') - .then((rustpy) => { - rp = rustpy; + .then((rustpython) => { + rp = rustpython; // so people can play around with it - window.rp = rustpy; + window.rp = rustpython; onReady(); }) .catch((e) => { diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs index f159c467d0..5a3ac4025b 100644 --- a/wasm/lib/src/js_module.rs +++ b/wasm/lib/src/js_module.rs @@ -575,7 +575,7 @@ mod _js { Some(prom) => { if val.is_some() { Err(vm.new_type_error( - "can't send non-None value to an awaitpromise".to_owned(), + "can't send non-None value to an AwaitPromise".to_owned(), )) } else { Ok(PyIterReturn::Return(prom)) diff --git a/wasm/notebook/src/index.js b/wasm/notebook/src/index.js index 422bc4d0d6..64b058a9ac 100644 --- a/wasm/notebook/src/index.js +++ b/wasm/notebook/src/index.js @@ -34,10 +34,10 @@ let rp; // A dependency graph that contains any wasm must be imported asynchronously. import('rustpython') - .then((rustpy) => { - rp = rustpy; + .then((rustpython) => { + rp = rustpython; // so people can play around with it - window.rp = rustpy; + window.rp = rustpython; onReady(); }) .catch((e) => { diff --git a/wtf8/src/lib.rs b/wtf8/src/lib.rs index 3ba28a5146..73b5168453 100644 --- a/wtf8/src/lib.rs +++ b/wtf8/src/lib.rs @@ -757,7 +757,7 @@ impl Wtf8 { /// Create a WTF-8 slice from a WTF-8 byte slice. // - // whooops! using WTF-8 for interchange! + // whoops! using WTF-8 for interchange! #[inline] pub fn from_bytes(b: &[u8]) -> Option<&Self> { let mut rest = b; From fd2764c7c7fc0f4ff4ec6fb89d38f807fe3c6c0f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 10 Apr 2025 19:11:12 -0700 Subject: [PATCH 213/317] Remove asynchat and asyncore (#5688) * try to remove asynchat and asyncore * remove associated tests * temp patch of test_ftplib --- Lib/asynchat.py | 307 -------------- Lib/asyncore.py | 642 ----------------------------- Lib/test/test_asynchat.py | 290 ------------- Lib/test/test_asyncore.py | 838 -------------------------------------- Lib/test/test_ftplib.py | 9 - 5 files changed, 2086 deletions(-) delete mode 100644 Lib/asynchat.py delete mode 100644 Lib/asyncore.py delete mode 100644 Lib/test/test_asynchat.py delete mode 100644 Lib/test/test_asyncore.py diff --git a/Lib/asynchat.py b/Lib/asynchat.py deleted file mode 100644 index fc1146adbb..0000000000 --- a/Lib/asynchat.py +++ /dev/null @@ -1,307 +0,0 @@ -# -*- Mode: Python; tab-width: 4 -*- -# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp -# Author: Sam Rushing - -# ====================================================================== -# Copyright 1996 by Sam Rushing -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Sam -# Rushing not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# ====================================================================== - -r"""A class supporting chat-style (command/response) protocols. - -This class adds support for 'chat' style protocols - where one side -sends a 'command', and the other sends a response (examples would be -the common internet protocols - smtp, nntp, ftp, etc..). - -The handle_read() method looks at the input stream for the current -'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' -for multi-line output), calling self.found_terminator() on its -receipt. - -for example: -Say you build an async nntp client using this class. At the start -of the connection, you'll have self.terminator set to '\r\n', in -order to process the single-line greeting. Just before issuing a -'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST -command will be accumulated (using your own 'collect_incoming_data' -method) up to the terminator, and then control will be returned to -you - by calling your self.found_terminator() method. -""" -import asyncore -from collections import deque - - -class async_chat(asyncore.dispatcher): - """This is an abstract class. You must derive from this class, and add - the two methods collect_incoming_data() and found_terminator()""" - - # these are overridable defaults - - ac_in_buffer_size = 65536 - ac_out_buffer_size = 65536 - - # we don't want to enable the use of encoding by default, because that is a - # sign of an application bug that we don't want to pass silently - - use_encoding = 0 - encoding = 'latin-1' - - def __init__(self, sock=None, map=None): - # for string terminator matching - self.ac_in_buffer = b'' - - # we use a list here rather than io.BytesIO for a few reasons... - # del lst[:] is faster than bio.truncate(0) - # lst = [] is faster than bio.truncate(0) - self.incoming = [] - - # we toss the use of the "simple producer" and replace it with - # a pure deque, which the original fifo was a wrapping of - self.producer_fifo = deque() - asyncore.dispatcher.__init__(self, sock, map) - - def collect_incoming_data(self, data): - raise NotImplementedError("must be implemented in subclass") - - def _collect_incoming_data(self, data): - self.incoming.append(data) - - def _get_data(self): - d = b''.join(self.incoming) - del self.incoming[:] - return d - - def found_terminator(self): - raise NotImplementedError("must be implemented in subclass") - - def set_terminator(self, term): - """Set the input delimiter. - - Can be a fixed string of any length, an integer, or None. - """ - if isinstance(term, str) and self.use_encoding: - term = bytes(term, self.encoding) - elif isinstance(term, int) and term < 0: - raise ValueError('the number of received bytes must be positive') - self.terminator = term - - def get_terminator(self): - return self.terminator - - # grab some more data from the socket, - # throw it to the collector method, - # check for the terminator, - # if found, transition to the next state. - - def handle_read(self): - - try: - data = self.recv(self.ac_in_buffer_size) - except BlockingIOError: - return - except OSError as why: - self.handle_error() - return - - if isinstance(data, str) and self.use_encoding: - data = bytes(str, self.encoding) - self.ac_in_buffer = self.ac_in_buffer + data - - # Continue to search for self.terminator in self.ac_in_buffer, - # while calling self.collect_incoming_data. The while loop - # is necessary because we might read several data+terminator - # combos with a single recv(4096). - - while self.ac_in_buffer: - lb = len(self.ac_in_buffer) - terminator = self.get_terminator() - if not terminator: - # no terminator, collect it all - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - elif isinstance(terminator, int): - # numeric terminator - n = terminator - if lb < n: - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - self.terminator = self.terminator - lb - else: - self.collect_incoming_data(self.ac_in_buffer[:n]) - self.ac_in_buffer = self.ac_in_buffer[n:] - self.terminator = 0 - self.found_terminator() - else: - # 3 cases: - # 1) end of buffer matches terminator exactly: - # collect data, transition - # 2) end of buffer matches some prefix: - # collect data to the prefix - # 3) end of buffer does not match any prefix: - # collect data - terminator_len = len(terminator) - index = self.ac_in_buffer.find(terminator) - if index != -1: - # we found the terminator - if index > 0: - # don't bother reporting the empty string - # (source of subtle bugs) - self.collect_incoming_data(self.ac_in_buffer[:index]) - self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] - # This does the Right Thing if the terminator - # is changed here. - self.found_terminator() - else: - # check for a prefix of the terminator - index = find_prefix_at_end(self.ac_in_buffer, terminator) - if index: - if index != lb: - # we found a prefix, collect up to the prefix - self.collect_incoming_data(self.ac_in_buffer[:-index]) - self.ac_in_buffer = self.ac_in_buffer[-index:] - break - else: - # no prefix, collect it all - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - - def handle_write(self): - self.initiate_send() - - def handle_close(self): - self.close() - - def push(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - sabs = self.ac_out_buffer_size - if len(data) > sabs: - for i in range(0, len(data), sabs): - self.producer_fifo.append(data[i:i+sabs]) - else: - self.producer_fifo.append(data) - self.initiate_send() - - def push_with_producer(self, producer): - self.producer_fifo.append(producer) - self.initiate_send() - - def readable(self): - "predicate for inclusion in the readable for select()" - # cannot use the old predicate, it violates the claim of the - # set_terminator method. - - # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) - return 1 - - def writable(self): - "predicate for inclusion in the writable for select()" - return self.producer_fifo or (not self.connected) - - def close_when_done(self): - "automatically close this channel once the outgoing queue is empty" - self.producer_fifo.append(None) - - def initiate_send(self): - while self.producer_fifo and self.connected: - first = self.producer_fifo[0] - # handle empty string/buffer or None entry - if not first: - del self.producer_fifo[0] - if first is None: - self.handle_close() - return - - # handle classic producer behavior - obs = self.ac_out_buffer_size - try: - data = first[:obs] - except TypeError: - data = first.more() - if data: - self.producer_fifo.appendleft(data) - else: - del self.producer_fifo[0] - continue - - if isinstance(data, str) and self.use_encoding: - data = bytes(data, self.encoding) - - # send the data - try: - num_sent = self.send(data) - except OSError: - self.handle_error() - return - - if num_sent: - if num_sent < len(data) or obs < len(first): - self.producer_fifo[0] = first[num_sent:] - else: - del self.producer_fifo[0] - # we tried to send some actual data - return - - def discard_buffers(self): - # Emergencies only! - self.ac_in_buffer = b'' - del self.incoming[:] - self.producer_fifo.clear() - - -class simple_producer: - - def __init__(self, data, buffer_size=512): - self.data = data - self.buffer_size = buffer_size - - def more(self): - if len(self.data) > self.buffer_size: - result = self.data[:self.buffer_size] - self.data = self.data[self.buffer_size:] - return result - else: - result = self.data - self.data = b'' - return result - - -# Given 'haystack', see if any prefix of 'needle' is at its end. This -# assumes an exact match has already been checked. Return the number of -# characters matched. -# for example: -# f_p_a_e("qwerty\r", "\r\n") => 1 -# f_p_a_e("qwertydkjf", "\r\n") => 0 -# f_p_a_e("qwerty\r\n", "\r\n") => - -# this could maybe be made faster with a computed regex? -# [answer: no; circa Python-2.0, Jan 2001] -# new python: 28961/s -# old python: 18307/s -# re: 12820/s -# regex: 14035/s - -def find_prefix_at_end(haystack, needle): - l = len(needle) - 1 - while l and not haystack.endswith(needle[:l]): - l -= 1 - return l diff --git a/Lib/asyncore.py b/Lib/asyncore.py deleted file mode 100644 index 0e92be3ad1..0000000000 --- a/Lib/asyncore.py +++ /dev/null @@ -1,642 +0,0 @@ -# -*- Mode: Python -*- -# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp -# Author: Sam Rushing - -# ====================================================================== -# Copyright 1996 by Sam Rushing -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Sam -# Rushing not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# ====================================================================== - -"""Basic infrastructure for asynchronous socket service clients and servers. - -There are only two ways to have a program on a single processor do "more -than one thing at a time". Multi-threaded programming is the simplest and -most popular way to do it, but there is another very different technique, -that lets you have nearly all the advantages of multi-threading, without -actually using multiple threads. it's really only practical if your program -is largely I/O bound. If your program is CPU bound, then pre-emptive -scheduled threads are probably what you really need. Network servers are -rarely CPU-bound, however. - -If your operating system supports the select() system call in its I/O -library (and nearly all do), then you can use it to juggle multiple -communication channels at once; doing other work while your I/O is taking -place in the "background." Although this strategy can seem strange and -complex, especially at first, it is in many ways easier to understand and -control than multi-threaded programming. The module documented here solves -many of the difficult problems for you, making the task of building -sophisticated high-performance network servers and clients a snap. -""" - -import select -import socket -import sys -import time -import warnings - -import os -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ - errorcode - -_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, - EBADF}) - -try: - socket_map -except NameError: - socket_map = {} - -def _strerror(err): - try: - return os.strerror(err) - except (ValueError, OverflowError, NameError): - if err in errorcode: - return errorcode[err] - return "Unknown error %s" %err - -class ExitNow(Exception): - pass - -_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) - -def read(obj): - try: - obj.handle_read_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def write(obj): - try: - obj.handle_write_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def _exception(obj): - try: - obj.handle_expt_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def readwrite(obj, flags): - try: - if flags & select.POLLIN: - obj.handle_read_event() - if flags & select.POLLOUT: - obj.handle_write_event() - if flags & select.POLLPRI: - obj.handle_expt_event() - if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): - obj.handle_close() - except OSError as e: - if e.args[0] not in _DISCONNECTED: - obj.handle_error() - else: - obj.handle_close() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def poll(timeout=0.0, map=None): - if map is None: - map = socket_map - if map: - r = []; w = []; e = [] - for fd, obj in list(map.items()): - is_r = obj.readable() - is_w = obj.writable() - if is_r: - r.append(fd) - # accepting sockets should not be writable - if is_w and not obj.accepting: - w.append(fd) - if is_r or is_w: - e.append(fd) - if [] == r == w == e: - time.sleep(timeout) - return - - r, w, e = select.select(r, w, e, timeout) - - for fd in r: - obj = map.get(fd) - if obj is None: - continue - read(obj) - - for fd in w: - obj = map.get(fd) - if obj is None: - continue - write(obj) - - for fd in e: - obj = map.get(fd) - if obj is None: - continue - _exception(obj) - -def poll2(timeout=0.0, map=None): - # Use the poll() support added to the select module in Python 2.0 - if map is None: - map = socket_map - if timeout is not None: - # timeout is in milliseconds - timeout = int(timeout*1000) - pollster = select.poll() - if map: - for fd, obj in list(map.items()): - flags = 0 - if obj.readable(): - flags |= select.POLLIN | select.POLLPRI - # accepting sockets should not be writable - if obj.writable() and not obj.accepting: - flags |= select.POLLOUT - if flags: - pollster.register(fd, flags) - - r = pollster.poll(timeout) - for fd, flags in r: - obj = map.get(fd) - if obj is None: - continue - readwrite(obj, flags) - -poll3 = poll2 # Alias for backward compatibility - -def loop(timeout=30.0, use_poll=False, map=None, count=None): - if map is None: - map = socket_map - - if use_poll and hasattr(select, 'poll'): - poll_fun = poll2 - else: - poll_fun = poll - - if count is None: - while map: - poll_fun(timeout, map) - - else: - while map and count > 0: - poll_fun(timeout, map) - count = count - 1 - -class dispatcher: - - debug = False - connected = False - accepting = False - connecting = False - closing = False - addr = None - ignore_log_types = frozenset({'warning'}) - - def __init__(self, sock=None, map=None): - if map is None: - self._map = socket_map - else: - self._map = map - - self._fileno = None - - if sock: - # Set to nonblocking just to make sure for cases where we - # get a socket from a blocking source. - sock.setblocking(0) - self.set_socket(sock, map) - self.connected = True - # The constructor no longer requires that the socket - # passed be connected. - try: - self.addr = sock.getpeername() - except OSError as err: - if err.args[0] in (ENOTCONN, EINVAL): - # To handle the case where we got an unconnected - # socket. - self.connected = False - else: - # The socket is broken in some unknown way, alert - # the user and remove it from the map (to prevent - # polling of broken sockets). - self.del_channel(map) - raise - else: - self.socket = None - - def __repr__(self): - status = [self.__class__.__module__+"."+self.__class__.__qualname__] - if self.accepting and self.addr: - status.append('listening') - elif self.connected: - status.append('connected') - if self.addr is not None: - try: - status.append('%s:%d' % self.addr) - except TypeError: - status.append(repr(self.addr)) - return '<%s at %#x>' % (' '.join(status), id(self)) - - def add_channel(self, map=None): - #self.log_info('adding channel %s' % self) - if map is None: - map = self._map - map[self._fileno] = self - - def del_channel(self, map=None): - fd = self._fileno - if map is None: - map = self._map - if fd in map: - #self.log_info('closing channel %d:%s' % (fd, self)) - del map[fd] - self._fileno = None - - def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): - self.family_and_type = family, type - sock = socket.socket(family, type) - sock.setblocking(0) - self.set_socket(sock) - - def set_socket(self, sock, map=None): - self.socket = sock - self._fileno = sock.fileno() - self.add_channel(map) - - def set_reuse_addr(self): - # try to re-use a server port if possible - try: - self.socket.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - self.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR) | 1 - ) - except OSError: - pass - - # ================================================== - # predicates for select() - # these are used as filters for the lists of sockets - # to pass to select(). - # ================================================== - - def readable(self): - return True - - def writable(self): - return True - - # ================================================== - # socket object methods. - # ================================================== - - def listen(self, num): - self.accepting = True - if os.name == 'nt' and num > 5: - num = 5 - return self.socket.listen(num) - - def bind(self, addr): - self.addr = addr - return self.socket.bind(addr) - - def connect(self, address): - self.connected = False - self.connecting = True - err = self.socket.connect_ex(address) - if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ - or err == EINVAL and os.name == 'nt': - self.addr = address - return - if err in (0, EISCONN): - self.addr = address - self.handle_connect_event() - else: - raise OSError(err, errorcode[err]) - - def accept(self): - # XXX can return either an address pair or None - try: - conn, addr = self.socket.accept() - except TypeError: - return None - except OSError as why: - if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): - return None - else: - raise - else: - return conn, addr - - def send(self, data): - try: - result = self.socket.send(data) - return result - except OSError as why: - if why.args[0] == EWOULDBLOCK: - return 0 - elif why.args[0] in _DISCONNECTED: - self.handle_close() - return 0 - else: - raise - - def recv(self, buffer_size): - try: - data = self.socket.recv(buffer_size) - if not data: - # a closed connection is indicated by signaling - # a read condition, and having recv() return 0. - self.handle_close() - return b'' - else: - return data - except OSError as why: - # winsock sometimes raises ENOTCONN - if why.args[0] in _DISCONNECTED: - self.handle_close() - return b'' - else: - raise - - def close(self): - self.connected = False - self.accepting = False - self.connecting = False - self.del_channel() - if self.socket is not None: - try: - self.socket.close() - except OSError as why: - if why.args[0] not in (ENOTCONN, EBADF): - raise - - # log and log_info may be overridden to provide more sophisticated - # logging and warning methods. In general, log is for 'hit' logging - # and 'log_info' is for informational, warning and error logging. - - def log(self, message): - sys.stderr.write('log: %s\n' % str(message)) - - def log_info(self, message, type='info'): - if type not in self.ignore_log_types: - print('%s: %s' % (type, message)) - - def handle_read_event(self): - if self.accepting: - # accepting sockets are never connected, they "spawn" new - # sockets that are connected - self.handle_accept() - elif not self.connected: - if self.connecting: - self.handle_connect_event() - self.handle_read() - else: - self.handle_read() - - def handle_connect_event(self): - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - raise OSError(err, _strerror(err)) - self.handle_connect() - self.connected = True - self.connecting = False - - def handle_write_event(self): - if self.accepting: - # Accepting sockets shouldn't get a write event. - # We will pretend it didn't happen. - return - - if not self.connected: - if self.connecting: - self.handle_connect_event() - self.handle_write() - - def handle_expt_event(self): - # handle_expt_event() is called if there might be an error on the - # socket, or if there is OOB data - # check for the error condition first - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - # we can get here when select.select() says that there is an - # exceptional condition on the socket - # since there is an error, we'll go ahead and close the socket - # like we would in a subclassed handle_read() that received no - # data - self.handle_close() - else: - self.handle_expt() - - def handle_error(self): - nil, t, v, tbinfo = compact_traceback() - - # sometimes a user repr method will crash. - try: - self_repr = repr(self) - except: - self_repr = '<__repr__(self) failed for object at %0x>' % id(self) - - self.log_info( - 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( - self_repr, - t, - v, - tbinfo - ), - 'error' - ) - self.handle_close() - - def handle_expt(self): - self.log_info('unhandled incoming priority event', 'warning') - - def handle_read(self): - self.log_info('unhandled read event', 'warning') - - def handle_write(self): - self.log_info('unhandled write event', 'warning') - - def handle_connect(self): - self.log_info('unhandled connect event', 'warning') - - def handle_accept(self): - pair = self.accept() - if pair is not None: - self.handle_accepted(*pair) - - def handle_accepted(self, sock, addr): - sock.close() - self.log_info('unhandled accepted event', 'warning') - - def handle_close(self): - self.log_info('unhandled close event', 'warning') - self.close() - -# --------------------------------------------------------------------------- -# adds simple buffered output capability, useful for simple clients. -# [for more sophisticated usage use asynchat.async_chat] -# --------------------------------------------------------------------------- - -class dispatcher_with_send(dispatcher): - - def __init__(self, sock=None, map=None): - dispatcher.__init__(self, sock, map) - self.out_buffer = b'' - - def initiate_send(self): - num_sent = 0 - num_sent = dispatcher.send(self, self.out_buffer[:65536]) - self.out_buffer = self.out_buffer[num_sent:] - - def handle_write(self): - self.initiate_send() - - def writable(self): - return (not self.connected) or len(self.out_buffer) - - def send(self, data): - if self.debug: - self.log_info('sending %s' % repr(data)) - self.out_buffer = self.out_buffer + data - self.initiate_send() - -# --------------------------------------------------------------------------- -# used for debugging. -# --------------------------------------------------------------------------- - -def compact_traceback(): - t, v, tb = sys.exc_info() - tbinfo = [] - if not tb: # Must have a traceback - raise AssertionError("traceback does not exist") - while tb: - tbinfo.append(( - tb.tb_frame.f_code.co_filename, - tb.tb_frame.f_code.co_name, - str(tb.tb_lineno) - )) - tb = tb.tb_next - - # just to be safe - del tb - - file, function, line = tbinfo[-1] - info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) - return (file, function, line), t, v, info - -def close_all(map=None, ignore_all=False): - if map is None: - map = socket_map - for x in list(map.values()): - try: - x.close() - except OSError as x: - if x.args[0] == EBADF: - pass - elif not ignore_all: - raise - except _reraised_exceptions: - raise - except: - if not ignore_all: - raise - map.clear() - -# Asynchronous File I/O: -# -# After a little research (reading man pages on various unixen, and -# digging through the linux kernel), I've determined that select() -# isn't meant for doing asynchronous file i/o. -# Heartening, though - reading linux/mm/filemap.c shows that linux -# supports asynchronous read-ahead. So _MOST_ of the time, the data -# will be sitting in memory for us already when we go to read it. -# -# What other OS's (besides NT) support async file i/o? [VMS?] -# -# Regardless, this is useful for pipes, and stdin/stdout... - -if os.name == 'posix': - class file_wrapper: - # Here we override just enough to make a file - # look like a socket for the purposes of asyncore. - # The passed fd is automatically os.dup()'d - - def __init__(self, fd): - self.fd = os.dup(fd) - - def __del__(self): - if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning, - source=self) - self.close() - - def recv(self, *args): - return os.read(self.fd, *args) - - def send(self, *args): - return os.write(self.fd, *args) - - def getsockopt(self, level, optname, buflen=None): - if (level == socket.SOL_SOCKET and - optname == socket.SO_ERROR and - not buflen): - return 0 - raise NotImplementedError("Only asyncore specific behaviour " - "implemented.") - - read = recv - write = send - - def close(self): - if self.fd < 0: - return - fd = self.fd - self.fd = -1 - os.close(fd) - - def fileno(self): - return self.fd - - class file_dispatcher(dispatcher): - - def __init__(self, fd, map=None): - dispatcher.__init__(self, None, map) - self.connected = True - try: - fd = fd.fileno() - except AttributeError: - pass - self.set_file(fd) - # set it to non-blocking mode - os.set_blocking(fd, False) - - def set_file(self, fd): - self.socket = file_wrapper(fd) - self._fileno = self.socket.fileno() - self.add_channel() diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py deleted file mode 100644 index 1fcc882ce6..0000000000 --- a/Lib/test/test_asynchat.py +++ /dev/null @@ -1,290 +0,0 @@ -# test asynchat - -from test import support -from test.support import socket_helper -from test.support import threading_helper - - -import asynchat -import asyncore -import errno -import socket -import sys -import threading -import time -import unittest -import unittest.mock - -HOST = socket_helper.HOST -SERVER_QUIT = b'QUIT\n' -TIMEOUT = 3.0 - - -class echo_server(threading.Thread): - # parameter to determine the number of bytes passed back to the - # client each send - chunk_size = 1 - - def __init__(self, event): - threading.Thread.__init__(self) - self.event = event - self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.port = socket_helper.bind_port(self.sock) - # This will be set if the client wants us to wait before echoing - # data back. - self.start_resend_event = None - - def run(self): - self.sock.listen() - self.event.set() - conn, client = self.sock.accept() - self.buffer = b"" - # collect data until quit message is seen - while SERVER_QUIT not in self.buffer: - data = conn.recv(1) - if not data: - break - self.buffer = self.buffer + data - - # remove the SERVER_QUIT message - self.buffer = self.buffer.replace(SERVER_QUIT, b'') - - if self.start_resend_event: - self.start_resend_event.wait() - - # re-send entire set of collected data - try: - # this may fail on some tests, such as test_close_when_done, - # since the client closes the channel when it's done sending - while self.buffer: - n = conn.send(self.buffer[:self.chunk_size]) - time.sleep(0.001) - self.buffer = self.buffer[n:] - except: - pass - - conn.close() - self.sock.close() - -class echo_client(asynchat.async_chat): - - def __init__(self, terminator, server_port): - asynchat.async_chat.__init__(self) - self.contents = [] - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.connect((HOST, server_port)) - self.set_terminator(terminator) - self.buffer = b"" - - def handle_connect(self): - pass - - if sys.platform == 'darwin': - # select.poll returns a select.POLLHUP at the end of the tests - # on darwin, so just ignore it - def handle_expt(self): - pass - - def collect_incoming_data(self, data): - self.buffer += data - - def found_terminator(self): - self.contents.append(self.buffer) - self.buffer = b"" - -def start_echo_server(): - event = threading.Event() - s = echo_server(event) - s.start() - event.wait() - event.clear() - time.sleep(0.01) # Give server time to start accepting. - return s, event - - -class TestAsynchat(unittest.TestCase): - usepoll = False - - def setUp(self): - self._threads = threading_helper.threading_setup() - - def tearDown(self): - threading_helper.threading_cleanup(*self._threads) - - def line_terminator_check(self, term, server_chunk): - event = threading.Event() - s = echo_server(event) - s.chunk_size = server_chunk - s.start() - event.wait() - event.clear() - time.sleep(0.01) # Give server time to start accepting. - c = echo_client(term, s.port) - c.push(b"hello ") - c.push(b"world" + term) - c.push(b"I'm not dead yet!" + term) - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) - - # the line terminator tests below check receiving variously-sized - # chunks back from the server in order to exercise all branches of - # async_chat.handle_read - - def test_line_terminator1(self): - # test one-character terminator - for l in (1, 2, 3): - self.line_terminator_check(b'\n', l) - - def test_line_terminator2(self): - # test two-character terminator - for l in (1, 2, 3): - self.line_terminator_check(b'\r\n', l) - - def test_line_terminator3(self): - # test three-character terminator - for l in (1, 2, 3): - self.line_terminator_check(b'qqq', l) - - def numeric_terminator_check(self, termlen): - # Try reading a fixed number of bytes - s, event = start_echo_server() - c = echo_client(termlen, s.port) - data = b"hello world, I'm not dead yet!\n" - c.push(data) - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [data[:termlen]]) - - def test_numeric_terminator1(self): - # check that ints & longs both work (since type is - # explicitly checked in async_chat.handle_read) - self.numeric_terminator_check(1) - - def test_numeric_terminator2(self): - self.numeric_terminator_check(6) - - def test_none_terminator(self): - # Try reading a fixed number of bytes - s, event = start_echo_server() - c = echo_client(None, s.port) - data = b"hello world, I'm not dead yet!\n" - c.push(data) - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, []) - self.assertEqual(c.buffer, data) - - def test_simple_producer(self): - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - data = b"hello world\nI'm not dead yet!\n" - p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8) - c.push_with_producer(p) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) - - def test_string_producer(self): - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - data = b"hello world\nI'm not dead yet!\n" - c.push_with_producer(data+SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) - - def test_empty_line(self): - # checks that empty lines are handled correctly - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - c.push(b"hello world\n\nI'm not dead yet!\n") - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - - self.assertEqual(c.contents, - [b"hello world", b"", b"I'm not dead yet!"]) - - def test_close_when_done(self): - s, event = start_echo_server() - s.start_resend_event = threading.Event() - c = echo_client(b'\n', s.port) - c.push(b"hello world\nI'm not dead yet!\n") - c.push(SERVER_QUIT) - c.close_when_done() - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - - # Only allow the server to start echoing data back to the client after - # the client has closed its connection. This prevents a race condition - # where the server echoes all of its data before we can check that it - # got any down below. - s.start_resend_event.set() - threading_helper.join_thread(s) - - self.assertEqual(c.contents, []) - # the server might have been able to send a byte or two back, but this - # at least checks that it received something and didn't just fail - # (which could still result in the client not having received anything) - self.assertGreater(len(s.buffer), 0) - - def test_push(self): - # Issue #12523: push() should raise a TypeError if it doesn't get - # a bytes string - s, event = start_echo_server() - c = echo_client(b'\n', s.port) - data = b'bytes\n' - c.push(data) - c.push(bytearray(data)) - c.push(memoryview(data)) - self.assertRaises(TypeError, c.push, 10) - self.assertRaises(TypeError, c.push, 'unicode') - c.push(SERVER_QUIT) - asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) - threading_helper.join_thread(s) - self.assertEqual(c.contents, [b'bytes', b'bytes', b'bytes']) - - -class TestAsynchat_WithPoll(TestAsynchat): - usepoll = True - - -class TestAsynchatMocked(unittest.TestCase): - def test_blockingioerror(self): - # Issue #16133: handle_read() must ignore BlockingIOError - sock = unittest.mock.Mock() - sock.recv.side_effect = BlockingIOError(errno.EAGAIN) - - dispatcher = asynchat.async_chat() - dispatcher.set_socket(sock) - self.addCleanup(dispatcher.del_channel) - - with unittest.mock.patch.object(dispatcher, 'handle_error') as error: - dispatcher.handle_read() - self.assertFalse(error.called) - - -class TestHelperFunctions(unittest.TestCase): - def test_find_prefix_at_end(self): - self.assertEqual(asynchat.find_prefix_at_end("qwerty\r", "\r\n"), 1) - self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0) - - -class TestNotConnected(unittest.TestCase): - def test_disallow_negative_terminator(self): - # Issue #11259 - client = asynchat.async_chat() - self.assertRaises(ValueError, client.set_terminator, -1) - - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py deleted file mode 100644 index bd43463da3..0000000000 --- a/Lib/test/test_asyncore.py +++ /dev/null @@ -1,838 +0,0 @@ -import asyncore -import unittest -import select -import os -import socket -import sys -import time -import errno -import struct -import threading - -from test import support -from test.support import os_helper -from test.support import socket_helper -from test.support import threading_helper -from test.support import warnings_helper -from io import BytesIO - -if support.PGO: - raise unittest.SkipTest("test is not helpful for PGO") - - -TIMEOUT = 3 -HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') - -class dummysocket: - def __init__(self): - self.closed = False - - def close(self): - self.closed = True - - def fileno(self): - return 42 - -class dummychannel: - def __init__(self): - self.socket = dummysocket() - - def close(self): - self.socket.close() - -class exitingdummy: - def __init__(self): - pass - - def handle_read_event(self): - raise asyncore.ExitNow() - - handle_write_event = handle_read_event - handle_close = handle_read_event - handle_expt_event = handle_read_event - -class crashingdummy: - def __init__(self): - self.error_handled = False - - def handle_read_event(self): - raise Exception() - - handle_write_event = handle_read_event - handle_close = handle_read_event - handle_expt_event = handle_read_event - - def handle_error(self): - self.error_handled = True - -# used when testing senders; just collects what it gets until newline is sent -def capture_server(evt, buf, serv): - try: - serv.listen() - conn, addr = serv.accept() - except socket.timeout: - pass - else: - n = 200 - start = time.monotonic() - while n > 0 and time.monotonic() - start < 3.0: - r, w, e = select.select([conn], [], [], 0.1) - if r: - n -= 1 - data = conn.recv(10) - # keep everything except for the newline terminator - buf.write(data.replace(b'\n', b'')) - if b'\n' in data: - break - time.sleep(0.01) - - conn.close() - finally: - serv.close() - evt.set() - -def bind_af_aware(sock, addr): - """Helper function to bind a socket according to its family.""" - if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: - # Make sure the path doesn't exist. - os_helper.unlink(addr) - socket_helper.bind_unix_socket(sock, addr) - else: - sock.bind(addr) - - -class HelperFunctionTests(unittest.TestCase): - def test_readwriteexc(self): - # Check exception handling behavior of read, write and _exception - - # check that ExitNow exceptions in the object handler method - # bubbles all the way up through asyncore read/write/_exception calls - tr1 = exitingdummy() - self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) - self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) - self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) - - # check that an exception other than ExitNow in the object handler - # method causes the handle_error method to get called - tr2 = crashingdummy() - asyncore.read(tr2) - self.assertEqual(tr2.error_handled, True) - - tr2 = crashingdummy() - asyncore.write(tr2) - self.assertEqual(tr2.error_handled, True) - - tr2 = crashingdummy() - asyncore._exception(tr2) - self.assertEqual(tr2.error_handled, True) - - # asyncore.readwrite uses constants in the select module that - # are not present in Windows systems (see this thread: - # http://mail.python.org/pipermail/python-list/2001-October/109973.html) - # These constants should be present as long as poll is available - - @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') - def test_readwrite(self): - # Check that correct methods are called by readwrite() - - attributes = ('read', 'expt', 'write', 'closed', 'error_handled') - - expected = ( - (select.POLLIN, 'read'), - (select.POLLPRI, 'expt'), - (select.POLLOUT, 'write'), - (select.POLLERR, 'closed'), - (select.POLLHUP, 'closed'), - (select.POLLNVAL, 'closed'), - ) - - class testobj: - def __init__(self): - self.read = False - self.write = False - self.closed = False - self.expt = False - self.error_handled = False - - def handle_read_event(self): - self.read = True - - def handle_write_event(self): - self.write = True - - def handle_close(self): - self.closed = True - - def handle_expt_event(self): - self.expt = True - - def handle_error(self): - self.error_handled = True - - for flag, expectedattr in expected: - tobj = testobj() - self.assertEqual(getattr(tobj, expectedattr), False) - asyncore.readwrite(tobj, flag) - - # Only the attribute modified by the routine we expect to be - # called should be True. - for attr in attributes: - self.assertEqual(getattr(tobj, attr), attr==expectedattr) - - # check that ExitNow exceptions in the object handler method - # bubbles all the way up through asyncore readwrite call - tr1 = exitingdummy() - self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) - - # check that an exception other than ExitNow in the object handler - # method causes the handle_error method to get called - tr2 = crashingdummy() - self.assertEqual(tr2.error_handled, False) - asyncore.readwrite(tr2, flag) - self.assertEqual(tr2.error_handled, True) - - def test_closeall(self): - self.closeall_check(False) - - def test_closeall_default(self): - self.closeall_check(True) - - def closeall_check(self, usedefault): - # Check that close_all() closes everything in a given map - - l = [] - testmap = {} - for i in range(10): - c = dummychannel() - l.append(c) - self.assertEqual(c.socket.closed, False) - testmap[i] = c - - if usedefault: - socketmap = asyncore.socket_map - try: - asyncore.socket_map = testmap - asyncore.close_all() - finally: - testmap, asyncore.socket_map = asyncore.socket_map, socketmap - else: - asyncore.close_all(testmap) - - self.assertEqual(len(testmap), 0) - - for c in l: - self.assertEqual(c.socket.closed, True) - - def test_compact_traceback(self): - try: - raise Exception("I don't like spam!") - except: - real_t, real_v, real_tb = sys.exc_info() - r = asyncore.compact_traceback() - else: - self.fail("Expected exception") - - (f, function, line), t, v, info = r - self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') - self.assertEqual(function, 'test_compact_traceback') - self.assertEqual(t, real_t) - self.assertEqual(v, real_v) - self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) - - -class DispatcherTests(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - asyncore.close_all() - - def test_basic(self): - d = asyncore.dispatcher() - self.assertEqual(d.readable(), True) - self.assertEqual(d.writable(), True) - - def test_repr(self): - d = asyncore.dispatcher() - self.assertEqual(repr(d), '' % id(d)) - - def test_log(self): - d = asyncore.dispatcher() - - # capture output of dispatcher.log() (to stderr) - l1 = "Lovely spam! Wonderful spam!" - l2 = "I don't like spam!" - with support.captured_stderr() as stderr: - d.log(l1) - d.log(l2) - - lines = stderr.getvalue().splitlines() - self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) - - def test_log_info(self): - d = asyncore.dispatcher() - - # capture output of dispatcher.log_info() (to stdout via print) - l1 = "Have you got anything without spam?" - l2 = "Why can't she have egg bacon spam and sausage?" - l3 = "THAT'S got spam in it!" - with support.captured_stdout() as stdout: - d.log_info(l1, 'EGGS') - d.log_info(l2) - d.log_info(l3, 'SPAM') - - lines = stdout.getvalue().splitlines() - expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] - self.assertEqual(lines, expected) - - def test_unhandled(self): - d = asyncore.dispatcher() - d.ignore_log_types = () - - # capture output of dispatcher.log_info() (to stdout via print) - with support.captured_stdout() as stdout: - d.handle_expt() - d.handle_read() - d.handle_write() - d.handle_connect() - - lines = stdout.getvalue().splitlines() - expected = ['warning: unhandled incoming priority event', - 'warning: unhandled read event', - 'warning: unhandled write event', - 'warning: unhandled connect event'] - self.assertEqual(lines, expected) - - def test_strerror(self): - # refers to bug #8573 - err = asyncore._strerror(errno.EPERM) - if hasattr(os, 'strerror'): - self.assertEqual(err, os.strerror(errno.EPERM)) - err = asyncore._strerror(-1) - self.assertTrue(err != "") - - -class dispatcherwithsend_noread(asyncore.dispatcher_with_send): - def readable(self): - return False - - def handle_connect(self): - pass - - -class DispatcherWithSendTests(unittest.TestCase): - def setUp(self): - pass - - def tearDown(self): - asyncore.close_all() - - @threading_helper.reap_threads - def test_send(self): - evt = threading.Event() - sock = socket.socket() - sock.settimeout(3) - port = socket_helper.bind_port(sock) - - cap = BytesIO() - args = (evt, cap, sock) - t = threading.Thread(target=capture_server, args=args) - t.start() - try: - # wait a little longer for the server to initialize (it sometimes - # refuses connections on slow machines without this wait) - time.sleep(0.2) - - data = b"Suppose there isn't a 16-ton weight?" - d = dispatcherwithsend_noread() - d.create_socket() - d.connect((socket_helper.HOST, port)) - - # give time for socket to connect - time.sleep(0.1) - - d.send(data) - d.send(data) - d.send(b'\n') - - n = 1000 - while d.out_buffer and n > 0: - asyncore.poll() - n -= 1 - - evt.wait() - - self.assertEqual(cap.getvalue(), data*2) - finally: - threading_helper.join_thread(t) - - -@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), - 'asyncore.file_wrapper required') -class FileWrapperTest(unittest.TestCase): - def setUp(self): - self.d = b"It's not dead, it's sleeping!" - with open(os_helper.TESTFN, 'wb') as file: - file.write(self.d) - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def test_recv(self): - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - w = asyncore.file_wrapper(fd) - os.close(fd) - - self.assertNotEqual(w.fd, fd) - self.assertNotEqual(w.fileno(), fd) - self.assertEqual(w.recv(13), b"It's not dead") - self.assertEqual(w.read(6), b", it's") - w.close() - self.assertRaises(OSError, w.read, 1) - - def test_send(self): - d1 = b"Come again?" - d2 = b"I want to buy some cheese." - fd = os.open(os_helper.TESTFN, os.O_WRONLY | os.O_APPEND) - w = asyncore.file_wrapper(fd) - os.close(fd) - - w.write(d1) - w.send(d2) - w.close() - with open(os_helper.TESTFN, 'rb') as file: - self.assertEqual(file.read(), self.d + d1 + d2) - - @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), - 'asyncore.file_dispatcher required') - def test_dispatcher(self): - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - data = [] - class FileDispatcher(asyncore.file_dispatcher): - def handle_read(self): - data.append(self.recv(29)) - s = FileDispatcher(fd) - os.close(fd) - asyncore.loop(timeout=0.01, use_poll=True, count=2) - self.assertEqual(b"".join(data), self.d) - - def test_resource_warning(self): - # Issue #11453 - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - f = asyncore.file_wrapper(fd) - - os.close(fd) - with warnings_helper.check_warnings(('', ResourceWarning)): - f = None - support.gc_collect() - - def test_close_twice(self): - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - f = asyncore.file_wrapper(fd) - os.close(fd) - - os.close(f.fd) # file_wrapper dupped fd - with self.assertRaises(OSError): - f.close() - - self.assertEqual(f.fd, -1) - # calling close twice should not fail - f.close() - - -class BaseTestHandler(asyncore.dispatcher): - - def __init__(self, sock=None): - asyncore.dispatcher.__init__(self, sock) - self.flag = False - - def handle_accept(self): - raise Exception("handle_accept not supposed to be called") - - def handle_accepted(self): - raise Exception("handle_accepted not supposed to be called") - - def handle_connect(self): - raise Exception("handle_connect not supposed to be called") - - def handle_expt(self): - raise Exception("handle_expt not supposed to be called") - - def handle_close(self): - raise Exception("handle_close not supposed to be called") - - def handle_error(self): - raise - - -class BaseServer(asyncore.dispatcher): - """A server which listens on an address and dispatches the - connection to a handler. - """ - - def __init__(self, family, addr, handler=BaseTestHandler): - asyncore.dispatcher.__init__(self) - self.create_socket(family) - self.set_reuse_addr() - bind_af_aware(self.socket, addr) - self.listen(5) - self.handler = handler - - @property - def address(self): - return self.socket.getsockname() - - def handle_accepted(self, sock, addr): - self.handler(sock) - - def handle_error(self): - raise - - -class BaseClient(BaseTestHandler): - - def __init__(self, family, address): - BaseTestHandler.__init__(self) - self.create_socket(family) - self.connect(address) - - def handle_connect(self): - pass - - -class BaseTestAPI: - - def tearDown(self): - asyncore.close_all(ignore_all=True) - - def loop_waiting_for_flag(self, instance, timeout=5): - timeout = float(timeout) / 100 - count = 100 - while asyncore.socket_map and count > 0: - asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) - if instance.flag: - return - count -= 1 - time.sleep(timeout) - self.fail("flag not set") - - def test_handle_connect(self): - # make sure handle_connect is called on connect() - - class TestClient(BaseClient): - def handle_connect(self): - self.flag = True - - server = BaseServer(self.family, self.addr) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_accept(self): - # make sure handle_accept() is called when a client connects - - class TestListener(BaseTestHandler): - - def __init__(self, family, addr): - BaseTestHandler.__init__(self) - self.create_socket(family) - bind_af_aware(self.socket, addr) - self.listen(5) - self.address = self.socket.getsockname() - - def handle_accept(self): - self.flag = True - - server = TestListener(self.family, self.addr) - client = BaseClient(self.family, server.address) - self.loop_waiting_for_flag(server) - - def test_handle_accepted(self): - # make sure handle_accepted() is called when a client connects - - class TestListener(BaseTestHandler): - - def __init__(self, family, addr): - BaseTestHandler.__init__(self) - self.create_socket(family) - bind_af_aware(self.socket, addr) - self.listen(5) - self.address = self.socket.getsockname() - - def handle_accept(self): - asyncore.dispatcher.handle_accept(self) - - def handle_accepted(self, sock, addr): - sock.close() - self.flag = True - - server = TestListener(self.family, self.addr) - client = BaseClient(self.family, server.address) - self.loop_waiting_for_flag(server) - - - def test_handle_read(self): - # make sure handle_read is called on data received - - class TestClient(BaseClient): - def handle_read(self): - self.flag = True - - class TestHandler(BaseTestHandler): - def __init__(self, conn): - BaseTestHandler.__init__(self, conn) - self.send(b'x' * 1024) - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_write(self): - # make sure handle_write is called - - class TestClient(BaseClient): - def handle_write(self): - self.flag = True - - server = BaseServer(self.family, self.addr) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_close(self): - # make sure handle_close is called when the other end closes - # the connection - - class TestClient(BaseClient): - - def handle_read(self): - # in order to make handle_close be called we are supposed - # to make at least one recv() call - self.recv(1024) - - def handle_close(self): - self.flag = True - self.close() - - class TestHandler(BaseTestHandler): - def __init__(self, conn): - BaseTestHandler.__init__(self, conn) - self.close() - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_close_after_conn_broken(self): - # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and - # #11265). - - data = b'\0' * 128 - - class TestClient(BaseClient): - - def handle_write(self): - self.send(data) - - def handle_close(self): - self.flag = True - self.close() - - def handle_expt(self): - self.flag = True - self.close() - - class TestHandler(BaseTestHandler): - - def handle_read(self): - self.recv(len(data)) - self.close() - - def writable(self): - return False - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - @unittest.skipIf(sys.platform.startswith("sunos"), - "OOB support is broken on Solaris") - def test_handle_expt(self): - # Make sure handle_expt is called on OOB data received. - # Note: this might fail on some platforms as OOB data is - # tenuously supported and rarely used. - if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: - self.skipTest("Not applicable to AF_UNIX sockets.") - - if sys.platform == "darwin" and self.use_poll: - self.skipTest("poll may fail on macOS; see issue #28087") - - class TestClient(BaseClient): - def handle_expt(self): - self.socket.recv(1024, socket.MSG_OOB) - self.flag = True - - class TestHandler(BaseTestHandler): - def __init__(self, conn): - BaseTestHandler.__init__(self, conn) - self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) - - server = BaseServer(self.family, self.addr, TestHandler) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_handle_error(self): - - class TestClient(BaseClient): - def handle_write(self): - 1.0 / 0 - def handle_error(self): - self.flag = True - try: - raise - except ZeroDivisionError: - pass - else: - raise Exception("exception not raised") - - server = BaseServer(self.family, self.addr) - client = TestClient(self.family, server.address) - self.loop_waiting_for_flag(client) - - def test_connection_attributes(self): - server = BaseServer(self.family, self.addr) - client = BaseClient(self.family, server.address) - - # we start disconnected - self.assertFalse(server.connected) - self.assertTrue(server.accepting) - # this can't be taken for granted across all platforms - #self.assertFalse(client.connected) - self.assertFalse(client.accepting) - - # execute some loops so that client connects to server - asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) - self.assertFalse(server.connected) - self.assertTrue(server.accepting) - self.assertTrue(client.connected) - self.assertFalse(client.accepting) - - # disconnect the client - client.close() - self.assertFalse(server.connected) - self.assertTrue(server.accepting) - self.assertFalse(client.connected) - self.assertFalse(client.accepting) - - # stop serving - server.close() - self.assertFalse(server.connected) - self.assertFalse(server.accepting) - - def test_create_socket(self): - s = asyncore.dispatcher() - s.create_socket(self.family) - self.assertEqual(s.socket.type, socket.SOCK_STREAM) - self.assertEqual(s.socket.family, self.family) - self.assertEqual(s.socket.gettimeout(), 0) - self.assertFalse(s.socket.get_inheritable()) - - def test_bind(self): - if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: - self.skipTest("Not applicable to AF_UNIX sockets.") - s1 = asyncore.dispatcher() - s1.create_socket(self.family) - s1.bind(self.addr) - s1.listen(5) - port = s1.socket.getsockname()[1] - - s2 = asyncore.dispatcher() - s2.create_socket(self.family) - # EADDRINUSE indicates the socket was correctly bound - self.assertRaises(OSError, s2.bind, (self.addr[0], port)) - - def test_set_reuse_addr(self): - if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: - self.skipTest("Not applicable to AF_UNIX sockets.") - - with socket.socket(self.family) as sock: - try: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except OSError: - unittest.skip("SO_REUSEADDR not supported on this platform") - else: - # if SO_REUSEADDR succeeded for sock we expect asyncore - # to do the same - s = asyncore.dispatcher(socket.socket(self.family)) - self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - s.socket.close() - s.create_socket(self.family) - s.set_reuse_addr() - self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR)) - - @threading_helper.reap_threads - def test_quick_connect(self): - # see: http://bugs.python.org/issue10340 - if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): - self.skipTest("test specific to AF_INET and AF_INET6") - - server = BaseServer(self.family, self.addr) - # run the thread 500 ms: the socket should be connected in 200 ms - t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, - count=5)) - t.start() - try: - with socket.socket(self.family, socket.SOCK_STREAM) as s: - s.settimeout(.2) - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) - - try: - s.connect(server.address) - except OSError: - pass - finally: - threading_helper.join_thread(t) - -class TestAPI_UseIPv4Sockets(BaseTestAPI): - family = socket.AF_INET - addr = (socket_helper.HOST, 0) - -@unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 support required') -class TestAPI_UseIPv6Sockets(BaseTestAPI): - family = socket.AF_INET6 - addr = (socket_helper.HOSTv6, 0) - -@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') -class TestAPI_UseUnixSockets(BaseTestAPI): - if HAS_UNIX_SOCKETS: - family = socket.AF_UNIX - addr = os_helper.TESTFN - - def tearDown(self): - os_helper.unlink(self.addr) - BaseTestAPI.tearDown(self) - -class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase): - use_poll = False - -@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase): - use_poll = True - -class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase): - use_poll = False - -@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase): - use_poll = True - -class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase): - use_poll = False - -@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase): - use_poll = True - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 7e632efa4c..b3e4e776a4 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -25,15 +25,6 @@ from test.support import asyncore from test.support.socket_helper import HOST, HOSTv6 -import sys -if sys.platform == 'win32': - raise unittest.SkipTest("test_ftplib not working on windows") -if getattr(sys, '_rustpython_debugbuild', False): - raise unittest.SkipTest("something's weird on debug builds") - -asynchat = warnings_helper.import_deprecated('asynchat') -asyncore = warnings_helper.import_deprecated('asyncore') - support.requires_working_socket(module=True) From 4ae2936a455b4f9c04928ad778daebdbe745d6af Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Fri, 11 Apr 2025 12:08:07 +0900 Subject: [PATCH 214/317] fix more cspell warnings (#5689) --- .cspell.dict/cpython.txt | 6 + .cspell.dict/python-more.txt | 2 + .cspell.json | 4 +- common/src/boxvec.rs | 2 +- common/src/linked_list.rs | 2 + common/src/lock.rs | 2 +- compiler/codegen/src/compile.rs | 30 ++-- src/lib.rs | 8 +- stdlib/src/binascii.rs | 235 ++++++++++++++++---------------- stdlib/src/syslog.rs | 2 +- vm/src/buffer.rs | 22 +-- vm/src/builtins/type.rs | 2 +- vm/src/exceptions.rs | 6 +- wtf8/src/core_char.rs | 1 + wtf8/src/core_str_count.rs | 1 + wtf8/src/lib.rs | 2 + 16 files changed, 171 insertions(+), 156 deletions(-) diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 1840918a4d..d28a4bb8c5 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -17,6 +17,7 @@ elts excepthandler fileutils finalbody +formatfloat freevar freevars fromlist @@ -25,10 +26,12 @@ HIGHRES IMMUTABLETYPE kwonlyarg kwonlyargs +lasti linearise maxdepth mult nkwargs +noraise numer orelse pathconfig @@ -36,9 +39,12 @@ patma posonlyarg posonlyargs prec +preinitialized PYTHREAD_NAME SA_ONSTACK stackdepth +stringlib +structseq tok_oldval unaryop unparse diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 0be3c0f45a..0404428324 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -16,6 +16,7 @@ baserepl basicsize bdfl bigcharset +bignum breakpointhook cformat chunksize @@ -153,6 +154,7 @@ ospath pendingcr phello platlibdir +popleft posixsubprocess posonly posonlyargcount diff --git a/.cspell.json b/.cspell.json index 69ace83483..98a03180fe 100644 --- a/.cspell.json +++ b/.cspell.json @@ -105,9 +105,9 @@ "reducelib", "richcompare", "RustPython", + "significand", "struc", - // plural of summand - "summands", + "summands", // plural of summand "sysmodule", "tracebacks", "typealiases", diff --git a/common/src/boxvec.rs b/common/src/boxvec.rs index 4501835477..f5dd622f58 100644 --- a/common/src/boxvec.rs +++ b/common/src/boxvec.rs @@ -1,4 +1,4 @@ -// cspell:ignore +// cspell:disable //! An unresizable vector backed by a `Box<[T]>` #![allow(clippy::needless_lifetimes)] diff --git a/common/src/linked_list.rs b/common/src/linked_list.rs index 83577b71d1..4e6e1b7000 100644 --- a/common/src/linked_list.rs +++ b/common/src/linked_list.rs @@ -1,3 +1,5 @@ +// cspell:disable + //! This module is modified from tokio::util::linked_list: //! Tokio is licensed under the MIT license: //! diff --git a/common/src/lock.rs b/common/src/lock.rs index 811c461112..ca5ffe8de3 100644 --- a/common/src/lock.rs +++ b/common/src/lock.rs @@ -39,4 +39,4 @@ pub type PyMappedRwLockReadGuard<'a, T> = MappedRwLockReadGuard<'a, RawRwLock, T pub type PyRwLockWriteGuard<'a, T> = RwLockWriteGuard<'a, RawRwLock, T>; pub type PyMappedRwLockWriteGuard<'a, T> = MappedRwLockWriteGuard<'a, RawRwLock, T>; -// can add fn const_{mutex,rwlock}() if necessary, but we probably won't need to +// can add fn const_{mutex,rw_lock}() if necessary, but we probably won't need to diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index c8474a537f..3c92146c13 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2143,8 +2143,8 @@ impl Compiler<'_> { attrs: &[Identifier], _patterns: &[Pattern], ) -> CompileResult<()> { - let nattrs = attrs.len(); - for i in 0..nattrs { + let n_attrs = attrs.len(); + for i in 0..n_attrs { let attr = attrs[i].as_str(); // Check if the attribute name is forbidden in a Store context. if self.forbidden_name(attr, NameUsage::Store)? { @@ -2152,7 +2152,7 @@ impl Compiler<'_> { return Err(self.compile_error_forbidden_name(attr)); } // Check for duplicates: compare with every subsequent attribute. - for ident in attrs.iter().take(nattrs).skip(i + 1) { + for ident in attrs.iter().take(n_attrs).skip(i + 1) { let other = ident.as_str(); if attr == other { todo!(); @@ -2184,20 +2184,20 @@ impl Compiler<'_> { } let nargs = patterns.len(); - let nattrs = kwd_attrs.len(); + let n_attrs = kwd_attrs.len(); let nkwd_patterns = kwd_patterns.len(); // Validate that keyword attribute names and patterns match in length. - if nattrs != nkwd_patterns { + if n_attrs != nkwd_patterns { let msg = format!( "kwd_attrs ({}) / kwd_patterns ({}) length mismatch in class pattern", - nattrs, nkwd_patterns + n_attrs, nkwd_patterns ); unreachable!("{}", msg); } // Check for too many sub-patterns. - if nargs > u32::MAX as usize || (nargs + nattrs).saturating_sub(1) > i32::MAX as usize { + if nargs > u32::MAX as usize || (nargs + n_attrs).saturating_sub(1) > i32::MAX as usize { let msg = format!( "too many sub-patterns in class pattern {:?}", match_class.cls @@ -2207,7 +2207,7 @@ impl Compiler<'_> { } // Validate keyword attributes if any. - if nattrs != 0 { + if n_attrs != 0 { self.validate_kwd_attrs(&kwd_attrs, &kwd_patterns)?; } @@ -2237,12 +2237,12 @@ impl Compiler<'_> { // 5. Compare with IS_OP 1. emit!(self, Instruction::IsOperation(true)); - // At this point the TOS is a tuple of (nargs + nattrs) attributes (or None). + // At this point the TOS is a tuple of (nargs + n_attrs) attributes (or None). pc.on_top += 1; self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; - // Unpack the tuple into (nargs + nattrs) items. - let total = nargs + nattrs; + // Unpack the tuple into (nargs + n_attrs) items. + let total = nargs + n_attrs; emit!( self, Instruction::UnpackSequence { @@ -2280,12 +2280,12 @@ impl Compiler<'_> { // let keys = &mapping.keys; // let patterns = &mapping.patterns; // let size = keys.len(); - // let npatterns = patterns.len(); + // let n_patterns = patterns.len(); - // if size != npatterns { - // panic!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, npatterns); + // if size != n_patterns { + // panic!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, n_patterns); // // return self.compiler_error( - // // &format!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, npatterns) + // // &format!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, n_patterns) // // ); // } diff --git a/src/lib.rs b/src/lib.rs index 98de3de5c6..e9c9c91294 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -233,14 +233,14 @@ fn write_profile(settings: &Settings) -> Result<(), Box> enum ProfileFormat { Html, Text, - SpeedScore, + SpeedScope, } let profile_output = settings.profile_output.as_deref(); let profile_format = match settings.profile_format.as_deref() { Some("html") => ProfileFormat::Html, Some("text") => ProfileFormat::Text, None if profile_output == Some("-".as_ref()) => ProfileFormat::Text, - Some("speedscope") | None => ProfileFormat::SpeedScore, + Some("speedscope") | None => ProfileFormat::SpeedScope, Some(other) => { error!("Unknown profile format {}", other); // TODO: Need to change to ExitCode or Termination @@ -251,7 +251,7 @@ fn write_profile(settings: &Settings) -> Result<(), Box> let profile_output = profile_output.unwrap_or_else(|| match profile_format { ProfileFormat::Html => "flame-graph.html".as_ref(), ProfileFormat::Text => "flame.txt".as_ref(), - ProfileFormat::SpeedScore => "flamescope.json".as_ref(), + ProfileFormat::SpeedScope => "flamescope.json".as_ref(), }); let profile_output: Box = if profile_output == "-" { @@ -265,7 +265,7 @@ fn write_profile(settings: &Settings) -> Result<(), Box> match profile_format { ProfileFormat::Html => flame::dump_html(profile_output)?, ProfileFormat::Text => flame::dump_text_to_writer(profile_output)?, - ProfileFormat::SpeedScore => flamescope::dump(profile_output)?, + ProfileFormat::SpeedScope => flamescope::dump(profile_output)?, } Ok(()) diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index a8df1fb60b..2510325b4e 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -1,4 +1,4 @@ -// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB +// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode pub(super) use decl::crc32; pub(crate) use decl::make_module; @@ -328,7 +328,7 @@ mod decl { idx += 1; } } else if buffer[idx] == b'=' { - // roken case from broken python qp + // broken case from broken python qp out_data.push(b'='); idx += 1; } else if idx + 1 < len @@ -382,183 +382,184 @@ mod decl { let header = args.header; s.with_ref(|buf| { let buflen = buf.len(); - let mut linelen = 0; - let mut odatalen = 0; + let mut line_len = 0; + let mut out_data_len = 0; let mut crlf = false; let mut ch; - let mut inidx; - let mut outidx; + let mut in_idx; + let mut out_idx; - inidx = 0; - while inidx < buflen { - if buf[inidx] == b'\n' { + in_idx = 0; + while in_idx < buflen { + if buf[in_idx] == b'\n' { break; } - inidx += 1; + in_idx += 1; } - if buflen > 0 && inidx < buflen && buf[inidx - 1] == b'\r' { + if buflen > 0 && in_idx < buflen && buf[in_idx - 1] == b'\r' { crlf = true; } - inidx = 0; - while inidx < buflen { + in_idx = 0; + while in_idx < buflen { let mut delta = 0; - if (buf[inidx] > 126) - || (buf[inidx] == b'=') - || (header && buf[inidx] == b'_') - || (buf[inidx] == b'.' - && linelen == 0 - && (inidx + 1 == buflen - || buf[inidx + 1] == b'\n' - || buf[inidx + 1] == b'\r' - || buf[inidx + 1] == 0)) - || (!istext && ((buf[inidx] == b'\r') || (buf[inidx] == b'\n'))) - || ((buf[inidx] == b'\t' || buf[inidx] == b' ') && (inidx + 1 == buflen)) - || ((buf[inidx] < 33) - && (buf[inidx] != b'\r') - && (buf[inidx] != b'\n') - && (quotetabs || ((buf[inidx] != b'\t') && (buf[inidx] != b' ')))) + if (buf[in_idx] > 126) + || (buf[in_idx] == b'=') + || (header && buf[in_idx] == b'_') + || (buf[in_idx] == b'.' + && line_len == 0 + && (in_idx + 1 == buflen + || buf[in_idx + 1] == b'\n' + || buf[in_idx + 1] == b'\r' + || buf[in_idx + 1] == 0)) + || (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n'))) + || ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen)) + || ((buf[in_idx] < 33) + && (buf[in_idx] != b'\r') + && (buf[in_idx] != b'\n') + && (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' ')))) { - if (linelen + 3) >= MAXLINESIZE { - linelen = 0; + if (line_len + 3) >= MAXLINESIZE { + line_len = 0; delta += if crlf { 3 } else { 2 }; } - linelen += 3; + line_len += 3; delta += 3; - inidx += 1; + in_idx += 1; } else if istext - && ((buf[inidx] == b'\n') - || ((inidx + 1 < buflen) - && (buf[inidx] == b'\r') - && (buf[inidx + 1] == b'\n'))) + && ((buf[in_idx] == b'\n') + || ((in_idx + 1 < buflen) + && (buf[in_idx] == b'\r') + && (buf[in_idx + 1] == b'\n'))) { - linelen = 0; + line_len = 0; // Protect against whitespace on end of line - if (inidx != 0) && ((buf[inidx - 1] == b' ') || (buf[inidx - 1] == b'\t')) { + if (in_idx != 0) && ((buf[in_idx - 1] == b' ') || (buf[in_idx - 1] == b'\t')) { delta += 2; } delta += if crlf { 2 } else { 1 }; - inidx += if buf[inidx] == b'\r' { 2 } else { 1 }; + in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 }; } else { - if (inidx + 1 != buflen) - && (buf[inidx + 1] != b'\n') - && (linelen + 1) >= MAXLINESIZE + if (in_idx + 1 != buflen) + && (buf[in_idx + 1] != b'\n') + && (line_len + 1) >= MAXLINESIZE { - linelen = 0; + line_len = 0; delta += if crlf { 3 } else { 2 }; } - linelen += 1; + line_len += 1; delta += 1; - inidx += 1; + in_idx += 1; } - odatalen += delta; + out_data_len += delta; } - let mut out_data = Vec::with_capacity(odatalen); - inidx = 0; - outidx = 0; - linelen = 0; - - while inidx < buflen { - if (buf[inidx] > 126) - || (buf[inidx] == b'=') - || (header && buf[inidx] == b'_') - || ((buf[inidx] == b'.') - && (linelen == 0) - && (inidx + 1 == buflen - || buf[inidx + 1] == b'\n' - || buf[inidx + 1] == b'\r' - || buf[inidx + 1] == 0)) - || (!istext && ((buf[inidx] == b'\r') || (buf[inidx] == b'\n'))) - || ((buf[inidx] == b'\t' || buf[inidx] == b' ') && (inidx + 1 == buflen)) - || ((buf[inidx] < 33) - && (buf[inidx] != b'\r') - && (buf[inidx] != b'\n') - && (quotetabs || ((buf[inidx] != b'\t') && (buf[inidx] != b' ')))) + let mut out_data = Vec::with_capacity(out_data_len); + in_idx = 0; + out_idx = 0; + line_len = 0; + + while in_idx < buflen { + if (buf[in_idx] > 126) + || (buf[in_idx] == b'=') + || (header && buf[in_idx] == b'_') + || ((buf[in_idx] == b'.') + && (line_len == 0) + && (in_idx + 1 == buflen + || buf[in_idx + 1] == b'\n' + || buf[in_idx + 1] == b'\r' + || buf[in_idx + 1] == 0)) + || (!istext && ((buf[in_idx] == b'\r') || (buf[in_idx] == b'\n'))) + || ((buf[in_idx] == b'\t' || buf[in_idx] == b' ') && (in_idx + 1 == buflen)) + || ((buf[in_idx] < 33) + && (buf[in_idx] != b'\r') + && (buf[in_idx] != b'\n') + && (quotetabs || ((buf[in_idx] != b'\t') && (buf[in_idx] != b' ')))) { - if (linelen + 3) >= MAXLINESIZE { + if (line_len + 3) >= MAXLINESIZE { // MAXLINESIZE = 76 out_data.push(b'='); - outidx += 1; + out_idx += 1; if crlf { out_data.push(b'\r'); - outidx += 1; + out_idx += 1; } out_data.push(b'\n'); - outidx += 1; - linelen = 0; + out_idx += 1; + line_len = 0; } out_data.push(b'='); - outidx += 1; + out_idx += 1; - ch = hex_nibble(buf[inidx] >> 4); + ch = hex_nibble(buf[in_idx] >> 4); if (b'a'..=b'f').contains(&ch) { ch -= b' '; } out_data.push(ch); - ch = hex_nibble(buf[inidx] & 0xf); + ch = hex_nibble(buf[in_idx] & 0xf); if (b'a'..=b'f').contains(&ch) { ch -= b' '; } out_data.push(ch); - outidx += 2; - inidx += 1; - linelen += 3; + out_idx += 2; + in_idx += 1; + line_len += 3; } else if istext - && ((buf[inidx] == b'\n') - || ((inidx + 1 < buflen) - && (buf[inidx] == b'\r') - && (buf[inidx + 1] == b'\n'))) + && ((buf[in_idx] == b'\n') + || ((in_idx + 1 < buflen) + && (buf[in_idx] == b'\r') + && (buf[in_idx + 1] == b'\n'))) { - linelen = 0; - if (outidx != 0) - && ((out_data[outidx - 1] == b' ') || (out_data[outidx - 1] == b'\t')) + line_len = 0; + if (out_idx != 0) + && ((out_data[out_idx - 1] == b' ') || (out_data[out_idx - 1] == b'\t')) { - ch = hex_nibble(out_data[outidx - 1] >> 4); + ch = hex_nibble(out_data[out_idx - 1] >> 4); if (b'a'..=b'f').contains(&ch) { ch -= b' '; } out_data.push(ch); - ch = hex_nibble(out_data[outidx - 1] & 0xf); + ch = hex_nibble(out_data[out_idx - 1] & 0xf); if (b'a'..=b'f').contains(&ch) { ch -= b' '; } out_data.push(ch); - out_data[outidx - 1] = b'='; - outidx += 2; + out_data[out_idx - 1] = b'='; + out_idx += 2; } if crlf { out_data.push(b'\r'); - outidx += 1; + out_idx += 1; } out_data.push(b'\n'); - outidx += 1; - inidx += if buf[inidx] == b'\r' { 2 } else { 1 }; + out_idx += 1; + in_idx += if buf[in_idx] == b'\r' { 2 } else { 1 }; } else { - if (inidx + 1 != buflen) && (buf[inidx + 1] != b'\n') && (linelen + 1) >= 76 { + if (in_idx + 1 != buflen) && (buf[in_idx + 1] != b'\n') && (line_len + 1) >= 76 + { // MAXLINESIZE = 76 out_data.push(b'='); - outidx += 1; + out_idx += 1; if crlf { out_data.push(b'\r'); - outidx += 1; + out_idx += 1; } out_data.push(b'\n'); - outidx += 1; - linelen = 0; + out_idx += 1; + line_len = 0; } - linelen += 1; - if header && buf[inidx] == b' ' { + line_len += 1; + if header && buf[in_idx] == b' ' { out_data.push(b'_'); - outidx += 1; - inidx += 1; + out_idx += 1; + in_idx += 1; } else { - out_data.push(buf[inidx]); - outidx += 1; - inidx += 1; + out_data.push(buf[in_idx]); + out_idx += 1; + in_idx += 1; } } } @@ -568,7 +569,7 @@ mod decl { #[pyfunction] fn rlecode_hqx(s: ArgAsciiBuffer) -> PyResult> { - const RUNCHAR: u8 = 0x90; // b'\x90' + const RUN_CHAR: u8 = 0x90; // b'\x90' s.with_ref(|buffer| { let len = buffer.len(); let mut out_data = Vec::::with_capacity((len * 2) + 2); @@ -577,20 +578,20 @@ mod decl { while idx < len { let ch = buffer[idx]; - if ch == RUNCHAR { - out_data.push(RUNCHAR); + if ch == RUN_CHAR { + out_data.push(RUN_CHAR); out_data.push(0); return Ok(out_data); } else { - let mut inend = idx + 1; - while inend < len && buffer[inend] == ch && inend < idx + 255 { - inend += 1; + let mut in_end = idx + 1; + while in_end < len && buffer[in_end] == ch && in_end < idx + 255 { + in_end += 1; } - if inend - idx > 3 { + if in_end - idx > 3 { out_data.push(ch); - out_data.push(RUNCHAR); - out_data.push(((inend - idx) % 256) as u8); - idx = inend - 1; + out_data.push(RUN_CHAR); + out_data.push(((in_end - idx) % 256) as u8); + idx = in_end - 1; } else { out_data.push(ch); } @@ -603,7 +604,7 @@ mod decl { #[pyfunction] fn rledecode_hqx(s: ArgAsciiBuffer) -> PyResult> { - const RUNCHAR: u8 = 0x90; //b'\x90' + const RUN_CHAR: u8 = 0x90; //b'\x90' s.with_ref(|buffer| { let len = buffer.len(); let mut out_data = Vec::::with_capacity(len); @@ -613,9 +614,9 @@ mod decl { idx += 1; while idx < len { - if buffer[idx] == RUNCHAR { + if buffer[idx] == RUN_CHAR { if buffer[idx + 1] == 0 { - out_data.push(RUNCHAR); + out_data.push(RUN_CHAR); } else { let ch = buffer[idx - 1]; let range = buffer[idx + 1]; diff --git a/stdlib/src/syslog.rs b/stdlib/src/syslog.rs index 69e9d1cb9e..dcdf317b02 100644 --- a/stdlib/src/syslog.rs +++ b/stdlib/src/syslog.rs @@ -1,4 +1,4 @@ -// spell-checker:ignore logoption openlog setlogmask upto NDELAY +// spell-checker:ignore logoption openlog setlogmask upto NDELAY ODELAY pub(crate) use syslog::make_module; diff --git a/vm/src/buffer.rs b/vm/src/buffer.rs index 3b76002d04..a07048757a 100644 --- a/vm/src/buffer.rs +++ b/vm/src/buffer.rs @@ -453,7 +453,7 @@ trait PackInt: PrimInt { fn unpack_int(data: &[u8]) -> Self; } -macro_rules! make_pack_primint { +macro_rules! make_pack_prim_int { ($T:ty) => { impl PackInt for $T { fn pack_int(self, data: &mut [u8]) { @@ -502,16 +502,16 @@ where .map_err(|_| new_struct_error(vm, "argument out of range".to_owned())) } -make_pack_primint!(i8); -make_pack_primint!(u8); -make_pack_primint!(i16); -make_pack_primint!(u16); -make_pack_primint!(i32); -make_pack_primint!(u32); -make_pack_primint!(i64); -make_pack_primint!(u64); -make_pack_primint!(usize); -make_pack_primint!(isize); +make_pack_prim_int!(i8); +make_pack_prim_int!(u8); +make_pack_prim_int!(i16); +make_pack_prim_int!(u16); +make_pack_prim_int!(i32); +make_pack_prim_int!(u32); +make_pack_prim_int!(i64); +make_pack_prim_int!(u64); +make_pack_prim_int!(usize); +make_pack_prim_int!(isize); macro_rules! make_pack_float { ($T:ty) => { diff --git a/vm/src/builtins/type.rs b/vm/src/builtins/type.rs index a0c7ee58ff..7351797dec 100644 --- a/vm/src/builtins/type.rs +++ b/vm/src/builtins/type.rs @@ -489,7 +489,7 @@ impl PyType { #[pygetset(setter, name = "__bases__")] fn set_bases(zelf: &Py, bases: Vec, vm: &VirtualMachine) -> PyResult<()> { // TODO: Assigning to __bases__ is only used in typing.NamedTupleMeta.__new__ - // Rather than correctly reinitializing the class, we are skipping a few steps for now + // Rather than correctly re-initializing the class, we are skipping a few steps for now if zelf.slots.flags.has_feature(PyTypeFlags::IMMUTABLETYPE) { return Err(vm.new_type_error(format!( "cannot set '__bases__' attribute of immutable type '{}'", diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 708a93fe61..6c4f97fe38 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -211,9 +211,9 @@ impl VirtualMachine { if let Some(text) = maybe_text { // if text ends with \n, remove it - let rtext = text.as_str().trim_end_matches('\n'); - let l_text = rtext.trim_start_matches([' ', '\n', '\x0c']); // \x0c is \f - let spaces = (rtext.len() - l_text.len()) as isize; + let r_text = text.as_str().trim_end_matches('\n'); + let l_text = r_text.trim_start_matches([' ', '\n', '\x0c']); // \x0c is \f + let spaces = (r_text.len() - l_text.len()) as isize; writeln!(output, " {}", l_text)?; diff --git a/wtf8/src/core_char.rs b/wtf8/src/core_char.rs index 1444e8e130..e2400430cd 100644 --- a/wtf8/src/core_char.rs +++ b/wtf8/src/core_char.rs @@ -1,3 +1,4 @@ +// cspell:disable //! Unstable functions from [`core::char`] use core::slice; diff --git a/wtf8/src/core_str_count.rs b/wtf8/src/core_str_count.rs index cff5a4b076..30fcd4645f 100644 --- a/wtf8/src/core_str_count.rs +++ b/wtf8/src/core_str_count.rs @@ -1,3 +1,4 @@ +// cspell:disable //! Modified from core::str::count use super::Wtf8; diff --git a/wtf8/src/lib.rs b/wtf8/src/lib.rs index 73b5168453..635ddf353b 100644 --- a/wtf8/src/lib.rs +++ b/wtf8/src/lib.rs @@ -1,3 +1,5 @@ +// cspell:disable + //! An implementation of [WTF-8], a utf8-compatible encoding that allows for //! unpaired surrogate codepoints. This implementation additionally allows for //! paired surrogates that are nonetheless treated as two separate codepoints. From 4094c5bfc904b7c726d7aca879fa8d731f089d45 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 13 Apr 2025 19:41:57 -0700 Subject: [PATCH 215/317] minor mark --- Lib/test/test_os.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f10de0b7af..3b19d3d3cd 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1980,7 +1980,6 @@ def get_urandom_subprocess(self, count): self.assertEqual(len(stdout), count) return stdout - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") def test_urandom_subprocess(self): data1 = self.get_urandom_subprocess(16) data2 = self.get_urandom_subprocess(16) From c7df3448052507c3c35c5696d86c14656abbbcdf Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 13 Apr 2025 23:39:36 -0700 Subject: [PATCH 216/317] update calendar and test_calendar to 3.13.2 --- Lib/calendar.py | 45 ++++--- Lib/test/test_calendar.py | 255 +++++++++++++++++++++++++++++++------- 2 files changed, 238 insertions(+), 62 deletions(-) diff --git a/Lib/calendar.py b/Lib/calendar.py index baab52a157..8c1c646da4 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -10,7 +10,6 @@ from enum import IntEnum, global_enum import locale as _locale from itertools import repeat -import warnings __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", @@ -28,7 +27,9 @@ error = ValueError # Exceptions raised for bad input -class IllegalMonthError(ValueError): +# This is trick for backward compatibility. Since 3.13, we will raise IllegalMonthError instead of +# IndexError for bad month number(out of 1-12). But we can't remove IndexError for backward compatibility. +class IllegalMonthError(ValueError, IndexError): def __init__(self, month): self.month = month def __str__(self): @@ -44,6 +45,7 @@ def __str__(self): def __getattr__(name): if name in ('January', 'February'): + import warnings warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead", DeprecationWarning, stacklevel=2) if name == 'January': @@ -158,11 +160,14 @@ def weekday(year, month, day): return Day(datetime.date(year, month, day).weekday()) -def monthrange(year, month): - """Return weekday (0-6 ~ Mon-Sun) and number of days (28-31) for - year, month.""" +def _validate_month(month): if not 1 <= month <= 12: raise IllegalMonthError(month) + +def monthrange(year, month): + """Return weekday of first day of month (0-6 ~ Mon-Sun) + and number of days (28-31) for year, month.""" + _validate_month(month) day1 = weekday(year, month, 1) ndays = mdays[month] + (month == FEBRUARY and isleap(year)) return day1, ndays @@ -370,6 +375,8 @@ def formatmonthname(self, theyear, themonth, width, withyear=True): """ Return a formatted month name. """ + _validate_month(themonth) + s = month_name[themonth] if withyear: s = "%s %r" % (s, theyear) @@ -500,6 +507,7 @@ def formatmonthname(self, theyear, themonth, withyear=True): """ Return a month name as a table row. """ + _validate_month(themonth) if withyear: s = '%s %s' % (month_name[themonth], theyear) else: @@ -585,8 +593,6 @@ def __enter__(self): _locale.setlocale(_locale.LC_TIME, self.locale) def __exit__(self, *args): - if self.oldlocale is None: - return _locale.setlocale(_locale.LC_TIME, self.oldlocale) @@ -690,7 +696,7 @@ def timegm(tuple): return seconds -def main(args): +def main(args=None): import argparse parser = argparse.ArgumentParser() textgroup = parser.add_argument_group('text only arguments') @@ -736,10 +742,15 @@ def main(args): choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "-f", "--first-weekday", + type=int, default=0, + help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + ) parser.add_argument( "year", nargs='?', type=int, - help="year number (1-9999)" + help="year number" ) parser.add_argument( "month", @@ -747,7 +758,7 @@ def main(args): help="month number (1-12, text only)" ) - options = parser.parse_args(args[1:]) + options = parser.parse_args(args) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -756,10 +767,14 @@ def main(args): locale = options.locale, options.encoding if options.type == "html": + if options.month: + parser.error("incorrect number of arguments") + sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: cal = HTMLCalendar() + cal.setfirstweekday(options.first_weekday) encoding = options.encoding if encoding is None: encoding = sys.getdefaultencoding() @@ -767,20 +782,20 @@ def main(args): write = sys.stdout.buffer.write if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif options.month is None: - write(cal.formatyearpage(options.year, **optdict)) else: - parser.error("incorrect number of arguments") - sys.exit(1) + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: cal = LocaleTextCalendar(locale=locale) else: cal = TextCalendar() + cal.setfirstweekday(options.first_weekday) optdict = dict(w=options.width, l=options.lines) if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months + if options.month is not None: + _validate_month(options.month) if options.year is None: result = cal.formatyear(datetime.date.today().year, **optdict) elif options.month is None: @@ -795,4 +810,4 @@ def main(args): if __name__ == "__main__": - main(sys.argv) + main() diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 24e472b5fe..df102fe198 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -3,12 +3,13 @@ from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -import time -import locale -import sys +import contextlib import datetime +import io +import locale import os -import warnings +import sys +import time # From https://en.wikipedia.org/wiki/Leap_year_starting_on_Saturday result_0_02_text = """\ @@ -456,6 +457,11 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + calendar.TextCalendar().formatmonth(2017, -1) def test_formatmonthname_with_year(self): self.assertEqual( @@ -550,26 +556,92 @@ def test_months(self): # verify it "acts like a sequence" in two forms of iteration self.assertEqual(value[::-1], list(reversed(value))) - def test_locale_calendars(self): + def test_locale_text_calendar(self): + try: + cal = calendar.LocaleTextCalendar(locale='') + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + cal = calendar.LocaleTextCalendar(locale=None) + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + cal = calendar.LocaleTextCalendar(locale='C') + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + def test_locale_html_calendar(self): + try: + cal = calendar.LocaleHTMLCalendar(locale='') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + cal = calendar.LocaleHTMLCalendar(locale=None) + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + cal = calendar.LocaleHTMLCalendar(locale='C') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + def test_locale_calendars_reset_locale_properly(self): # ensure that Locale{Text,HTML}Calendar resets the locale properly # (it is still not thread-safe though) old_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) try: cal = calendar.LocaleTextCalendar(locale='') local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) local_month = cal.formatmonthname(2010, 10, 10) except locale.Error: # cannot set the system default locale -- skip rest of test raise unittest.SkipTest('cannot set the system default locale') self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) self.assertIsInstance(local_month, str) self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) self.assertGreaterEqual(len(local_month), 10) + cal = calendar.LocaleHTMLCalendar(locale='') local_weekday = cal.formatweekday(1) local_month = cal.formatmonthname(2010, 10) self.assertIsInstance(local_weekday, str) self.assertIsInstance(local_month, str) + new_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) self.assertEqual(old_october, new_october) @@ -590,6 +662,21 @@ def test_locale_calendar_formatweekday(self): except locale.Error: raise unittest.SkipTest('cannot set the en_US locale') + def test_locale_calendar_formatmonthname(self): + try: + # formatmonthname uses the same month names regardless of the width argument. + cal = calendar.LocaleTextCalendar(locale='en_US') + # For too short widths, a full name (with year) is used. + self.assertEqual(cal.formatmonthname(2022, 6, 2, withyear=False), "June") + self.assertEqual(cal.formatmonthname(2022, 6, 2, withyear=True), "June 2022") + self.assertEqual(cal.formatmonthname(2022, 6, 3, withyear=False), "June") + self.assertEqual(cal.formatmonthname(2022, 6, 3, withyear=True), "June 2022") + # For long widths, a centered name is used. + self.assertEqual(cal.formatmonthname(2022, 6, 10, withyear=False), " June ") + self.assertEqual(cal.formatmonthname(2022, 6, 15, withyear=True), " June 2022 ") + except locale.Error: + raise unittest.SkipTest('cannot set the en_US locale') + def test_locale_html_calendar_custom_css_class_month_name(self): try: cal = calendar.LocaleHTMLCalendar(locale='') @@ -848,46 +935,104 @@ def conv(s): return s.replace('\n', os.linesep).encode() class CommandLineTestCase(unittest.TestCase): - def run_ok(self, *args): + def setUp(self): + self.runners = [self.run_cli_ok, self.run_cmd_ok] + + @contextlib.contextmanager + def captured_stdout_with_buffer(self): + orig_stdout = sys.stdout + buffer = io.BytesIO() + sys.stdout = io.TextIOWrapper(buffer) + try: + yield sys.stdout + finally: + sys.stdout.flush() + sys.stdout.buffer.seek(0) + sys.stdout = orig_stdout + + @contextlib.contextmanager + def captured_stderr_with_buffer(self): + orig_stderr = sys.stderr + buffer = io.BytesIO() + sys.stderr = io.TextIOWrapper(buffer) + try: + yield sys.stderr + finally: + sys.stderr.flush() + sys.stderr.buffer.seek(0) + sys.stderr = orig_stderr + + def run_cli_ok(self, *args): + with self.captured_stdout_with_buffer() as stdout: + calendar.main(args) + return stdout.buffer.read() + + def run_cmd_ok(self, *args): return assert_python_ok('-m', 'calendar', *args)[1] - def assertFailure(self, *args): + def assertCLIFails(self, *args): + with self.captured_stderr_with_buffer() as stderr: + self.assertRaises(SystemExit, calendar.main, args) + stderr = stderr.buffer.read() + self.assertIn(b'usage:', stderr) + return stderr + + def assertCmdFails(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) + return rc, stdout, stderr + + def assertFailure(self, *args): + self.assertCLIFails(*args) + self.assertCmdFails(*args) def test_help(self): - stdout = self.run_ok('-h') + stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) self.assertIn(b'calendar.py', stdout) self.assertIn(b'--help', stdout) + # special case: stdout but sys.exit() + with self.captured_stdout_with_buffer() as output: + self.assertRaises(SystemExit, calendar.main, ['-h']) + output = output.buffer.read() + self.assertIn(b'usage:', output) + self.assertIn(b'--help', output) + def test_illegal_arguments(self): self.assertFailure('-z') self.assertFailure('spam') self.assertFailure('2004', 'spam') + self.assertFailure('2004', '1', 'spam') + self.assertFailure('2004', '1', '1') + self.assertFailure('2004', '1', '1', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): - stdout = self.run_ok() - year = datetime.datetime.now().year - self.assertIn((' %s' % year).encode(), stdout) - self.assertIn(b'January', stdout) - self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) + for run in self.runners: + output = run() + year = datetime.datetime.now().year + self.assertIn(conv(' %s' % year), output) + self.assertIn(b'January', output) + self.assertIn(b'Mo Tu We Th Fr Sa Su', output) def test_output_year(self): - stdout = self.run_ok('2004') - self.assertEqual(stdout, conv(result_2004_text)) + for run in self.runners: + output = run('2004') + self.assertEqual(output, conv(result_2004_text)) def test_output_month(self): - stdout = self.run_ok('2004', '1') - self.assertEqual(stdout, conv(result_2004_01_text)) + for run in self.runners: + output = run('2004', '1') + self.assertEqual(output, conv(result_2004_01_text)) def test_option_encoding(self): self.assertFailure('-e') self.assertFailure('--encoding') - stdout = self.run_ok('--encoding', 'utf-16-le', '2004') - self.assertEqual(stdout, result_2004_text.encode('utf-16-le')) + for run in self.runners: + output = run('--encoding', 'utf-16-le', '2004') + self.assertEqual(output, result_2004_text.encode('utf-16-le')) def test_option_locale(self): self.assertFailure('-L') @@ -905,66 +1050,75 @@ def test_option_locale(self): locale.setlocale(locale.LC_TIME, oldlocale) except (locale.Error, ValueError): self.skipTest('cannot set the system default locale') - stdout = self.run_ok('--locale', lang, '--encoding', enc, '2004') - self.assertIn('2004'.encode(enc), stdout) + for run in self.runners: + for type in ('text', 'html'): + output = run( + '--type', type, '--locale', lang, '--encoding', enc, '2004' + ) + self.assertIn('2004'.encode(enc), output) def test_option_width(self): self.assertFailure('-w') self.assertFailure('--width') self.assertFailure('-w', 'spam') - stdout = self.run_ok('--width', '3', '2004') - self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) + for run in self.runners: + output = run('--width', '3', '2004') + self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', output) def test_option_lines(self): self.assertFailure('-l') self.assertFailure('--lines') self.assertFailure('-l', 'spam') - stdout = self.run_ok('--lines', '2', '2004') - self.assertIn(conv('December\n\nMo Tu We'), stdout) + for run in self.runners: + output = run('--lines', '2', '2004') + self.assertIn(conv('December\n\nMo Tu We'), output) def test_option_spacing(self): self.assertFailure('-s') self.assertFailure('--spacing') self.assertFailure('-s', 'spam') - stdout = self.run_ok('--spacing', '8', '2004') - self.assertIn(b'Su Mo', stdout) + for run in self.runners: + output = run('--spacing', '8', '2004') + self.assertIn(b'Su Mo', output) def test_option_months(self): self.assertFailure('-m') self.assertFailure('--month') self.assertFailure('-m', 'spam') - stdout = self.run_ok('--months', '1', '2004') - self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) + for run in self.runners: + output = run('--months', '1', '2004') + self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), output) def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') self.assertFailure('-t', 'spam') - stdout = self.run_ok('--type', 'text', '2004') - self.assertEqual(stdout, conv(result_2004_text)) - stdout = self.run_ok('--type', 'html', '2004') - self.assertEqual(stdout[:6], b'Calendar for 2004', stdout) + for run in self.runners: + output = run('--type', 'text', '2004') + self.assertEqual(output, conv(result_2004_text)) + output = run('--type', 'html', '2004') + self.assertEqual(output[:6], b'Calendar for 2004', output) def test_html_output_current_year(self): - stdout = self.run_ok('--type', 'html') - year = datetime.datetime.now().year - self.assertIn(('Calendar for %s' % year).encode(), - stdout) - self.assertIn(b'January', - stdout) + for run in self.runners: + output = run('--type', 'html') + year = datetime.datetime.now().year + self.assertIn(('Calendar for %s' % year).encode(), output) + self.assertIn(b'January', output) def test_html_output_year_encoding(self): - stdout = self.run_ok('-t', 'html', '--encoding', 'ascii', '2004') - self.assertEqual(stdout, - result_2004_html.format(**default_format).encode('ascii')) + for run in self.runners: + output = run('-t', 'html', '--encoding', 'ascii', '2004') + self.assertEqual(output, result_2004_html.format(**default_format).encode('ascii')) def test_html_output_year_css(self): self.assertFailure('-t', 'html', '-c') self.assertFailure('-t', 'html', '--css') - stdout = self.run_ok('-t', 'html', '--css', 'custom.css', '2004') - self.assertIn(b'', stdout) + for run in self.runners: + output = run('-t', 'html', '--css', 'custom.css', '2004') + self.assertIn(b'', output) class MiscTestCase(unittest.TestCase): @@ -972,7 +1126,7 @@ def test__all__(self): not_exported = { 'mdays', 'January', 'February', 'EPOCH', 'different_locale', 'c', 'prweek', 'week', 'format', - 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth'} + 'formatstring', 'main', 'monthlen', 'prevmonth', 'nextmonth', ""} support.check__all__(self, calendar, not_exported=not_exported) @@ -1000,6 +1154,13 @@ def test_formatmonth(self): self.assertIn('class="text-center month"', self.cal.formatmonth(2017, 5)) + def test_formatmonth_with_invalid_month(self): + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, 13) + with self.assertRaises(calendar.IllegalMonthError): + self.cal.formatmonth(2017, -1) + + def test_formatweek(self): weeks = self.cal.monthdays2calendar(2017, 5) self.assertIn('class="wed text-nowrap"', self.cal.formatweek(weeks[0])) From c2c20758c93e2046b419594ba25d36be9cd8d995 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 14 Apr 2025 00:56:22 -0700 Subject: [PATCH 217/317] Remove imp (#5693) --- Lib/imp.py | 346 --------------------------- Lib/pkgutil.py | 182 -------------- Lib/test/test_imp.py | 498 --------------------------------------- Lib/test/test_pkgutil.py | 27 --- 4 files changed, 1053 deletions(-) delete mode 100644 Lib/imp.py delete mode 100644 Lib/test/test_imp.py diff --git a/Lib/imp.py b/Lib/imp.py deleted file mode 100644 index fc42c15765..0000000000 --- a/Lib/imp.py +++ /dev/null @@ -1,346 +0,0 @@ -"""This module provides the components needed to build your own __import__ -function. Undocumented functions are obsolete. - -In most cases it is preferred you consider using the importlib module's -functionality over this module. - -""" -# (Probably) need to stay in _imp -from _imp import (lock_held, acquire_lock, release_lock, - get_frozen_object, is_frozen_package, - init_frozen, is_builtin, is_frozen, - _fix_co_filename, _frozen_module_names) -try: - from _imp import create_dynamic -except ImportError: - # Platform doesn't support dynamic loading. - create_dynamic = None - -from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name -from importlib._bootstrap_external import SourcelessFileLoader - -from importlib import machinery -from importlib import util -import importlib -import os -import sys -import tokenize -import types -import warnings - -warnings.warn("the imp module is deprecated in favour of importlib and slated " - "for removal in Python 3.12; " - "see the module's documentation for alternative uses", - DeprecationWarning, stacklevel=2) - -# DEPRECATED -SEARCH_ERROR = 0 -PY_SOURCE = 1 -PY_COMPILED = 2 -C_EXTENSION = 3 -PY_RESOURCE = 4 -PKG_DIRECTORY = 5 -C_BUILTIN = 6 -PY_FROZEN = 7 -PY_CODERESOURCE = 8 -IMP_HOOK = 9 - - -def new_module(name): - """**DEPRECATED** - - Create a new module. - - The module is not entered into sys.modules. - - """ - return types.ModuleType(name) - - -def get_magic(): - """**DEPRECATED** - - Return the magic number for .pyc files. - """ - return util.MAGIC_NUMBER - - -def get_tag(): - """Return the magic tag for .pyc files.""" - return sys.implementation.cache_tag - - -def cache_from_source(path, debug_override=None): - """**DEPRECATED** - - Given the path to a .py file, return the path to its .pyc file. - - The .py file does not need to exist; this simply returns the path to the - .pyc file calculated as if the .py file were imported. - - If debug_override is not None, then it must be a boolean and is used in - place of sys.flags.optimize. - - If sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - return util.cache_from_source(path, debug_override) - - -def source_from_cache(path): - """**DEPRECATED** - - Given the path to a .pyc. file, return the path to its .py file. - - The .pyc file does not need to exist; this simply returns the path to - the .py file calculated to correspond to the .pyc file. If path does - not conform to PEP 3147 format, ValueError will be raised. If - sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - return util.source_from_cache(path) - - -def get_suffixes(): - """**DEPRECATED**""" - extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES] - source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] - bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] - - return extensions + source + bytecode - - -class NullImporter: - - """**DEPRECATED** - - Null import object. - - """ - - def __init__(self, path): - if path == '': - raise ImportError('empty pathname', path='') - elif os.path.isdir(path): - raise ImportError('existing directory', path=path) - - def find_module(self, fullname): - """Always returns None.""" - return None - - -class _HackedGetData: - - """Compatibility support for 'file' arguments of various load_*() - functions.""" - - def __init__(self, fullname, path, file=None): - super().__init__(fullname, path) - self.file = file - - def get_data(self, path): - """Gross hack to contort loader to deal w/ load_*()'s bad API.""" - if self.file and path == self.path: - # The contract of get_data() requires us to return bytes. Reopen the - # file in binary mode if needed. - if not self.file.closed: - file = self.file - if 'b' not in file.mode: - file.close() - if self.file.closed: - self.file = file = open(self.path, 'rb') - - with file: - return file.read() - else: - return super().get_data(path) - - -class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader): - - """Compatibility support for implementing load_source().""" - - -def load_source(name, pathname, file=None): - loader = _LoadSourceCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = machinery.SourceFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader): - - """Compatibility support for implementing load_compiled().""" - - -def load_compiled(name, pathname, file=None): - """**DEPRECATED**""" - loader = _LoadCompiledCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = SourcelessFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -def load_package(name, path): - """**DEPRECATED**""" - if os.path.isdir(path): - extensions = (machinery.SOURCE_SUFFIXES[:] + - machinery.BYTECODE_SUFFIXES[:]) - for extension in extensions: - init_path = os.path.join(path, '__init__' + extension) - if os.path.exists(init_path): - path = init_path - break - else: - raise ValueError('{!r} is not a package'.format(path)) - spec = util.spec_from_file_location(name, path, - submodule_search_locations=[]) - if name in sys.modules: - return _exec(spec, sys.modules[name]) - else: - return _load(spec) - - -def load_module(name, file, filename, details): - """**DEPRECATED** - - Load a module, given information returned by find_module(). - - The module name must include the full package name, if any. - - """ - suffix, mode, type_ = details - if mode and (not mode.startswith('r') or '+' in mode): - raise ValueError('invalid file open mode {!r}'.format(mode)) - elif file is None and type_ in {PY_SOURCE, PY_COMPILED}: - msg = 'file object required for import (type code {})'.format(type_) - raise ValueError(msg) - elif type_ == PY_SOURCE: - return load_source(name, filename, file) - elif type_ == PY_COMPILED: - return load_compiled(name, filename, file) - elif type_ == C_EXTENSION and load_dynamic is not None: - if file is None: - with open(filename, 'rb') as opened_file: - return load_dynamic(name, filename, opened_file) - else: - return load_dynamic(name, filename, file) - elif type_ == PKG_DIRECTORY: - return load_package(name, filename) - elif type_ == C_BUILTIN: - return init_builtin(name) - elif type_ == PY_FROZEN: - return init_frozen(name) - else: - msg = "Don't know how to import {} (type code {})".format(name, type_) - raise ImportError(msg, name=name) - - -def find_module(name, path=None): - """**DEPRECATED** - - Search for a module. - - If path is omitted or None, search for a built-in, frozen or special - module and continue search in sys.path. The module name cannot - contain '.'; to search for a submodule of a package, pass the - submodule name and the package's __path__. - - """ - if not isinstance(name, str): - raise TypeError("'name' must be a str, not {}".format(type(name))) - elif not isinstance(path, (type(None), list)): - # Backwards-compatibility - raise RuntimeError("'path' must be None or a list, " - "not {}".format(type(path))) - - if path is None: - if is_builtin(name): - return None, None, ('', '', C_BUILTIN) - elif is_frozen(name): - return None, None, ('', '', PY_FROZEN) - else: - path = sys.path - - for entry in path: - package_directory = os.path.join(entry, name) - for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: - package_file_name = '__init__' + suffix - file_path = os.path.join(package_directory, package_file_name) - if os.path.isfile(file_path): - return None, package_directory, ('', '', PKG_DIRECTORY) - for suffix, mode, type_ in get_suffixes(): - file_name = name + suffix - file_path = os.path.join(entry, file_name) - if os.path.isfile(file_path): - break - else: - continue - break # Break out of outer loop when breaking out of inner loop. - else: - raise ImportError(_ERR_MSG.format(name), name=name) - - encoding = None - if 'b' not in mode: - with open(file_path, 'rb') as file: - encoding = tokenize.detect_encoding(file.readline)[0] - file = open(file_path, mode, encoding=encoding) - return file, file_path, (suffix, mode, type_) - - -def reload(module): - """**DEPRECATED** - - Reload the module and return it. - - The module must have been successfully imported before. - - """ - return importlib.reload(module) - - -def init_builtin(name): - """**DEPRECATED** - - Load and return a built-in module by name, or None is such module doesn't - exist - """ - try: - return _builtin_from_name(name) - except ImportError: - return None - - -if create_dynamic: - def load_dynamic(name, path, file=None): - """**DEPRECATED** - - Load an extension module. - """ - import importlib.machinery - loader = importlib.machinery.ExtensionFileLoader(name, path) - - # Issue #24748: Skip the sys.modules check in _load_module_shim; - # always load new extension - spec = importlib.machinery.ModuleSpec( - name=name, loader=loader, origin=path) - return _load(spec) - -else: - load_dynamic = None diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 8e010c79c1..a4c474006b 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -184,188 +184,6 @@ def _iter_file_finder_modules(importer, prefix=''): iter_importer_modules.register( importlib.machinery.FileFinder, _iter_file_finder_modules) - -def _import_imp(): - global imp - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - imp = importlib.import_module('imp') - -class ImpImporter: - """PEP 302 Finder that wraps Python's "classic" import algorithm - - ImpImporter(dirname) produces a PEP 302 finder that searches that - directory. ImpImporter(None) produces a PEP 302 finder that searches - the current sys.path, plus any modules that are frozen or built-in. - - Note that ImpImporter does not currently support being used by placement - on sys.meta_path. - """ - - def __init__(self, path=None): - global imp - warnings.warn("This emulation is deprecated and slated for removal " - "in Python 3.12; use 'importlib' instead", - DeprecationWarning) - _import_imp() - self.path = path - - def find_module(self, fullname, path=None): - # Note: we ignore 'path' argument since it is only used via meta_path - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [os.path.realpath(self.path)] - try: - file, filename, etc = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(fullname, file, filename, etc) - - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - import inspect - try: - filenames = os.listdir(self.path) - except OSError: - # ignore unreadable directories like import does - filenames = [] - filenames.sort() # handle packages before same-named modules - - for fn in filenames: - modname = inspect.getmodulename(fn) - if modname=='__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - try: - dircontents = os.listdir(path) - except OSError: - # ignore unreadable directories like import does - dircontents = [] - for fn in dircontents: - subname = inspect.getmodulename(fn) - if subname=='__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -class ImpLoader: - """PEP 302 Loader that wraps Python's "classic" import algorithm - """ - code = source = None - - def __init__(self, fullname, file, filename, etc): - warnings.warn("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning) - _import_imp() - self.file = file - self.filename = filename - self.fullname = fullname - self.etc = etc - - def load_module(self, fullname): - self._reopen() - try: - mod = imp.load_module(fullname, self.file, self.filename, self.etc) - finally: - if self.file: - self.file.close() - # Note: we don't set __loader__ because we want the module to look - # normal; i.e. this is just a wrapper for standard import machinery - return mod - - def get_data(self, pathname): - with open(pathname, "rb") as file: - return file.read() - - def _reopen(self): - if self.file and self.file.closed: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self.file = open(self.filename, 'r') - elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): - self.file = open(self.filename, 'rb') - - def _fix_name(self, fullname): - if fullname is None: - fullname = self.fullname - elif fullname != self.fullname: - raise ImportError("Loader for module %s cannot handle " - "module %s" % (self.fullname, fullname)) - return fullname - - def is_package(self, fullname): - fullname = self._fix_name(fullname) - return self.etc[2]==imp.PKG_DIRECTORY - - def get_code(self, fullname=None): - fullname = self._fix_name(fullname) - if self.code is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - source = self.get_source(fullname) - self.code = compile(source, self.filename, 'exec') - elif mod_type==imp.PY_COMPILED: - self._reopen() - try: - self.code = read_code(self.file) - finally: - self.file.close() - elif mod_type==imp.PKG_DIRECTORY: - self.code = self._get_delegate().get_code() - return self.code - - def get_source(self, fullname=None): - fullname = self._fix_name(fullname) - if self.source is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self._reopen() - try: - self.source = self.file.read() - finally: - self.file.close() - elif mod_type==imp.PY_COMPILED: - if os.path.exists(self.filename[:-1]): - with open(self.filename[:-1], 'r') as f: - self.source = f.read() - elif mod_type==imp.PKG_DIRECTORY: - self.source = self._get_delegate().get_source() - return self.source - - def _get_delegate(self): - finder = ImpImporter(self.filename) - spec = _get_spec(finder, '__init__') - return spec.loader - - def get_filename(self, fullname=None): - fullname = self._fix_name(fullname) - mod_type = self.etc[2] - if mod_type==imp.PKG_DIRECTORY: - return self._get_delegate().get_filename() - elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): - return self.filename - return None - - try: import zipimport from zipimport import zipimporter diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py deleted file mode 100644 index c4aba69197..0000000000 --- a/Lib/test/test_imp.py +++ /dev/null @@ -1,498 +0,0 @@ -import gc -import importlib -import importlib.util -import os -import os.path -import py_compile -import sys -from test import support -from test.support import import_helper -from test.support import os_helper -from test.support import script_helper -from test.support import warnings_helper -import unittest -import warnings -imp = warnings_helper.import_deprecated('imp') -import _imp - - -OS_PATH_NAME = os.path.__name__ - - -def requires_load_dynamic(meth): - """Decorator to skip a test if not running under CPython or lacking - imp.load_dynamic().""" - meth = support.cpython_only(meth) - return unittest.skipIf(getattr(imp, 'load_dynamic', None) is None, - 'imp.load_dynamic() required')(meth) - - -class LockTests(unittest.TestCase): - - """Very basic test of import lock functions.""" - - def verify_lock_state(self, expected): - self.assertEqual(imp.lock_held(), expected, - "expected imp.lock_held() to be %r" % expected) - def testLock(self): - LOOPS = 50 - - # The import lock may already be held, e.g. if the test suite is run - # via "import test.autotest". - lock_held_at_start = imp.lock_held() - self.verify_lock_state(lock_held_at_start) - - for i in range(LOOPS): - imp.acquire_lock() - self.verify_lock_state(True) - - for i in range(LOOPS): - imp.release_lock() - - # The original state should be restored now. - self.verify_lock_state(lock_held_at_start) - - if not lock_held_at_start: - try: - imp.release_lock() - except RuntimeError: - pass - else: - self.fail("release_lock() without lock should raise " - "RuntimeError") - -class ImportTests(unittest.TestCase): - def setUp(self): - mod = importlib.import_module('test.encoded_modules') - self.test_strings = mod.test_strings - self.test_path = mod.__path__ - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_import_encoded_module(self): - for modname, encoding, teststr in self.test_strings: - mod = importlib.import_module('test.encoded_modules.' - 'module_' + modname) - self.assertEqual(teststr, mod.test) - - def test_find_module_encoding(self): - for mod, encoding, _ in self.test_strings: - with imp.find_module('module_' + mod, self.test_path)[0] as fd: - self.assertEqual(fd.encoding, encoding) - - path = [os.path.dirname(__file__)] - with self.assertRaises(SyntaxError): - imp.find_module('badsyntax_pep3120', path) - - def test_issue1267(self): - for mod, encoding, _ in self.test_strings: - fp, filename, info = imp.find_module('module_' + mod, - self.test_path) - with fp: - self.assertNotEqual(fp, None) - self.assertEqual(fp.encoding, encoding) - self.assertEqual(fp.tell(), 0) - self.assertEqual(fp.readline(), '# test %s encoding\n' - % encoding) - - fp, filename, info = imp.find_module("tokenize") - with fp: - self.assertNotEqual(fp, None) - self.assertEqual(fp.encoding, "utf-8") - self.assertEqual(fp.tell(), 0) - self.assertEqual(fp.readline(), - '"""Tokenization help for Python programs.\n') - - def test_issue3594(self): - temp_mod_name = 'test_imp_helper' - sys.path.insert(0, '.') - try: - with open(temp_mod_name + '.py', 'w', encoding="latin-1") as file: - file.write("# coding: cp1252\nu = 'test.test_imp'\n") - file, filename, info = imp.find_module(temp_mod_name) - file.close() - self.assertEqual(file.encoding, 'cp1252') - finally: - del sys.path[0] - os_helper.unlink(temp_mod_name + '.py') - os_helper.unlink(temp_mod_name + '.pyc') - - def test_issue5604(self): - # Test cannot cover imp.load_compiled function. - # Martin von Loewis note what shared library cannot have non-ascii - # character because init_xxx function cannot be compiled - # and issue never happens for dynamic modules. - # But sources modified to follow generic way for processing paths. - - # the return encoding could be uppercase or None - fs_encoding = sys.getfilesystemencoding() - - # covers utf-8 and Windows ANSI code pages - # one non-space symbol from every page - # (http://en.wikipedia.org/wiki/Code_page) - known_locales = { - 'utf-8' : b'\xc3\xa4', - 'cp1250' : b'\x8C', - 'cp1251' : b'\xc0', - 'cp1252' : b'\xc0', - 'cp1253' : b'\xc1', - 'cp1254' : b'\xc0', - 'cp1255' : b'\xe0', - 'cp1256' : b'\xe0', - 'cp1257' : b'\xc0', - 'cp1258' : b'\xc0', - } - - if sys.platform == 'darwin': - self.assertEqual(fs_encoding, 'utf-8') - # Mac OS X uses the Normal Form D decomposition - # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html - special_char = b'a\xcc\x88' - else: - special_char = known_locales.get(fs_encoding) - - if not special_char: - self.skipTest("can't run this test with %s as filesystem encoding" - % fs_encoding) - decoded_char = special_char.decode(fs_encoding) - temp_mod_name = 'test_imp_helper_' + decoded_char - test_package_name = 'test_imp_helper_package_' + decoded_char - init_file_name = os.path.join(test_package_name, '__init__.py') - try: - # if the curdir is not in sys.path the test fails when run with - # ./python ./Lib/test/regrtest.py test_imp - sys.path.insert(0, os.curdir) - with open(temp_mod_name + '.py', 'w', encoding="utf-8") as file: - file.write('a = 1\n') - file, filename, info = imp.find_module(temp_mod_name) - with file: - self.assertIsNotNone(file) - self.assertTrue(filename[:-3].endswith(temp_mod_name)) - self.assertEqual(info[0], '.py') - self.assertEqual(info[1], 'r') - self.assertEqual(info[2], imp.PY_SOURCE) - - mod = imp.load_module(temp_mod_name, file, filename, info) - self.assertEqual(mod.a, 1) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') - self.assertEqual(mod.a, 1) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - if not sys.dont_write_bytecode: - mod = imp.load_compiled( - temp_mod_name, - imp.cache_from_source(temp_mod_name + '.py')) - self.assertEqual(mod.a, 1) - - if not os.path.exists(test_package_name): - os.mkdir(test_package_name) - with open(init_file_name, 'w', encoding="utf-8") as file: - file.write('b = 2\n') - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - package = imp.load_package(test_package_name, test_package_name) - self.assertEqual(package.b, 2) - finally: - del sys.path[0] - for ext in ('.py', '.pyc'): - os_helper.unlink(temp_mod_name + ext) - os_helper.unlink(init_file_name + ext) - os_helper.rmtree(test_package_name) - os_helper.rmtree('__pycache__') - - def test_issue9319(self): - path = os.path.dirname(__file__) - self.assertRaises(SyntaxError, - imp.find_module, "badsyntax_pep3120", [path]) - - def test_load_from_source(self): - # Verify that the imp module can correctly load and find .py files - # XXX (ncoghlan): It would be nice to use import_helper.CleanImport - # here, but that breaks because the os module registers some - # handlers in copy_reg on import. Since CleanImport doesn't - # revert that registration, the module is left in a broken - # state after reversion. Reinitialising the module contents - # and just reverting os.environ to its previous state is an OK - # workaround - with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME): - import os - orig_path = os.path - orig_getenv = os.getenv - with os_helper.EnvironmentVarGuard(): - x = imp.find_module("os") - self.addCleanup(x[0].close) - new_os = imp.load_module("os", *x) - self.assertIs(os, new_os) - self.assertIs(orig_path, new_os.path) - self.assertIsNot(orig_getenv, new_os.getenv) - - @requires_load_dynamic - def test_issue15828_load_extensions(self): - # Issue 15828 picked up that the adapter between the old imp API - # and importlib couldn't handle C extensions - example = "_heapq" - x = imp.find_module(example) - file_ = x[0] - if file_ is not None: - self.addCleanup(file_.close) - mod = imp.load_module(example, *x) - self.assertEqual(mod.__name__, example) - - @requires_load_dynamic - def test_issue16421_multiple_modules_in_one_dll(self): - # Issue 16421: loading several modules from the same compiled file fails - m = '_testimportmultiple' - fileobj, pathname, description = imp.find_module(m) - fileobj.close() - mod0 = imp.load_dynamic(m, pathname) - mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname) - mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname) - self.assertEqual(mod0.__name__, m) - self.assertEqual(mod1.__name__, '_testimportmultiple_foo') - self.assertEqual(mod2.__name__, '_testimportmultiple_bar') - with self.assertRaises(ImportError): - imp.load_dynamic('nonexistent', pathname) - - @requires_load_dynamic - def test_load_dynamic_ImportError_path(self): - # Issue #1559549 added `name` and `path` attributes to ImportError - # in order to provide better detail. Issue #10854 implemented those - # attributes on import failures of extensions on Windows. - path = 'bogus file path' - name = 'extension' - with self.assertRaises(ImportError) as err: - imp.load_dynamic(name, path) - self.assertIn(path, err.exception.path) - self.assertEqual(name, err.exception.name) - - @requires_load_dynamic - def test_load_module_extension_file_is_None(self): - # When loading an extension module and the file is None, open one - # on the behalf of imp.load_dynamic(). - # Issue #15902 - name = '_testimportmultiple' - found = imp.find_module(name) - if found[0] is not None: - found[0].close() - if found[2][2] != imp.C_EXTENSION: - self.skipTest("found module doesn't appear to be a C extension") - imp.load_module(name, None, *found[1:]) - - @requires_load_dynamic - def test_issue24748_load_module_skips_sys_modules_check(self): - name = 'test.imp_dummy' - try: - del sys.modules[name] - except KeyError: - pass - try: - module = importlib.import_module(name) - spec = importlib.util.find_spec('_testmultiphase') - module = imp.load_dynamic(name, spec.origin) - self.assertEqual(module.__name__, name) - self.assertEqual(module.__spec__.name, name) - self.assertEqual(module.__spec__.origin, spec.origin) - self.assertRaises(AttributeError, getattr, module, 'dummy_name') - self.assertEqual(module.int_const, 1969) - self.assertIs(sys.modules[name], module) - finally: - try: - del sys.modules[name] - except KeyError: - pass - - @unittest.skipIf(sys.dont_write_bytecode, - "test meaningful only when writing bytecode") - def test_bug7732(self): - with os_helper.temp_cwd(): - source = os_helper.TESTFN + '.py' - os.mkdir(source) - self.assertRaisesRegex(ImportError, '^No module', - imp.find_module, os_helper.TESTFN, ["."]) - - def test_multiple_calls_to_get_data(self): - # Issue #18755: make sure multiple calls to get_data() can succeed. - loader = imp._LoadSourceCompatibility('imp', imp.__file__, - open(imp.__file__, encoding="utf-8")) - loader.get_data(imp.__file__) # File should be closed - loader.get_data(imp.__file__) # Will need to create a newly opened file - - def test_load_source(self): - # Create a temporary module since load_source(name) modifies - # sys.modules[name] attributes like __loader___ - modname = f"tmp{__name__}" - mod = type(sys.modules[__name__])(modname) - with support.swap_item(sys.modules, modname, mod): - with self.assertRaisesRegex(ValueError, 'embedded null'): - imp.load_source(modname, __file__ + "\0") - - @support.cpython_only - def test_issue31315(self): - # There shouldn't be an assertion failure in imp.create_dynamic(), - # when spec.name is not a string. - create_dynamic = support.get_attribute(imp, 'create_dynamic') - class BadSpec: - name = None - origin = 'foo' - with self.assertRaises(TypeError): - create_dynamic(BadSpec()) - - def test_issue_35321(self): - # Both _frozen_importlib and _frozen_importlib_external - # should have a spec origin of "frozen" and - # no need to clean up imports in this case. - - import _frozen_importlib_external - self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen") - - import _frozen_importlib - self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_source_hash(self): - self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~') - self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2') - - def test_pyc_invalidation_mode_from_cmdline(self): - cases = [ - ([], "default"), - (["--check-hash-based-pycs", "default"], "default"), - (["--check-hash-based-pycs", "always"], "always"), - (["--check-hash-based-pycs", "never"], "never"), - ] - for interp_args, expected in cases: - args = interp_args + [ - "-c", - "import _imp; print(_imp.check_hash_based_pycs)", - ] - res = script_helper.assert_python_ok(*args) - self.assertEqual(res.out.strip().decode('utf-8'), expected) - - def test_find_and_load_checked_pyc(self): - # issue 34056 - with os_helper.temp_cwd(): - with open('mymod.py', 'wb') as fp: - fp.write(b'x = 42\n') - py_compile.compile( - 'mymod.py', - doraise=True, - invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, - ) - file, path, description = imp.find_module('mymod', path=['.']) - mod = imp.load_module('mymod', file, path, description) - self.assertEqual(mod.x, 42) - - - @support.cpython_only - def test_create_builtin_subinterp(self): - # gh-99578: create_builtin() behavior changes after the creation of the - # first sub-interpreter. Test both code paths, before and after the - # creation of a sub-interpreter. Previously, create_builtin() had - # a reference leak after the creation of the first sub-interpreter. - - import builtins - create_builtin = support.get_attribute(_imp, "create_builtin") - class Spec: - name = "builtins" - spec = Spec() - - def check_get_builtins(): - refcnt = sys.getrefcount(builtins) - mod = _imp.create_builtin(spec) - self.assertIs(mod, builtins) - self.assertEqual(sys.getrefcount(builtins), refcnt + 1) - # Check that a GC collection doesn't crash - gc.collect() - - check_get_builtins() - - ret = support.run_in_subinterp("import builtins") - self.assertEqual(ret, 0) - - check_get_builtins() - - -class ReloadTests(unittest.TestCase): - - """Very basic tests to make sure that imp.reload() operates just like - reload().""" - - def test_source(self): - # XXX (ncoghlan): It would be nice to use test.import_helper.CleanImport - # here, but that breaks because the os module registers some - # handlers in copy_reg on import. Since CleanImport doesn't - # revert that registration, the module is left in a broken - # state after reversion. Reinitialising the module contents - # and just reverting os.environ to its previous state is an OK - # workaround - with os_helper.EnvironmentVarGuard(): - import os - imp.reload(os) - - def test_extension(self): - with import_helper.CleanImport('time'): - import time - imp.reload(time) - - def test_builtin(self): - with import_helper.CleanImport('marshal'): - import marshal - imp.reload(marshal) - - def test_with_deleted_parent(self): - # see #18681 - from html import parser - html = sys.modules.pop('html') - def cleanup(): - sys.modules['html'] = html - self.addCleanup(cleanup) - with self.assertRaisesRegex(ImportError, 'html'): - imp.reload(parser) - - -class PEP3147Tests(unittest.TestCase): - """Tests of PEP 3147.""" - - tag = imp.get_tag() - - @unittest.skipUnless(sys.implementation.cache_tag is not None, - 'requires sys.implementation.cache_tag not be None') - def test_cache_from_source(self): - # Given the path to a .py file, return the path to its PEP 3147 - # defined .pyc file (i.e. under __pycache__). - path = os.path.join('foo', 'bar', 'baz', 'qux.py') - expect = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) - self.assertEqual(imp.cache_from_source(path, True), expect) - - @unittest.skipUnless(sys.implementation.cache_tag is not None, - 'requires sys.implementation.cache_tag to not be ' - 'None') - def test_source_from_cache(self): - # Given the path to a PEP 3147 defined .pyc file, return the path to - # its source. This tests the good path. - path = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) - expect = os.path.join('foo', 'bar', 'baz', 'qux.py') - self.assertEqual(imp.source_from_cache(path), expect) - - -class NullImporterTests(unittest.TestCase): - @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, - "Need an undecodeable filename") - def test_unencodeable(self): - name = os_helper.TESTFN_UNENCODABLE - os.mkdir(name) - try: - self.assertRaises(ImportError, imp.NullImporter, name) - finally: - os.rmdir(name) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 3a10ec8fc3..ece4cf2d05 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -491,33 +491,6 @@ def test_nested(self): self.assertEqual(c, 1) self.assertEqual(d, 2) - -class ImportlibMigrationTests(unittest.TestCase): - # With full PEP 302 support in the standard import machinery, the - # PEP 302 emulation in this module is in the process of being - # deprecated in favour of importlib proper - - def check_deprecated(self): - return check_warnings( - ("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning)) - - def test_importer_deprecated(self): - with self.check_deprecated(): - pkgutil.ImpImporter("") - - def test_loader_deprecated(self): - with self.check_deprecated(): - pkgutil.ImpLoader("", "", "", "") - - def test_get_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader("sys")) - self.assertIsNotNone(pkgutil.get_loader("os")) - self.assertIsNotNone(pkgutil.get_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') def test_get_loader_handles_missing_loader_attribute(self): global __loader__ From a7ad84827023508684fd57de1f18781082b57060 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 14 Apr 2025 16:50:29 -0700 Subject: [PATCH 218/317] Rust dependency updates (#5651) --- Cargo.lock | 246 +++++++++++++++++++++++++++---------- Cargo.toml | 10 +- benches/execution.rs | 8 +- benches/microbenchmarks.rs | 8 +- stdlib/Cargo.toml | 2 +- stdlib/src/binascii.rs | 13 +- 6 files changed, 205 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60f0c44ce1..9ffadb13f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.97" @@ -112,9 +162,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "base64" -version = "0.13.1" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bindgen" @@ -125,7 +175,7 @@ dependencies = [ "bitflags 2.9.0", "cexpr", "clang-sys", - "itertools 0.11.0", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", @@ -238,9 +288,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.17" +version = "1.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" dependencies = [ "shlex", ] @@ -311,6 +361,12 @@ dependencies = [ "error-code", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "compact_str" version = "0.8.1" @@ -435,7 +491,7 @@ dependencies = [ "rustc-hash", "serde", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", ] [[package]] @@ -481,7 +537,7 @@ dependencies = [ "cranelift-codegen", "log", "smallvec", - "target-lexicon 0.13.2", + "target-lexicon", ] [[package]] @@ -505,7 +561,7 @@ dependencies = [ "libc", "log", "region", - "target-lexicon 0.13.2", + "target-lexicon", "wasmtime-jit-icache-coherence", "windows-sys 0.59.0", ] @@ -529,7 +585,7 @@ checksum = "e5b09bdd6407bf5d89661b80cf926ce731c9e8cc184bf49102267a2369a8358e" dependencies = [ "cranelift-codegen", "libc", - "target-lexicon 0.13.2", + "target-lexicon", ] [[package]] @@ -728,15 +784,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" -version = "0.9.3" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ - "atty", + "anstream", + "anstyle", + "env_filter", + "jiff", "log", - "termcolor", ] [[package]] @@ -747,9 +815,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -780,7 +848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix 1.0.3", + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -822,12 +890,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "libz-rs-sys", + "libz-rs-sys 0.5.0", "miniz_oxide", ] @@ -874,7 +942,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed7131e57abbde63513e0e6636f76668a1ca9798dcae2df4e283cae9ee83859e" dependencies = [ - "rustix 1.0.3", + "rustix 1.0.5", "windows-targets 0.52.6", ] @@ -1024,9 +1092,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown", @@ -1063,6 +1131,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1081,6 +1155,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -1096,6 +1179,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1127,9 +1234,9 @@ dependencies = [ [[package]] name = "lambert_w" -version = "1.2.5" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eeec1be8d026f51b1cf70ed28442b9f0ece61ff196cd3a99d8b4492a83a864b" +checksum = "fd4d9b9fa6582f5d77f954729c91c32a7c85834332e470b014d12e1678fd1793" dependencies = [ "num-complex", "num-traits", @@ -1251,7 +1358,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" dependencies = [ - "zlib-rs", + "zlib-rs 0.4.2", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs 0.5.0", ] [[package]] @@ -1418,9 +1534,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "ff70ce3e48ae43fa075863cef62e8b43b71a4f2382229920e0df362592919430" dependencies = [ "adler2", ] @@ -1623,7 +1739,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "smallvec", "windows-targets 0.52.6", ] @@ -1743,6 +1859,15 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1754,9 +1879,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", "syn 2.0.100", @@ -1783,9 +1908,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.6" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" dependencies = [ "cfg-if", "indoc", @@ -1801,19 +1926,19 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.6" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" dependencies = [ "once_cell", - "target-lexicon 0.12.16", + "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.22.6" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" dependencies = [ "libc", "pyo3-build-config", @@ -1821,9 +1946,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.6" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1833,9 +1958,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.6" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" dependencies = [ "heck", "proc-macro2", @@ -1963,9 +2088,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags 2.9.0", ] @@ -1983,9 +2108,9 @@ dependencies = [ [[package]] name = "regalloc2" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3" +checksum = "dc06e6b318142614e4a48bc725abbf08ff166694835c43c9dae5a9009704639a" dependencies = [ "allocator-api2", "bumpalo", @@ -2139,9 +2264,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags 2.9.0", "errno", @@ -2371,7 +2496,7 @@ dependencies = [ "junction", "libc", "libsqlite3-sys", - "libz-rs-sys", + "libz-rs-sys 0.4.2", "mac_address", "malachite-bigint", "md-5", @@ -2691,9 +2816,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "socket2" @@ -2796,12 +2921,6 @@ dependencies = [ "libc", ] -[[package]] -name = "target-lexicon" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" - [[package]] name = "target-lexicon" version = "0.13.2" @@ -2817,15 +2936,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "termios" version = "0.3.3" @@ -3662,3 +3772,9 @@ name = "zlib-rs" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" + +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" diff --git a/Cargo.toml b/Cargo.toml index 271c11e782..74fdf7464e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,8 +36,8 @@ log = { workspace = true } flame = { workspace = true, optional = true } lexopt = "0.3" -dirs = { package = "dirs-next", version = "2.0.0" } -env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] } +dirs = { package = "dirs-next", version = "2.0" } +env_logger = "0.11" flamescope = { version = "0.1.2", optional = true } [target.'cfg(windows)'.dependencies] @@ -48,7 +48,7 @@ rustyline = { workspace = true } [dev-dependencies] criterion = { workspace = true } -pyo3 = { version = "0.22", features = ["auto-initialize"] } +pyo3 = { version = "0.24", features = ["auto-initialize"] } [[bench]] name = "execution" @@ -162,13 +162,13 @@ getrandom = { version = "0.3", features = ["std"] } glob = "0.3" hex = "0.4.3" indexmap = { version = "2.2.6", features = ["std"] } -insta = "1.38.0" +insta = "1.42" itertools = "0.14.0" is-macro = "0.3.7" junction = "1.2.0" libc = "0.2.169" libffi = "3.2" -log = "0.4.25" +log = "0.4.27" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } malachite-bigint = "0.5" malachite-q = "0.5" diff --git a/benches/execution.rs b/benches/execution.rs index 88e5e3650b..956975c22f 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -9,9 +9,11 @@ use std::collections::HashMap; use std::path::Path; fn bench_cpython_code(b: &mut Bencher, source: &str) { + let c_str_source_head = std::ffi::CString::new(source).unwrap(); + let c_str_source = c_str_source_head.as_c_str(); pyo3::Python::with_gil(|py| { b.iter(|| { - let module = pyo3::types::PyModule::from_code_bound(py, source, "", "") + let module = pyo3::types::PyModule::from_code(py, c_str_source, c"", c"") .expect("Error running source"); black_box(module); }) @@ -53,8 +55,8 @@ pub fn benchmark_file_parsing(group: &mut BenchmarkGroup, name: &str, group.bench_function(BenchmarkId::new("cpython", name), |b| { use pyo3::types::PyAnyMethods; pyo3::Python::with_gil(|py| { - let builtins = pyo3::types::PyModule::import_bound(py, "builtins") - .expect("Failed to import builtins"); + let builtins = + pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let compile = builtins.getattr("compile").expect("no compile in builtins"); b.iter(|| { let x = compile diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index c62ecf239a..5f04f4bbf8 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -45,7 +45,7 @@ fn bench_cpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchma // Grab the exec function in advance so we don't have lookups in the hot code let builtins = - pyo3::types::PyModule::import_bound(py, "builtins").expect("Failed to import builtins"); + pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let exec = builtins.getattr("exec").expect("no exec in builtins"); let bench_func = |(globals, locals): &mut ( @@ -60,8 +60,8 @@ fn bench_cpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchma }; let bench_setup = |iterations| { - let globals = pyo3::types::PyDict::new_bound(py); - let locals = pyo3::types::PyDict::new_bound(py); + let globals = pyo3::types::PyDict::new(py); + let locals = pyo3::types::PyDict::new(py); if let Some(idx) = iterations { globals.set_item("ITERATIONS", idx).unwrap(); } @@ -99,7 +99,7 @@ fn cpy_compile_code<'a>( name: &str, ) -> pyo3::PyResult> { let builtins = - pyo3::types::PyModule::import_bound(py, "builtins").expect("Failed to import builtins"); + pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let compile = builtins.getattr("compile").expect("no compile in builtins"); compile.call1((code, name, "exec"))?.extract() } diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 26c85103e5..d4b99ebc92 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -43,7 +43,7 @@ num_enum = { workspace = true } parking_lot = { workspace = true } memchr = { workspace = true } -base64 = "0.13.0" +base64 = "0.22" csv-core = "0.1.11" dyn-clone = "1.0.10" puruspe = "0.4.0" diff --git a/stdlib/src/binascii.rs b/stdlib/src/binascii.rs index 2510325b4e..1c88477035 100644 --- a/stdlib/src/binascii.rs +++ b/stdlib/src/binascii.rs @@ -16,6 +16,7 @@ mod decl { convert::ToPyException, function::{ArgAsciiBuffer, ArgBytesLike, OptionalArg}, }; + use base64::Engine; use itertools::Itertools; #[pyattr(name = "Error", once)] @@ -263,7 +264,7 @@ mod decl { decoded.len() / 3 * 4 + 1, 0, )), - _ => Err(base64::DecodeError::InvalidLength), + _ => Err(base64::DecodeError::InvalidLength(quad_pos)), } }) .map_err(|err| super::Base64DecodeError(err).to_pyexception(vm)) @@ -272,7 +273,9 @@ mod decl { #[pyfunction] fn b2a_base64(data: ArgBytesLike, NewlineArg { newline }: NewlineArg) -> Vec { // https://stackoverflow.com/questions/63916821 - let mut encoded = data.with_ref(|b| base64::encode(b)).into_bytes(); + let mut encoded = data + .with_ref(|b| base64::engine::general_purpose::STANDARD.encode(b)) + .into_bytes(); if newline { encoded.push(b'\n'); } @@ -746,7 +749,7 @@ fn new_binascii_error(msg: String, vm: &VirtualMachine) -> PyBaseExceptionRef { impl ToPyException for Base64DecodeError { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { use base64::DecodeError::*; - let message = match self.0 { + let message = match &self.0 { InvalidByte(0, PAD) => "Leading padding not allowed".to_owned(), InvalidByte(_, PAD) => "Discontinuous padding not allowed".to_owned(), InvalidByte(_, _) => "Only base64 data is allowed".to_owned(), @@ -757,7 +760,9 @@ impl ToPyException for Base64DecodeError { length ) } - InvalidLength => "Incorrect padding".to_owned(), + // TODO: clean up errors + InvalidLength(_) => "Incorrect padding".to_owned(), + InvalidPadding => "Incorrect padding".to_owned(), }; new_binascii_error(format!("error decoding base64: {message}"), vm) } From 49b348cc7e6291c1e982183a8c76294024b26eca Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 14 Apr 2025 22:33:55 -0700 Subject: [PATCH 219/317] Remove Instruction::IsOperation --- compiler/codegen/src/compile.rs | 6 ++++-- .../rustpython_codegen__compile__tests__match.snap | 2 +- compiler/core/src/bytecode.rs | 7 +------ vm/src/frame.rs | 8 -------- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 3c92146c13..8aa968b51a 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2223,6 +2223,8 @@ impl Compiler<'_> { }); } + use bytecode::TestOperator::*; + // Emit instructions: // 1. Load the new tuple of attribute names. self.emit_load_const(ConstantData::Tuple { @@ -2235,7 +2237,7 @@ impl Compiler<'_> { // 4. Load None. self.emit_load_const(ConstantData::None); // 5. Compare with IS_OP 1. - emit!(self, Instruction::IsOperation(true)); + emit!(self, Instruction::TestOperation { op: IsNot }); // At this point the TOS is a tuple of (nargs + n_attrs) attributes (or None). pc.on_top += 1; @@ -2351,7 +2353,7 @@ impl Compiler<'_> { // emit!(self, Instruction::CopyItem { index: 1_u32 }); // self.emit_load_const(ConstantData::None); // // TODO: should be is - // emit!(self, Instruction::IsOperation(true)); + // emit!(self, Instruction::TestOperation::IsNot); // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; // // Unpack the tuple of values. diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap index f09f0f5eaf..3f6f4495d7 100644 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap @@ -30,7 +30,7 @@ expression: "compile_exec(\"\\\nclass Test:\n pass\n\nt = Test()\nmatch t:\n 14 MatchClass (0) 15 CopyItem (1) 16 LoadConst (None) - 17 IsOperation (true) + 17 TestOperation (IsNot) 18 JumpIfFalse (27) 19 UnpackSequence (0) 20 Pop diff --git a/compiler/core/src/bytecode.rs b/compiler/core/src/bytecode.rs index 81dd591ad1..e00ca28a58 100644 --- a/compiler/core/src/bytecode.rs +++ b/compiler/core/src/bytecode.rs @@ -437,9 +437,6 @@ pub enum Instruction { TestOperation { op: Arg, }, - /// If the argument is true, perform IS NOT. Otherwise perform the IS operation. - // TODO: duplication of TestOperator::{Is,IsNot}. Fix later. - IsOperation(Arg), CompareOperation { op: Arg, }, @@ -1227,8 +1224,7 @@ impl Instruction { BinaryOperation { .. } | BinaryOperationInplace { .. } | TestOperation { .. } - | CompareOperation { .. } - | IsOperation(..) => -1, + | CompareOperation { .. } => -1, BinarySubscript => -1, CopyItem { .. } => 1, Pop => -1, @@ -1436,7 +1432,6 @@ impl Instruction { BinarySubscript => w!(BinarySubscript), LoadAttr { idx } => w!(LoadAttr, name = idx), TestOperation { op } => w!(TestOperation, ?op), - IsOperation(neg) => w!(IsOperation, neg), CompareOperation { op } => w!(CompareOperation, ?op), CopyItem { index } => w!(CopyItem, index), Pop => w!(Pop), diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 6ffb454f54..dbe5cb077a 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -851,14 +851,6 @@ impl ExecutingFrame<'_> { bytecode::Instruction::UnaryOperation { op } => self.execute_unary_op(vm, op.get(arg)), bytecode::Instruction::TestOperation { op } => self.execute_test(vm, op.get(arg)), bytecode::Instruction::CompareOperation { op } => self.execute_compare(vm, op.get(arg)), - bytecode::Instruction::IsOperation(neg) => { - let a = self.pop_value(); - let b = self.pop_value(); - // xor with neg to invert the result if needed - let result = vm.ctx.new_bool(a.is(b.as_ref()) ^ neg.get(arg)); - self.push_value(result.into()); - Ok(None) - } bytecode::Instruction::ReturnValue => { let value = self.pop_value(); self.unwind_blocks(vm, UnwindReason::Returning { value }) From fb0c4b6b3c53da9a6852f79b102ed5348c48a682 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 14 Apr 2025 21:18:19 -0700 Subject: [PATCH 220/317] add test_winapi --- Lib/test/test_winapi.py | 179 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 Lib/test/test_winapi.py diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py new file mode 100644 index 0000000000..7a33f90698 --- /dev/null +++ b/Lib/test/test_winapi.py @@ -0,0 +1,179 @@ +# Test the Windows-only _winapi module + +import os +import pathlib +import random +import re +import threading +import time +import unittest +from test.support import import_helper, os_helper + +_winapi = import_helper.import_module('_winapi', required_on=['win']) + +MAXIMUM_WAIT_OBJECTS = 64 +MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): + def _events_waitall_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + + # Ensure no errors raised when all are triggered + for e in evts: + _winapi.SetEvent(e) + try: + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + except TimeoutError: + self.fail("expected wait to complete immediately") + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Replace events with invalid handles to make sure we fail + for i in chosen: + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, True, 100) + evts[i] = old_evt + + + def _events_waitany_test(self, n): + evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + + with self.assertRaises(TimeoutError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + + # Choose 8 events to set, distributed throughout the list, to make sure + # we don't always have them in the first chunk + chosen = [i * (len(evts) // 8) for i in range(8)] + + # Trigger one by one. They are auto-reset events, so will only trigger once + for i in chosen: + with self.subTest(f"trigger event {i} of {len(evts)}"): + _winapi.SetEvent(evts[i]) + triggered = _winapi.BatchedWaitForMultipleObjects(evts, False, 10000) + self.assertSetEqual(set(triggered), {i}) + + # Trigger all at once. This may require multiple calls + for i in chosen: + _winapi.SetEvent(evts[i]) + triggered = set() + while len(triggered) < len(chosen): + triggered.update(_winapi.BatchedWaitForMultipleObjects(evts, False, 10000)) + self.assertSetEqual(triggered, set(chosen)) + + # Replace events with invalid handles to make sure we fail + for i in chosen: + with self.subTest(f"corrupt event {i} of {len(evts)}"): + old_evt = evts[i] + evts[i] = -1 + with self.assertRaises(OSError): + _winapi.BatchedWaitForMultipleObjects(evts, False, 100) + evts[i] = old_evt + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_few_events_waitall(self): + self._events_waitall_test(16) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_many_events_waitall(self): + self._events_waitall_test(256) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_max_events_waitall(self): + self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_few_events_waitany(self): + self._events_waitany_test(16) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_many_events_waitany(self): + self._events_waitany_test(256) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_max_events_waitany(self): + self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) + + +class WinAPITests(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getlongpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" + if not os.path.isdir(testfn): + raise unittest.SkipTest("require x:\\PROGRA~1 to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetLongPathName(testfn) + + actual = _winapi.GetLongPathName(os.fsdecode(testfn)) + + # Can't assume that PROGRA~1 expands to any particular variation, so + # ensure it matches any one of them. + candidates = set(testfn.parent.glob("Progra*")) + self.assertIn(pathlib.Path(actual), candidates) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getshortpathname(self): + testfn = pathlib.Path(os.getenv("ProgramFiles")) + if not os.path.isdir(testfn): + raise unittest.SkipTest("require '%ProgramFiles%' to test") + + # pathlib.Path will be rejected - only str is accepted + with self.assertRaises(TypeError): + _winapi.GetShortPathName(testfn) + + actual = _winapi.GetShortPathName(os.fsdecode(testfn)) + + # Should contain "PROGRA~" but we can't predict the number + self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_namedpipe(self): + pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" + + # Pipe does not exist, so this raises + with self.assertRaises(FileNotFoundError): + _winapi.WaitNamedPipe(pipe_name, 0) + + pipe = _winapi.CreateNamedPipe( + pipe_name, + _winapi.PIPE_ACCESS_DUPLEX, + 8, # 8=PIPE_REJECT_REMOTE_CLIENTS + 2, # two instances available + 32, 32, 0, 0) + self.addCleanup(_winapi.CloseHandle, pipe) + + # Pipe instance is available, so this passes + _winapi.WaitNamedPipe(pipe_name, 0) + + with open(pipe_name, 'w+b') as pipe2: + # No instances available, so this times out + # (WinError 121 does not get mapped to TimeoutError) + with self.assertRaises(OSError): + _winapi.WaitNamedPipe(pipe_name, 0) + + _winapi.WriteFile(pipe, b'testdata') + self.assertEqual(b'testdata', pipe2.read(8)) + + self.assertEqual((b'', 0), _winapi.PeekNamedPipe(pipe, 8)[:2]) + pipe2.write(b'testdata') + pipe2.flush() + self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) \ No newline at end of file From a917da3b1a0b025f4d82708eb11e802377128dd0 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 14 Apr 2025 21:29:32 -0700 Subject: [PATCH 221/317] update test_float to 3.13.3 --- Lib/test/test_float.py | 45 +++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 30d27072fb..f65eb55ca5 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -25,7 +25,7 @@ #locate file with float format test values test_dir = os.path.dirname(__file__) or os.curdir -format_testfile = os.path.join(test_dir, 'formatfloat_testcases.txt') +format_testfile = os.path.join(test_dir, 'mathdata', 'formatfloat_testcases.txt') class FloatSubclass(float): pass @@ -131,7 +131,7 @@ def check(s): with self.assertRaises(ValueError, msg='float(%r)' % (s,)) as cm: float(s) self.assertEqual(str(cm.exception), - 'could not convert string to float: %r' % (s,)) + 'could not convert string to float: %r' % (s,)) check('\xbd') check('123\xbd') @@ -290,11 +290,11 @@ def test_is_integer(self): def test_floatasratio(self): for f, ratio in [ - (0.875, (7, 8)), - (-0.875, (-7, 8)), - (0.0, (0, 1)), - (11.5, (23, 2)), - ]: + (0.875, (7, 8)), + (-0.875, (-7, 8)), + (0.0, (0, 1)), + (11.5, (23, 2)), + ]: self.assertEqual(f.as_integer_ratio(), ratio) for i in range(10000): @@ -337,7 +337,7 @@ def test_float_containment(self): self.assertTrue((f,) == (f,), "(%r,) != (%r,)" % (f, f)) self.assertTrue({f} == {f}, "{%r} != {%r}" % (f, f)) self.assertTrue({f : None} == {f: None}, "{%r : None} != " - "{%r : None}" % (f, f)) + "{%r : None}" % (f, f)) # identical containers l, t, s, d = [f], (f,), {f}, {f: None} @@ -400,9 +400,9 @@ def test_float_mod(self): self.assertEqualAndEqualSign(mod(1e-100, -1.0), -1.0) self.assertEqualAndEqualSign(mod(1.0, -1.0), -0.0) + @support.requires_IEEE_754 # TODO: RUSTPYTHON @unittest.expectedFailure - @support.requires_IEEE_754 def test_float_pow(self): # test builtin pow and ** operator for IEEE 754 special cases. # Special cases taken from section F.9.4.4 of the C99 specification @@ -728,6 +728,8 @@ def test_format(self): self.assertEqual(format(INF, 'F'), 'INF') @support.requires_IEEE_754 + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_format_testfile(self): with open(format_testfile, encoding="utf-8") as testfile: for line in testfile: @@ -772,9 +774,12 @@ def test_issue35560(self): self.assertEqual(format(-123.34, '00.10g'), '-123.34') class ReprTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repr(self): with open(os.path.join(os.path.split(__file__)[0], - 'floating_points.txt'), encoding="utf-8") as floats_file: + 'mathdata', + 'floating_points.txt'), encoding="utf-8") as floats_file: for line in floats_file: line = line.strip() if not line or line.startswith('#'): @@ -824,7 +829,7 @@ def test_short_repr(self): '2.86438000439698e+28', '8.89142905246179e+28', '3.08578087079232e+35', - ] + ] for s in test_strings: negs = '-'+s @@ -874,14 +879,14 @@ def test_overflow(self): self.assertRaises(OverflowError, round, 1.6e308, -308) self.assertRaises(OverflowError, round, -1.7e308, -308) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short', "applies only when using short float repr style") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_previous_round_bugs(self): # particular cases that have occurred in bug reports self.assertEqual(round(562949953421312.5, 1), - 562949953421312.5) + 562949953421312.5) self.assertEqual(round(56294995342131.5, 3), 56294995342131.5) # round-half-even @@ -894,10 +899,10 @@ def test_previous_round_bugs(self): self.assertEqual(round(85.0, -1), 80.0) self.assertEqual(round(95.0, -1), 100.0) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short', "applies only when using short float repr style") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_matches_float_format(self): # round should give the same results as float formatting for i in range(500): @@ -1131,7 +1136,7 @@ def test_invalid_inputs(self): '0x1.\uff10p0', '0x1p0 \n 0x2p0', '0x1p0\0 0x1p0', # embedded null byte is not end of string - ] + ] for x in invalid_inputs: try: result = fromHex(x) @@ -1150,7 +1155,7 @@ def test_whitespace(self): ('1.0', 1.0), ('-0x.2', -0.125), ('-0.0', -0.0) - ] + ] whitespace = [ '', ' ', @@ -1160,7 +1165,7 @@ def test_whitespace(self): '\f', '\v', '\r' - ] + ] for inp, expected in value_pairs: for lead in whitespace: for trail in whitespace: @@ -1510,4 +1515,4 @@ def __init__(self, value): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file From c7042fd8470ad8d9ca4c33246d6b390b74855b85 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 08:39:58 -0700 Subject: [PATCH 222/317] remove unneeded validation --- compiler/codegen/src/compile.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 8aa968b51a..052bd1b290 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2185,16 +2185,6 @@ impl Compiler<'_> { let nargs = patterns.len(); let n_attrs = kwd_attrs.len(); - let nkwd_patterns = kwd_patterns.len(); - - // Validate that keyword attribute names and patterns match in length. - if n_attrs != nkwd_patterns { - let msg = format!( - "kwd_attrs ({}) / kwd_patterns ({}) length mismatch in class pattern", - n_attrs, nkwd_patterns - ); - unreachable!("{}", msg); - } // Check for too many sub-patterns. if nargs > u32::MAX as usize || (nargs + n_attrs).saturating_sub(1) > i32::MAX as usize { From 456e555e8b173cde5ed16f2d784ee56a4601843c Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 09:25:06 -0700 Subject: [PATCH 223/317] better error --- compiler/codegen/src/ir.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 39857e6fc5..9106d3fdff 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -244,7 +244,8 @@ impl CodeInfo { let instr_display = instr.display(display_arg, self); eprint!("{instr_display}: {depth} {effect:+} => "); } - let new_depth = depth.checked_add_signed(effect).unwrap(); + // Either it's less than 0 or more than u32. Option 1 is far more likely unless something is seriously wrong. + let new_depth = depth.checked_add_signed(effect).expect("The stack is detected to have likely been reduced to less than 0 elements, this is a bug."); if DEBUG { eprintln!("{new_depth}"); } From d47944b2fd74db8da80e8965273df2fabcf2ec36 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 10:02:32 -0700 Subject: [PATCH 224/317] error handling --- compiler/codegen/src/ir.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 9106d3fdff..662ba81dbe 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -244,8 +244,10 @@ impl CodeInfo { let instr_display = instr.display(display_arg, self); eprint!("{instr_display}: {depth} {effect:+} => "); } - // Either it's less than 0 or more than u32. Option 1 is far more likely unless something is seriously wrong. - let new_depth = depth.checked_add_signed(effect).expect("The stack is detected to have likely been reduced to less than 0 elements, this is a bug."); + if effect < 0 && depth < effect.abs() as u32 { + panic!("The stack will underflow at {depth} with {effect} effect on {instr:?}"); + } + let new_depth = depth.checked_add_signed(effect).unwrap(); if DEBUG { eprintln!("{new_depth}"); } From 09c199a1bab2a292fe4370fcde255d89dbefef51 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 10:03:29 -0700 Subject: [PATCH 225/317] match cleanup --- compiler/codegen/src/compile.rs | 99 ++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 052bd1b290..297feef67a 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -1944,32 +1944,35 @@ impl Compiler<'_> { n: Option<&Identifier>, pc: &mut PatternContext, ) -> CompileResult<()> { - // If no name is provided, simply pop the top of the stack. - if n.is_none() { - emit!(self, Instruction::Pop); - return Ok(()); - } - let name = n.unwrap(); - - // Check if the name is forbidden for storing. - if self.forbidden_name(name.as_str(), NameUsage::Store)? { - return Err(self.compile_error_forbidden_name(name.as_str())); - } + match n { + // If no name is provided, simply pop the top of the stack. + None => { + emit!(self, Instruction::Pop); + return Ok(()); + } + Some(name) => { + // Check if the name is forbidden for storing. + if self.forbidden_name(name.as_str(), NameUsage::Store)? { + return Err(self.compile_error_forbidden_name(name.as_str())); + } - // Ensure we don't store the same name twice. - if pc.stores.contains(&name.to_string()) { - return Err(self.error(CodegenErrorType::DuplicateStore(name.as_str().to_string()))); - } + // Ensure we don't store the same name twice. + // TODO: maybe pc.stores should be a set? + if pc.stores.contains(&name.to_string()) { + return Err( + self.error(CodegenErrorType::DuplicateStore(name.as_str().to_string())) + ); + } - // Calculate how many items to rotate: - // the count is the number of items to preserve on top plus the current stored names, - // plus one for the new value. - let rotations = pc.on_top + pc.stores.len() + 1; - self.pattern_helper_rotate(rotations)?; + // Calculate how many items to rotate: + let rotations = pc.on_top + pc.stores.len() + 1; + self.pattern_helper_rotate(rotations)?; - // Append the name to the captured stores. - pc.stores.push(name.to_string()); - Ok(()) + // Append the name to the captured stores. + pc.stores.push(name.to_string()); + Ok(()) + } + } } fn pattern_unpack_helper(&mut self, elts: &[Pattern]) -> CompileResult<()> { @@ -2245,20 +2248,12 @@ impl Compiler<'_> { pc.on_top -= 1; // Process each sub-pattern. - for i in 0..total { - // Decrement the on_top counter as each sub-pattern is processed. + for subpattern in patterns.iter().chain(kwd_patterns.iter()) { + // Decrement the on_top counter as each sub-pattern is processed + // (on_top should be zero at the end of the algorithm as a sanity check). pc.on_top -= 1; - let subpattern = if i < nargs { - // Positional sub-pattern. - &patterns[i] - } else { - // Keyword sub-pattern. - &kwd_patterns[i - nargs] - }; if subpattern.is_wildcard() { - // For wildcard patterns, simply pop the top of the stack. emit!(self, Instruction::Pop); - continue; } // Compile the subpattern without irrefutability checks. self.compile_pattern_subpattern(subpattern, pc)?; @@ -2479,7 +2474,7 @@ impl Compiler<'_> { self.pattern_helper_rotate(nrots)?; let name = &control.as_ref().unwrap()[i]; // Check for duplicate binding. - if pc.stores.iter().any(|n| n == name) { + if pc.stores.contains(name) { return Err(self.error(CodegenErrorType::DuplicateStore(name.to_string()))); } pc.stores.push(name.clone()); @@ -4607,17 +4602,33 @@ for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): #[test] fn test_match() { assert_dis_snapshot!(compile_exec( - "\ -class Test: + r#"\ +class Shape: pass -t = Test() -match t: - case Test(): - assert True - case _: - assert False -" +class Circle(Shape): + def __init__(self, radius): + self.radius = radius + +class Rectangle(Shape): + def __init__(self, width, height): + self.width = width + self.height = height + +def describe_shape(shape): + match shape: + case Circle(radius=r): + return f"A circle with radius {r}" + case Rectangle(width=w, height=h): + return f"A rectangle {w} by {h}" + case _: + return "Unknown shape" + +# Test it out +shapes = [Circle(5), Rectangle(4, 6), "not a shape"] +for s in shapes: + print(describe_shape(s)) +"# )); } } From e949c9aa3f070b4909f81d1ac68ad3442aa9fba5 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 11:41:27 -0700 Subject: [PATCH 226/317] rename --- compiler/codegen/src/compile.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 297feef67a..286f318ccb 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2467,11 +2467,11 @@ impl Compiler<'_> { self.switch_to_block(end); // Adjust the final captures. - let nstores = control.as_ref().unwrap().len(); - let nrots = nstores + 1 + pc.on_top + pc.stores.len(); - for i in 0..nstores { + let n_stores = control.as_ref().unwrap().len(); + let n_rots = n_stores + 1 + pc.on_top + pc.stores.len(); + for i in 0..n_stores { // Rotate the capture to its proper place. - self.pattern_helper_rotate(nrots)?; + self.pattern_helper_rotate(n_rots)?; let name = &control.as_ref().unwrap()[i]; // Check for duplicate binding. if pc.stores.contains(name) { From 628287c14e293224d5c1890bb343cbe3f763a1a9 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 11:42:13 -0700 Subject: [PATCH 227/317] update snapshot --- ...python_codegen__compile__tests__match.snap | 191 ++++++++++++++---- 1 file changed, 149 insertions(+), 42 deletions(-) diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap index 3f6f4495d7..379d42f18b 100644 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap @@ -1,53 +1,160 @@ --- source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nclass Test:\n pass\n\nt = Test()\nmatch t:\n case Test():\n assert True\n case _:\n assert False\n\")" +expression: "compile_exec(r#\"\\\nclass Shape:\n pass\n\nclass Circle(Shape):\n def __init__(self, radius):\n self.radius = radius\n\nclass Rectangle(Shape):\n def __init__(self, width, height):\n self.width = width\n self.height = height\n\ndef describe_shape(shape):\n match shape:\n case Circle(radius=r):\n return f\"A circle with radius {r}\"\n case Rectangle(width=w, height=h):\n return f\"A rectangle {w} by {h}\"\n case _:\n return \"Unknown shape\"\n\n# Test it out\nshapes = [Circle(5), Rectangle(4, 6), \"not a shape\"]\nfor s in shapes:\n print(describe_shape(s))\n\"#)" --- - 2 0 LoadBuildClass - 1 LoadConst (): 1 0 LoadGlobal (0, __name__) + 3 0 LoadBuildClass + 1 LoadConst (): 2 0 LoadGlobal (0, __name__) 1 StoreLocal (1, __module__) - 2 LoadConst ("Test") + 2 LoadConst ("Shape") 3 StoreLocal (2, __qualname__) 4 LoadConst (None) 5 StoreLocal (3, __doc__) - 2 6 ReturnConst (None) + 3 6 ReturnConst (None) - 2 LoadConst ("Test") + 2 LoadConst ("Shape") 3 MakeFunction (MakeFunctionFlags(0x0)) - 4 LoadConst ("Test") + 4 LoadConst ("Shape") 5 CallFunctionPositional(2) - 6 StoreLocal (0, Test) - - 4 7 LoadNameAny (0, Test) - 8 CallFunctionPositional(0) - 9 StoreLocal (1, t) - - 5 10 LoadNameAny (1, t) - 11 CopyItem (1) - - 6 12 LoadNameAny (0, Test) - 13 LoadConst (()) - 14 MatchClass (0) - 15 CopyItem (1) - 16 LoadConst (None) - 17 TestOperation (IsNot) - 18 JumpIfFalse (27) - 19 UnpackSequence (0) - 20 Pop - - 7 21 LoadConst (True) - 22 JumpIfTrue (26) - 23 LoadGlobal (2, AssertionError) - 24 CallFunctionPositional(0) - 25 Raise (Raise) - >> 26 Jump (35) - >> 27 Pop - 28 Pop - - 9 29 LoadConst (False) - 30 JumpIfTrue (34) - 31 LoadGlobal (2, AssertionError) - 32 CallFunctionPositional(0) - 33 Raise (Raise) - >> 34 Jump (35) - >> 35 ReturnConst (None) + 6 StoreLocal (0, Shape) + + 7 7 LoadBuildClass + 8 LoadConst (): 5 0 LoadGlobal (0, __name__) + 1 StoreLocal (1, __module__) + 2 LoadConst ("Circle") + 3 StoreLocal (2, __qualname__) + 4 LoadConst (None) + 5 StoreLocal (3, __doc__) + + 7 6 LoadConst (): 7 0 LoadFast (1, radius) + 1 LoadFast (0, self) + 2 StoreAttr (0, radius) + 3 ReturnConst (None) + + 7 LoadConst ("Circle.__init__") + 8 MakeFunction (MakeFunctionFlags(0x0)) + 9 StoreLocal (4, __init__) + 10 ReturnConst (None) + + 9 LoadConst ("Circle") + 10 MakeFunction (MakeFunctionFlags(0x0)) + 11 LoadConst ("Circle") + + 5 12 LoadNameAny (0, Shape) + 13 CallFunctionPositional(3) + 14 StoreLocal (1, Circle) + + 12 15 LoadBuildClass + 16 LoadConst (): 9 0 LoadGlobal (0, __name__) + 1 StoreLocal (1, __module__) + 2 LoadConst ("Rectangle") + 3 StoreLocal (2, __qualname__) + 4 LoadConst (None) + 5 StoreLocal (3, __doc__) + + 12 6 LoadConst (): 11 0 LoadFast (1, width) + 1 LoadFast (0, self) + 2 StoreAttr (0, width) + + 12 3 LoadFast (2, height) + 4 LoadFast (0, self) + 5 StoreAttr (1, height) + 6 ReturnConst (None) + + 7 LoadConst ("Rectangle.__init__") + 8 MakeFunction (MakeFunctionFlags(0x0)) + 9 StoreLocal (4, __init__) + 10 ReturnConst (None) + + 17 LoadConst ("Rectangle") + 18 MakeFunction (MakeFunctionFlags(0x0)) + 19 LoadConst ("Rectangle") + + 9 20 LoadNameAny (0, Shape) + 21 CallFunctionPositional(3) + 22 StoreLocal (2, Rectangle) + + 21 23 LoadConst (): 15 0 LoadFast (0, shape) + 1 CopyItem (1) + + 16 2 LoadGlobal (0, Circle) + 3 LoadConst (("radius")) + 4 MatchClass (0) + 5 CopyItem (1) + 6 LoadConst (None) + 7 TestOperation (IsNot) + 8 JumpIfFalse (19) + 9 UnpackSequence (1) + 10 Pop + 11 Pop + + 17 12 LoadConst ("A circle with radius ") + 13 LoadConst ("") + 14 LoadFast (1, r) + 15 FormatValue (None) + 16 BuildString (2) + 17 ReturnValue + 18 Jump (47) + >> 19 Pop + 20 CopyItem (1) + + 18 21 LoadGlobal (1, Rectangle) + 22 LoadConst (("width", "height")) + 23 MatchClass (0) + 24 CopyItem (1) + 25 LoadConst (None) + 26 TestOperation (IsNot) + 27 JumpIfFalse (43) + 28 UnpackSequence (2) + 29 Pop + 30 Pop + 31 Pop + + 19 32 LoadConst ("A rectangle ") + 33 LoadConst ("") + 34 LoadFast (2, w) + 35 FormatValue (None) + 36 LoadConst (" by ") + 37 LoadConst ("") + 38 LoadFast (3, h) + 39 FormatValue (None) + 40 BuildString (4) + 41 ReturnValue + 42 Jump (47) + >> 43 Pop + 44 Pop + + 21 45 ReturnConst ("Unknown shape") + 46 Jump (47) + >> 47 ReturnConst (None) + + 24 LoadConst ("describe_shape") + 25 MakeFunction (MakeFunctionFlags(0x0)) + 26 StoreLocal (3, describe_shape) + + 24 27 LoadNameAny (1, Circle) + 28 LoadConst (5) + 29 CallFunctionPositional(1) + 30 LoadNameAny (2, Rectangle) + 31 LoadConst (4) + 32 LoadConst (6) + 33 CallFunctionPositional(2) + 34 LoadConst ("not a shape") + 35 BuildList (3) + 36 StoreLocal (4, shapes) + + 25 37 SetupLoop + 38 LoadNameAny (4, shapes) + 39 GetIter + >> 40 ForIter (49) + 41 StoreLocal (5, s) + + 26 42 LoadNameAny (6, print) + 43 LoadNameAny (3, describe_shape) + 44 LoadNameAny (5, s) + 45 CallFunctionPositional(1) + 46 CallFunctionPositional(1) + 47 Pop + 48 Jump (40) + >> 49 PopBlock + 50 ReturnConst (None) From d44324d4d0d522df8e6b77151b996c50bdd8d348 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 11:44:04 -0700 Subject: [PATCH 228/317] clippy --- compiler/codegen/src/compile.rs | 2 +- compiler/codegen/src/ir.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 286f318ccb..d32c3f2691 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -1948,7 +1948,7 @@ impl Compiler<'_> { // If no name is provided, simply pop the top of the stack. None => { emit!(self, Instruction::Pop); - return Ok(()); + Ok(()) } Some(name) => { // Check if the name is forbidden for storing. diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 662ba81dbe..ee29c65a16 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -244,7 +244,7 @@ impl CodeInfo { let instr_display = instr.display(display_arg, self); eprint!("{instr_display}: {depth} {effect:+} => "); } - if effect < 0 && depth < effect.abs() as u32 { + if effect < 0 && depth < effect.unsigned_abs() { panic!("The stack will underflow at {depth} with {effect} effect on {instr:?}"); } let new_depth = depth.checked_add_signed(effect).unwrap(); From 21272025c29f17b640016c78faf640212e26a4c4 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 11:54:46 -0700 Subject: [PATCH 229/317] improve error handling --- compiler/codegen/src/compile.rs | 14 ++++++-------- compiler/codegen/src/error.rs | 8 ++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index d32c3f2691..0a355c4a40 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -2158,10 +2158,7 @@ impl Compiler<'_> { for ident in attrs.iter().take(n_attrs).skip(i + 1) { let other = ident.as_str(); if attr == other { - todo!(); - // return Err(self.compiler_error( - // &format!("attribute name repeated in class pattern: {}", attr), - // )); + return Err(self.error(CodegenErrorType::RepeatedAttributePattern)); } } } @@ -2415,15 +2412,16 @@ impl Compiler<'_> { } else { let control_vec = control.as_ref().unwrap(); if nstores != control_vec.len() { - todo!(); - // return self.compiler_error("alternative patterns bind different names"); + return Err(self.error(CodegenErrorType::ConflictingNameBindPattern)); } else if nstores > 0 { // Check that the names occur in the same order. for icontrol in (0..nstores).rev() { let name = &control_vec[icontrol]; // Find the index of `name` in the current stores. - let istores = pc.stores.iter().position(|n| n == name).unwrap(); - // .ok_or_else(|| self.compiler_error("alternative patterns bind different names"))?; + let istores = + pc.stores.iter().position(|n| n == name).ok_or_else(|| { + self.error(CodegenErrorType::ConflictingNameBindPattern) + })?; if icontrol != istores { // The orders differ; we must reorder. assert!(istores < icontrol, "expected istores < icontrol"); diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index b1b4f9379f..4394b936d2 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -65,6 +65,8 @@ pub enum CodegenErrorType { ForbiddenName, DuplicateStore(String), UnreachablePattern(PatternUnreachableReason), + RepeatedAttributePattern, + ConflictingNameBindPattern, NotImplementedYet, // RustPython marker for unimplemented features } @@ -119,6 +121,12 @@ impl fmt::Display for CodegenErrorType { UnreachablePattern(reason) => { write!(f, "{reason} makes remaining patterns unreachable") } + RepeatedAttributePattern => { + write!(f, "attribute name repeated in class pattern") + } + ConflictingNameBindPattern => { + write!(f, "alternative patterns bind different names") + } NotImplementedYet => { write!(f, "RustPython does not implement this feature yet") } From 4d53f5925cea220701312a1851783abb992f8c45 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 11:56:29 -0700 Subject: [PATCH 230/317] remove match test --- compiler/codegen/src/compile.rs | 33 ---- ...python_codegen__compile__tests__match.snap | 160 ------------------ 2 files changed, 193 deletions(-) delete mode 100644 compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 0a355c4a40..762585783c 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -4596,37 +4596,4 @@ for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')): " )); } - - #[test] - fn test_match() { - assert_dis_snapshot!(compile_exec( - r#"\ -class Shape: - pass - -class Circle(Shape): - def __init__(self, radius): - self.radius = radius - -class Rectangle(Shape): - def __init__(self, width, height): - self.width = width - self.height = height - -def describe_shape(shape): - match shape: - case Circle(radius=r): - return f"A circle with radius {r}" - case Rectangle(width=w, height=h): - return f"A rectangle {w} by {h}" - case _: - return "Unknown shape" - -# Test it out -shapes = [Circle(5), Rectangle(4, 6), "not a shape"] -for s in shapes: - print(describe_shape(s)) -"# - )); - } } diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap deleted file mode 100644 index 379d42f18b..0000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__match.snap +++ /dev/null @@ -1,160 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(r#\"\\\nclass Shape:\n pass\n\nclass Circle(Shape):\n def __init__(self, radius):\n self.radius = radius\n\nclass Rectangle(Shape):\n def __init__(self, width, height):\n self.width = width\n self.height = height\n\ndef describe_shape(shape):\n match shape:\n case Circle(radius=r):\n return f\"A circle with radius {r}\"\n case Rectangle(width=w, height=h):\n return f\"A rectangle {w} by {h}\"\n case _:\n return \"Unknown shape\"\n\n# Test it out\nshapes = [Circle(5), Rectangle(4, 6), \"not a shape\"]\nfor s in shapes:\n print(describe_shape(s))\n\"#)" ---- - 3 0 LoadBuildClass - 1 LoadConst (): 2 0 LoadGlobal (0, __name__) - 1 StoreLocal (1, __module__) - 2 LoadConst ("Shape") - 3 StoreLocal (2, __qualname__) - 4 LoadConst (None) - 5 StoreLocal (3, __doc__) - - 3 6 ReturnConst (None) - - 2 LoadConst ("Shape") - 3 MakeFunction (MakeFunctionFlags(0x0)) - 4 LoadConst ("Shape") - 5 CallFunctionPositional(2) - 6 StoreLocal (0, Shape) - - 7 7 LoadBuildClass - 8 LoadConst (): 5 0 LoadGlobal (0, __name__) - 1 StoreLocal (1, __module__) - 2 LoadConst ("Circle") - 3 StoreLocal (2, __qualname__) - 4 LoadConst (None) - 5 StoreLocal (3, __doc__) - - 7 6 LoadConst (): 7 0 LoadFast (1, radius) - 1 LoadFast (0, self) - 2 StoreAttr (0, radius) - 3 ReturnConst (None) - - 7 LoadConst ("Circle.__init__") - 8 MakeFunction (MakeFunctionFlags(0x0)) - 9 StoreLocal (4, __init__) - 10 ReturnConst (None) - - 9 LoadConst ("Circle") - 10 MakeFunction (MakeFunctionFlags(0x0)) - 11 LoadConst ("Circle") - - 5 12 LoadNameAny (0, Shape) - 13 CallFunctionPositional(3) - 14 StoreLocal (1, Circle) - - 12 15 LoadBuildClass - 16 LoadConst (): 9 0 LoadGlobal (0, __name__) - 1 StoreLocal (1, __module__) - 2 LoadConst ("Rectangle") - 3 StoreLocal (2, __qualname__) - 4 LoadConst (None) - 5 StoreLocal (3, __doc__) - - 12 6 LoadConst (): 11 0 LoadFast (1, width) - 1 LoadFast (0, self) - 2 StoreAttr (0, width) - - 12 3 LoadFast (2, height) - 4 LoadFast (0, self) - 5 StoreAttr (1, height) - 6 ReturnConst (None) - - 7 LoadConst ("Rectangle.__init__") - 8 MakeFunction (MakeFunctionFlags(0x0)) - 9 StoreLocal (4, __init__) - 10 ReturnConst (None) - - 17 LoadConst ("Rectangle") - 18 MakeFunction (MakeFunctionFlags(0x0)) - 19 LoadConst ("Rectangle") - - 9 20 LoadNameAny (0, Shape) - 21 CallFunctionPositional(3) - 22 StoreLocal (2, Rectangle) - - 21 23 LoadConst (): 15 0 LoadFast (0, shape) - 1 CopyItem (1) - - 16 2 LoadGlobal (0, Circle) - 3 LoadConst (("radius")) - 4 MatchClass (0) - 5 CopyItem (1) - 6 LoadConst (None) - 7 TestOperation (IsNot) - 8 JumpIfFalse (19) - 9 UnpackSequence (1) - 10 Pop - 11 Pop - - 17 12 LoadConst ("A circle with radius ") - 13 LoadConst ("") - 14 LoadFast (1, r) - 15 FormatValue (None) - 16 BuildString (2) - 17 ReturnValue - 18 Jump (47) - >> 19 Pop - 20 CopyItem (1) - - 18 21 LoadGlobal (1, Rectangle) - 22 LoadConst (("width", "height")) - 23 MatchClass (0) - 24 CopyItem (1) - 25 LoadConst (None) - 26 TestOperation (IsNot) - 27 JumpIfFalse (43) - 28 UnpackSequence (2) - 29 Pop - 30 Pop - 31 Pop - - 19 32 LoadConst ("A rectangle ") - 33 LoadConst ("") - 34 LoadFast (2, w) - 35 FormatValue (None) - 36 LoadConst (" by ") - 37 LoadConst ("") - 38 LoadFast (3, h) - 39 FormatValue (None) - 40 BuildString (4) - 41 ReturnValue - 42 Jump (47) - >> 43 Pop - 44 Pop - - 21 45 ReturnConst ("Unknown shape") - 46 Jump (47) - >> 47 ReturnConst (None) - - 24 LoadConst ("describe_shape") - 25 MakeFunction (MakeFunctionFlags(0x0)) - 26 StoreLocal (3, describe_shape) - - 24 27 LoadNameAny (1, Circle) - 28 LoadConst (5) - 29 CallFunctionPositional(1) - 30 LoadNameAny (2, Rectangle) - 31 LoadConst (4) - 32 LoadConst (6) - 33 CallFunctionPositional(2) - 34 LoadConst ("not a shape") - 35 BuildList (3) - 36 StoreLocal (4, shapes) - - 25 37 SetupLoop - 38 LoadNameAny (4, shapes) - 39 GetIter - >> 40 ForIter (49) - 41 StoreLocal (5, s) - - 26 42 LoadNameAny (6, print) - 43 LoadNameAny (3, describe_shape) - 44 LoadNameAny (5, s) - 45 CallFunctionPositional(1) - 46 CallFunctionPositional(1) - 47 Pop - 48 Jump (40) - >> 49 PopBlock - 50 ReturnConst (None) From 213506d9aea0143e0da5c92b332fd5b20f729304 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 14:48:09 -0700 Subject: [PATCH 231/317] update part of test.support.__init__ to 3.13.2 --- Lib/test/support/__init__.py | 143 +++++++++++++++++++++++------------ 1 file changed, 93 insertions(+), 50 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 1efe5bddb1..a05ef26631 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -74,13 +74,7 @@ # # The timeout should be long enough for connect(), recv() and send() methods # of socket.socket. -LOOPBACK_TIMEOUT = 5.0 -if sys.platform == 'win32' and ' 32 bit (ARM)' in sys.version: - # bpo-37553: test_socket.SendfileUsingSendTest is taking longer than 2 - # seconds on Windows ARM32 buildbot - LOOPBACK_TIMEOUT = 10 -elif sys.platform == 'vxworks': - LOOPBACK_TIMEOUT = 10 +LOOPBACK_TIMEOUT = 10.0 # Timeout in seconds for network requests going to the internet. The timeout is # short enough to prevent a test to wait for too long if the internet request @@ -113,7 +107,6 @@ STDLIB_DIR = os.path.dirname(TEST_HOME_DIR) REPO_ROOT = os.path.dirname(STDLIB_DIR) - class Error(Exception): """Base class for regression test exceptions.""" @@ -259,22 +252,16 @@ class USEROBJECTFLAGS(ctypes.Structure): # process not running under the same user id as the current console # user. To avoid that, raise an exception if the window manager # connection is not available. - from ctypes import cdll, c_int, pointer, Structure - from ctypes.util import find_library - - app_services = cdll.LoadLibrary(find_library("ApplicationServices")) - - if app_services.CGMainDisplayID() == 0: - reason = "gui tests cannot run without OS X window manager" + import subprocess + try: + rc = subprocess.run(["launchctl", "managername"], + capture_output=True, check=True) + managername = rc.stdout.decode("utf-8").strip() + except subprocess.CalledProcessError: + reason = "unable to detect macOS launchd job manager" else: - class ProcessSerialNumber(Structure): - _fields_ = [("highLongOfPSN", c_int), - ("lowLongOfPSN", c_int)] - psn = ProcessSerialNumber() - psn_p = pointer(psn) - if ( (app_services.GetCurrentProcess(psn_p) < 0) or - (app_services.SetFrontProcess(psn_p) < 0) ): - reason = "cannot run without OS X gui process" + if managername != "Aqua": + reason = f"{managername=} -- can only run in a macOS GUI session" # check on every platform whether tkinter can actually do anything if not reason: @@ -391,11 +378,12 @@ def wrapper(*args, **kw): def skip_if_buildbot(reason=None): """Decorator raising SkipTest if running on a buildbot.""" + import getpass if not reason: reason = 'not suitable for buildbots' try: isbuildbot = getpass.getuser().lower() == 'buildbot' - except (KeyError, EnvironmentError) as err: + except (KeyError, OSError) as err: warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) isbuildbot = False return unittest.skipIf(isbuildbot, reason) @@ -409,35 +397,48 @@ def check_sanitizer(*, address=False, memory=False, ub=False, thread=False): cflags = sysconfig.get_config_var('CFLAGS') or '' config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' memory_sanitizer = ( - '-fsanitize=memory' in cflags or - '--with-memory-sanitizer' in config_args + '-fsanitize=memory' in cflags or + '--with-memory-sanitizer' in config_args ) address_sanitizer = ( - '-fsanitize=address' in cflags or - '--with-address-sanitizer' in config_args + '-fsanitize=address' in cflags or + '--with-address-sanitizer' in config_args ) ub_sanitizer = ( - '-fsanitize=undefined' in cflags or - '--with-undefined-behavior-sanitizer' in config_args + '-fsanitize=undefined' in cflags or + '--with-undefined-behavior-sanitizer' in config_args ) thread_sanitizer = ( - '-fsanitize=thread' in cflags or - '--with-thread-sanitizer' in config_args + '-fsanitize=thread' in cflags or + '--with-thread-sanitizer' in config_args ) return ( - (memory and memory_sanitizer) or - (address and address_sanitizer) or - (ub and ub_sanitizer) or - (thread and thread_sanitizer) + (memory and memory_sanitizer) or + (address and address_sanitizer) or + (ub and ub_sanitizer) or + (thread and thread_sanitizer) ) + def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False): """Decorator raising SkipTest if running with a sanitizer active.""" if not reason: reason = 'not working with sanitizers active' - skip = check_sanitizer(address=address, memory=memory, ub=ub) + skip = check_sanitizer(address=address, memory=memory, ub=ub, thread=thread) return unittest.skipIf(skip, reason) +# gh-89363: True if fork() can hang if Python is built with Address Sanitizer +# (ASAN): libasan race condition, dead lock in pthread_create(). +HAVE_ASAN_FORK_BUG = check_sanitizer(address=True) + + +def set_sanitizer_env_var(env, option): + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS', 'TSAN_OPTIONS'): + if name in env: + env[name] += f':{option}' + else: + env[name] = option + def system_must_validate_cert(f): """Skip the test on TLS certificate validation failures.""" @@ -510,21 +511,42 @@ def has_no_debug_ranges(): def requires_debug_ranges(reason='requires co_positions / debug_ranges'): return unittest.skipIf(has_no_debug_ranges(), reason) -def requires_legacy_unicode_capi(): +@contextlib.contextmanager +def suppress_immortalization(suppress=True): + """Suppress immortalization of deferred objects.""" + try: + import _testinternalcapi + except ImportError: + yield + return + + if not suppress: + yield + return + + _testinternalcapi.suppress_immortalization(True) try: - from _testcapi import unicode_legacy_string + yield + finally: + _testinternalcapi.suppress_immortalization(False) + +def skip_if_suppress_immortalization(): + try: + import _testinternalcapi except ImportError: - unicode_legacy_string = None + return + return unittest.skipUnless(_testinternalcapi.get_immortalize_deferred(), + "requires immortalization of deferred objects") - return unittest.skipUnless(unicode_legacy_string, - 'requires legacy Unicode C API') + +MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. is_jython = sys.platform.startswith('java') -is_android = hasattr(sys, 'getandroidapilevel') +is_android = sys.platform == "android" -if sys.platform not in ('win32', 'vxworks'): +if sys.platform not in {"win32", "vxworks", "ios", "tvos", "watchos"}: unix_shell = '/system/bin/sh' if is_android else '/bin/sh' else: unix_shell = None @@ -534,23 +556,44 @@ def requires_legacy_unicode_capi(): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" -has_fork_support = hasattr(os, "fork") and not is_emscripten and not is_wasi +is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} +is_apple = is_apple_mobile or sys.platform == "darwin" + +has_fork_support = hasattr(os, "fork") and not ( + # WASM and Apple mobile platforms do not support subprocesses. + is_emscripten + or is_wasi + or is_apple_mobile -# From python 3.12.6 -is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' -skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') + # Although Android supports fork, it's unsafe to call it from Python because + # all Android apps are multi-threaded. + or is_android +) def requires_fork(): return unittest.skipUnless(has_fork_support, "requires working os.fork()") -has_subprocess_support = not is_emscripten and not is_wasi +has_subprocess_support = not ( + # WASM and Apple mobile platforms do not support subprocesses. + is_emscripten + or is_wasi + or is_apple_mobile + + # Although Android supports subproceses, they're almost never useful in + # practice (see PEP 738). And most of the tests that use them are calling + # sys.executable, which won't work when Python is embedded in an Android app. + or is_android +) def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") # Emscripten's socket emulation and WASI sockets have limitations. -has_socket_support = not is_emscripten and not is_wasi +has_socket_support = not ( + is_emscripten + or is_wasi +) def requires_working_socket(*, module=False): """Skip tests or modules that require working sockets From 844090b0b8413d25d94a80be0c683e23d347f33a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 14:48:20 -0700 Subject: [PATCH 232/317] update test_struct to 3.13.3 --- Lib/test/test_struct.py | 85 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 6cb7b610e5..bc801a08d6 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -9,7 +9,7 @@ import weakref from test import support -from test.support import import_helper +from test.support import import_helper, suppress_immortalization from test.support.script_helper import assert_python_ok ISBIGENDIAN = sys.byteorder == "big" @@ -96,6 +96,13 @@ def test_new_features(self): ('10s', b'helloworld', b'helloworld', b'helloworld', 0), ('11s', b'helloworld', b'helloworld\0', b'helloworld\0', 1), ('20s', b'helloworld', b'helloworld'+10*b'\0', b'helloworld'+10*b'\0', 1), + ('0p', b'helloworld', b'', b'', 1), + ('1p', b'helloworld', b'\x00', b'\x00', 1), + ('2p', b'helloworld', b'\x01h', b'\x01h', 1), + ('10p', b'helloworld', b'\x09helloworl', b'\x09helloworl', 1), + ('11p', b'helloworld', b'\x0Ahelloworld', b'\x0Ahelloworld', 0), + ('12p', b'helloworld', b'\x0Ahelloworld\0', b'\x0Ahelloworld\0', 1), + ('20p', b'helloworld', b'\x0Ahelloworld'+9*b'\0', b'\x0Ahelloworld'+9*b'\0', 1), ('b', 7, b'\7', b'\7', 0), ('b', -7, b'\371', b'\371', 0), ('B', 7, b'\7', b'\7', 0), @@ -339,6 +346,7 @@ def assertStructError(func, *args, **kwargs): def test_p_code(self): # Test p ("Pascal string") code. for code, input, expected, expectedback in [ + ('0p', b'abc', b'', b''), ('p', b'abc', b'\x00', b''), ('1p', b'abc', b'\x00', b''), ('2p', b'abc', b'\x01a', b'a'), @@ -523,6 +531,9 @@ def __bool__(self): for c in [b'\x01', b'\x7f', b'\xff', b'\x0f', b'\xf0']: self.assertTrue(struct.unpack('>?', c)[0]) + self.assertTrue(struct.unpack('': + for int_type in 'BHILQ': + test_error_msg(prefix, int_type, True) + for int_type in 'bhilq': + test_error_msg(prefix, int_type, False) + + int_type = 'N' + test_error_msg('@', int_type, True) + + int_type = 'n' + test_error_msg('@', int_type, False) + + @support.cpython_only + def test_issue98248_error_propagation(self): + class Div0: + def __index__(self): + 1 / 0 + + def test_error_propagation(fmt_str): + with self.subTest(format_str=fmt_str, exception="ZeroDivisionError"): + with self.assertRaises(ZeroDivisionError): + struct.pack(fmt_str, Div0()) + + for prefix in '@=<>': + for int_type in 'BHILQbhilq': + test_error_propagation(prefix + int_type) + + test_error_propagation('N') + test_error_propagation('n') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_struct_subclass_instantiation(self): + # Regression test for https://github.com/python/cpython/issues/112358 + class MyStruct(struct.Struct): + def __init__(self): + super().__init__('>h') + + my_struct = MyStruct() + self.assertEqual(my_struct.pack(12345), b'\x30\x39') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_repr(self): + s = struct.Struct('=i2H') + self.assertEqual(repr(s), f'Struct({s.format!r})') class UnpackIteratorTest(unittest.TestCase): """ @@ -895,4 +976,4 @@ def test_half_float(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file From 4336b9e787cc88c48d79e1714c706e75d86aa450 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 15:55:09 -0700 Subject: [PATCH 233/317] fix test_decimal --- Lib/test/test_decimal.py | 75 ++-------------------------------------- 1 file changed, 2 insertions(+), 73 deletions(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 04f4fc4a01..0493d6a41d 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -34,10 +34,10 @@ import locale from test.support import (is_resource_enabled, requires_IEEE_754, requires_docstrings, - requires_legacy_unicode_capi, check_sanitizer) + check_disallow_instantiation) from test.support import (TestFailed, run_with_locale, cpython_only, - darwin_malloc_err_warning, is_emscripten) + darwin_malloc_err_warning) from test.support.import_helper import import_fresh_module from test.support import threading_helper from test.support import warnings_helper @@ -586,18 +586,6 @@ def test_explicit_from_string(self): # underscores don't prevent errors self.assertRaises(InvalidOperation, Decimal, "1_2_\u00003") - @cpython_only - @requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_from_legacy_strings(self): - import _testcapi - Decimal = self.decimal.Decimal - context = self.decimal.Context() - - s = _testcapi.unicode_legacy_string('9.999999') - self.assertEqual(str(Decimal(s)), '9.999999') - self.assertEqual(str(context.create_decimal(s)), '9.999999') - def test_explicit_from_tuples(self): Decimal = self.decimal.Decimal @@ -2928,23 +2916,6 @@ def test_none_args(self): assert_signals(self, c, 'traps', [InvalidOperation, DivisionByZero, Overflow]) - @cpython_only - @requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_from_legacy_strings(self): - import _testcapi - c = self.decimal.Context() - - for rnd in RoundingModes: - c.rounding = _testcapi.unicode_legacy_string(rnd) - self.assertEqual(c.rounding, rnd) - - s = _testcapi.unicode_legacy_string('') - self.assertRaises(TypeError, setattr, c, 'rounding', s) - - s = _testcapi.unicode_legacy_string('ROUND_\x00UP') - self.assertRaises(TypeError, setattr, c, 'rounding', s) - def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -5654,48 +5625,6 @@ def __abs__(self): self.assertEqual(Decimal.from_float(cls(101.1)), Decimal.from_float(101.1)) - # Issue 41540: - @unittest.skipIf(sys.platform.startswith("aix"), - "AIX: default ulimit: test is flaky because of extreme over-allocation") - @unittest.skipIf(is_emscripten, "Test is unstable on Emscripten") - @unittest.skipIf(check_sanitizer(address=True, memory=True), - "ASAN/MSAN sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_maxcontext_exact_arith(self): - - # Make sure that exact operations do not raise MemoryError due - # to huge intermediate values when the context precision is very - # large. - - # The following functions fill the available precision and are - # therefore not suitable for large precisions (by design of the - # specification). - MaxContextSkip = ['logical_invert', 'next_minus', 'next_plus', - 'logical_and', 'logical_or', 'logical_xor', - 'next_toward', 'rotate', 'shift'] - - Decimal = C.Decimal - Context = C.Context - localcontext = C.localcontext - - # Here only some functions that are likely candidates for triggering a - # MemoryError are tested. deccheck.py has an exhaustive test. - maxcontext = Context(prec=C.MAX_PREC, Emin=C.MIN_EMIN, Emax=C.MAX_EMAX) - with localcontext(maxcontext): - self.assertEqual(Decimal(0).exp(), 1) - self.assertEqual(Decimal(1).ln(), 0) - self.assertEqual(Decimal(1).log10(), 0) - self.assertEqual(Decimal(10**2).log10(), 2) - self.assertEqual(Decimal(10**223).log10(), 223) - self.assertEqual(Decimal(10**19).logb(), 19) - self.assertEqual(Decimal(4).sqrt(), 2) - self.assertEqual(Decimal("40E9").sqrt(), Decimal('2.0E+5')) - self.assertEqual(divmod(Decimal(10), 3), (3, 1)) - self.assertEqual(Decimal(10) // 3, 3) - self.assertEqual(Decimal(4) / 2, 2) - self.assertEqual(Decimal(400) ** -1, Decimal('0.0025')) - - def test_c_signaldict_segfault(self): # See gh-106263 for details. SignalDict = type(C.Context().flags) From a0226df16628bf801c6058de013bb43142d108cc Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 16:11:32 -0700 Subject: [PATCH 234/317] fix test_csv --- Lib/test/test_csv.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 2646be086c..9a1743da6d 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -291,18 +291,6 @@ def test_writerows_errors(self): self.assertRaises(TypeError, writer.writerows, None) self.assertRaises(OSError, writer.writerows, BadIterable()) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @warnings_helper.ignore_warnings(category=DeprecationWarning) - def test_writerows_legacy_strings(self): - import _testcapi - c = _testcapi.unicode_legacy_string('a') - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - writer.writerows([[c]]) - fileobj.seek(0) - self.assertEqual(fileobj.read(), "a\r\n") - def _read_test(self, input, expect, **kwargs): reader = csv.reader(input, **kwargs) result = list(reader) From 3a1a5d3cd02ed6c10f00e6c4f4d84524c955ff31 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 16:12:37 -0700 Subject: [PATCH 235/317] fix test_unicode --- Lib/test/test_unicode.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 4da63c54d4..0eeb1ae936 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -818,16 +818,6 @@ def test_isidentifier(self): self.assertFalse("©".isidentifier()) self.assertFalse("0".isidentifier()) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_isidentifier_legacy(self): - u = '𝖀𝖓𝖎𝖈𝖔𝖉𝖊' - self.assertTrue(u.isidentifier()) - with warnings_helper.check_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertTrue(_testcapi.unicode_legacy_string(u).isidentifier()) - def test_isprintable(self): self.assertTrue("".isprintable()) self.assertTrue(" ".isprintable()) @@ -2522,26 +2512,6 @@ def test_getnewargs(self): self.assertEqual(args[0], text) self.assertEqual(len(args), 1) - @support.cpython_only - @support.requires_legacy_unicode_capi() - @unittest.skipIf(_testcapi is None, 'need _testcapi module') - def test_resize(self): - for length in range(1, 100, 7): - # generate a fresh string (refcount=1) - text = 'a' * length + 'b' - - # fill wstr internal field - with self.assertWarns(DeprecationWarning): - abc = _testcapi.getargs_u(text) - self.assertEqual(abc, text) - - # resize text: wstr field must be cleared and then recomputed - text += 'c' - with self.assertWarns(DeprecationWarning): - abcdef = _testcapi.getargs_u(text) - self.assertNotEqual(abc, abcdef) - self.assertEqual(abcdef, text) - def test_compare(self): # Issue #17615 N = 10 From 662d3a1b4ad0e6a33f8cc52be9e8a420ae1b778f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 15 Apr 2025 16:16:06 -0700 Subject: [PATCH 236/317] fix test_zlib --- Lib/test/support/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a05ef26631..106265ab2f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2594,7 +2594,8 @@ def adjust_int_max_str_digits(max_digits): # The default C recursion limit (from Include/cpython/pystate.h). C_RECURSION_LIMIT = 1500 -#Windows doesn't have os.uname() but it doesn't support s390x. +# Windows doesn't have os.uname() but it doesn't support s390x. +is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', 'skipped on s390x') HAVE_ASAN_FORK_BUG = check_sanitizer(address=True) From 99a384a3c7f4f580eef406d0cdca16fe3ff6b7ea Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 16 Apr 2025 13:35:30 -0500 Subject: [PATCH 237/317] Upgrade radium --- Cargo.lock | 7 +++++-- common/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ffadb13f6..95e1c19256 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1986,9 +1986,12 @@ checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "radium" -version = "0.7.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +checksum = "db0b76288902db304c864a12046b73d2d895cc34a4bb8137baaeebe9978a072c" +dependencies = [ + "cfg-if", +] [[package]] name = "radix_trie" diff --git a/common/Cargo.toml b/common/Cargo.toml index 299c2875b2..87e1ee118d 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -32,7 +32,7 @@ parking_lot = { workspace = true, optional = true } unicode_names2 = { workspace = true } lock_api = "0.4" -radium = "0.7" +radium = "1.1" siphasher = "1" volatile = "0.3" From 2e260761ae9cc6e3d1bd6c2ec3336cf1e2505c26 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 16 Apr 2025 13:43:58 -0500 Subject: [PATCH 238/317] Upgrade criterion --- Cargo.lock | 155 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 2 +- 2 files changed, 79 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95e1c19256..d66e1bb33b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.18" @@ -143,17 +149,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.4.0" @@ -330,6 +325,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -343,15 +365,29 @@ dependencies = [ [[package]] name = "clap" -version = "2.34.0" +version = "4.5.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" dependencies = [ - "bitflags 1.3.2", - "textwrap 0.11.0", - "unicode-width 0.1.14", + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" +dependencies = [ + "anstyle", + "clap_lex", ] +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + [[package]] name = "clipboard-win" version = "5.4.0" @@ -599,24 +635,24 @@ dependencies = [ [[package]] name = "criterion" -version = "0.3.6" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ - "atty", + "anes", "cast", + "ciborium", "clap", "criterion-plot", - "csv", + "is-terminal", "itertools 0.10.5", - "lazy_static 1.5.0", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", "regex", "serde", - "serde_cbor", "serde_derive", "serde_json", "tinytemplate", @@ -625,9 +661,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.4.5" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", "itertools 0.10.5", @@ -674,18 +710,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - [[package]] name = "csv-core" version = "0.1.12" @@ -858,7 +882,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc2706461e1ee94f55cab2ed2e3d34ae9536cfa830358ef80acff1a3dacab30" dependencies = [ - "lazy_static 0.2.11", + "lazy_static", "serde", "serde_derive", "serde_json", @@ -999,12 +1023,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - [[package]] name = "half" version = "2.5.0" @@ -1032,18 +1050,15 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" [[package]] name = "hex" @@ -1131,6 +1146,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi 0.5.0", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1248,12 +1274,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "lexical-parse-float" version = "1.0.5" @@ -2414,7 +2434,7 @@ dependencies = [ "rustpython-doc", "syn 2.0.100", "syn-ext", - "textwrap 0.16.2", + "textwrap", ] [[package]] @@ -2565,7 +2585,7 @@ dependencies = [ "flamer", "getrandom 0.3.2", "glob", - "half 2.5.0", + "half", "hex", "indexmap", "is-macro", @@ -2734,16 +2754,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half 1.8.3", - "serde", -] - [[package]] name = "serde_derive" version = "1.0.219" @@ -2948,15 +2958,6 @@ dependencies = [ "libc", ] -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width 0.1.14", -] - [[package]] name = "textwrap" version = "0.16.2" diff --git a/Cargo.toml b/Cargo.toml index 74fdf7464e..fe444b1963 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ bitflags = "2.4.2" bstr = "1" cfg-if = "1.0" chrono = "0.4.39" -criterion = { version = "0.3.5", features = ["html_reports"] } +criterion = { version = "0.5", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" getrandom = { version = "0.3", features = ["std"] } From 7eb361c92fabd1e2a5c5405c0d551cd9eca676ef Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 16 Apr 2025 13:48:21 -0500 Subject: [PATCH 239/317] Upgrade which and rustix --- Cargo.lock | 43 +++++++++++++++---------------------------- Cargo.toml | 2 +- stdlib/src/select.rs | 26 +++++++++++++++----------- vm/Cargo.toml | 2 +- 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d66e1bb33b..ccd224f4f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -818,6 +818,12 @@ dependencies = [ "regex", ] +[[package]] +name = "env_home" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" + [[package]] name = "env_logger" version = "0.11.8" @@ -872,7 +878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix 1.0.5", + "rustix", "windows-sys 0.59.0", ] @@ -966,7 +972,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed7131e57abbde63513e0e6636f76668a1ca9798dcae2df4e283cae9ee83859e" dependencies = [ - "rustix 1.0.5", + "rustix", "windows-targets 0.52.6", ] @@ -1396,12 +1402,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.9.3" @@ -2272,19 +2272,6 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags 2.9.0", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.0.5" @@ -2294,7 +2281,7 @@ dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.9.3", + "linux-raw-sys", "windows-sys 0.59.0", ] @@ -2539,7 +2526,7 @@ dependencies = [ "paste", "puruspe", "rand_core 0.9.3", - "rustix 0.38.44", + "rustix", "rustpython-common", "rustpython-derive", "rustpython-vm", @@ -2613,7 +2600,7 @@ dependencies = [ "ruff_python_parser", "ruff_source_file", "ruff_text_size", - "rustix 0.38.44", + "rustix", "rustpython-codegen", "rustpython-common", "rustpython-compiler", @@ -3427,13 +3414,13 @@ dependencies = [ [[package]] name = "which" -version = "6.0.3" +version = "7.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" +checksum = "24d643ce3fd3e5b54854602a080f34fb10ab75e0b813ee32d00ca2b44fa74762" dependencies = [ "either", - "home", - "rustix 0.38.44", + "env_home", + "rustix", "winsafe", ] diff --git a/Cargo.toml b/Cargo.toml index fe444b1963..621c255175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,7 +186,7 @@ proc-macro2 = "1.0.93" quote = "1.0.38" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } -rustix = { version = "0.38", features = ["event"] } +rustix = { version = "1.0", features = ["event"] } rustyline = "15.0.0" serde = { version = "1.0.133", default-features = false } schannel = "0.1.27" diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index f89a6c4f03..f600d60ede 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -519,7 +519,7 @@ mod decl { use rustix::event::epoll::{self, EventData, EventFlags}; use std::ops::Deref; use std::os::fd::{AsRawFd, IntoRawFd, OwnedFd}; - use std::time::{Duration, Instant}; + use std::time::Instant; #[pyclass(module = "select", name = "epoll")] #[derive(Debug, rustpython_vm::PyPayload)] @@ -636,12 +636,11 @@ mod decl { let poll::TimeoutArg(timeout) = args.timeout; let maxevents = args.maxevents; - let make_poll_timeout = |d: Duration| i32::try_from(d.as_millis()); - let mut poll_timeout = match timeout { - Some(d) => make_poll_timeout(d) - .map_err(|_| vm.new_overflow_error("timeout is too large".to_owned()))?, - None => -1, - }; + let mut poll_timeout = + timeout + .map(rustix::event::Timespec::try_from) + .transpose() + .map_err(|_| vm.new_overflow_error("timeout is too large".to_owned()))?; let deadline = timeout.map(|d| Instant::now() + d); let maxevents = match maxevents { @@ -654,19 +653,24 @@ mod decl { _ => maxevents as usize, }; - let mut events = epoll::EventVec::with_capacity(maxevents); + let mut events = Vec::::with_capacity(maxevents); let epoll = &*self.get_epoll(vm)?; loop { - match epoll::wait(epoll, &mut events, poll_timeout) { - Ok(()) => break, + events.clear(); + match epoll::wait( + epoll, + rustix::buffer::spare_capacity(&mut events), + poll_timeout.as_ref(), + ) { + Ok(_) => break, Err(rustix::io::Errno::INTR) => vm.check_signals()?, Err(e) => return Err(e.into_pyexception(vm)), } if let Some(deadline) = deadline { if let Some(new_timeout) = deadline.checked_duration_since(Instant::now()) { - poll_timeout = make_poll_timeout(new_timeout).unwrap(); + poll_timeout = Some(new_timeout.try_into().unwrap()); } else { break; } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 125c263da9..1fdd700b75 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -101,7 +101,7 @@ uname = "0.1.1" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] rustyline = { workspace = true } -which = "6" +which = "7" errno = "0.3" widestring = { workspace = true } From 2e14b7b5e8c9e4a16bd9e2a60356927e709d6d2b Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 16 Apr 2025 13:52:23 -0500 Subject: [PATCH 240/317] Upgrade system-configuration and libz-rs-sys --- Cargo.lock | 31 ++++++++----------------------- stdlib/Cargo.toml | 4 ++-- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccd224f4f7..941b1f97fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -925,7 +925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "libz-rs-sys 0.5.0", + "libz-rs-sys", "miniz_oxide", ] @@ -1378,22 +1378,13 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-rs-sys" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" -dependencies = [ - "zlib-rs 0.4.2", -] - [[package]] name = "libz-rs-sys" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" dependencies = [ - "zlib-rs 0.5.0", + "zlib-rs", ] [[package]] @@ -2506,7 +2497,7 @@ dependencies = [ "junction", "libc", "libsqlite3-sys", - "libz-rs-sys 0.4.2", + "libz-rs-sys", "mac_address", "malachite-bigint", "md-5", @@ -2902,20 +2893,20 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.0", "core-foundation", "system-configuration-sys", ] [[package]] name = "system-configuration-sys" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ "core-foundation-sys", "libc", @@ -3758,12 +3749,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "zlib-rs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" - [[package]] name = "zlib-rs" version = "0.5.0" diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index d4b99ebc92..eb6b9fe4dd 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -79,7 +79,7 @@ ucd = "0.1.1" adler32 = "1.2.0" crc32fast = "1.3.2" flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } -libz-sys = { package = "libz-rs-sys", version = "0.4" } +libz-sys = { package = "libz-rs-sys", version = "0.5" } bzip2 = { version = "0.4", optional = true } # tkinter @@ -132,7 +132,7 @@ features = [ ] [target.'cfg(target_os = "macos")'.dependencies] -system-configuration = "0.5.1" +system-configuration = "0.6.1" [lints] workspace = true From 55998c9e3b53407461cfbae87b31a7f43c6e0b0f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 17 Apr 2025 04:01:43 -0700 Subject: [PATCH 241/317] Merge pull request #5701 from arihant2math/more-tests Add more independent tests at 3.13.3 --- Lib/test/test___all__.py | 54 ++--- Lib/test/test_android.py | 448 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 476 insertions(+), 26 deletions(-) create mode 100644 Lib/test/test_android.py diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index a620dd5b4c..7b5356ea02 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -5,17 +5,21 @@ import sys import types -try: - import _multiprocessing -except ModuleNotFoundError: - _multiprocessing = None - if support.check_sanitizer(address=True, memory=True): - # bpo-46633: test___all__ is skipped because importing some modules - # directly can trigger known problems with ASAN (like tk or crypt). - raise unittest.SkipTest("workaround ASAN build issues on loading tests " - "like tk or crypt") + SKIP_MODULES = frozenset(( + # gh-90791: Tests involving libX11 can SEGFAULT on ASAN/MSAN builds. + # Skip modules, packages and tests using '_tkinter'. + '_tkinter', + 'tkinter', + 'test_tkinter', + 'test_ttk', + 'test_ttk_textonly', + 'idlelib', + 'test_idle', + )) +else: + SKIP_MODULES = () class NoAll(RuntimeError): @@ -27,17 +31,6 @@ class FailedImport(RuntimeError): class AllTest(unittest.TestCase): - def setUp(self): - # concurrent.futures uses a __getattr__ hook. Its __all__ triggers - # import of a submodule, which fails when _multiprocessing is not - # available. - if _multiprocessing is None: - sys.modules["_multiprocessing"] = types.ModuleType("_multiprocessing") - - def tearDown(self): - if _multiprocessing is None: - sys.modules.pop("_multiprocessing") - def check_all(self, modname): names = {} with warnings_helper.check_warnings( @@ -83,16 +76,24 @@ def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): path = os.path.join(basedir, fn) if os.path.isdir(path): + if fn in SKIP_MODULES: + continue pkg_init = os.path.join(path, '__init__.py') if os.path.exists(pkg_init): yield pkg_init, modpath + fn for p, m in self.walk_modules(path, modpath + fn + "."): yield p, m continue - if not fn.endswith('.py') or fn == '__init__.py': + + if fn == '__init__.py': continue - yield path, modpath + fn[:-3] - + if not fn.endswith('.py'): + continue + modname = fn.removesuffix('.py') + if modname in SKIP_MODULES: + continue + yield path, modpath + modname + # TODO: RUSTPYTHON @unittest.expectedFailure def test_all(self): @@ -103,7 +104,8 @@ def test_all(self): ]) # In case _socket fails to build, make this test fail more gracefully - # than an AttributeError somewhere deep in CGIHTTPServer. + # than an AttributeError somewhere deep in concurrent.futures, email + # or unittest. import _socket ignored = [] @@ -120,14 +122,14 @@ def test_all(self): if denied: continue if support.verbose: - print(modname) + print(f"Check {modname}", flush=True) try: # This heuristic speeds up the process by removing, de facto, # most test modules (and avoiding the auto-executing ones). with open(path, "rb") as f: if b"__all__" not in f.read(): raise NoAll(modname) - self.check_all(modname) + self.check_all(modname) except NoAll: ignored.append(modname) except FailedImport: diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 0000000000..076190f757 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,448 @@ +import io +import platform +import queue +import re +import subprocess +import sys +import unittest +from _android_support import TextLogStream +from array import array +from contextlib import ExitStack, contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time +from unittest.mock import patch + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + +# (name, level, fileno) +STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] + + +# Test redirection of stdout and stderr to the Android log. +@unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + self.logcat_thread = Thread(target=logcat_thread) + self.logcat_thread.start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + self.logcat_thread.join(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. + def stream_context(self, stream_name, level): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] + + stack = ExitStack() + stack.enter_context(self.subTest(stream_name)) + stream = getattr(sys, stream_name) + native_stream = getattr(sys, f"__{stream_name}__") + if isinstance(stream, io.StringIO): + stack.enter_context( + patch( + f"sys.{stream_name}", + TextLogStream( + prio, f"python.{stream_name}", native_stream.fileno(), + errors="backslashreplace" + ), + ) + ) + return stack + + def test_str(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + self.assertEqual("UTF-8", stream.encoding) + self.assertEqual("backslashreplace", stream.errors) + self.assertIs(stream.line_buffering, True) + self.assertIs(stream.write_through, False) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + # Since this is a line-based logging system, line buffering + # cannot be turned off, i.e. a newline always causes a flush. + stream.reconfigure(line_buffering=False) + self.assertIs(stream.line_buffering, True) + + # However, buffering can be turned off completely if you want a + # flush after every write. + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", # 4200 bytes in + [("foobar" * 666) + "foob", # 4000 bytes out + "ar" + ("foobar" * 33)]) # 200 bytes out + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", # 4500 bytes in + [s * 100, # 3000 bytes out + s * 50]) # 1500 bytes out + + s = "0123456789" + write(s * 200, []) # 2000 bytes in + write(s * 150, []) # 1500 bytes in + write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out + write("\n", [s * 51]) # 0 bytes in, 510 bytes out + + def test_bytes(self): + for stream_name, level, fileno in STREAM_INFO: + with self.stream_context(stream_name, level): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertIs(stream.writable(), True) + self.assertIs(stream.readable(), False) + self.assertEqual(stream.fileno(), fileno) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + +class TestAndroidRateLimit(unittest.TestCase): + def test_rate_limit(self): + # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39 + PER_MESSAGE_OVERHEAD = 28 + + # https://developer.android.com/ndk/reference/group/logging + ANDROID_LOG_DEBUG = 3 + + # To avoid flooding the test script output, use a different tag rather + # than stdout or stderr. + tag = "python.rate_limit" + stream = TextLogStream(ANDROID_LOG_DEBUG, tag) + + # Make a test message which consumes 1 KB of the logcat buffer. + message = "Line {:03d} " + message += "." * ( + 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0)) + ) + "\n" + + # To avoid depending on the performance of the test device, we mock the + # passage of time. + mock_now = time() + + def mock_time(): + # Avoid division by zero by simulating a small delay. + mock_sleep(0.0001) + return mock_now + + def mock_sleep(duration): + nonlocal mock_now + mock_now += duration + + # See _android_support.py. The default values of these parameters work + # well across a wide range of devices, but we'll use smaller values to + # ensure a quick and reliable test that doesn't flood the log too much. + MAX_KB_PER_SECOND = 100 + BUCKET_KB = 10 + with ( + patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024), + patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024), + patch("_android_support.sleep", mock_sleep), + patch("_android_support.time", mock_time), + ): + # Make sure the token bucket is full. + stream.write("Initial message to reset _prev_write_time") + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + line_num = 0 + + # Write BUCKET_KB messages, and return the rate at which they were + # accepted in KB per second. + def write_bucketful(): + nonlocal line_num + start = mock_time() + max_line_num = line_num + BUCKET_KB + while line_num < max_line_num: + stream.write(message.format(line_num)) + line_num += 1 + return BUCKET_KB / (mock_time() - start) + + # The first bucketful should be written with minimal delay. The + # factor of 2 here is not arbitrary: it verifies that the system can + # write fast enough to empty the bucket within two bucketfuls, which + # the next part of the test depends on. + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) + + # Write another bucketful to empty the token bucket completely. + write_bucketful() + + # The next bucketful should be written at the rate limit. + self.assertAlmostEqual( + write_bucketful(), MAX_KB_PER_SECOND, + delta=MAX_KB_PER_SECOND * 0.1 + ) + + # Once the token bucket refills, we should go back to full speed. + mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND) + self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2) From b72f3a4928746d66f1d5d67d9a13abb03064ce82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Apr 2025 20:23:32 +0900 Subject: [PATCH 242/317] Bump http-proxy-middleware from 2.0.7 to 2.0.9 in /wasm/demo (#5708) Bumps [http-proxy-middleware](https://github.com/chimurai/http-proxy-middleware) from 2.0.7 to 2.0.9. - [Release notes](https://github.com/chimurai/http-proxy-middleware/releases) - [Changelog](https://github.com/chimurai/http-proxy-middleware/blob/v2.0.9/CHANGELOG.md) - [Commits](https://github.com/chimurai/http-proxy-middleware/compare/v2.0.7...v2.0.9) --- updated-dependencies: - dependency-name: http-proxy-middleware dependency-version: 2.0.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- wasm/demo/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wasm/demo/package-lock.json b/wasm/demo/package-lock.json index 01753cf48f..7bbb974322 100644 --- a/wasm/demo/package-lock.json +++ b/wasm/demo/package-lock.json @@ -2733,9 +2733,9 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", - "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", + "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", "dev": true, "license": "MIT", "dependencies": { From 10fd02e42e7bbae6574cf2d66e8d273c6ffaba58 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 16 Apr 2025 17:01:21 -0500 Subject: [PATCH 243/317] Use constant_time_eq instead of our bespoke implementation --- Cargo.lock | 14 ++++++------ Cargo.toml | 1 + common/Cargo.toml | 1 - common/src/cmp.rs | 48 --------------------------------------- common/src/lib.rs | 1 - vm/Cargo.toml | 1 + vm/src/stdlib/operator.rs | 8 +++---- 7 files changed, 12 insertions(+), 62 deletions(-) delete mode 100644 common/src/cmp.rs diff --git a/Cargo.lock b/Cargo.lock index 941b1f97fe..87cfb55109 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.9.4" @@ -2348,7 +2354,6 @@ dependencies = [ "rustpython-wtf8", "siphasher", "unicode_names2", - "volatile", "widestring", "windows-sys 0.59.0", ] @@ -2556,6 +2561,7 @@ dependencies = [ "caseless", "cfg-if", "chrono", + "constant_time_eq", "crossbeam-utils", "errno", "exitcode", @@ -3279,12 +3285,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "volatile" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6" - [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 621c255175..21fb9de990 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,6 +155,7 @@ bitflags = "2.4.2" bstr = "1" cfg-if = "1.0" chrono = "0.4.39" +constant_time_eq = "0.4" criterion = { version = "0.5", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" diff --git a/common/Cargo.toml b/common/Cargo.toml index 87e1ee118d..0649c2a509 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -34,7 +34,6 @@ unicode_names2 = { workspace = true } lock_api = "0.4" radium = "1.1" siphasher = "1" -volatile = "0.3" [target.'cfg(windows)'.dependencies] widestring = { workspace = true } diff --git a/common/src/cmp.rs b/common/src/cmp.rs deleted file mode 100644 index d182340a98..0000000000 --- a/common/src/cmp.rs +++ /dev/null @@ -1,48 +0,0 @@ -use volatile::Volatile; - -/// Compare 2 byte slices in a way that ensures that the timing of the operation can't be used to -/// glean any information about the data. -#[inline(never)] -#[cold] -pub fn timing_safe_cmp(a: &[u8], b: &[u8]) -> bool { - // we use raw pointers here to keep faithful to the C implementation and - // to try to avoid any optimizations rustc might do with slices - let len_a = a.len(); - let a = a.as_ptr(); - let len_b = b.len(); - let b = b.as_ptr(); - /* The volatile type declarations make sure that the compiler has no - * chance to optimize and fold the code in any way that may change - * the timing. - */ - let mut result: u8 = 0; - /* loop count depends on length of b */ - let length: Volatile = Volatile::new(len_b); - let mut left: Volatile<*const u8> = Volatile::new(std::ptr::null()); - let mut right: Volatile<*const u8> = Volatile::new(b); - - /* don't use else here to keep the amount of CPU instructions constant, - * volatile forces re-evaluation - * */ - if len_a == length.read() { - left.write(Volatile::new(a).read()); - result = 0; - } - if len_a != length.read() { - left.write(b); - result = 1; - } - - for _ in 0..length.read() { - let l = left.read(); - left.write(l.wrapping_add(1)); - let r = right.read(); - right.write(r.wrapping_add(1)); - // safety: the 0..length range will always be either: - // * as long as the length of both a and b, if len_a and len_b are equal - // * as long as b, and both `left` and `right` are b - result |= unsafe { l.read_volatile() ^ r.read_volatile() }; - } - - result == 0 -} diff --git a/common/src/lib.rs b/common/src/lib.rs index c75451802a..a62c9517b4 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -10,7 +10,6 @@ pub mod atomic; pub mod borrow; pub mod boxvec; pub mod cformat; -pub mod cmp; #[cfg(any(unix, windows, target_os = "wasi"))] pub mod crt_fd; pub mod encodings; diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 1fdd700b75..9c85c4cc1a 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -51,6 +51,7 @@ bstr = { workspace = true } cfg-if = { workspace = true } crossbeam-utils = { workspace = true } chrono = { workspace = true, features = ["wasmbind"] } +constant_time_eq = { workspace = true } flame = { workspace = true, optional = true } getrandom = { workspace = true } hex = { workspace = true } diff --git a/vm/src/stdlib/operator.rs b/vm/src/stdlib/operator.rs index fbb8147e9f..2404b0c337 100644 --- a/vm/src/stdlib/operator.rs +++ b/vm/src/stdlib/operator.rs @@ -2,7 +2,6 @@ pub(crate) use _operator::make_module; #[pymodule] mod _operator { - use crate::common::cmp; use crate::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyInt, PyIntRef, PyStr, PyStrRef, PyTupleRef, PyTypeRef}, @@ -13,6 +12,7 @@ mod _operator { recursion::ReprGuard, types::{Callable, Constructor, PyComparisonOp, Representable}, }; + use constant_time_eq::constant_time_eq; #[pyfunction] fn lt(a: PyObjectRef, b: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -328,11 +328,9 @@ mod _operator { "comparing strings with non-ASCII characters is not supported".to_owned(), )); } - cmp::timing_safe_cmp(a.as_bytes(), b.as_bytes()) - } - (Either::B(a), Either::B(b)) => { - a.with_ref(|a| b.with_ref(|b| cmp::timing_safe_cmp(a, b))) + constant_time_eq(a.as_bytes(), b.as_bytes()) } + (Either::B(a), Either::B(b)) => a.with_ref(|a| b.with_ref(|b| constant_time_eq(a, b))), _ => { return Err(vm.new_type_error( "unsupported operand types(s) or combination of types".to_owned(), From fd665f6bb254004299d6bcde6aa3c1f140a5e1fb Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 17 Apr 2025 19:31:42 -0700 Subject: [PATCH 244/317] fix _suggestions module init renames the module init from suggestions to _suggestions Signed-off-by: Ashwin Naren --- stdlib/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index e9d10dfde4..1e81606f3c 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -134,7 +134,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "unicodedata" => unicodedata::make_module, "zlib" => zlib::make_module, "_statistics" => statistics::make_module, - "suggestions" => suggestions::make_module, + "_suggestions" => suggestions::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, } #[cfg(any(unix, target_os = "wasi"))] From dabd93c255ba325d742c748e3f6e90c7aa4bfb38 Mon Sep 17 00:00:00 2001 From: Daniel Stuart Date: Fri, 18 Apr 2025 14:10:25 -0500 Subject: [PATCH 245/317] Make stdio a feature (#5420) --- Cargo.toml | 3 ++- vm/Cargo.toml | 3 ++- vm/src/vm/mod.rs | 11 ++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 21fb9de990..0a3e3b89ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,9 +10,10 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "importlib"] +default = ["threading", "stdlib", "stdio", "importlib"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] +stdio = ["rustpython-vm/stdio"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 9c85c4cc1a..5a4b0df2a1 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -10,7 +10,8 @@ repository.workspace = true license.workspace = true [features] -default = ["compiler", "wasmbind"] +default = ["compiler", "wasmbind", "stdio"] +stdio = [] importlib = [] encodings = ["importlib"] vm-tracing-logging = [] diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 7baaae7770..08fbff94f9 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -14,6 +14,8 @@ mod vm_new; mod vm_object; mod vm_ops; +#[cfg(not(feature = "stdio"))] +use crate::builtins::PyNone; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, builtins::{ @@ -301,7 +303,8 @@ impl VirtualMachine { #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] { let io = import::import_builtin(self, "_io")?; - let set_stdio = |name, fd, write| { + #[cfg(feature = "stdio")] + let make_stdio = |name, fd, write| { let buffered_stdio = self.state.settings.buffered_stdio; let unbuffered = write && !buffered_stdio; let buf = crate::stdlib::io::open( @@ -332,7 +335,13 @@ impl VirtualMachine { )?; let mode = if write { "w" } else { "r" }; stdio.set_attr("mode", self.ctx.new_str(mode), self)?; + Ok(stdio) + }; + #[cfg(not(feature = "stdio"))] + let make_stdio = |_name, _fd, _write| Ok(PyNone.into_pyobject(self)); + let set_stdio = |name, fd, write| { + let stdio = make_stdio(name, fd, write)?; let dunder_name = self.ctx.intern_str(format!("__{name}__")); self.sys_module.set_attr( dunder_name, // e.g. __stdin__ From 960954eba57ccc1321a53a7f76e27c2e708c07b8 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 18 Apr 2025 14:15:10 -0500 Subject: [PATCH 246/317] Fix CI --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index afd3201e28..4e51d48953 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,7 +16,7 @@ concurrency: cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. # test_glob: many failing tests # test_io: many failing tests From a4d1bba74e555bfb3fc17cfd541cbbd315bbe6f8 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 18 Apr 2025 21:03:15 -0500 Subject: [PATCH 247/317] Update tarfile to 3.12.3 (#5714) * Update tarfile to 3.12.3 * Unmark tests --- Lib/tarfile.py | 436 +++++++++++++++++++++++++++++++++------- Lib/test/test_shutil.py | 4 - 2 files changed, 361 insertions(+), 79 deletions(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index dea150e8db..3bbbcaa621 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -46,6 +46,7 @@ import struct import copy import re +import warnings try: import pwd @@ -57,19 +58,19 @@ grp = None # os.symlink on Windows prior to 6.0 raises NotImplementedError -symlink_exception = (AttributeError, NotImplementedError) -try: - # OSError (winerror=1314) will be raised if the caller does not hold the - # SeCreateSymbolicLinkPrivilege privilege - symlink_exception += (OSError,) -except NameError: - pass +# OSError (winerror=1314) will be raised if the caller does not hold the +# SeCreateSymbolicLinkPrivilege privilege +symlink_exception = (AttributeError, NotImplementedError, OSError) # from tarfile import * __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", "CompressionError", "StreamError", "ExtractError", "HeaderError", "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", - "DEFAULT_FORMAT", "open"] + "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", + "tar_filter", "FilterError", "AbsoluteLinkError", + "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", + "LinkOutsideDestinationError"] + #--------------------------------------------------------- # tar constants @@ -158,6 +159,8 @@ def stn(s, length, encoding, errors): """Convert a string to a null-terminated bytes object. """ + if s is None: + raise ValueError("metadata cannot contain None") s = s.encode(encoding, errors) return s[:length] + (length - len(s)) * NUL @@ -328,15 +331,17 @@ def write(self, s): class _Stream: """Class that serves as an adapter between TarFile and a stream-like object. The stream-like object only - needs to have a read() or write() method and is accessed - blockwise. Use of gzip or bzip2 compression is possible. - A stream-like object could be for example: sys.stdin, - sys.stdout, a socket, a tape device etc. + needs to have a read() or write() method that works with bytes, + and the method is accessed blockwise. + Use of gzip or bzip2 compression is possible. + A stream-like object could be for example: sys.stdin.buffer, + sys.stdout.buffer, a socket, a tape device etc. _Stream is intended to be used only internally. """ - def __init__(self, name, mode, comptype, fileobj, bufsize): + def __init__(self, name, mode, comptype, fileobj, bufsize, + compresslevel): """Construct a _Stream object. """ self._extfileobj = True @@ -368,10 +373,10 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": - self._init_read_gz() self.exception = zlib.error + self._init_read_gz() else: - self._init_write_gz() + self._init_write_gz(compresslevel) elif comptype == "bz2": try: @@ -383,7 +388,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.cmp = bz2.BZ2Decompressor() self.exception = OSError else: - self.cmp = bz2.BZ2Compressor() + self.cmp = bz2.BZ2Compressor(compresslevel) elif comptype == "xz": try: @@ -410,13 +415,14 @@ def __del__(self): if hasattr(self, "closed") and not self.closed: self.close() - def _init_write_gz(self): + def _init_write_gz(self, compresslevel): """Initialize for writing with gzip compression. """ - self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED, - -self.zlib.MAX_WBITS, - self.zlib.DEF_MEM_LEVEL, - 0) + self.cmp = self.zlib.compressobj(compresslevel, + self.zlib.DEFLATED, + -self.zlib.MAX_WBITS, + self.zlib.DEF_MEM_LEVEL, + 0) timestamp = struct.pack("" % (self.__class__.__name__,self.name,id(self)) + def replace(self, *, + name=_KEEP, mtime=_KEEP, mode=_KEEP, linkname=_KEEP, + uid=_KEEP, gid=_KEEP, uname=_KEEP, gname=_KEEP, + deep=True, _KEEP=_KEEP): + """Return a deep copy of self with the given attributes replaced. + """ + if deep: + result = copy.deepcopy(self) + else: + result = copy.copy(self) + if name is not _KEEP: + result.name = name + if mtime is not _KEEP: + result.mtime = mtime + if mode is not _KEEP: + result.mode = mode + if linkname is not _KEEP: + result.linkname = linkname + if uid is not _KEEP: + result.uid = uid + if gid is not _KEEP: + result.gid = gid + if uname is not _KEEP: + result.uname = uname + if gname is not _KEEP: + result.gname = gname + return result + def get_info(self): """Return the TarInfo's attributes as a dictionary. """ + if self.mode is None: + mode = None + else: + mode = self.mode & 0o7777 info = { "name": self.name, - "mode": self.mode & 0o7777, + "mode": mode, "uid": self.uid, "gid": self.gid, "size": self.size, @@ -820,6 +983,9 @@ def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="surrogateescap """Return a tar header as a string of 512 byte blocks. """ info = self.get_info() + for name, value in info.items(): + if value is None: + raise ValueError("%s may not be None" % name) if format == USTAR_FORMAT: return self.create_ustar_header(info, encoding, errors) @@ -950,6 +1116,12 @@ def _create_header(info, format, encoding, errors): devmajor = stn("", 8, encoding, errors) devminor = stn("", 8, encoding, errors) + # None values in metadata should cause ValueError. + # itn()/stn() do this for all fields except type. + filetype = info.get("type", REGTYPE) + if filetype is None: + raise ValueError("TarInfo.type must not be None") + parts = [ stn(info.get("name", ""), 100, encoding, errors), itn(info.get("mode", 0) & 0o7777, 8, format), @@ -958,7 +1130,7 @@ def _create_header(info, format, encoding, errors): itn(info.get("size", 0), 12, format), itn(info.get("mtime", 0), 12, format), b" ", # checksum field - info.get("type", REGTYPE), + filetype, stn(info.get("linkname", ""), 100, encoding, errors), info.get("magic", POSIX_MAGIC), stn(info.get("uname", ""), 32, encoding, errors), @@ -1264,11 +1436,7 @@ def _proc_pax(self, tarfile): # the newline. keyword and value are both UTF-8 encoded strings. regex = re.compile(br"(\d+) ([^=]+)=") pos = 0 - while True: - match = regex.match(buf, pos) - if not match: - break - + while match := regex.match(buf, pos): length, keyword = match.groups() length = int(length) if length == 0: @@ -1468,6 +1636,8 @@ class TarFile(object): fileobject = ExFileObject # The file-object for extractfile(). + extraction_filter = None # The default filter for extraction. + def __init__(self, name=None, mode="r", fileobj=None, format=None, tarinfo=None, dereference=None, ignore_zeros=None, encoding=None, errors="surrogateescape", pax_headers=None, debug=None, @@ -1659,7 +1829,9 @@ def not_compressed(comptype): if filemode not in ("r", "w"): raise ValueError("mode must be 'r' or 'w'") - stream = _Stream(name, filemode, comptype, fileobj, bufsize) + compresslevel = kwargs.pop("compresslevel", 9) + stream = _Stream(name, filemode, comptype, fileobj, bufsize, + compresslevel) try: t = cls(name, filemode, stream, **kwargs) except: @@ -1940,7 +2112,10 @@ def list(self, verbose=True, *, members=None): members = self for tarinfo in members: if verbose: - _safe_print(stat.filemode(tarinfo.mode)) + if tarinfo.mode is None: + _safe_print("??????????") + else: + _safe_print(stat.filemode(tarinfo.mode)) _safe_print("%s/%s" % (tarinfo.uname or tarinfo.uid, tarinfo.gname or tarinfo.gid)) if tarinfo.ischr() or tarinfo.isblk(): @@ -1948,8 +2123,11 @@ def list(self, verbose=True, *, members=None): ("%d,%d" % (tarinfo.devmajor, tarinfo.devminor))) else: _safe_print("%10d" % tarinfo.size) - _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ - % time.localtime(tarinfo.mtime)[:6]) + if tarinfo.mtime is None: + _safe_print("????-??-?? ??:??:??") + else: + _safe_print("%d-%02d-%02d %02d:%02d:%02d" \ + % time.localtime(tarinfo.mtime)[:6]) _safe_print(tarinfo.name + ("/" if tarinfo.isdir() else "")) @@ -2036,32 +2214,63 @@ def addfile(self, tarinfo, fileobj=None): self.members.append(tarinfo) - def extractall(self, path=".", members=None, *, numeric_owner=False): + def _get_filter_function(self, filter): + if filter is None: + filter = self.extraction_filter + if filter is None: + warnings.warn( + 'Python 3.14 will, by default, filter extracted tar ' + + 'archives and reject files or modify their metadata. ' + + 'Use the filter argument to control this behavior.', + DeprecationWarning) + return fully_trusted_filter + if isinstance(filter, str): + raise TypeError( + 'String names are not supported for ' + + 'TarFile.extraction_filter. Use a function such as ' + + 'tarfile.data_filter directly.') + return filter + if callable(filter): + return filter + try: + return _NAMED_FILTERS[filter] + except KeyError: + raise ValueError(f"filter {filter!r} not found") from None + + def extractall(self, path=".", members=None, *, numeric_owner=False, + filter=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on directories afterwards. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by getmembers(). If `numeric_owner` is True, only the numbers for user/group names are used and not the names. + + The `filter` function will be called on each member just + before extraction. + It can return a changed TarInfo or None to skip the member. + String names of common filters are accepted. """ directories = [] + filter_function = self._get_filter_function(filter) if members is None: members = self - for tarinfo in members: + for member in members: + tarinfo = self._get_extract_tarinfo(member, filter_function, path) + if tarinfo is None: + continue if tarinfo.isdir(): - # Extract directories with a safe mode. + # For directories, delay setting attributes until later, + # since permissions can interfere with extraction and + # extracting contents can reset mtime. directories.append(tarinfo) - tarinfo = copy.copy(tarinfo) - tarinfo.mode = 0o700 - # Do not set_attrs directories, as we will do that further down - self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), + numeric_owner=numeric_owner) # Reverse sort directories. - directories.sort(key=lambda a: a.name) - directories.reverse() + directories.sort(key=lambda a: a.name, reverse=True) # Set correct owner, mtime and filemode on directories. for tarinfo in directories: @@ -2071,12 +2280,10 @@ def extractall(self, path=".", members=None, *, numeric_owner=False): self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: - if self.errorlevel > 1: - raise - else: - self._dbg(1, "tarfile: %s" % e) + self._handle_nonfatal_error(e) - def extract(self, member, path="", set_attrs=True, *, numeric_owner=False): + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, + filter=None): """Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. `member' may be a filename or a TarInfo object. You can @@ -2084,35 +2291,70 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False): mtime, mode) are set unless `set_attrs' is False. If `numeric_owner` is True, only the numbers for user/group names are used and not the names. + + The `filter` function will be called before extraction. + It can return a changed TarInfo or None to skip the member. + String names of common filters are accepted. """ - self._check("r") + filter_function = self._get_filter_function(filter) + tarinfo = self._get_extract_tarinfo(member, filter_function, path) + if tarinfo is not None: + self._extract_one(tarinfo, path, set_attrs, numeric_owner) + def _get_extract_tarinfo(self, member, filter_function, path): + """Get filtered TarInfo (or None) from member, which might be a str""" if isinstance(member, str): tarinfo = self.getmember(member) else: tarinfo = member + unfiltered = tarinfo + try: + tarinfo = filter_function(tarinfo, path) + except (OSError, FilterError) as e: + self._handle_fatal_error(e) + except ExtractError as e: + self._handle_nonfatal_error(e) + if tarinfo is None: + self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) + return None # Prepare the link target for makelink(). if tarinfo.islnk(): + tarinfo = copy.copy(tarinfo) tarinfo._link_target = os.path.join(path, tarinfo.linkname) + return tarinfo + + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): + """Extract from filtered tarinfo to disk""" + self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, numeric_owner=numeric_owner) except OSError as e: - if self.errorlevel > 0: - raise - else: - if e.filename is None: - self._dbg(1, "tarfile: %s" % e.strerror) - else: - self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) + self._handle_fatal_error(e) except ExtractError as e: - if self.errorlevel > 1: - raise + self._handle_nonfatal_error(e) + + def _handle_nonfatal_error(self, e): + """Handle non-fatal error (ExtractError) according to errorlevel""" + if self.errorlevel > 1: + raise + else: + self._dbg(1, "tarfile: %s" % e) + + def _handle_fatal_error(self, e): + """Handle "fatal" error according to self.errorlevel""" + if self.errorlevel > 0: + raise + elif isinstance(e, OSError): + if e.filename is None: + self._dbg(1, "tarfile: %s" % e.strerror) else: - self._dbg(1, "tarfile: %s" % e) + self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename)) + else: + self._dbg(1, "tarfile: %s %s" % (type(e).__name__, e)) def extractfile(self, member): """Extract a member from the archive as a file object. `member' may be @@ -2199,11 +2441,16 @@ def makedir(self, tarinfo, targetpath): """Make a directory called targetpath. """ try: - # Use a safe mode for the directory, the real mode is set - # later in _extract_member(). - os.mkdir(targetpath, 0o700) + if tarinfo.mode is None: + # Use the system's default mode + os.mkdir(targetpath) + else: + # Use a safe mode for the directory, the real mode is set + # later in _extract_member(). + os.mkdir(targetpath, 0o700) except FileExistsError: - pass + if not os.path.isdir(targetpath): + raise def makefile(self, tarinfo, targetpath): """Make a file called targetpath. @@ -2244,6 +2491,9 @@ def makedev(self, tarinfo, targetpath): raise ExtractError("special devices not supported by system") mode = tarinfo.mode + if mode is None: + # Use mknod's default + mode = 0o600 if tarinfo.isblk(): mode |= stat.S_IFBLK else: @@ -2265,7 +2515,6 @@ def makelink(self, tarinfo, targetpath): os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) else: - # See extract(). if os.path.exists(tarinfo._link_target): os.link(tarinfo._link_target, targetpath) else: @@ -2290,15 +2539,19 @@ def chown(self, tarinfo, targetpath, numeric_owner): u = tarinfo.uid if not numeric_owner: try: - if grp: + if grp and tarinfo.gname: g = grp.getgrnam(tarinfo.gname)[2] except KeyError: pass try: - if pwd: + if pwd and tarinfo.uname: u = pwd.getpwnam(tarinfo.uname)[2] except KeyError: pass + if g is None: + g = -1 + if u is None: + u = -1 try: if tarinfo.issym() and hasattr(os, "lchown"): os.lchown(targetpath, u, g) @@ -2310,6 +2563,8 @@ def chown(self, tarinfo, targetpath, numeric_owner): def chmod(self, tarinfo, targetpath): """Set file permissions of targetpath according to tarinfo. """ + if tarinfo.mode is None: + return try: os.chmod(targetpath, tarinfo.mode) except OSError as e: @@ -2318,10 +2573,13 @@ def chmod(self, tarinfo, targetpath): def utime(self, tarinfo, targetpath): """Set modification time of targetpath according to tarinfo. """ + mtime = tarinfo.mtime + if mtime is None: + return if not hasattr(os, 'utime'): return try: - os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime)) + os.utime(targetpath, (mtime, mtime)) except OSError as e: raise ExtractError("could not change modification time") from e @@ -2339,6 +2597,8 @@ def next(self): # Advance the file pointer. if self.offset != self.fileobj.tell(): + if self.offset == 0: + return None self.fileobj.seek(self.offset - 1) if not self.fileobj.read(1): raise ReadError("unexpected end of data") @@ -2397,13 +2657,26 @@ def _getmember(self, name, tarinfo=None, normalize=False): members = self.getmembers() # Limit the member search list up to tarinfo. + skipping = False if tarinfo is not None: - members = members[:members.index(tarinfo)] + try: + index = members.index(tarinfo) + except ValueError: + # The given starting point might be a (modified) copy. + # We'll later skip members until we find an equivalent. + skipping = True + else: + # Happy fast path + members = members[:index] if normalize: name = os.path.normpath(name) for member in reversed(members): + if skipping: + if tarinfo.offset == member.offset: + skipping = False + continue if normalize: member_name = os.path.normpath(member.name) else: @@ -2412,14 +2685,16 @@ def _getmember(self, name, tarinfo=None, normalize=False): if name == member_name: return member + if skipping: + # Starting point was not found + raise ValueError(tarinfo) + def _load(self): """Read through the entire archive file and look for readable members. """ - while True: - tarinfo = self.next() - if tarinfo is None: - break + while self.next() is not None: + pass self._loaded = True def _check(self, mode=None): @@ -2504,6 +2779,7 @@ def __exit__(self, type, value, traceback): #-------------------- # exported functions #-------------------- + def is_tarfile(name): """Return True if name points to a tar archive that we are able to handle, else return False. @@ -2512,7 +2788,9 @@ def is_tarfile(name): """ try: if hasattr(name, "read"): + pos = name.tell() t = open(fileobj=name) + name.seek(pos) else: t = open(name) t.close() @@ -2530,6 +2808,10 @@ def main(): parser = argparse.ArgumentParser(description=description) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') + parser.add_argument('--filter', metavar='', + choices=_NAMED_FILTERS, + help='Filter for extraction') + group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='', help='Show listing of a tarfile') @@ -2541,8 +2823,12 @@ def main(): help='Create tarfile from sources') group.add_argument('-t', '--test', metavar='', help='Test if a tarfile is valid') + args = parser.parse_args() + if args.filter and args.extract is None: + parser.exit(1, '--filter is only valid for extraction\n') + if args.test is not None: src = args.test if is_tarfile(src): @@ -2573,7 +2859,7 @@ def main(): if is_tarfile(src): with TarFile.open(src, 'r:*') as tf: - tf.extractall(path=curdir) + tf.extractall(path=curdir, filter=args.filter) if args.verbose: if curdir == '.': msg = '{!r} file is extracted.'.format(src) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 16416547c1..b64ccb37a5 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2000,13 +2000,9 @@ def check_unpack_tarball(self, format): ('Python 3.14', DeprecationWarning)): self.check_unpack_archive(format) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unpack_archive_tar(self): self.check_unpack_tarball('tar') - # TODO: RUSTPYTHON - @unittest.expectedFailure @support.requires_zlib() def test_unpack_archive_gztar(self): self.check_unpack_tarball('gztar') From 974c54ede2682b22f539ddc1cecf3f1341b12af9 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 19 Apr 2025 10:08:50 +0800 Subject: [PATCH 248/317] Updated test_baseexception from 3.13.2 (#5638) Signed-off-by: Hanif Ariffin --- Lib/test/test_baseexception.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index a73711c416..e19162a6ab 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -79,9 +79,10 @@ def test_inheritance(self): finally: inheritance_tree.close() + # Underscore-prefixed (private) exceptions don't need to be documented + exc_set = set(e for e in exc_set if not e.startswith('_')) # RUSTPYTHON specific exc_set.discard("JitError") - self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) interface_tests = ("length", "args", "str", "repr") @@ -135,7 +136,7 @@ class Value(str): d[HashThisKeyWillClearTheDict()] = Value() # refcount of Value() is 1 now - # Exception.__setstate__ should aquire a strong reference of key and + # Exception.__setstate__ should acquire a strong reference of key and # value in the dict. Otherwise, Value()'s refcount would go below # zero in the tp_hash call in PyObject_SetAttr(), and it would cause # crash in GC. From 12ceb9695c553047eadab5be47ffcfd4be81a8e9 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 18 Apr 2025 22:31:29 -0700 Subject: [PATCH 249/317] cleaner panic output Signed-off-by: Ashwin Naren --- compiler/codegen/src/compile.rs | 65 +++++++++++++++++++++++++-------- compiler/codegen/src/error.rs | 18 +++++++++ compiler/codegen/src/ir.rs | 25 ++++++++++--- compiler/codegen/src/lib.rs | 2 + 4 files changed, 88 insertions(+), 22 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 762585783c..67b5d3b67e 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -42,7 +42,9 @@ use rustpython_compiler_core::{ use rustpython_compiler_source::SourceCode; // use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; use std::borrow::Cow; +use crate::error::InternalError; +pub(crate) type InternalResult = Result; type CompileResult = Result; #[derive(PartialEq, Eq, Clone, Copy)] @@ -210,6 +212,43 @@ macro_rules! emit { }; } +fn print_location(zelf: &mut Compiler) { + let start = zelf.source_code.source_location(zelf.current_source_range.start()); + let end = zelf.source_code.source_location(zelf.current_source_range.end()); + eprintln!("LOCATION: {} from {}:{} to {}:{}", zelf.source_code.path.to_owned(), start.row, start.column, end.row, end.column); +} + +/// Better traceback for internal error +fn unwrap_internal(zelf: &mut Compiler, r: InternalResult) -> T { + if let Err(r_err) = r { + eprintln!("=== CODEGEN PANIC INFO ==="); + eprintln!("This IS an internal error: {}", r_err); + print_location(zelf); + eprintln!("=== END PANIC INFO ==="); + } + r.unwrap() +} + +fn compiler_unwrap_option(zelf: &mut Compiler, o: Option) -> T { + if let None = o { + eprintln!("=== CODEGEN PANIC INFO ==="); + eprintln!("This IS an internal error, an option was unwrapped during codegen"); + print_location(zelf); + eprintln!("=== END PANIC INFO ==="); + } + o.unwrap() +} + +fn compiler_result_unwrap(zelf: &mut Compiler, result: Result) -> T { + if let Err(r_err) = r { + eprintln!("=== CODEGEN PANIC INFO ==="); + eprintln!("This IS an internal error, an result was unwrapped during codegen"); + print_location(zelf); + eprintln!("=== END PANIC INFO ==="); + } + r.unwrap() +} + /// The pattern context holds information about captured names and jump targets. #[derive(Clone)] pub struct PatternContext { @@ -372,9 +411,8 @@ impl Compiler<'_> { fn pop_code_object(&mut self) -> CodeObject { let table = self.pop_symbol_table(); assert!(table.sub_tables.is_empty()); - self.code_stack - .pop() - .unwrap() + unwrap_internal(self, self.code_stack + .pop()) .finalize_code(self.opts.optimize) } @@ -482,7 +520,7 @@ impl Compiler<'_> { self.current_block().instructions.pop(); // pop Instruction::Pop } Stmt::FunctionDef(_) | Stmt::ClassDef(_) => { - let store_inst = self.current_block().instructions.pop().unwrap(); // pop Instruction::Store + let store_inst = compiler_unwrap_option(zelf, self.current_block().instructions.pop()); // pop Instruction::Store emit!(self, Instruction::Duplicate); self.current_block().instructions.push(store_inst); } @@ -540,9 +578,9 @@ impl Compiler<'_> { self.check_forbidden_name(&name, usage)?; let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = symbol_table.lookup(name.as_ref()).unwrap_or_else(|| - unreachable!("the symbol '{name}' should be present in the symbol table, even when it is undefined in python."), - ); + let symbol = unwrap_internal(symbol_table.lookup(name.as_ref()).ok_or_else(|| + InternalError::MissingSymbol(name.to_owned()) + )); let info = self.code_stack.last_mut().unwrap(); let mut cache = &mut info.name_cache; enum NameOpType { @@ -1476,12 +1514,9 @@ impl Compiler<'_> { } for var in &*code.freevars { let table = self.symbol_table_stack.last().unwrap(); - let symbol = table.lookup(var).unwrap_or_else(|| { - panic!( - "couldn't look up var {} in {} in {}", - var, code.obj_name, self.source_code.path - ) - }); + let symbol = unwrap_internal(table.lookup(var).ok_or_else(|| { + InternalError::MissingSymbol(var) + })); let parent_code = self.code_stack.last().unwrap(); let vars = match symbol.scope { SymbolScope::Free => &parent_code.freevar_cache, @@ -1564,9 +1599,7 @@ impl Compiler<'_> { // Check if the class is declared global let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = symbol_table.lookup(name.as_ref()).expect( - "The symbol must be present in the symbol table, even when it is undefined in python.", - ); + let symbol = unwrap_internal(symbol_table.lookup(name.as_ref()).ok_or_else(|| Err(InternalError::MissingSymbol(name.to_owned())))); let mut global_path_prefix = Vec::new(); if symbol.scope == SymbolScope::GlobalExplicit { global_path_prefix.append(&mut self.qualified_path); diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index 4394b936d2..69eab61183 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -34,6 +34,24 @@ impl fmt::Display for CodegenError { } } +#[derive(Debug)] +#[non_exhaustive] +pub enum InternalError { + StackOverflow, + StackUnderflow, + MissingSymbol(String), +} + +impl Display for InternalError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StackOverflow => write!(f, "stack overflow"), + Self::StackUnderflow => write!(f, "stack underflow"), + Self::MissingSymbol(s) => write!(f, "The symbol '{s}' must be present in the symbol table, even when it is undefined in python.") + } + } +} + #[derive(Debug)] #[non_exhaustive] pub enum CodegenErrorType { diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index ee29c65a16..00a74954ae 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -5,6 +5,7 @@ use ruff_source_file::{OneIndexed, SourceLocation}; use rustpython_compiler_core::bytecode::{ CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg, }; +use crate::error::InternalError; // use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -82,12 +83,12 @@ pub struct CodeInfo { pub freevar_cache: IndexSet, } impl CodeInfo { - pub fn finalize_code(mut self, optimize: u8) -> CodeObject { + pub fn finalize_code(mut self, optimize: u8) -> crate::InternalResult { if optimize > 0 { self.dce(); } - let max_stackdepth = self.max_stackdepth(); + let max_stackdepth = self.max_stackdepth()?; let cell2arg = self.cell2arg(); let CodeInfo { @@ -219,7 +220,7 @@ impl CodeInfo { } } - fn max_stackdepth(&self) -> u32 { + fn max_stackdepth(&self) -> crate::InternalResult { let mut maxdepth = 0u32; let mut stack = Vec::with_capacity(self.blocks.len()); let mut start_depths = vec![u32::MAX; self.blocks.len()]; @@ -247,7 +248,13 @@ impl CodeInfo { if effect < 0 && depth < effect.unsigned_abs() { panic!("The stack will underflow at {depth} with {effect} effect on {instr:?}"); } - let new_depth = depth.checked_add_signed(effect).unwrap(); + let new_depth = depth.checked_add_signed(effect).ok_or_else(|| { + if effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; if DEBUG { eprintln!("{new_depth}"); } @@ -264,7 +271,13 @@ impl CodeInfo { ) { let effect = instr.stack_effect(ins.arg, true); - let target_depth = depth.checked_add_signed(effect).unwrap(); + let target_depth = depth.checked_add_signed(effect).ok_or_else(|| { + if effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; if target_depth > maxdepth { maxdepth = target_depth } @@ -280,7 +293,7 @@ impl CodeInfo { if DEBUG { eprintln!("DONE: {maxdepth}"); } - maxdepth + Ok(maxdepth) } } diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs index 90e11c5b84..3ef6a7456f 100644 --- a/compiler/codegen/src/lib.rs +++ b/compiler/codegen/src/lib.rs @@ -18,6 +18,8 @@ mod unparse; pub use compile::CompileOpts; use ruff_python_ast::Expr; +pub(crate) use compile::InternalResult; + pub trait ToPythonName { /// Returns a short name for the node suitable for use in error messages. fn python_name(&self) -> &'static str; From fc331a154fdc8a08650773e4a621fbccdde9efac Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Fri, 18 Apr 2025 22:41:48 -0700 Subject: [PATCH 250/317] fix errors and formatting Signed-off-by: Ashwin Naren --- compiler/codegen/src/compile.rs | 45 +++++++++++++++++---------------- compiler/codegen/src/ir.rs | 4 +-- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 67b5d3b67e..b8cf205316 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -212,15 +212,15 @@ macro_rules! emit { }; } -fn print_location(zelf: &mut Compiler) { +fn print_location(zelf: &Compiler<'_>) { let start = zelf.source_code.source_location(zelf.current_source_range.start()); let end = zelf.source_code.source_location(zelf.current_source_range.end()); eprintln!("LOCATION: {} from {}:{} to {}:{}", zelf.source_code.path.to_owned(), start.row, start.column, end.row, end.column); } /// Better traceback for internal error -fn unwrap_internal(zelf: &mut Compiler, r: InternalResult) -> T { - if let Err(r_err) = r { +fn unwrap_internal(zelf: &Compiler<'_>, r: InternalResult) -> T { + if let Err(ref r_err) = r { eprintln!("=== CODEGEN PANIC INFO ==="); eprintln!("This IS an internal error: {}", r_err); print_location(zelf); @@ -229,7 +229,7 @@ fn unwrap_internal(zelf: &mut Compiler, r: InternalResult) -> T { r.unwrap() } -fn compiler_unwrap_option(zelf: &mut Compiler, o: Option) -> T { +fn compiler_unwrap_option(zelf: &Compiler<'_>, o: Option) -> T { if let None = o { eprintln!("=== CODEGEN PANIC INFO ==="); eprintln!("This IS an internal error, an option was unwrapped during codegen"); @@ -239,15 +239,15 @@ fn compiler_unwrap_option(zelf: &mut Compiler, o: Option) -> T { o.unwrap() } -fn compiler_result_unwrap(zelf: &mut Compiler, result: Result) -> T { - if let Err(r_err) = r { - eprintln!("=== CODEGEN PANIC INFO ==="); - eprintln!("This IS an internal error, an result was unwrapped during codegen"); - print_location(zelf); - eprintln!("=== END PANIC INFO ==="); - } - r.unwrap() -} +// fn compiler_result_unwrap(zelf: &Compiler<'_>, result: Result) -> T { +// if result.is_err() { +// eprintln!("=== CODEGEN PANIC INFO ==="); +// eprintln!("This IS an internal error, an result was unwrapped during codegen"); +// print_location(zelf); +// eprintln!("=== END PANIC INFO ==="); +// } +// result.unwrap() +// } /// The pattern context holds information about captured names and jump targets. #[derive(Clone)] @@ -411,9 +411,9 @@ impl Compiler<'_> { fn pop_code_object(&mut self) -> CodeObject { let table = self.pop_symbol_table(); assert!(table.sub_tables.is_empty()); - unwrap_internal(self, self.code_stack - .pop()) - .finalize_code(self.opts.optimize) + let pop = self.code_stack.pop(); + let stack_top = compiler_unwrap_option(self, pop); + unwrap_internal(self, stack_top.finalize_code(self.opts.optimize)) } // could take impl Into>, but everything is borrowed from ast structs; we never @@ -520,7 +520,8 @@ impl Compiler<'_> { self.current_block().instructions.pop(); // pop Instruction::Pop } Stmt::FunctionDef(_) | Stmt::ClassDef(_) => { - let store_inst = compiler_unwrap_option(zelf, self.current_block().instructions.pop()); // pop Instruction::Store + let pop_instructions = self.current_block().instructions.pop(); + let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store emit!(self, Instruction::Duplicate); self.current_block().instructions.push(store_inst); } @@ -578,8 +579,8 @@ impl Compiler<'_> { self.check_forbidden_name(&name, usage)?; let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal(symbol_table.lookup(name.as_ref()).ok_or_else(|| - InternalError::MissingSymbol(name.to_owned()) + let symbol = unwrap_internal(self, symbol_table.lookup(name.as_ref()).ok_or_else(|| + InternalError::MissingSymbol(name.to_string()) )); let info = self.code_stack.last_mut().unwrap(); let mut cache = &mut info.name_cache; @@ -1514,8 +1515,8 @@ impl Compiler<'_> { } for var in &*code.freevars { let table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal(table.lookup(var).ok_or_else(|| { - InternalError::MissingSymbol(var) + let symbol = unwrap_internal(self, table.lookup(var).ok_or_else(|| { + InternalError::MissingSymbol(var.to_owned()) })); let parent_code = self.code_stack.last().unwrap(); let vars = match symbol.scope { @@ -1599,7 +1600,7 @@ impl Compiler<'_> { // Check if the class is declared global let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal(symbol_table.lookup(name.as_ref()).ok_or_else(|| Err(InternalError::MissingSymbol(name.to_owned())))); + let symbol = unwrap_internal(self, symbol_table.lookup(name.as_ref()).ok_or_else(|| InternalError::MissingSymbol(name.to_owned()))); let mut global_path_prefix = Vec::new(); if symbol.scope == SymbolScope::GlobalExplicit { global_path_prefix.append(&mut self.qualified_path); diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 00a74954ae..915de373b1 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -155,7 +155,7 @@ impl CodeInfo { locations.clear() } - CodeObject { + Ok(CodeObject { flags, posonlyarg_count, arg_count, @@ -173,7 +173,7 @@ impl CodeInfo { cellvars: cellvar_cache.into_iter().collect(), freevars: freevar_cache.into_iter().collect(), cell2arg, - } + }) } fn cell2arg(&self) -> Option> { From 783e45f8acb29277a50d6c8416649c2894b16af7 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 19 Apr 2025 09:48:20 -0700 Subject: [PATCH 251/317] Apply review Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- compiler/codegen/src/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index b8cf205316..52710eca01 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -212,7 +212,7 @@ macro_rules! emit { }; } -fn print_location(zelf: &Compiler<'_>) { +fn eprint_location(zelf: &Compiler<'_>) { let start = zelf.source_code.source_location(zelf.current_source_range.start()); let end = zelf.source_code.source_location(zelf.current_source_range.end()); eprintln!("LOCATION: {} from {}:{} to {}:{}", zelf.source_code.path.to_owned(), start.row, start.column, end.row, end.column); From 397a1968c8a31fba3c3f3b979f123a8d263de215 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 19 Apr 2025 11:15:01 -0700 Subject: [PATCH 252/317] fix clippy Signed-off-by: Ashwin Naren --- compiler/codegen/src/compile.rs | 52 +++++++++++++++++++++++---------- compiler/codegen/src/error.rs | 5 +++- compiler/codegen/src/ir.rs | 9 ++---- 3 files changed, 44 insertions(+), 22 deletions(-) diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs index 52710eca01..18215003ee 100644 --- a/compiler/codegen/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -41,8 +41,8 @@ use rustpython_compiler_core::{ }; use rustpython_compiler_source::SourceCode; // use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; -use std::borrow::Cow; use crate::error::InternalError; +use std::borrow::Cow; pub(crate) type InternalResult = Result; type CompileResult = Result; @@ -213,9 +213,20 @@ macro_rules! emit { } fn eprint_location(zelf: &Compiler<'_>) { - let start = zelf.source_code.source_location(zelf.current_source_range.start()); - let end = zelf.source_code.source_location(zelf.current_source_range.end()); - eprintln!("LOCATION: {} from {}:{} to {}:{}", zelf.source_code.path.to_owned(), start.row, start.column, end.row, end.column); + let start = zelf + .source_code + .source_location(zelf.current_source_range.start()); + let end = zelf + .source_code + .source_location(zelf.current_source_range.end()); + eprintln!( + "LOCATION: {} from {}:{} to {}:{}", + zelf.source_code.path.to_owned(), + start.row, + start.column, + end.row, + end.column + ); } /// Better traceback for internal error @@ -223,17 +234,17 @@ fn unwrap_internal(zelf: &Compiler<'_>, r: InternalResult) -> T { if let Err(ref r_err) = r { eprintln!("=== CODEGEN PANIC INFO ==="); eprintln!("This IS an internal error: {}", r_err); - print_location(zelf); + eprint_location(zelf); eprintln!("=== END PANIC INFO ==="); } r.unwrap() } fn compiler_unwrap_option(zelf: &Compiler<'_>, o: Option) -> T { - if let None = o { + if o.is_none() { eprintln!("=== CODEGEN PANIC INFO ==="); eprintln!("This IS an internal error, an option was unwrapped during codegen"); - print_location(zelf); + eprint_location(zelf); eprintln!("=== END PANIC INFO ==="); } o.unwrap() @@ -243,7 +254,7 @@ fn compiler_unwrap_option(zelf: &Compiler<'_>, o: Option) -> T { // if result.is_err() { // eprintln!("=== CODEGEN PANIC INFO ==="); // eprintln!("This IS an internal error, an result was unwrapped during codegen"); -// print_location(zelf); +// eprint_location(zelf); // eprintln!("=== END PANIC INFO ==="); // } // result.unwrap() @@ -579,9 +590,12 @@ impl Compiler<'_> { self.check_forbidden_name(&name, usage)?; let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal(self, symbol_table.lookup(name.as_ref()).ok_or_else(|| - InternalError::MissingSymbol(name.to_string()) - )); + let symbol = unwrap_internal( + self, + symbol_table + .lookup(name.as_ref()) + .ok_or_else(|| InternalError::MissingSymbol(name.to_string())), + ); let info = self.code_stack.last_mut().unwrap(); let mut cache = &mut info.name_cache; enum NameOpType { @@ -1515,9 +1529,12 @@ impl Compiler<'_> { } for var in &*code.freevars { let table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal(self, table.lookup(var).ok_or_else(|| { - InternalError::MissingSymbol(var.to_owned()) - })); + let symbol = unwrap_internal( + self, + table + .lookup(var) + .ok_or_else(|| InternalError::MissingSymbol(var.to_owned())), + ); let parent_code = self.code_stack.last().unwrap(); let vars = match symbol.scope { SymbolScope::Free => &parent_code.freevar_cache, @@ -1600,7 +1617,12 @@ impl Compiler<'_> { // Check if the class is declared global let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal(self, symbol_table.lookup(name.as_ref()).ok_or_else(|| InternalError::MissingSymbol(name.to_owned()))); + let symbol = unwrap_internal( + self, + symbol_table + .lookup(name.as_ref()) + .ok_or_else(|| InternalError::MissingSymbol(name.to_owned())), + ); let mut global_path_prefix = Vec::new(); if symbol.scope == SymbolScope::GlobalExplicit { global_path_prefix.append(&mut self.qualified_path); diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs index 69eab61183..5e0ac12934 100644 --- a/compiler/codegen/src/error.rs +++ b/compiler/codegen/src/error.rs @@ -47,7 +47,10 @@ impl Display for InternalError { match self { Self::StackOverflow => write!(f, "stack overflow"), Self::StackUnderflow => write!(f, "stack underflow"), - Self::MissingSymbol(s) => write!(f, "The symbol '{s}' must be present in the symbol table, even when it is undefined in python.") + Self::MissingSymbol(s) => write!( + f, + "The symbol '{s}' must be present in the symbol table, even when it is undefined in python." + ), } } } diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs index 915de373b1..bb1f8b7564 100644 --- a/compiler/codegen/src/ir.rs +++ b/compiler/codegen/src/ir.rs @@ -1,11 +1,11 @@ use std::ops; use crate::IndexSet; +use crate::error::InternalError; use ruff_source_file::{OneIndexed, SourceLocation}; use rustpython_compiler_core::bytecode::{ CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg, }; -use crate::error::InternalError; // use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -245,10 +245,7 @@ impl CodeInfo { let instr_display = instr.display(display_arg, self); eprint!("{instr_display}: {depth} {effect:+} => "); } - if effect < 0 && depth < effect.unsigned_abs() { - panic!("The stack will underflow at {depth} with {effect} effect on {instr:?}"); - } - let new_depth = depth.checked_add_signed(effect).ok_or_else(|| { + let new_depth = depth.checked_add_signed(effect).ok_or({ if effect < 0 { InternalError::StackUnderflow } else { @@ -271,7 +268,7 @@ impl CodeInfo { ) { let effect = instr.stack_effect(ins.arg, true); - let target_depth = depth.checked_add_signed(effect).ok_or_else(|| { + let target_depth = depth.checked_add_signed(effect).ok_or({ if effect < 0 { InternalError::StackUnderflow } else { From e6404875726549f28eec66e9f42538dba8c8028b Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 17 Apr 2025 12:10:17 -0500 Subject: [PATCH 253/317] Use libbz2-rs-sys for bzip2 implementation Co-authored-by: Ashwin Naren --- Cargo.lock | 12 +++++++++--- stdlib/Cargo.toml | 2 +- stdlib/src/bz2.rs | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87cfb55109..0255535c1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,12 +239,12 @@ checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" [[package]] name = "bzip2" -version = "0.4.4" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ "bzip2-sys", - "libc", + "libbz2-rs-sys", ] [[package]] @@ -1322,6 +1322,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" +[[package]] +name = "libbz2-rs-sys" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0864a00c8d019e36216b69c2c4ce50b83b7bd966add3cf5ba554ec44f8bebcf5" + [[package]] name = "libc" version = "0.2.171" diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index eb6b9fe4dd..ac1a6c41ee 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -80,7 +80,7 @@ adler32 = "1.2.0" crc32fast = "1.3.2" flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } libz-sys = { package = "libz-rs-sys", version = "0.5" } -bzip2 = { version = "0.4", optional = true } +bzip2 = { version = "0.5", optional = true, features = ["libbz2-rs-sys"] } # tkinter tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true } diff --git a/stdlib/src/bz2.rs b/stdlib/src/bz2.rs index ba74a38db1..e580cbd713 100644 --- a/stdlib/src/bz2.rs +++ b/stdlib/src/bz2.rs @@ -33,7 +33,7 @@ mod _bz2 { } impl fmt::Debug for BZ2Decompressor { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "_bz2.BZ2Decompressor") } } @@ -178,7 +178,7 @@ mod _bz2 { } impl fmt::Debug for BZ2Compressor { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "_bz2.BZ2Compressor") } } From e377e43f057ae26f1390ab399879c081208854b6 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 17 Apr 2025 12:10:56 -0500 Subject: [PATCH 254/317] Remove bz2 feature --- Cargo.toml | 1 - stdlib/Cargo.toml | 3 +-- stdlib/src/lib.rs | 6 +----- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0a3e3b89ff..fb066e5dfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] -bz2 = ["stdlib", "rustpython-stdlib/bz2"] sqlite = ["rustpython-stdlib/sqlite"] ssl = ["rustpython-stdlib/ssl"] ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"] diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index ac1a6c41ee..d29bd3b21e 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -14,7 +14,6 @@ license.workspace = true default = ["compiler"] compiler = ["rustpython-vm/compiler"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] -bz2 = ["bzip2"] sqlite = ["dep:libsqlite3-sys"] ssl = ["openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] ssl-vendor = ["ssl", "openssl/vendored"] @@ -80,7 +79,7 @@ adler32 = "1.2.0" crc32fast = "1.3.2" flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } libz-sys = { package = "libz-rs-sys", version = "0.5" } -bzip2 = { version = "0.5", optional = true, features = ["libbz2-rs-sys"] } +bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] } # tkinter tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true } diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 1e81606f3c..254aa39322 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -36,7 +36,6 @@ mod statistics; mod suggestions; // TODO: maybe make this an extension module, if we ever get those // mod re; -#[cfg(feature = "bz2")] mod bz2; #[cfg(not(target_arch = "wasm32"))] pub mod socket; @@ -112,6 +111,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "array" => array::make_module, "binascii" => binascii::make_module, "_bisect" => bisect::make_module, + "_bz2" => bz2::make_module, "cmath" => cmath::make_module, "_contextvars" => contextvars::make_module, "_csv" => csv::make_module, @@ -158,10 +158,6 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit { "_ssl" => ssl::make_module, } - #[cfg(feature = "bz2")] - { - "_bz2" => bz2::make_module, - } #[cfg(windows)] { "_overlapped" => overlapped::make_module, From f0d46bfeaa8188ab4cdf939adc771b035fc3be93 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 17 Apr 2025 14:00:15 -0500 Subject: [PATCH 255/317] Finish _bz2 implementation --- Lib/test/test_bz2.py | 6 + stdlib/src/bz2.rs | 147 +++++----------- stdlib/src/zlib.rs | 347 +++++++++++++++++++++++++++----------- vm/src/function/number.rs | 2 +- 4 files changed, 301 insertions(+), 201 deletions(-) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 1f0b9adc36..b716d6016b 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -676,6 +676,8 @@ def testCompress4G(self, size): finally: data = None + # TODO: RUSTPYTHON + @unittest.expectedFailure def testPickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): @@ -734,6 +736,8 @@ def testDecompress4G(self, size): compressed = None decompressed = None + # TODO: RUSTPYTHON + @unittest.expectedFailure def testPickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): @@ -1001,6 +1005,8 @@ def test_encoding_error_handler(self): as f: self.assertEqual(f.read(), "foobar") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = self.TEXT.decode("ascii") diff --git a/stdlib/src/bz2.rs b/stdlib/src/bz2.rs index e580cbd713..6339a44a24 100644 --- a/stdlib/src/bz2.rs +++ b/stdlib/src/bz2.rs @@ -12,24 +12,44 @@ mod _bz2 { object::{PyPayload, PyResult}, types::Constructor, }; + use crate::zlib::{ + DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, + }; use bzip2::{Decompress, Status, write::BzEncoder}; + use rustpython_vm::convert::ToPyException; use std::{fmt, io::Write}; - // const BUFSIZ: i32 = 8192; - - struct DecompressorState { - decoder: Decompress, - eof: bool, - needs_input: bool, - // input_buffer: Vec, - // output_buffer: Vec, - } + const BUFSIZ: usize = 8192; #[pyattr] #[pyclass(name = "BZ2Decompressor")] #[derive(PyPayload)] struct BZ2Decompressor { - state: PyMutex, + state: PyMutex>, + } + + impl Decompressor for Decompress { + type Flush = (); + type Status = Status; + type Error = bzip2::Error; + + fn total_in(&self) -> u64 { + self.total_in() + } + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + (): Self::Flush, + ) -> Result { + self.decompress_vec(input, output) + } + } + + impl DecompressStatus for Status { + fn is_stream_end(&self) -> bool { + *self == Status::StreamEnd + } } impl fmt::Debug for BZ2Decompressor { @@ -43,13 +63,7 @@ mod _bz2 { fn py_new(cls: PyTypeRef, _: Self::Args, vm: &VirtualMachine) -> PyResult { Self { - state: PyMutex::new(DecompressorState { - decoder: Decompress::new(false), - eof: false, - needs_input: true, - // input_buffer: Vec::new(), - // output_buffer: Vec::new(), - }), + state: PyMutex::new(DecompressState::new(Decompress::new(false), vm)), } .into_ref_with_type(vm, cls) .map(Into::into) @@ -59,107 +73,34 @@ mod _bz2 { #[pyclass(with(Constructor))] impl BZ2Decompressor { #[pymethod] - fn decompress( - &self, - data: ArgBytesLike, - // TODO: PyIntRef - max_length: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult { - let max_length = max_length.unwrap_or(-1); - if max_length >= 0 { - return Err(vm.new_not_implemented_error( - "the max_value argument is not implemented yet".to_owned(), - )); - } - // let max_length = if max_length < 0 || max_length >= BUFSIZ { - // BUFSIZ - // } else { - // max_length - // }; + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { + let max_length = args.max_length(); + let data = &*args.data(); let mut state = self.state.lock(); - let DecompressorState { - decoder, - eof, - .. - // needs_input, - // input_buffer, - // output_buffer, - } = &mut *state; - - if *eof { - return Err(vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "End of stream already reached".to_owned(), - )); - } - - // data.with_ref(|data| input_buffer.extend(data)); - - // If max_length is negative: - // read the input X bytes at a time, compress it and append it to output. - // Once you're out of input, setting needs_input to true and return the - // output as bytes. - // - // TODO: - // If max_length is non-negative: - // Read the input X bytes at a time, compress it and append it to - // the output. If output reaches `max_length` in size, return - // it (up to max_length), and store the rest of the output - // for later. - - // TODO: arbitrary choice, not the right way to do it. - let mut buf = Vec::with_capacity(data.len() * 32); - - let before = decoder.total_in(); - let res = data.with_ref(|data| decoder.decompress_vec(data, &mut buf)); - let _written = (decoder.total_in() - before) as usize; - - let res = match res { - Ok(x) => x, - // TODO: error message - _ => return Err(vm.new_os_error("Invalid data stream".to_owned())), - }; - - if res == Status::StreamEnd { - *eof = true; - } - Ok(vm.ctx.new_bytes(buf.to_vec())) + state + .decompress(data, max_length, BUFSIZ, vm) + .map_err(|e| match e { + DecompressError::Decompress(err) => vm.new_os_error(err.to_string()), + DecompressError::Eof(err) => err.to_pyexception(vm), + }) } #[pygetset] fn eof(&self) -> bool { - let state = self.state.lock(); - state.eof + self.state.lock().eof() } #[pygetset] - fn unused_data(&self, vm: &VirtualMachine) -> PyBytesRef { - // Data found after the end of the compressed stream. - // If this attribute is accessed before the end of the stream - // has been reached, its value will be b''. - vm.ctx.new_bytes(b"".to_vec()) - // alternatively, be more honest: - // Err(vm.new_not_implemented_error( - // "unused_data isn't implemented yet".to_owned(), - // )) - // - // TODO - // let state = self.state.lock(); - // if state.eof { - // vm.ctx.new_bytes(state.input_buffer.to_vec()) - // else { - // vm.ctx.new_bytes(b"".to_vec()) - // } + fn unused_data(&self) -> PyBytesRef { + self.state.lock().unused_data() } #[pygetset] fn needs_input(&self) -> bool { // False if the decompress() method can provide more // decompressed data before requiring new uncompressed input. - let state = self.state.lock(); - state.needs_input + self.state.lock().needs_input() } // TODO: mro()? diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 9c19b74066..0578f20c86 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -1,14 +1,17 @@ // spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker -pub(crate) use zlib::make_module; +pub(crate) use zlib::{DecompressArgs, make_module}; #[pymodule] mod zlib { + use super::generic::{ + DecompressError, DecompressState, DecompressStatus, Decompressor, FlushKind, flush_sync, + }; use crate::vm::{ PyObject, PyPayload, PyResult, VirtualMachine, builtins::{PyBaseExceptionRef, PyBytesRef, PyIntRef, PyTypeRef}, common::lock::PyMutex, - convert::TryFromBorrowedObject, + convert::{ToPyException, TryFromBorrowedObject}, function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, types::Constructor, }; @@ -142,18 +145,18 @@ mod zlib { } #[derive(Clone)] - struct Chunker<'a> { + pub(crate) struct Chunker<'a> { data1: &'a [u8], data2: &'a [u8], } impl<'a> Chunker<'a> { - fn new(data: &'a [u8]) -> Self { + pub(crate) fn new(data: &'a [u8]) -> Self { Self { data1: data, data2: &[], } } - fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { + pub(crate) fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { if data1.is_empty() { Self { data1: data2, @@ -163,19 +166,19 @@ mod zlib { Self { data1, data2 } } } - fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { self.data1.len() + self.data2.len() } - fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.data1.is_empty() } - fn to_vec(&self) -> Vec { + pub(crate) fn to_vec(&self) -> Vec { [self.data1, self.data2].concat() } - fn chunk(&self) -> &'a [u8] { + pub(crate) fn chunk(&self) -> &'a [u8] { self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) } - fn advance(&mut self, consumed: usize) { + pub(crate) fn advance(&mut self, consumed: usize) { self.data1 = &self.data1[consumed..]; if self.data1.is_empty() { self.data1 = std::mem::take(&mut self.data2); @@ -183,28 +186,24 @@ mod zlib { } } - fn _decompress( + fn _decompress( data: &[u8], - d: &mut Decompress, + d: &mut D, bufsize: usize, max_length: Option, - is_flush: bool, - zdict: Option<&ArgBytesLike>, - vm: &VirtualMachine, - ) -> PyResult<(Vec, bool)> { + calc_flush: impl Fn(bool) -> D::Flush, + ) -> Result<(Vec, bool), D::Error> { let mut data = Chunker::new(data); - _decompress_chunks(&mut data, d, bufsize, max_length, is_flush, zdict, vm) + _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) } - fn _decompress_chunks( + pub(super) fn _decompress_chunks( data: &mut Chunker<'_>, - d: &mut Decompress, + d: &mut D, bufsize: usize, max_length: Option, - is_flush: bool, - zdict: Option<&ArgBytesLike>, - vm: &VirtualMachine, - ) -> PyResult<(Vec, bool)> { + calc_flush: impl Fn(bool) -> D::Flush, + ) -> Result<(Vec, bool), D::Error> { if data.is_empty() { return Ok((Vec::new(), true)); } @@ -213,16 +212,7 @@ mod zlib { 'outer: loop { let chunk = data.chunk(); - let flush = if is_flush { - // if this is the final chunk, finish it - if chunk.len() == data.len() { - FlushDecompress::Finish - } else { - FlushDecompress::None - } - } else { - FlushDecompress::Sync - }; + let flush = calc_flush(chunk.len() == data.len()); loop { let additional = std::cmp::min(bufsize, max_length - buf.capacity()); if additional == 0 { @@ -238,7 +228,7 @@ mod zlib { match res { Ok(status) => { - let stream_end = status == Status::StreamEnd; + let stream_end = status.is_stream_end(); if stream_end || data.is_empty() { // we've reached the end of the stream, we're done buf.shrink_to_fit(); @@ -252,11 +242,7 @@ mod zlib { } } Err(e) => { - let Some(zdict) = e.needs_dictionary().and(zdict) else { - return Err(new_zlib_error(&e.to_string(), vm)); - }; - d.set_dictionary(&zdict.borrow_buf()) - .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; + d.maybe_set_dict(e)?; // now try the next chunk continue 'outer; } @@ -285,8 +271,8 @@ mod zlib { } = args; data.with_ref(|data| { let mut d = InitOptions::new(wbits.value, vm)?.decompress(); - let (buf, stream_end) = - _decompress(data, &mut d, bufsize.value, None, false, None, vm)?; + let (buf, stream_end) = _decompress(data, &mut d, bufsize.value, None, flush_sync) + .map_err(|e| new_zlib_error(e.to_string(), vm))?; if !stream_end { return Err(new_zlib_error( "Error -5 while decompressing data: incomplete or truncated stream", @@ -316,9 +302,8 @@ mod zlib { } } let inner = PyDecompressInner { - decompress: Some(decompress), + decompress: Some(DecompressWithDict { decompress, zdict }), eof: false, - zdict, unused_data: vm.ctx.empty_bytes.clone(), unconsumed_tail: vm.ctx.empty_bytes.clone(), }; @@ -329,8 +314,7 @@ mod zlib { #[derive(Debug)] struct PyDecompressInner { - decompress: Option, - zdict: Option, + decompress: Option, eof: bool, unused_data: PyBytesRef, unconsumed_tail: PyBytesRef, @@ -370,14 +354,25 @@ mod zlib { return Err(new_zlib_error(USE_AFTER_FINISH_ERR, vm)); }; - let zdict = if is_flush { None } else { inner.zdict.as_ref() }; - let prev_in = d.total_in(); - let (ret, stream_end) = - match _decompress(data, d, bufsize, max_length, is_flush, zdict, vm) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), + let res = if is_flush { + // if is_flush: ignore zdict, finish if final chunk + let calc_flush = |final_chunk| { + if final_chunk { + FlushDecompress::Finish + } else { + FlushDecompress::None + } }; + _decompress(data, &mut d.decompress, bufsize, max_length, calc_flush) + } else { + _decompress(data, d, bufsize, max_length, flush_sync) + } + .map_err(|e| new_zlib_error(e.to_string(), vm)); + let (ret, stream_end) = match res { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; let consumed = (d.total_in() - prev_in) as usize; // save unused input @@ -404,7 +399,7 @@ mod zlib { .try_into() .map_err(|_| vm.new_value_error("must be non-negative".to_owned()))?; let max_length = (max_length != 0).then_some(max_length); - let data = &*args.data.borrow_buf(); + let data = &*args.data(); let inner = &mut *self.inner.lock(); @@ -440,13 +435,24 @@ mod zlib { } #[derive(FromArgs)] - struct DecompressArgs { + pub(crate) struct DecompressArgs { #[pyarg(positional)] data: ArgBytesLike, #[pyarg(any, optional)] max_length: OptionalArg, } + impl DecompressArgs { + pub(crate) fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { + self.data.borrow_buf() + } + pub(crate) fn max_length(&self) -> Option { + self.max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()) + } + } + #[derive(FromArgs)] #[allow(dead_code)] // FIXME: use args struct CompressobjArgs { @@ -588,8 +594,8 @@ mod zlib { } } - fn new_zlib_error(message: &str, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_exception_msg(vm.class("zlib", "error"), message.to_owned()) + fn new_zlib_error(message: impl Into, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_exception_msg(vm.class("zlib", "error"), message.into()) } const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; @@ -626,19 +632,68 @@ mod zlib { #[pyclass(name = "_ZlibDecompressor")] #[derive(Debug, PyPayload)] struct ZlibDecompressor { - inner: PyMutex, + inner: PyMutex>, } #[derive(Debug)] - struct ZlibDecompressorInner { + struct DecompressWithDict { decompress: Decompress, - unused_data: PyBytesRef, - input_buffer: Vec, zdict: Option, - eof: bool, - needs_input: bool, } + impl DecompressStatus for Status { + fn is_stream_end(&self) -> bool { + *self == Status::StreamEnd + } + } + + impl FlushKind for FlushDecompress { + const SYNC: Self = FlushDecompress::Sync; + } + + impl Decompressor for Decompress { + type Flush = FlushDecompress; + type Status = Status; + type Error = flate2::DecompressError; + + fn total_in(&self) -> u64 { + self.total_in() + } + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result { + self.decompress_vec(input, output, flush) + } + } + + impl Decompressor for DecompressWithDict { + type Flush = FlushDecompress; + type Status = Status; + type Error = flate2::DecompressError; + + fn total_in(&self) -> u64 { + self.decompress.total_in() + } + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result { + self.decompress.decompress_vec(input, output, flush) + } + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + let zdict = err.needs_dictionary().and(self.zdict.as_ref()).ok_or(err)?; + self.decompress.set_dictionary(&zdict.borrow_buf())?; + Ok(()) + } + } + + // impl Deconstruct + impl Constructor for ZlibDecompressor { type Args = DecompressobjArgs; @@ -651,14 +706,7 @@ mod zlib { .map_err(|_| new_zlib_error("failed to set dictionary", vm))?; } } - let inner = ZlibDecompressorInner { - decompress, - unused_data: vm.ctx.empty_bytes.clone(), - input_buffer: Vec::new(), - zdict, - eof: false, - needs_input: true, - }; + let inner = DecompressState::new(DecompressWithDict { decompress, zdict }, vm); Self { inner: PyMutex::new(inner), } @@ -671,61 +719,151 @@ mod zlib { impl ZlibDecompressor { #[pygetset] fn eof(&self) -> bool { - self.inner.lock().eof + self.inner.lock().eof() } #[pygetset] fn unused_data(&self) -> PyBytesRef { - self.inner.lock().unused_data.clone() + self.inner.lock().unused_data() } #[pygetset] fn needs_input(&self) -> bool { - self.inner.lock().needs_input + self.inner.lock().needs_input() } #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length = args - .max_length - .into_option() - .and_then(|ArgSize { value }| usize::try_from(value).ok()); - let data = &*args.data.borrow_buf(); + let max_length = args.max_length(); + let data = &*args.data(); let inner = &mut *self.inner.lock(); - if inner.eof { - return Err(vm.new_eof_error("End of stream already reached".to_owned())); + inner + .decompress(data, max_length, DEF_BUF_SIZE, vm) + .map_err(|e| match e { + DecompressError::Decompress(err) => new_zlib_error(err.to_string(), vm), + DecompressError::Eof(err) => err.to_pyexception(vm), + }) + } + + // TODO: Wait for getstate pyslot to be fixed + // #[pyslot] + // fn getstate(zelf: &PyObject, vm: &VirtualMachine) -> PyResult { + // Err(vm.new_type_error("cannot serialize '_ZlibDecompressor' object".to_owned())) + // } + } +} + +mod generic { + use super::zlib::{_decompress_chunks, Chunker}; + use crate::vm::{ + VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef}, + convert::ToPyException, + }; + + pub(crate) trait Decompressor { + type Flush: FlushKind; + type Status: DecompressStatus; + type Error; + + fn total_in(&self) -> u64; + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result; + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + Err(err) + } + } + + pub(crate) trait DecompressStatus { + fn is_stream_end(&self) -> bool; + } + + pub(crate) trait FlushKind: Copy { + const SYNC: Self; + } + + impl FlushKind for () { + const SYNC: Self = (); + } + + pub(super) fn flush_sync(_final_chunk: bool) -> T { + T::SYNC + } + + #[derive(Debug)] + pub(crate) struct DecompressState { + decompress: D, + unused_data: PyBytesRef, + input_buffer: Vec, + eof: bool, + needs_input: bool, + } + + impl DecompressState { + pub(crate) fn new(decompress: D, vm: &VirtualMachine) -> Self { + Self { + decompress, + unused_data: vm.ctx.empty_bytes.clone(), + input_buffer: Vec::new(), + eof: false, + needs_input: true, + } + } + + pub(crate) fn eof(&self) -> bool { + self.eof + } + + pub(crate) fn unused_data(&self) -> PyBytesRef { + self.unused_data.clone() + } + + pub(crate) fn needs_input(&self) -> bool { + self.needs_input + } + + pub(crate) fn decompress( + &mut self, + data: &[u8], + max_length: Option, + bufsize: usize, + vm: &VirtualMachine, + ) -> Result, DecompressError> { + if self.eof { + return Err(DecompressError::Eof(EofError)); } - let input_buffer = &mut inner.input_buffer; - let d = &mut inner.decompress; + let input_buffer = &mut self.input_buffer; + let d = &mut self.decompress; let mut chunks = Chunker::chain(input_buffer, data); - let zdict = inner.zdict.as_ref(); - let bufsize = DEF_BUF_SIZE; - let prev_len = chunks.len(); let (ret, stream_end) = - match _decompress_chunks(&mut chunks, d, bufsize, max_length, false, zdict, vm) { + match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { Ok((buf, stream_end)) => (Ok(buf), stream_end), Err(err) => (Err(err), false), }; let consumed = prev_len - chunks.len(); - inner.eof |= stream_end; + self.eof |= stream_end; - if inner.eof { - inner.needs_input = false; + if self.eof { + self.needs_input = false; if !chunks.is_empty() { - inner.unused_data = vm.ctx.new_bytes(chunks.to_vec()); + self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); } } else if chunks.is_empty() { input_buffer.clear(); - inner.needs_input = true; + self.needs_input = true; } else { - inner.needs_input = false; + self.needs_input = false; if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { input_buffer.clear(); input_buffer.extend_from_slice(&data[n_consumed_from_data..]); @@ -735,13 +873,28 @@ mod zlib { } } - ret + ret.map_err(DecompressError::Decompress) } + } - // TODO: Wait for getstate pyslot to be fixed - // #[pyslot] - // fn getstate(zelf: &PyObject, vm: &VirtualMachine) -> PyResult { - // Err(vm.new_type_error("cannot serialize '_ZlibDecompressor' object".to_owned())) - // } + pub(crate) enum DecompressError { + Decompress(E), + Eof(EofError), + } + + impl From for DecompressError { + fn from(err: E) -> Self { + Self::Decompress(err) + } + } + + pub(crate) struct EofError; + + impl ToPyException for EofError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_eof_error("End of stream already reached".to_owned()) + } } } + +pub(crate) use generic::{DecompressError, DecompressState, DecompressStatus, Decompressor}; diff --git a/vm/src/function/number.rs b/vm/src/function/number.rs index 0e36f57ad1..bead82123e 100644 --- a/vm/src/function/number.rs +++ b/vm/src/function/number.rs @@ -158,7 +158,7 @@ impl TryFromObject for ArgIndex { } } -#[derive(Debug)] +#[derive(Debug, Copy, Clone)] #[repr(transparent)] pub struct ArgPrimitiveIndex { pub value: T, From e434ff5f6ecefc9d0653beadac20bd412d5cc3aa Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 20 Apr 2025 19:23:35 +0900 Subject: [PATCH 256/317] basic wasip2 support --- .cspell.dict/rust-more.txt | 2 ++ common/src/lib.rs | 1 + src/lib.rs | 2 ++ stdlib/src/lib.rs | 2 ++ 4 files changed, 7 insertions(+) diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index e92f1ff8c1..e5d5ac86c4 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -74,6 +74,8 @@ unic unistd unraw unsync +wasip1 +wasip2 wasmbind wasmtime widestring diff --git a/common/src/lib.rs b/common/src/lib.rs index a62c9517b4..a66d3ad9c5 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,5 +1,6 @@ //! A crate to hold types and functions common to all rustpython components. +#![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] #![cfg_attr(target_os = "redox", feature(byte_slice_trim_ascii, new_uninit))] #[macro_use] diff --git a/src/lib.rs b/src/lib.rs index e9c9c91294..262904c1cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,8 @@ //! //! The binary will have all the standard arguments of a python interpreter (including a REPL!) but //! it will have your modules loaded into the vm. + +#![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] #![allow(clippy::needless_doctest_main)] #[macro_use] diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 1e81606f3c..ed35b4db7c 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -1,6 +1,8 @@ // to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of // how `mod` works, but we want this sometimes for pymodule declarations + #![allow(clippy::module_inception)] +#![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] #![cfg_attr(target_os = "redox", feature(raw_ref_op))] #[macro_use] From 82a62382d0e3fc24934c40536d741c55ffc4e21d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 20 Apr 2025 16:05:36 -0700 Subject: [PATCH 257/317] add wave at 3.13.2 Signed-off-by: Ashwin Naren --- Lib/test/test_wave.py | 227 +++++++++++++++ Lib/wave.py | 663 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 890 insertions(+) create mode 100644 Lib/test/test_wave.py create mode 100644 Lib/wave.py diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py new file mode 100644 index 0000000000..5e771c8de9 --- /dev/null +++ b/Lib/test/test_wave.py @@ -0,0 +1,227 @@ +import unittest +from test import audiotests +from test import support +import io +import struct +import sys +import wave + + +class WaveTest(audiotests.AudioWriteTests, + audiotests.AudioTestsWithSourceFile): + module = wave + + +class WavePCM8Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm8.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 1 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 827F CB80 B184 0088 4B86 C883 3F81 837E 387A 3473 A96B 9A66 \ + 6D64 4662 8E60 6F60 D762 7B68 936F 5877 177B 757C 887B 5F7B \ + 917A BE7B 3C7C E67F 4F84 C389 418E D192 6E97 0296 FF94 0092 \ + C98E D28D 6F8F 4E8F 648C E38A 888A AB8B D18E 0B91 368E C48A \ + """) + + +class WavePCM16Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm16.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 2 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022EFFEA 4B5C00F9 311404EF 80DC0843 CBDF06B2 48AA03F3 BFE701B2 036BFE7C \ + B857FA3E B4B2F34F 2999EBCA 1A5FE6D7 EDFCE491 C626E279 0E05E0B8 EF27E02D \ + 5754E275 FB31E843 1373EF89 D827F72C 978BFB7A F5F7FC11 0866FB9C DF30FB42 \ + 117FFA36 3EE4FB5D BC75FCB6 66D5FF5F CF16040E 43220978 C1BC0EC8 511F12A4 \ + EEDF1755 82061666 7FFF1446 80001296 499C0EB2 52BA0DB9 EFB70F5C CE400FBC \ + E4B50CEB 63440A5A 08CA0A1F 2BBA0B0B 51460E47 8BCB113C B6F50EEA 44150A59 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 2) + + +class WavePCM24Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm24.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 3 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022D65FFEB9D 4B5A0F00FA54 3113C304EE2B 80DCD6084303 \ + CBDEC006B261 48A99803F2F8 BFE82401B07D 036BFBFE7B5D \ + B85756FA3EC9 B4B055F3502B 299830EBCB62 1A5CA7E6D99A \ + EDFA3EE491BD C625EBE27884 0E05A9E0B6CF EF2929E02922 \ + 5758D8E27067 FB3557E83E16 1377BFEF8402 D82C5BF7272A \ + 978F16FB7745 F5F865FC1013 086635FB9C4E DF30FCFB40EE \ + 117FE0FA3438 3EE6B8FB5AC3 BC77A3FCB2F4 66D6DAFF5F32 \ + CF13B9041275 431D69097A8C C1BB600EC74E 5120B912A2BA \ + EEDF641754C0 8207001664B7 7FFFFF14453F 8000001294E6 \ + 499C1B0EB3B2 52B73E0DBCA0 EFB2B20F5FD8 CE3CDB0FBE12 \ + E4B49C0CEA2D 6344A80A5A7C 08C8FE0A1FFE 2BB9860B0A0E \ + 51486F0E44E1 8BCC64113B05 B6F4EC0EEB36 4413170A5B48 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 3) + + +class WavePCM24ExtTest(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm24-ext.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 3 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022D65FFEB9D 4B5A0F00FA54 3113C304EE2B 80DCD6084303 \ + CBDEC006B261 48A99803F2F8 BFE82401B07D 036BFBFE7B5D \ + B85756FA3EC9 B4B055F3502B 299830EBCB62 1A5CA7E6D99A \ + EDFA3EE491BD C625EBE27884 0E05A9E0B6CF EF2929E02922 \ + 5758D8E27067 FB3557E83E16 1377BFEF8402 D82C5BF7272A \ + 978F16FB7745 F5F865FC1013 086635FB9C4E DF30FCFB40EE \ + 117FE0FA3438 3EE6B8FB5AC3 BC77A3FCB2F4 66D6DAFF5F32 \ + CF13B9041275 431D69097A8C C1BB600EC74E 5120B912A2BA \ + EEDF641754C0 8207001664B7 7FFFFF14453F 8000001294E6 \ + 499C1B0EB3B2 52B73E0DBCA0 EFB2B20F5FD8 CE3CDB0FBE12 \ + E4B49C0CEA2D 6344A80A5A7C 08C8FE0A1FFE 2BB9860B0A0E \ + 51486F0E44E1 8BCC64113B05 B6F4EC0EEB36 4413170A5B48 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 3) + + +class WavePCM32Test(WaveTest, unittest.TestCase): + sndfilename = 'pluck-pcm32.wav' + sndfilenframes = 3307 + nchannels = 2 + sampwidth = 4 + framerate = 11025 + nframes = 48 + comptype = 'NONE' + compname = 'not compressed' + frames = bytes.fromhex("""\ + 022D65BCFFEB9D92 4B5A0F8000FA549C 3113C34004EE2BC0 80DCD680084303E0 \ + CBDEC0C006B26140 48A9980003F2F8FC BFE8248001B07D92 036BFB60FE7B5D34 \ + B8575600FA3EC920 B4B05500F3502BC0 29983000EBCB6240 1A5CA7A0E6D99A60 \ + EDFA3E80E491BD40 C625EB80E27884A0 0E05A9A0E0B6CFE0 EF292940E0292280 \ + 5758D800E2706700 FB3557D8E83E1640 1377BF00EF840280 D82C5B80F7272A80 \ + 978F1600FB774560 F5F86510FC101364 086635A0FB9C4E20 DF30FC40FB40EE28 \ + 117FE0A0FA3438B0 3EE6B840FB5AC3F0 BC77A380FCB2F454 66D6DA80FF5F32B4 \ + CF13B980041275B0 431D6980097A8C00 C1BB60000EC74E00 5120B98012A2BAA0 \ + EEDF64C01754C060 820700001664B780 7FFFFFFF14453F40 800000001294E6E0 \ + 499C1B000EB3B270 52B73E000DBCA020 EFB2B2E00F5FD880 CE3CDB400FBE1270 \ + E4B49CC00CEA2D90 6344A8800A5A7CA0 08C8FE800A1FFEE0 2BB986C00B0A0E00 \ + 51486F800E44E190 8BCC6480113B0580 B6F4EC000EEB3630 441317800A5B48A0 \ + """) + if sys.byteorder != 'big': + frames = wave._byteswap(frames, 4) + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + not_exported = {'WAVE_FORMAT_PCM', 'WAVE_FORMAT_EXTENSIBLE', 'KSDATAFORMAT_SUBTYPE_PCM'} + support.check__all__(self, wave, not_exported=not_exported) + + def test_read_deprecations(self): + filename = support.findfile('pluck-pcm8.wav', subdir='audiodata') + with wave.open(filename) as reader: + with self.assertWarns(DeprecationWarning): + with self.assertRaises(wave.Error): + reader.getmark('mark') + with self.assertWarns(DeprecationWarning): + self.assertIsNone(reader.getmarkers()) + + def test_write_deprecations(self): + with io.BytesIO(b'') as tmpfile: + with wave.open(tmpfile, 'wb') as writer: + writer.setnchannels(1) + writer.setsampwidth(1) + writer.setframerate(1) + writer.setcomptype('NONE', 'not compressed') + + with self.assertWarns(DeprecationWarning): + with self.assertRaises(wave.Error): + writer.setmark(0, 0, 'mark') + with self.assertWarns(DeprecationWarning): + with self.assertRaises(wave.Error): + writer.getmark('mark') + with self.assertWarns(DeprecationWarning): + self.assertIsNone(writer.getmarkers()) + + +class WaveLowLevelTest(unittest.TestCase): + + def test_read_no_chunks(self): + b = b'SPAM' + with self.assertRaises(EOFError): + wave.open(io.BytesIO(b)) + + def test_read_no_riff_chunk(self): + b = b'SPAM' + struct.pack(' self.chunksize: + raise RuntimeError + self.file.seek(self.offset + pos, 0) + self.size_read = pos + + def tell(self): + if self.closed: + raise ValueError("I/O operation on closed file") + return self.size_read + + def read(self, size=-1): + """Read at most size bytes from the chunk. + If size is omitted or negative, read until the end + of the chunk. + """ + + if self.closed: + raise ValueError("I/O operation on closed file") + if self.size_read >= self.chunksize: + return b'' + if size < 0: + size = self.chunksize - self.size_read + if size > self.chunksize - self.size_read: + size = self.chunksize - self.size_read + data = self.file.read(size) + self.size_read = self.size_read + len(data) + if self.size_read == self.chunksize and \ + self.align and \ + (self.chunksize & 1): + dummy = self.file.read(1) + self.size_read = self.size_read + len(dummy) + return data + + def skip(self): + """Skip the rest of the chunk. + If you are not interested in the contents of the chunk, + this method should be called so that the file points to + the start of the next chunk. + """ + + if self.closed: + raise ValueError("I/O operation on closed file") + if self.seekable: + try: + n = self.chunksize - self.size_read + # maybe fix alignment + if self.align and (self.chunksize & 1): + n = n + 1 + self.file.seek(n, 1) + self.size_read = self.size_read + n + return + except OSError: + pass + while self.size_read < self.chunksize: + n = min(8192, self.chunksize - self.size_read) + dummy = self.read(n) + if not dummy: + raise EOFError + + +class Wave_read: + """Variables used in this class: + + These variables are available to the user though appropriate + methods of this class: + _file -- the open file with methods read(), close(), and seek() + set through the __init__() method + _nchannels -- the number of audio channels + available through the getnchannels() method + _nframes -- the number of audio frames + available through the getnframes() method + _sampwidth -- the number of bytes per audio sample + available through the getsampwidth() method + _framerate -- the sampling frequency + available through the getframerate() method + _comptype -- the AIFF-C compression type ('NONE' if AIFF) + available through the getcomptype() method + _compname -- the human-readable AIFF-C compression type + available through the getcomptype() method + _soundpos -- the position in the audio stream + available through the tell() method, set through the + setpos() method + + These variables are used internally only: + _fmt_chunk_read -- 1 iff the FMT chunk has been read + _data_seek_needed -- 1 iff positioned correctly in audio + file for readframes() + _data_chunk -- instantiation of a chunk class for the DATA chunk + _framesize -- size of one frame in the file + """ + + def initfp(self, file): + self._convert = None + self._soundpos = 0 + self._file = _Chunk(file, bigendian = 0) + if self._file.getname() != b'RIFF': + raise Error('file does not start with RIFF id') + if self._file.read(4) != b'WAVE': + raise Error('not a WAVE file') + self._fmt_chunk_read = 0 + self._data_chunk = None + while 1: + self._data_seek_needed = 1 + try: + chunk = _Chunk(self._file, bigendian = 0) + except EOFError: + break + chunkname = chunk.getname() + if chunkname == b'fmt ': + self._read_fmt_chunk(chunk) + self._fmt_chunk_read = 1 + elif chunkname == b'data': + if not self._fmt_chunk_read: + raise Error('data chunk before fmt chunk') + self._data_chunk = chunk + self._nframes = chunk.chunksize // self._framesize + self._data_seek_needed = 0 + break + chunk.skip() + if not self._fmt_chunk_read or not self._data_chunk: + raise Error('fmt chunk and/or data chunk missing') + + def __init__(self, f): + self._i_opened_the_file = None + if isinstance(f, str): + f = builtins.open(f, 'rb') + self._i_opened_the_file = f + # else, assume it is an open file object already + try: + self.initfp(f) + except: + if self._i_opened_the_file: + f.close() + raise + + def __del__(self): + self.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + # + # User visible methods. + # + def getfp(self): + return self._file + + def rewind(self): + self._data_seek_needed = 1 + self._soundpos = 0 + + def close(self): + self._file = None + file = self._i_opened_the_file + if file: + self._i_opened_the_file = None + file.close() + + def tell(self): + return self._soundpos + + def getnchannels(self): + return self._nchannels + + def getnframes(self): + return self._nframes + + def getsampwidth(self): + return self._sampwidth + + def getframerate(self): + return self._framerate + + def getcomptype(self): + return self._comptype + + def getcompname(self): + return self._compname + + def getparams(self): + return _wave_params(self.getnchannels(), self.getsampwidth(), + self.getframerate(), self.getnframes(), + self.getcomptype(), self.getcompname()) + + def getmarkers(self): + import warnings + warnings._deprecated("Wave_read.getmarkers", remove=(3, 15)) + return None + + def getmark(self, id): + import warnings + warnings._deprecated("Wave_read.getmark", remove=(3, 15)) + raise Error('no marks') + + def setpos(self, pos): + if pos < 0 or pos > self._nframes: + raise Error('position not in range') + self._soundpos = pos + self._data_seek_needed = 1 + + def readframes(self, nframes): + if self._data_seek_needed: + self._data_chunk.seek(0, 0) + pos = self._soundpos * self._framesize + if pos: + self._data_chunk.seek(pos, 0) + self._data_seek_needed = 0 + if nframes == 0: + return b'' + data = self._data_chunk.read(nframes * self._framesize) + if self._sampwidth != 1 and sys.byteorder == 'big': + data = _byteswap(data, self._sampwidth) + if self._convert and data: + data = self._convert(data) + self._soundpos = self._soundpos + len(data) // (self._nchannels * self._sampwidth) + return data + + # + # Internal methods. + # + + def _read_fmt_chunk(self, chunk): + try: + wFormatTag, self._nchannels, self._framerate, dwAvgBytesPerSec, wBlockAlign = struct.unpack_from(' 4: + raise Error('bad sample width') + self._sampwidth = sampwidth + + def getsampwidth(self): + if not self._sampwidth: + raise Error('sample width not set') + return self._sampwidth + + def setframerate(self, framerate): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if framerate <= 0: + raise Error('bad frame rate') + self._framerate = int(round(framerate)) + + def getframerate(self): + if not self._framerate: + raise Error('frame rate not set') + return self._framerate + + def setnframes(self, nframes): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + self._nframes = nframes + + def getnframes(self): + return self._nframeswritten + + def setcomptype(self, comptype, compname): + if self._datawritten: + raise Error('cannot change parameters after starting to write') + if comptype not in ('NONE',): + raise Error('unsupported compression type') + self._comptype = comptype + self._compname = compname + + def getcomptype(self): + return self._comptype + + def getcompname(self): + return self._compname + + def setparams(self, params): + nchannels, sampwidth, framerate, nframes, comptype, compname = params + if self._datawritten: + raise Error('cannot change parameters after starting to write') + self.setnchannels(nchannels) + self.setsampwidth(sampwidth) + self.setframerate(framerate) + self.setnframes(nframes) + self.setcomptype(comptype, compname) + + def getparams(self): + if not self._nchannels or not self._sampwidth or not self._framerate: + raise Error('not all parameters set') + return _wave_params(self._nchannels, self._sampwidth, self._framerate, + self._nframes, self._comptype, self._compname) + + def setmark(self, id, pos, name): + import warnings + warnings._deprecated("Wave_write.setmark", remove=(3, 15)) + raise Error('setmark() not supported') + + def getmark(self, id): + import warnings + warnings._deprecated("Wave_write.getmark", remove=(3, 15)) + raise Error('no marks') + + def getmarkers(self): + import warnings + warnings._deprecated("Wave_write.getmarkers", remove=(3, 15)) + return None + + def tell(self): + return self._nframeswritten + + def writeframesraw(self, data): + if not isinstance(data, (bytes, bytearray)): + data = memoryview(data).cast('B') + self._ensure_header_written(len(data)) + nframes = len(data) // (self._sampwidth * self._nchannels) + if self._convert: + data = self._convert(data) + if self._sampwidth != 1 and sys.byteorder == 'big': + data = _byteswap(data, self._sampwidth) + self._file.write(data) + self._datawritten += len(data) + self._nframeswritten = self._nframeswritten + nframes + + def writeframes(self, data): + self.writeframesraw(data) + if self._datalength != self._datawritten: + self._patchheader() + + def close(self): + try: + if self._file: + self._ensure_header_written(0) + if self._datalength != self._datawritten: + self._patchheader() + self._file.flush() + finally: + self._file = None + file = self._i_opened_the_file + if file: + self._i_opened_the_file = None + file.close() + + # + # Internal methods. + # + + def _ensure_header_written(self, datasize): + if not self._headerwritten: + if not self._nchannels: + raise Error('# channels not specified') + if not self._sampwidth: + raise Error('sample width not specified') + if not self._framerate: + raise Error('sampling rate not specified') + self._write_header(datasize) + + def _write_header(self, initlength): + assert not self._headerwritten + self._file.write(b'RIFF') + if not self._nframes: + self._nframes = initlength // (self._nchannels * self._sampwidth) + self._datalength = self._nframes * self._nchannels * self._sampwidth + try: + self._form_length_pos = self._file.tell() + except (AttributeError, OSError): + self._form_length_pos = None + self._file.write(struct.pack(' Date: Sun, 20 Apr 2025 21:15:29 -0700 Subject: [PATCH 258/317] update webbrowser and test_webbrowser to 3.13.3 Signed-off-by: Ashwin Naren --- Lib/test/test_webbrowser.py | 230 ++++++++++++++++++++--- Lib/webbrowser.py | 357 ++++++++++++++++++------------------ 2 files changed, 376 insertions(+), 211 deletions(-) diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 673cc995d3..4fcbc5c2e5 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -1,15 +1,22 @@ -import webbrowser -import unittest import os -import sys +import re +import shlex import subprocess -from unittest import mock +import sys +import unittest +import webbrowser from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper +from test.support import requires_subprocess +from test.support import threading_helper +from unittest import mock +# The webbrowser module uses threading locks +threading_helper.requires_working_threading(module=True) -URL = 'http://www.example.com' +URL = 'https://www.example.com' CMD_NAME = 'test' @@ -22,6 +29,7 @@ def wait(self, seconds=None): return 0 +@requires_subprocess() class CommandTestMixin: def _test(self, meth, *, args=[URL], kw={}, options, arguments): @@ -92,10 +100,19 @@ def test_open_new_tab(self): options=[], arguments=[URL]) + def test_open_bad_new_parameter(self): + with self.assertRaisesRegex(webbrowser.Error, + re.escape("Bad 'new' parameter to open(); " + "expected 0, 1, or 2, got 999")): + self._test('open', + options=[], + arguments=[URL], + kw=dict(new=999)) -class MozillaCommandTest(CommandTestMixin, unittest.TestCase): - browser_class = webbrowser.Mozilla +class EdgeCommandTest(CommandTestMixin, unittest.TestCase): + + browser_class = webbrowser.Edge def test_open(self): self._test('open', @@ -109,43 +126,43 @@ def test_open_with_autoraise_false(self): def test_open_new(self): self._test('open_new', - options=[], - arguments=['-new-window', URL]) + options=['--new-window'], + arguments=[URL]) def test_open_new_tab(self): self._test('open_new_tab', options=[], - arguments=['-new-tab', URL]) + arguments=[URL]) -class NetscapeCommandTest(CommandTestMixin, unittest.TestCase): +class MozillaCommandTest(CommandTestMixin, unittest.TestCase): - browser_class = webbrowser.Netscape + browser_class = webbrowser.Mozilla def test_open(self): self._test('open', - options=['-raise', '-remote'], - arguments=['openURL({})'.format(URL)]) + options=[], + arguments=[URL]) def test_open_with_autoraise_false(self): self._test('open', kw=dict(autoraise=False), - options=['-noraise', '-remote'], - arguments=['openURL({})'.format(URL)]) + options=[], + arguments=[URL]) def test_open_new(self): self._test('open_new', - options=['-raise', '-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + options=[], + arguments=['-new-window', URL]) def test_open_new_tab(self): self._test('open_new_tab', - options=['-raise', '-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + options=[], + arguments=['-new-tab', URL]) -class GaleonCommandTest(CommandTestMixin, unittest.TestCase): +class EpiphanyCommandTest(CommandTestMixin, unittest.TestCase): - browser_class = webbrowser.Galeon + browser_class = webbrowser.Epiphany def test_open(self): self._test('open', @@ -199,22 +216,89 @@ class ELinksCommandTest(CommandTestMixin, unittest.TestCase): def test_open(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_with_autoraise_false(self): self._test('open', options=['-remote'], - arguments=['openURL({})'.format(URL)]) + arguments=[f'openURL({URL})']) def test_open_new(self): self._test('open_new', options=['-remote'], - arguments=['openURL({},new-window)'.format(URL)]) + arguments=[f'openURL({URL},new-window)']) def test_open_new_tab(self): self._test('open_new_tab', options=['-remote'], - arguments=['openURL({},new-tab)'.format(URL)]) + arguments=[f'openURL({URL},new-tab)']) + + +@unittest.skipUnless(sys.platform == "ios", "Test only applicable to iOS") +class IOSBrowserTest(unittest.TestCase): + def _obj_ref(self, *args): + # Construct a string representation of the arguments that can be used + # as a proxy for object instance references + return "|".join(str(a) for a in args) + + @unittest.skipIf(getattr(webbrowser, "objc", None) is None, + "iOS Webbrowser tests require ctypes") + def setUp(self): + # Intercept the objc library. Wrap the calls to get the + # references to classes and selectors to return strings, and + # wrap msgSend to return stringified object references + self.orig_objc = webbrowser.objc + + webbrowser.objc = mock.Mock() + webbrowser.objc.objc_getClass = lambda cls: f"C#{cls.decode()}" + webbrowser.objc.sel_registerName = lambda sel: f"S#{sel.decode()}" + webbrowser.objc.objc_msgSend.side_effect = self._obj_ref + + def tearDown(self): + webbrowser.objc = self.orig_objc + + def _test(self, meth, **kwargs): + # The browser always gets focus, there's no concept of separate browser + # windows, and there's no API-level control over creating a new tab. + # Therefore, all calls to webbrowser are effectively the same. + getattr(webbrowser, meth)(URL, **kwargs) + + # The ObjC String version of the URL is created with UTF-8 encoding + url_string_args = [ + "C#NSString", + "S#stringWithCString:encoding:", + b'https://www.example.com', + 4, + ] + # The NSURL version of the URL is created from that string + url_obj_args = [ + "C#NSURL", + "S#URLWithString:", + self._obj_ref(*url_string_args), + ] + # The openURL call is invoked on the shared application + shared_app_args = ["C#UIApplication", "S#sharedApplication"] + + # Verify that the last call is the one that opens the URL. + webbrowser.objc.objc_msgSend.assert_called_with( + self._obj_ref(*shared_app_args), + "S#openURL:options:completionHandler:", + self._obj_ref(*url_obj_args), + None, + None + ) + + def test_open(self): + self._test('open') + + def test_open_with_autoraise_false(self): + self._test('open', autoraise=False) + + def test_open_new(self): + self._test('open_new') + + def test_open_new_tab(self): + self._test('open_new_tab') class BrowserRegistrationTest(unittest.TestCase): @@ -269,6 +353,16 @@ def test_register_default(self): def test_register_preferred(self): self._check_registration(preferred=True) + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") + def test_no_xdg_settings_on_macOS(self): + # On macOS webbrowser should not use xdg-settings to + # look for X11 based browsers (for those users with + # XQuartz installed) + with mock.patch("subprocess.check_output") as ck_o: + webbrowser.register_standard_browsers() + + ck_o.assert_not_called() + class ImportTest(unittest.TestCase): def test_register(self): @@ -294,29 +388,38 @@ def test_get(self): webbrowser.get('fakebrowser') self.assertIsNotNone(webbrowser._tryorder) + @unittest.skipIf(" " in sys.executable, "test assumes no space in path (GH-114452)") def test_synthesize(self): webbrowser = import_helper.import_fresh_module('webbrowser') name = os.path.basename(sys.executable).lower() webbrowser.register(name, None, webbrowser.GenericBrowser(name)) webbrowser.get(sys.executable) + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: browser = webbrowser.get().name - except (webbrowser.Error, AttributeError) as err: + except webbrowser.Error as err: self.skipTest(str(err)) with os_helper.EnvironmentVarGuard() as env: env["BROWSER"] = browser webbrowser = import_helper.import_fresh_module('webbrowser') webbrowser.get() + @unittest.skipIf( + is_apple_mobile, + "Apple mobile doesn't allow modifying browser with environment" + ) def test_environment_preferred(self): webbrowser = import_helper.import_fresh_module('webbrowser') try: webbrowser.get() least_preferred_browser = webbrowser.get(webbrowser._tryorder[-1]).name - except (webbrowser.Error, AttributeError, IndexError) as err: + except (webbrowser.Error, IndexError) as err: self.skipTest(str(err)) with os_helper.EnvironmentVarGuard() as env: @@ -330,5 +433,74 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) -if __name__=='__main__': +class CliTest(unittest.TestCase): + def test_parse_args(self): + for command, url, new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + args = webbrowser.parse_args(shlex.split(command)) + + self.assertEqual(args.url, url) + self.assertEqual(args.new_win, new_win) + + def test_parse_args_error(self): + for command in [ + # Arguments must not both be given + "https://example.com -n -t", + "https://example.com --new-window --new-tab", + "https://example.com -n --new-tab", + "https://example.com --new-window -t", + ]: + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split(command)) + self.assertIn( + 'error: argument -t/--new-tab: not allowed with argument -n/--new-window', + stderr.getvalue(), + ) + + # Ensure ambiguous shortening fails + with support.captured_stderr() as stderr: + with self.assertRaises(SystemExit): + webbrowser.parse_args(shlex.split("https://example.com --new")) + self.assertIn( + 'error: ambiguous option: --new could match --new-window, --new-tab', + stderr.getvalue() + ) + + def test_main(self): + for command, expected_url, expected_new_win in [ + # No optional arguments + ("https://example.com", "https://example.com", 0), + # Each optional argument + ("https://example.com -n", "https://example.com", 1), + ("-n https://example.com", "https://example.com", 1), + ("https://example.com -t", "https://example.com", 2), + ("-t https://example.com", "https://example.com", 2), + # Long form + ("https://example.com --new-window", "https://example.com", 1), + ("--new-window https://example.com", "https://example.com", 1), + ("https://example.com --new-tab", "https://example.com", 2), + ("--new-tab https://example.com", "https://example.com", 2), + ]: + with ( + mock.patch("webbrowser.open", return_value=None) as mock_open, + mock.patch("builtins.print", return_value=None), + ): + webbrowser.main(shlex.split(command)) + mock_open.assert_called_once_with(expected_url, expected_new_win) + + +if __name__ == '__main__': unittest.main() diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index ec3cece48c..2f9555ad60 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -11,14 +11,17 @@ __all__ = ["Error", "open", "open_new", "open_new_tab", "get", "register"] + class Error(Exception): pass + _lock = threading.RLock() _browsers = {} # Dictionary of available browser controllers _tryorder = None # Preference order of available browsers _os_preferred_browser = None # The preferred browser + def register(name, klass, instance=None, *, preferred=False): """Register a browser connector.""" with _lock: @@ -29,11 +32,12 @@ def register(name, klass, instance=None, *, preferred=False): # Preferred browsers go to the front of the list. # Need to match to the default browser returned by xdg-settings, which # may be of the form e.g. "firefox.desktop". - if preferred or (_os_preferred_browser and name in _os_preferred_browser): + if preferred or (_os_preferred_browser and f'{name}.desktop' == _os_preferred_browser): _tryorder.insert(0, name) else: _tryorder.append(name) + def get(using=None): """Return a browser launcher instance appropriate for the environment.""" if _tryorder is None: @@ -64,6 +68,7 @@ def get(using=None): return command[0]() raise Error("could not locate runnable browser") + # Please note: the following definition hides a builtin function. # It is recommended one does "import webbrowser" and uses webbrowser.open(url) # instead of "from webbrowser import *". @@ -76,6 +81,9 @@ def open(url, new=0, autoraise=True): - 1: a new browser window. - 2: a new browser page ("tab"). If possible, autoraise raises the window (the default) or not. + + If opening the browser succeeds, return True. + If there is a problem, return False. """ if _tryorder is None: with _lock: @@ -87,6 +95,7 @@ def open(url, new=0, autoraise=True): return True return False + def open_new(url): """Open url in a new window of the default browser. @@ -94,6 +103,7 @@ def open_new(url): """ return open(url, 1) + def open_new_tab(url): """Open url in a new page ("tab") of the default browser. @@ -136,7 +146,7 @@ def _synthesize(browser, *, preferred=False): # General parent classes -class BaseBrowser(object): +class BaseBrowser: """Parent class for all browsers. Do not use directly.""" args = ['%s'] @@ -197,7 +207,7 @@ def open(self, url, new=0, autoraise=True): else: p = subprocess.Popen(cmdline, close_fds=True, start_new_session=True) - return (p.poll() is None) + return p.poll() is None except OSError: return False @@ -225,7 +235,8 @@ def _invoke(self, args, remote, autoraise, url=None): # use autoraise argument only for remote invocation autoraise = int(autoraise) opt = self.raise_opts[autoraise] - if opt: raise_opt = [opt] + if opt: + raise_opt = [opt] cmdline = [self.name] + raise_opt + args @@ -266,8 +277,8 @@ def open(self, url, new=0, autoraise=True): else: action = self.remote_action_newtab else: - raise Error("Bad 'new' parameter to open(); " + - "expected 0, 1, or 2, got %s" % new) + raise Error("Bad 'new' parameter to open(); " + f"expected 0, 1, or 2, got {new}") args = [arg.replace("%s", url).replace("%action", action) for arg in self.remote_args] @@ -291,19 +302,8 @@ class Mozilla(UnixBrowser): background = True -class Netscape(UnixBrowser): - """Launcher class for Netscape browser.""" - - raise_opts = ["-noraise", "-raise"] - remote_args = ['-remote', 'openURL(%s%action)'] - remote_action = "" - remote_action_newwin = ",new-window" - remote_action_newtab = ",new-tab" - background = True - - -class Galeon(UnixBrowser): - """Launcher class for Galeon/Epiphany browsers.""" +class Epiphany(UnixBrowser): + """Launcher class for Epiphany browser.""" raise_opts = ["-noraise", ""] remote_args = ['%action', '%s'] @@ -313,7 +313,7 @@ class Galeon(UnixBrowser): class Chrome(UnixBrowser): - "Launcher class for Google Chrome browser." + """Launcher class for Google Chrome browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -321,11 +321,12 @@ class Chrome(UnixBrowser): remote_action_newtab = "" background = True + Chromium = Chrome class Opera(UnixBrowser): - "Launcher class for Opera browser." + """Launcher class for Opera browser.""" remote_args = ['%action', '%s'] remote_action = "" @@ -335,7 +336,7 @@ class Opera(UnixBrowser): class Elinks(UnixBrowser): - "Launcher class for Elinks browsers." + """Launcher class for Elinks browsers.""" remote_args = ['-remote', 'openURL(%s%action)'] remote_action = "" @@ -398,54 +399,17 @@ def open(self, url, new=0, autoraise=True): except OSError: return False else: - return (p.poll() is None) - - -class Grail(BaseBrowser): - # There should be a way to maintain a connection to Grail, but the - # Grail remote control protocol doesn't really allow that at this - # point. It probably never will! - def _find_grail_rc(self): - import glob - import pwd - import socket - import tempfile - tempdir = os.path.join(tempfile.gettempdir(), - ".grail-unix") - user = pwd.getpwuid(os.getuid())[0] - filename = os.path.join(glob.escape(tempdir), glob.escape(user) + "-*") - maybes = glob.glob(filename) - if not maybes: - return None - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - for fn in maybes: - # need to PING each one until we find one that's live - try: - s.connect(fn) - except OSError: - # no good; attempt to clean it out, but don't fail: - try: - os.unlink(fn) - except OSError: - pass - else: - return s + return p.poll() is None - def _remote(self, action): - s = self._find_grail_rc() - if not s: - return 0 - s.send(action) - s.close() - return 1 - def open(self, url, new=0, autoraise=True): - sys.audit("webbrowser.open", url) - if new: - ok = self._remote("LOADNEW " + url) - else: - ok = self._remote("LOAD " + url) - return ok +class Edge(UnixBrowser): + """Launcher class for Microsoft Edge browser.""" + + remote_args = ['%action', '%s'] + remote_action = "" + remote_action_newwin = "--new-window" + remote_action_newtab = "" + background = True # @@ -461,47 +425,44 @@ def register_X_browsers(): if shutil.which("xdg-open"): register("xdg-open", None, BackgroundBrowser("xdg-open")) + # Opens an appropriate browser for the URL scheme according to + # freedesktop.org settings (GNOME, KDE, XFCE, etc.) + if shutil.which("gio"): + register("gio", None, BackgroundBrowser(["gio", "open", "--", "%s"])) + + xdg_desktop = os.getenv("XDG_CURRENT_DESKTOP", "").split(":") + # The default GNOME3 browser - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gvfs-open"): + if (("GNOME" in xdg_desktop or + "GNOME_DESKTOP_SESSION_ID" in os.environ) and + shutil.which("gvfs-open")): register("gvfs-open", None, BackgroundBrowser("gvfs-open")) - # The default GNOME browser - if "GNOME_DESKTOP_SESSION_ID" in os.environ and shutil.which("gnome-open"): - register("gnome-open", None, BackgroundBrowser("gnome-open")) - # The default KDE browser - if "KDE_FULL_SESSION" in os.environ and shutil.which("kfmclient"): + if (("KDE" in xdg_desktop or + "KDE_FULL_SESSION" in os.environ) and + shutil.which("kfmclient")): register("kfmclient", Konqueror, Konqueror("kfmclient")) + # Common symbolic link for the default X11 browser if shutil.which("x-www-browser"): register("x-www-browser", None, BackgroundBrowser("x-www-browser")) # The Mozilla browsers - for browser in ("firefox", "iceweasel", "iceape", "seamonkey"): + for browser in ("firefox", "iceweasel", "seamonkey", "mozilla-firefox", + "mozilla"): if shutil.which(browser): register(browser, None, Mozilla(browser)) - # The Netscape and old Mozilla browsers - for browser in ("mozilla-firefox", - "mozilla-firebird", "firebird", - "mozilla", "netscape"): - if shutil.which(browser): - register(browser, None, Netscape(browser)) - # Konqueror/kfm, the KDE browser. if shutil.which("kfm"): register("kfm", Konqueror, Konqueror("kfm")) elif shutil.which("konqueror"): register("konqueror", Konqueror, Konqueror("konqueror")) - # Gnome's Galeon and Epiphany - for browser in ("galeon", "epiphany"): - if shutil.which(browser): - register(browser, None, Galeon(browser)) - - # Skipstone, another Gtk/Mozilla based browser - if shutil.which("skipstone"): - register("skipstone", None, BackgroundBrowser("skipstone")) + # Gnome's Epiphany + if shutil.which("epiphany"): + register("epiphany", None, Epiphany("epiphany")) # Google Chrome/Chromium browsers for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"): @@ -512,13 +473,9 @@ def register_X_browsers(): if shutil.which("opera"): register("opera", None, Opera("opera")) - # Next, Mosaic -- old but still in use. - if shutil.which("mosaic"): - register("mosaic", None, BackgroundBrowser("mosaic")) + if shutil.which("microsoft-edge"): + register("microsoft-edge", None, Edge("microsoft-edge")) - # Grail, the Python browser. Does anybody still use it? - if shutil.which("grail"): - register("grail", Grail, None) def register_standard_browsers(): global _tryorder @@ -532,6 +489,9 @@ def register_standard_browsers(): # OS X can use below Unix support (but we prefer using the OS X # specific stuff) + if sys.platform == "ios": + register("iosbrowser", None, IOSBrowser(), preferred=True) + if sys.platform == "serenityos": # SerenityOS webbrowser, simply called "Browser". register("Browser", None, BackgroundBrowser("Browser")) @@ -540,21 +500,33 @@ def register_standard_browsers(): # First try to use the default Windows browser register("windows-default", WindowsDefault) - # Detect some common Windows browsers, fallback to IE - iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), - "Internet Explorer\\IEXPLORE.EXE") - for browser in ("firefox", "firebird", "seamonkey", "mozilla", - "netscape", "opera", iexplore): + # Detect some common Windows browsers, fallback to Microsoft Edge + # location in 64-bit Windows + edge64 = os.path.join(os.environ.get("PROGRAMFILES(x86)", "C:\\Program Files (x86)"), + "Microsoft\\Edge\\Application\\msedge.exe") + # location in 32-bit Windows + edge32 = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"), + "Microsoft\\Edge\\Application\\msedge.exe") + for browser in ("firefox", "seamonkey", "mozilla", "chrome", + "opera", edge64, edge32): if shutil.which(browser): register(browser, None, BackgroundBrowser(browser)) + if shutil.which("MicrosoftEdge.exe"): + register("microsoft-edge", None, Edge("MicrosoftEdge.exe")) else: # Prefer X browsers if present - if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): + # + # NOTE: Do not check for X11 browser on macOS, + # XQuartz installation sets a DISPLAY environment variable and will + # autostart when someone tries to access the display. Mac users in + # general don't need an X11 browser. + if sys.platform != "darwin" and (os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError, PermissionError, NotADirectoryError) : + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError, NotADirectoryError): pass else: global _os_preferred_browser @@ -564,14 +536,15 @@ def register_standard_browsers(): # Also try console browsers if os.environ.get("TERM"): + # Common symbolic link for the default text-based browser if shutil.which("www-browser"): register("www-browser", None, GenericBrowser("www-browser")) - # The Links/elinks browsers + # The Links/elinks browsers if shutil.which("links"): register("links", None, GenericBrowser("links")) if shutil.which("elinks"): register("elinks", None, Elinks("elinks")) - # The Lynx browser , + # The Lynx browser , if shutil.which("lynx"): register("lynx", None, GenericBrowser("lynx")) # The w3m browser @@ -613,105 +586,125 @@ def open(self, url, new=0, autoraise=True): return True # -# Platform support for MacOS +# Platform support for macOS # if sys.platform == 'darwin': - # Adapted from patch submitted to SourceForge by Steven J. Burr - class MacOSX(BaseBrowser): - """Launcher class for Aqua browsers on Mac OS X - - Optionally specify a browser name on instantiation. Note that this - will not work for Aqua browsers if the user has moved the application - package after installation. - - If no browser is specified, the default browser, as specified in the - Internet System Preferences panel, will be used. - """ - def __init__(self, name): - self.name = name + class MacOSXOSAScript(BaseBrowser): + def __init__(self, name='default'): + super().__init__(name) def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) - assert "'" not in url - # hack for local urls - if not ':' in url: - url = 'file:'+url - - # new must be 0 or 1 - new = int(bool(new)) - if self.name == "default": - # User called open, open_new or get without a browser parameter - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser + url = url.replace('"', '%22') + if self.name == 'default': + script = f'open location "{url}"' # opens in default browser else: - # User called get and chose a browser - if self.name == "OmniWeb": - toWindow = "" - else: - # Include toWindow parameter of OpenURL command for browsers - # that support it. 0 == new window; -1 == existing - toWindow = "toWindow %d" % (new - 1) - cmd = 'OpenURL "%s"' % url.replace('"', '%22') - script = '''tell application "%s" - activate - %s %s - end tell''' % (self.name, cmd, toWindow) - # Open pipe to AppleScript through osascript command + script = f''' + tell application "{self.name}" + activate + open location "{url}" + end + ''' + osapipe = os.popen("osascript", "w") if osapipe is None: return False - # Write script to osascript's stdin + osapipe.write(script) rc = osapipe.close() return not rc - class MacOSXOSAScript(BaseBrowser): - def __init__(self, name): - self._name = name +# +# Platform support for iOS +# +if sys.platform == "ios": + from _ios_support import objc + if objc: + # If objc exists, we know ctypes is also importable. + from ctypes import c_void_p, c_char_p, c_ulong + class IOSBrowser(BaseBrowser): def open(self, url, new=0, autoraise=True): - if self._name == 'default': - script = 'open location "%s"' % url.replace('"', '%22') # opens in default browser - else: - script = ''' - tell application "%s" - activate - open location "%s" - end - '''%(self._name, url.replace('"', '%22')) - - osapipe = os.popen("osascript", "w") - if osapipe is None: + sys.audit("webbrowser.open", url) + # If ctypes isn't available, we can't open a browser + if objc is None: return False - osapipe.write(script) - rc = osapipe.close() - return not rc + # All the messages in this call return object references. + objc.objc_msgSend.restype = c_void_p + + # This is the equivalent of: + # NSString url_string = + # [NSString stringWithCString:url.encode("utf-8") + # encoding:NSUTF8StringEncoding]; + NSString = objc.objc_getClass(b"NSString") + constructor = objc.sel_registerName(b"stringWithCString:encoding:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_char_p, c_ulong] + url_string = objc.objc_msgSend( + NSString, + constructor, + url.encode("utf-8"), + 4, # NSUTF8StringEncoding = 4 + ) + + # Create an NSURL object representing the URL + # This is the equivalent of: + # NSURL *nsurl = [NSURL URLWithString:url]; + NSURL = objc.objc_getClass(b"NSURL") + urlWithString_ = objc.sel_registerName(b"URLWithString:") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p, c_void_p] + ns_url = objc.objc_msgSend(NSURL, urlWithString_, url_string) + + # Get the shared UIApplication instance + # This code is the equivalent of: + # UIApplication shared_app = [UIApplication sharedApplication] + UIApplication = objc.objc_getClass(b"UIApplication") + sharedApplication = objc.sel_registerName(b"sharedApplication") + objc.objc_msgSend.argtypes = [c_void_p, c_void_p] + shared_app = objc.objc_msgSend(UIApplication, sharedApplication) + + # Open the URL on the shared application + # This code is the equivalent of: + # [shared_app openURL:ns_url + # options:NIL + # completionHandler:NIL]; + openURL_ = objc.sel_registerName(b"openURL:options:completionHandler:") + objc.objc_msgSend.argtypes = [ + c_void_p, c_void_p, c_void_p, c_void_p, c_void_p + ] + # Method returns void + objc.objc_msgSend.restype = None + objc.objc_msgSend(shared_app, openURL_, ns_url, None, None) + return True -def main(): - import getopt - usage = """Usage: %s [-n | -t] url - -n: open new window - -t: open new tab""" % sys.argv[0] - try: - opts, args = getopt.getopt(sys.argv[1:], 'ntd') - except getopt.error as msg: - print(msg, file=sys.stderr) - print(usage, file=sys.stderr) - sys.exit(1) - new_win = 0 - for o, a in opts: - if o == '-n': new_win = 1 - elif o == '-t': new_win = 2 - if len(args) != 1: - print(usage, file=sys.stderr) - sys.exit(1) - - url = args[0] - open(url, new_win) + +def parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser(description="Open URL in a web browser.") + parser.add_argument("url", help="URL to open") + + group = parser.add_mutually_exclusive_group() + group.add_argument("-n", "--new-window", action="store_const", + const=1, default=0, dest="new_win", + help="open new window") + group.add_argument("-t", "--new-tab", action="store_const", + const=2, default=0, dest="new_win", + help="open new tab") + + args = parser.parse_args(arg_list) + + return args + + +def main(arg_list: list[str] | None = None): + args = parse_args(arg_list) + + open(args.url, args.new_win) print("\a") + if __name__ == "__main__": main() From 5ce860443c064fa70c2294458548a77d4f0b6775 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 20 Apr 2025 23:14:52 -0700 Subject: [PATCH 259/317] implement nt.getlogin Signed-off-by: Ashwin Naren --- vm/src/stdlib/nt.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 083824bcd0..cdab9e2f71 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -388,6 +388,27 @@ pub(crate) mod module { } } + #[pyfunction] + fn getlogin(vm: &VirtualMachine) -> PyResult { + let mut buffer = [0u16; 257]; + let mut size = buffer.len() as u32; + + let success = unsafe { + windows_sys::Win32::System::WindowsProgramming::GetUserNameW( + buffer.as_mut_ptr(), + &mut size, + ) + }; + + if success != 0 { + // Convert the buffer (which is UTF-16) to a Rust String + let username = std::ffi::OsString::from_wide(&buffer[..(size - 1) as usize]); + Ok(username.to_str().unwrap().to_string()) + } else { + Err(vm.new_os_error(format!("Error code: {success}"))) + } + } + pub fn raw_set_handle_inheritable(handle: intptr_t, inheritable: bool) -> std::io::Result<()> { let flags = if inheritable { Foundation::HANDLE_FLAG_INHERIT From ecbc6f70441ca54065a254f87023720ae40b29a1 Mon Sep 17 00:00:00 2001 From: Reagan Bohan Date: Mon, 21 Apr 2025 08:44:43 +0000 Subject: [PATCH 260/317] Fix mmap aborting with invalid fd in debug mode --- stdlib/src/mmap.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stdlib/src/mmap.rs b/stdlib/src/mmap.rs index bca367ae4d..9319bab64c 100644 --- a/stdlib/src/mmap.rs +++ b/stdlib/src/mmap.rs @@ -23,13 +23,15 @@ mod mmap { }; use crossbeam_utils::atomic::AtomicCell; use memmap2::{Advice, Mmap, MmapMut, MmapOptions}; + #[cfg(unix)] + use nix::sys::stat::fstat; use nix::unistd; use num_traits::Signed; use std::fs::File; - use std::io::Write; + use std::io::{self, Write}; use std::ops::{Deref, DerefMut}; #[cfg(unix)] - use std::os::unix::io::{FromRawFd, IntoRawFd, RawFd}; + use std::os::unix::io::{FromRawFd, RawFd}; fn advice_try_from_i32(vm: &VirtualMachine, i: i32) -> PyResult { Ok(match i { @@ -299,7 +301,7 @@ mod mmap { fn py_new( cls: PyTypeRef, MmapNewArgs { - fileno: mut fd, + fileno: fd, length, flags, prot, @@ -348,12 +350,10 @@ mod mmap { }; if fd != -1 { - let file = unsafe { File::from_raw_fd(fd) }; - let metadata = file.metadata().map_err(|err| err.to_pyexception(vm))?; - let file_len: libc::off_t = metadata.len().try_into().expect("file size overflow"); - // File::from_raw_fd will consume the fd, so we - // have to get it again. - fd = file.into_raw_fd(); + let metadata = fstat(fd) + .map_err(|err| io::Error::from_raw_os_error(err as i32).to_pyexception(vm))?; + let file_len = metadata.st_size; + if map_size == 0 { if file_len == 0 { return Err(vm.new_value_error("cannot mmap an empty file".to_owned())); From 494918d9fe6dd519b50a1e0e5a8d8ed498c9b619 Mon Sep 17 00:00:00 2001 From: Noa Date: Mon, 21 Apr 2025 13:41:48 -0500 Subject: [PATCH 261/317] Remove cfg_attr features for redox --- common/src/lib.rs | 1 - stdlib/src/lib.rs | 1 - vm/src/lib.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/common/src/lib.rs b/common/src/lib.rs index a66d3ad9c5..c99ba0286a 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -1,7 +1,6 @@ //! A crate to hold types and functions common to all rustpython components. #![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] -#![cfg_attr(target_os = "redox", feature(byte_slice_trim_ascii, new_uninit))] #[macro_use] mod macros; diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 149ee97853..7b3124c50d 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -3,7 +3,6 @@ #![allow(clippy::module_inception)] #![cfg_attr(all(target_os = "wasi", target_env = "p2"), feature(wasip2))] -#![cfg_attr(target_os = "redox", feature(raw_ref_op))] #[macro_use] extern crate rustpython_derive; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index de0042c619..e854518dc2 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -14,7 +14,6 @@ #![allow(clippy::upper_case_acronyms)] #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] #![doc(html_root_url = "https://docs.rs/rustpython-vm/")] -#![cfg_attr(target_os = "redox", feature(raw_ref_op))] #[cfg(feature = "flame-it")] #[macro_use] From a85a84330faf68c47477679ecef2427f951c9704 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 20:43:43 -0700 Subject: [PATCH 262/317] update pprint to 3.13.3 --- Lib/pprint.py | 3 + Lib/test/test_pprint.py | 298 ++++++++++++++-------------------------- 2 files changed, 103 insertions(+), 198 deletions(-) diff --git a/Lib/pprint.py b/Lib/pprint.py index 34ed12637e..9314701db3 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -128,6 +128,9 @@ def __init__(self, indent=1, width=80, depth=None, stream=None, *, sort_dicts If true, dict keys are sorted. + underscore_numbers + If true, digit groups are separated with underscores. + """ indent = int(indent) width = int(width) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 6ea7e7db2c..4e6fed1ab9 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -7,8 +7,8 @@ import itertools import pprint import random +import re import test.support -import test.test_set import types import unittest @@ -535,7 +535,10 @@ def test_dataclass_with_repr(self): def test_dataclass_no_repr(self): dc = dataclass3() formatted = pprint.pformat(dc, width=10) - self.assertRegex(formatted, r"") + self.assertRegex( + formatted, + fr"<{re.escape(__name__)}.dataclass3 object at \w+>", + ) def test_recursive_dataclass(self): dc = dataclass4(None) @@ -619,9 +622,6 @@ def test_set_reprs(self): self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), 'frozenset3({0, 1, 2, 3, 4, 5, 6})') - @unittest.expectedFailure - #See http://bugs.python.org/issue13907 - @test.support.cpython_only def test_set_of_sets_reprs(self): # This test creates a complex arrangement of frozensets and # compares the pretty-printed repr against a string hard-coded in @@ -632,204 +632,106 @@ def test_set_of_sets_reprs(self): # partial ordering (subset relationships), the output of the # list.sort() method is undefined for lists of sets." # - # In a nutshell, the test assumes frozenset({0}) will always - # sort before frozenset({1}), but: - # # >>> frozenset({0}) < frozenset({1}) # False # >>> frozenset({1}) < frozenset({0}) # False # - # Consequently, this test is fragile and - # implementation-dependent. Small changes to Python's sort - # algorithm cause the test to fail when it should pass. - # XXX Or changes to the dictionary implementation... - - cube_repr_tgt = """\ -{frozenset(): frozenset({frozenset({2}), frozenset({0}), frozenset({1})}), - frozenset({0}): frozenset({frozenset(), - frozenset({0, 2}), - frozenset({0, 1})}), - frozenset({1}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 1})}), - frozenset({2}): frozenset({frozenset(), - frozenset({1, 2}), - frozenset({0, 2})}), - frozenset({1, 2}): frozenset({frozenset({2}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 2}): frozenset({frozenset({2}), - frozenset({0}), - frozenset({0, 1, 2})}), - frozenset({0, 1}): frozenset({frozenset({0}), - frozenset({1}), - frozenset({0, 1, 2})}), - frozenset({0, 1, 2}): frozenset({frozenset({1, 2}), - frozenset({0, 2}), - frozenset({0, 1})})}""" - cube = test.test_set.cube(3) - self.assertEqual(pprint.pformat(cube), cube_repr_tgt) - cubo_repr_tgt = """\ -{frozenset({frozenset({0, 2}), frozenset({0})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1}), frozenset({1})}): frozenset({frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})})}), - frozenset({frozenset({1, 2}), frozenset({1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({1, 2}), frozenset({2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({0})}): frozenset({frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset(), frozenset({1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({1}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset()}): frozenset({frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset(), - frozenset({1})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 1})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 1})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({0}), frozenset({0, 1})}): frozenset({frozenset({frozenset(), - frozenset({0})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({1}), - frozenset({0, - 1})})}), - frozenset({frozenset({2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset(), - frozenset({2})})}), - frozenset({frozenset({0, 1, 2}), frozenset({0, 2})}): frozenset({frozenset({frozenset({1, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0}), - frozenset({0, - 2})}), - frozenset({frozenset({2}), - frozenset({0, - 2})})}), - frozenset({frozenset({1, 2}), frozenset({0, 1, 2})}): frozenset({frozenset({frozenset({0, - 2}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({0, - 1}), - frozenset({0, - 1, - 2})}), - frozenset({frozenset({2}), - frozenset({1, - 2})}), - frozenset({frozenset({1}), - frozenset({1, - 2})})})}""" - - cubo = test.test_set.linegraph(cube) - self.assertEqual(pprint.pformat(cubo), cubo_repr_tgt) + # In this test we list all possible invariants of the result + # for unordered frozensets. + # + # This test has a long history, see: + # - https://github.com/python/cpython/commit/969fe57baa0eb80332990f9cda936a33e13fabef + # - https://github.com/python/cpython/issues/58115 + # - https://github.com/python/cpython/issues/111147 + + import textwrap + + # Single-line, always ordered: + fs0 = frozenset() + fs1 = frozenset(('abc', 'xyz')) + data = frozenset((fs0, fs1)) + self.assertEqual(pprint.pformat(data), + 'frozenset({%r, %r})' % (fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + fs2 = frozenset(('one', 'two')) + data = {fs2: frozenset((fs0, fs1))} + self.assertEqual(pprint.pformat(data), + "{%r: frozenset({%r, %r})}" % (fs2, fs0, fs1)) + self.assertEqual(pprint.pformat(data), repr(data)) + + # Single-line, unordered: + fs1 = frozenset(("xyz", "qwerty")) + fs2 = frozenset(("abcd", "spam")) + fs = frozenset((fs1, fs2)) + self.assertEqual(pprint.pformat(fs), repr(fs)) + + # Multiline, unordered: + def check(res, invariants): + self.assertIn(res, [textwrap.dedent(i).strip() for i in invariants]) + + # Inner-most frozensets are singleline, result is multiline, unordered: + fs1 = frozenset(('regular string', 'other string')) + fs2 = frozenset(('third string', 'one more string')) + check( + pprint.pformat(frozenset((fs1, fs2))), + [ + """ + frozenset({%r, + %r}) + """ % (fs1, fs2), + """ + frozenset({%r, + %r}) + """ % (fs2, fs1), + ], + ) + + # Everything is multiline, unordered: + check( + pprint.pformat( + frozenset(( + frozenset(( + "xyz very-very long string", + "qwerty is also absurdly long", + )), + frozenset(( + "abcd is even longer that before", + "spam is not so long", + )), + )), + ), + [ + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'})}) + """, + + """ + frozenset({frozenset({'abcd is even longer that before', + 'spam is not so long'}), + frozenset({'xyz very-very long string', + 'qwerty is also absurdly long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'abcd is even longer that before', + 'spam is not so long'})}) + """, + + """ + frozenset({frozenset({'qwerty is also absurdly long', + 'xyz very-very long string'}), + frozenset({'spam is not so long', + 'abcd is even longer that before'})}) + """, + ], + ) def test_depth(self): nested_tuple = (1, (2, (3, (4, (5, 6))))) From a5214a0de75182fa802669f53946d0be2f8950d4 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 20:45:06 -0700 Subject: [PATCH 263/317] update colorsys to 3.13.3 --- Lib/colorsys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/colorsys.py b/Lib/colorsys.py index bc897bd0f9..e97f91718a 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -24,7 +24,7 @@ __all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb", "rgb_to_hsv","hsv_to_rgb"] -# Some floating point constants +# Some floating-point constants ONE_THIRD = 1.0/3.0 ONE_SIXTH = 1.0/6.0 From 6567d1d6ec50b3e63ebde4d45297d9c5736bacbe Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 20:48:13 -0700 Subject: [PATCH 264/317] update queue to 3.13.3 --- Lib/queue.py | 60 +++++- Lib/test/test_queue.py | 416 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 475 insertions(+), 1 deletion(-) diff --git a/Lib/queue.py b/Lib/queue.py index 55f5008846..25beb46e30 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -10,7 +10,15 @@ except ImportError: SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] +__all__ = [ + 'Empty', + 'Full', + 'ShutDown', + 'Queue', + 'PriorityQueue', + 'LifoQueue', + 'SimpleQueue', +] try: @@ -25,6 +33,10 @@ class Full(Exception): pass +class ShutDown(Exception): + '''Raised when put/get with shut-down queue.''' + + class Queue: '''Create a queue object with a given maximum size. @@ -54,6 +66,9 @@ def __init__(self, maxsize=0): self.all_tasks_done = threading.Condition(self.mutex) self.unfinished_tasks = 0 + # Queue shutdown state + self.is_shutdown = False + def task_done(self): '''Indicate that a formerly enqueued task is complete. @@ -65,6 +80,9 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -129,8 +147,12 @@ def put(self, item, block=True, timeout=None): Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down. ''' with self.not_full: + if self.is_shutdown: + raise ShutDown if self.maxsize > 0: if not block: if self._qsize() >= self.maxsize: @@ -138,6 +160,8 @@ def put(self, item, block=True, timeout=None): elif timeout is None: while self._qsize() >= self.maxsize: self.not_full.wait() + if self.is_shutdown: + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -147,6 +171,8 @@ def put(self, item, block=True, timeout=None): if remaining <= 0.0: raise Full self.not_full.wait(remaining) + if self.is_shutdown: + raise ShutDown self._put(item) self.unfinished_tasks += 1 self.not_empty.notify() @@ -161,14 +187,21 @@ def get(self, block=True, timeout=None): Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). + + Raises ShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. ''' with self.not_empty: + if self.is_shutdown and not self._qsize(): + raise ShutDown if not block: if not self._qsize(): raise Empty elif timeout is None: while not self._qsize(): self.not_empty.wait() + if self.is_shutdown and not self._qsize(): + raise ShutDown elif timeout < 0: raise ValueError("'timeout' must be a non-negative number") else: @@ -178,6 +211,8 @@ def get(self, block=True, timeout=None): if remaining <= 0.0: raise Empty self.not_empty.wait(remaining) + if self.is_shutdown and not self._qsize(): + raise ShutDown item = self._get() self.not_full.notify() return item @@ -198,6 +233,29 @@ def get_nowait(self): ''' return self.get(block=False) + def shutdown(self, immediate=False): + '''Shut-down the queue, making queue gets and puts raise ShutDown. + + By default, gets will only raise once the queue is empty. Set + 'immediate' to True to make gets raise immediately instead. + + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). + ''' + with self.mutex: + self.is_shutdown = True + if immediate: + while self._qsize(): + self._get() + if self.unfinished_tasks > 0: + self.unfinished_tasks -= 1 + # release all blocked threads in `join()` + self.all_tasks_done.notify_all() + # All getters need to re-check queue-empty to raise ShutDown + self.not_empty.notify_all() + self.not_full.notify_all() + # Override these methods to implement other queue organizations # (e.g. stack or priority queue). # These will only be called with appropriate locks held diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index cfa6003a86..93cbe1fe23 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -2,6 +2,7 @@ # to ensure the Queue locks remain stable. import itertools import random +import sys import threading import time import unittest @@ -10,6 +11,8 @@ from test.support import import_helper from test.support import threading_helper +# queue module depends on threading primitives +threading_helper.requires_working_threading(module=True) py_queue = import_helper.import_fresh_module('queue', blocked=['_queue']) c_queue = import_helper.import_fresh_module('queue', fresh=['_queue']) @@ -239,6 +242,418 @@ def test_shrinking_queue(self): with self.assertRaises(self.queue.Full): q.put_nowait(4) + def test_shutdown_empty(self): + q = self.type2test() + q.shutdown() + with self.assertRaises(self.queue.ShutDown): + q.put("data") + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_nonempty(self): + q = self.type2test() + q.put("data") + q.shutdown() + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_immediate(self): + q = self.type2test() + q.put("data") + q.shutdown(immediate=True) + with self.assertRaises(self.queue.ShutDown): + q.get() + + def test_shutdown_allowed_transitions(self): + # allowed transitions would be from alive via shutdown to immediate + q = self.type2test() + self.assertFalse(q.is_shutdown) + + q.shutdown() + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=True) + self.assertTrue(q.is_shutdown) + + q.shutdown(immediate=False) + + def _shutdown_all_methods_in_one_thread(self, immediate): + q = self.type2test(2) + q.put("L") + q.put_nowait("O") + q.shutdown(immediate) + + with self.assertRaises(self.queue.ShutDown): + q.put("E") + with self.assertRaises(self.queue.ShutDown): + q.put_nowait("W") + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get() + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + with self.assertRaises(ValueError): + q.task_done() + q.join() + else: + self.assertIn(q.get(), "LO") + q.task_done() + self.assertIn(q.get(), "LO") + q.task_done() + q.join() + # on shutdown(immediate=False) + # when queue is empty, should raise ShutDown Exception + with self.assertRaises(self.queue.ShutDown): + q.get() # p.get(True) + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() # p.get(False) + with self.assertRaises(self.queue.ShutDown): + q.get(True, 1.0) + + def test_shutdown_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(False) + + def test_shutdown_immediate_all_methods_in_one_thread(self): + return self._shutdown_all_methods_in_one_thread(True) + + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): + try: + q.put((i, "YDLO")) + except self.queue.ShutDown: + results.append(False) + break + + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: + try: + q.get(False) + q.task_done() + except self.queue.ShutDown: + results.append(True) + break + except self.queue.Empty: + pass + + def _shutdown_thread(self, q, results, event_end, immediate): + event_end.wait() + q.shutdown(immediate) + results.append(q.qsize() == 0) + + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() + q.join() + + def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. + q = self.type2test() + ps = [] + res_puts = [] + res_gets = [] + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. + for func, n, args in lprocs: + for i in range(n): + ps.append(threading.Thread(target=func, args=args)) + ps[-1].start() + for thread in ps: + thread.join() + + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + + def test_shutdown_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(False) + + def test_shutdown_immediate_all_methods_in_many_threads(self): + return self._shutdown_all_methods_in_many_threads(True) + + def _get(self, q, go, results, shutdown=False): + go.wait() + try: + msg = q.get() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _get_shutdown(self, q, go, results): + return self._get(q, go, results, True) + + def _get_task_done(self, q, go, results): + go.wait() + try: + msg = q.get() + q.task_done() + results.append(True) + return msg + except self.queue.ShutDown: + results.append(False) + return False + + def _put(self, q, msg, go, results, shutdown=False): + go.wait() + try: + q.put(msg) + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _put_shutdown(self, q, msg, go, results): + return self._put(q, msg, go, results, True) + + def _join(self, q, results, shutdown=False): + try: + q.join() + results.append(not shutdown) + return not shutdown + except self.queue.ShutDown: + results.append(shutdown) + return shutdown + + def _join_shutdown(self, q, results): + return self._join(q, results, True) + + def _shutdown_get(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue full + + if immediate: + thrds = ( + (self._get_shutdown, (q, go, results)), + (self._get_shutdown, (q, go, results)), + ) + else: + thrds = ( + # on shutdown(immediate=False) + # one of these threads should raise Shutdown + (self._get, (q, go, results)), + (self._get, (q, go, results)), + (self._get, (q, go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + if immediate: + self.assertListEqual(results, [True, True]) + else: + self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) + + def test_shutdown_get(self): + return self._shutdown_get(False) + + def test_shutdown_immediate_get(self): + return self._shutdown_get(True) + + def _shutdown_put(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + # queue fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._put_shutdown, (q, "W", go, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + q.shutdown() + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_put(self): + return self._shutdown_put(False) + + def test_shutdown_immediate_put(self): + return self._shutdown_put(True) + + def _shutdown_join(self, immediate): + q = self.type2test() + results = [] + q.put("Y") + go = threading.Event() + nb = q.qsize() + + thrds = ( + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + if not immediate: + res = [] + for i in range(nb): + threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) + threads[-1].start() + q.shutdown(immediate) + go.set() + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_join(self): + return self._shutdown_join(True) + + def test_shutdown_join(self): + return self._shutdown_join(False) + + def _shutdown_put_join(self, immediate): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + # queue not fulled + + thrds = ( + (self._put_shutdown, (q, "E", go, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + self.assertEqual(q.unfinished_tasks, 1) + + q.shutdown(immediate) + go.set() + + if immediate: + with self.assertRaises(self.queue.ShutDown): + q.get_nowait() + else: + result = q.get() + self.assertEqual(result, "Y") + q.task_done() + + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_immediate_put_join(self): + return self._shutdown_put_join(True) + + def test_shutdown_put_join(self): + return self._shutdown_put_join(False) + + def test_shutdown_get_task_done_join(self): + q = self.type2test(2) + results = [] + go = threading.Event() + q.put("Y") + q.put("D") + self.assertEqual(q.unfinished_tasks, q.qsize()) + + thrds = ( + (self._get_task_done, (q, go, results)), + (self._get_task_done, (q, go, results)), + (self._join, (q, results)), + (self._join, (q, results)), + ) + threads = [] + for func, params in thrds: + threads.append(threading.Thread(target=func, args=params)) + threads[-1].start() + go.set() + q.shutdown(False) + for t in threads: + t.join() + + self.assertEqual(results, [True]*len(thrds)) + + def test_shutdown_pending_get(self): + def get(): + try: + results.append(q.get()) + except Exception as e: + results.append(e) + + q = self.type2test() + results = [] + get_thread = threading.Thread(target=get) + get_thread.start() + q.shutdown(immediate=False) + get_thread.join(timeout=10.0) + self.assertFalse(get_thread.is_alive()) + self.assertEqual(len(results), 1) + self.assertIsInstance(results[0], self.queue.ShutDown) + + class QueueTest(BaseQueueTestMixin): def setUp(self): @@ -289,6 +704,7 @@ class CPriorityQueueTest(PriorityQueueTest, unittest.TestCase): # A Queue subclass that can provoke failure at a moment's notice :) class FailingQueueException(Exception): pass + class FailingQueueTest(BlockingTestMixin): def setUp(self): From 70f3aec5527a8e3dfb672d033eb98f55b352391f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 21:08:15 -0700 Subject: [PATCH 265/317] update linecache to 3.13.3 --- Lib/linecache.py | 91 +++++++++++++++++++++++++++++--------- Lib/test/test_linecache.py | 80 +++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 21 deletions(-) diff --git a/Lib/linecache.py b/Lib/linecache.py index 97644a8e37..dc02de19eb 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,17 +5,13 @@ that name. """ -import functools -import sys -import os -import tokenize - __all__ = ["getline", "clearcache", "checkcache", "lazycache"] # The cache. Maps filenames to either a thunk which will provide source code, # or a tuple (size, mtime, lines, fullname) once loaded. cache = {} +_interactive_cache = {} def clearcache(): @@ -49,28 +45,54 @@ def getlines(filename, module_globals=None): return [] +def _getline_from_code(filename, lineno): + lines = _getlines_from_code(filename) + if 1 <= lineno <= len(lines): + return lines[lineno - 1] + return '' + +def _make_key(code): + return (code.co_filename, code.co_qualname, code.co_firstlineno) + +def _getlines_from_code(code): + code_id = _make_key(code) + if code_id in _interactive_cache: + entry = _interactive_cache[code_id] + if len(entry) != 1: + return _interactive_cache[code_id][2] + return [] + + def checkcache(filename=None): """Discard cache entries that are out of date. (This is not checked upon each call!)""" if filename is None: - filenames = list(cache.keys()) - elif filename in cache: - filenames = [filename] + # get keys atomically + filenames = cache.copy().keys() else: - return + filenames = [filename] for filename in filenames: - entry = cache[filename] + try: + entry = cache[filename] + except KeyError: + continue + if len(entry) == 1: # lazy cache entry, leave it lazy. continue size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ + try: + # This import can fail if the interpreter is shutting down + import os + except ImportError: + return try: stat = os.stat(fullname) - except OSError: + except (OSError, ValueError): cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: @@ -82,6 +104,17 @@ def updatecache(filename, module_globals=None): If something's wrong, print a message, discard the cache entry, and return an empty list.""" + # These imports are not at top level because linecache is in the critical + # path of the interpreter startup and importing os and sys take a lot of time + # and slows down the startup sequence. + try: + import os + import sys + import tokenize + except ImportError: + # These import can fail if the interpreter is shutting down + return [] + if filename in cache: if len(cache[filename]) != 1: cache.pop(filename, None) @@ -128,16 +161,20 @@ def updatecache(filename, module_globals=None): try: stat = os.stat(fullname) break - except OSError: + except (OSError, ValueError): pass else: return [] + except ValueError: # may be raised by os.stat() + return [] try: with tokenize.open(fullname) as fp: lines = fp.readlines() except (OSError, UnicodeDecodeError, SyntaxError): return [] - if lines and not lines[-1].endswith('\n'): + if not lines: + lines = ['\n'] + elif not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname @@ -166,17 +203,29 @@ def lazycache(filename, module_globals): return False # Try for a __loader__, if available if module_globals and '__name__' in module_globals: - name = module_globals['__name__'] - if (loader := module_globals.get('__loader__')) is None: - if spec := module_globals.get('__spec__'): - try: - loader = spec.loader - except AttributeError: - pass + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + loader = getattr(spec, 'loader', None) + if loader is None: + loader = module_globals.get('__loader__') get_source = getattr(loader, 'get_source', None) if name and get_source: - get_lines = functools.partial(get_source, name) + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) cache[filename] = (get_lines,) return True return False + +def _register_code(code, string, name): + entry = (len(string), + None, + [line + '\n' for line in string.splitlines()], + name) + stack = [code] + while stack: + code = stack.pop() + for const in code.co_consts: + if isinstance(const, type(code)): + stack.append(const) + _interactive_cache[_make_key(code)] = entry diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 72dd40136c..e23e1cc942 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -5,8 +5,10 @@ import os.path import tempfile import tokenize +from importlib.machinery import ModuleSpec from test import support from test.support import os_helper +from test.support.script_helper import assert_python_ok FILENAME = linecache.__file__ @@ -82,6 +84,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] @@ -97,6 +103,16 @@ class BadUnicode_WithDeclaration(GetLineTestsBadData, unittest.TestCase): file_byte_string = b'# coding=utf-8\n\x80abc' +class FakeLoader: + def get_source(self, fullname): + return f'source for {fullname}' + + +class NoSourceLoader: + def get_source(self, fullname): + return None + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -238,6 +254,70 @@ def raise_memoryerror(*args, **kwargs): self.assertEqual(lines3, []) self.assertEqual(linecache.getlines(FILENAME), lines) + def test_loader(self): + filename = 'scheme://path' + + for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': loader} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + for spec in (None, object(), ModuleSpec('', FakeLoader())): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + + linecache.clearcache() + spec = ModuleSpec('x.y.z', FakeLoader()) + module_globals = {'__name__': 'a.b.c', '__loader__': spec.loader, + '__spec__': spec} + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for x.y.z\n']) + + def test_invalid_names(self): + for name, desc in [ + ('\x00', 'NUL bytes filename'), + (__file__ + '\x00', 'filename with embedded NUL bytes'), + # A filename with surrogate codes. A UnicodeEncodeError is raised + # by os.stat() upon querying, which is a subclass of ValueError. + ("\uD834\uDD1E.py", 'surrogate codes (MUSICAL SYMBOL G CLEF)'), + # For POSIX platforms, an OSError will be raised but for Windows + # platforms, a ValueError is raised due to the path_t converter. + # See: https://github.com/python/cpython/issues/122170 + ('a' * 1_000_000, 'very long filename'), + ]: + with self.subTest(f'updatecache: {desc}'): + linecache.clearcache() + lines = linecache.updatecache(name) + self.assertListEqual(lines, []) + self.assertNotIn(name, linecache.cache) + + # hack into the cache (it shouldn't be allowed + # but we never know what people do...) + for key, fullname in [(name, 'ok'), ('key', name), (name, name)]: + with self.subTest(f'checkcache: {desc}', + key=key, fullname=fullname): + linecache.clearcache() + linecache.cache[key] = (0, 1234, [], fullname) + linecache.checkcache(key) + self.assertNotIn(key, linecache.cache) + + # just to be sure that we did not mess with cache + linecache.clearcache() + + def test_linecache_python_string(self): + cmdline = "import linecache;assert len(linecache.cache) == 0" + retcode, stdout, stderr = assert_python_ok('-c', cmdline) + self.assertEqual(retcode, 0) + self.assertEqual(stdout, b'') + self.assertEqual(stderr, b'') class LineCacheInvalidationTests(unittest.TestCase): def setUp(self): From 1c64bde0ee11e3f457425565d87c88b9f048197d Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 21:09:52 -0700 Subject: [PATCH 266/317] update sched to 3.13.3 --- Lib/sched.py | 2 +- Lib/test/test_sched.py | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Lib/sched.py b/Lib/sched.py index 14613cf298..fb20639d45 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -11,7 +11,7 @@ implement simulated time by writing your own functions. This can also be used to integrate scheduling with STDWIN events; the delay function is allowed to modify the queue. Time can be expressed as -integers or floating point numbers, as long as it is consistent. +integers or floating-point numbers, as long as it is consistent. Events are specified by tuples (time, priority, action, argument, kwargs). As in UNIX, lower priority numbers mean higher priority; in this diff --git a/Lib/test/test_sched.py b/Lib/test/test_sched.py index 7ae7baae85..eb52ac7983 100644 --- a/Lib/test/test_sched.py +++ b/Lib/test/test_sched.py @@ -58,6 +58,7 @@ def test_enterabs(self): scheduler.run() self.assertEqual(l, [0.01, 0.02, 0.03, 0.04, 0.05]) + @threading_helper.requires_working_threading() def test_enter_concurrent(self): q = queue.Queue() fun = q.put @@ -91,10 +92,23 @@ def test_priority(self): l = [] fun = lambda x: l.append(x) scheduler = sched.scheduler(time.time, time.sleep) - for priority in [1, 2, 3, 4, 5]: - z = scheduler.enterabs(0.01, priority, fun, (priority,)) - scheduler.run() - self.assertEqual(l, [1, 2, 3, 4, 5]) + + cases = [ + ([1, 2, 3, 4, 5], [1, 2, 3, 4, 5]), + ([5, 4, 3, 2, 1], [1, 2, 3, 4, 5]), + ([2, 5, 3, 1, 4], [1, 2, 3, 4, 5]), + ([1, 2, 3, 2, 1], [1, 1, 2, 2, 3]), + ] + for priorities, expected in cases: + with self.subTest(priorities=priorities, expected=expected): + for priority in priorities: + scheduler.enterabs(0.01, priority, fun, (priority,)) + scheduler.run() + self.assertEqual(l, expected) + + # Cleanup: + self.assertTrue(scheduler.empty()) + l.clear() def test_cancel(self): l = [] @@ -111,6 +125,7 @@ def test_cancel(self): scheduler.run() self.assertEqual(l, [0.02, 0.03, 0.04]) + @threading_helper.requires_working_threading() def test_cancel_concurrent(self): q = queue.Queue() fun = q.put From 8603cd93871f7fa0c934e27b411f0599a2dc09f4 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 21:12:12 -0700 Subject: [PATCH 267/317] update heapq to 3.13.3 --- Lib/test/test_heapq.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index cb1e4505b0..1aa8e4e289 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -4,7 +4,6 @@ import unittest import doctest -from test import support from test.support import import_helper from unittest import TestCase, skipUnless from operator import itemgetter From 92e72aabdcb458217b0cf8ca353520d9b5e356d1 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 21 Apr 2025 21:21:10 -0700 Subject: [PATCH 268/317] update graphlib to 3.13.3 --- Lib/graphlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/graphlib.py b/Lib/graphlib.py index 636545648e..9512865a8e 100644 --- a/Lib/graphlib.py +++ b/Lib/graphlib.py @@ -154,7 +154,7 @@ def done(self, *nodes): This method unblocks any successor of each node in *nodes* for being returned in the future by a call to "get_ready". - Raises :exec:`ValueError` if any node in *nodes* has already been marked as + Raises ValueError if any node in *nodes* has already been marked as processed by a previous call to this method, if a node was not added to the graph by using "add" or if called without calling "prepare" previously or if node has not yet been returned by "get_ready". From c97f4d1daf02fe08d73f8571787377ca28a0cdaa Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 21:28:38 -0700 Subject: [PATCH 269/317] Failure marker (#5695) * initial auto-upgrader * platform support * use ast walking * detect testname automatically * handle classes properly * add instructions to fix_test.py --- scripts/fix_test.py | 137 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 scripts/fix_test.py diff --git a/scripts/fix_test.py b/scripts/fix_test.py new file mode 100644 index 0000000000..99dfa2699a --- /dev/null +++ b/scripts/fix_test.py @@ -0,0 +1,137 @@ +""" +An automated script to mark failures in python test suite. +It adds @unittest.expectedFailure to the test functions that are failing in RustPython, but not in CPython. +As well as marking the test with a TODO comment. + +How to use: +1. Copy a specific test from the CPython repository to the RustPython repository. +2. Remove all unexpected failures from the test and skip the tests that hang +3. Run python ./scripts/fix_test.py --test test_venv --path ./Lib/test/test_venv.py or equivalent for the test from the project root. +4. Ensure that there are no unexpected successes in the test. +5. Actually fix the test. +""" +import argparse +import ast +import itertools +import platform +from pathlib import Path + +def parse_args(): + parser = argparse.ArgumentParser(description="Fix test.") + parser.add_argument("--path", type=Path, help="Path to test file") + parser.add_argument("--force", action="store_true", help="Force modification") + parser.add_argument("--platform", action="store_true", help="Platform specific failure") + + args = parser.parse_args() + return args + +class Test: + name: str = "" + path: str = "" + result: str = "" + + def __str__(self): + return f"Test(name={self.name}, path={self.path}, result={self.result})" + +class TestResult: + tests_result: str = "" + tests = [] + stdout = "" + + def __str__(self): + return f"TestResult(tests_result={self.tests_result},tests={len(self.tests)})" + + +def parse_results(result): + lines = result.stdout.splitlines() + test_results = TestResult() + test_results.stdout = result.stdout + in_test_results = False + for line in lines: + if line == "Run tests sequentially": + in_test_results = True + elif line.startswith("-----------"): + in_test_results = False + if in_test_results and not line.startswith("tests") and not line.startswith("["): + line = line.split(" ") + if line != [] and len(line) > 3: + test = Test() + test.name = line[0] + test.path = line[1].strip("(").strip(")") + test.result = " ".join(line[3:]).lower() + test_results.tests.append(test) + else: + if "== Tests result: " in line: + res = line.split("== Tests result: ")[1] + res = res.split(" ")[0] + test_results.tests_result = res + return test_results + +def path_to_test(path) -> list[str]: + return path.split(".")[2:] + +def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: + a = ast.parse(file) + lines = file.splitlines() + fixture = "@unittest.expectedFailure" + for node in ast.walk(a): + if isinstance(node, ast.FunctionDef): + if node.name == test[-1]: + assert not for_platform + indent = " " * node.col_offset + lines.insert(node.lineno - 1, indent + fixture) + lines.insert(node.lineno - 1, indent + "# TODO: RUSTPYTHON") + break + return "\n".join(lines) + +def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> str: + a = ast.parse(file) + lines = file.splitlines() + fixture = "@unittest.expectedFailure" + for key, node in ast.iter_fields(a): + if key == "body": + for i, n in enumerate(node): + match n: + case ast.ClassDef(): + if len(test) == 2 and test[0] == n.name: + # look through body for function def + for i, fn in enumerate(n.body): + match fn: + case ast.FunctionDef(): + if fn.name == test[-1]: + assert not for_platform + indent = " " * fn.col_offset + lines.insert(fn.lineno - 1, indent + fixture) + lines.insert(fn.lineno - 1, indent + "# TODO: RUSTPYTHON") + break + case ast.FunctionDef(): + if n.name == test[0] and len(test) == 1: + assert not for_platform + indent = " " * n.col_offset + lines.insert(n.lineno - 1, indent + fixture) + lines.insert(n.lineno - 1, indent + "# TODO: RUSTPYTHON") + break + if i > 500: + exit() + return "\n".join(lines) + +def run_test(test_name): + print(f"Running test: {test_name}") + rustpython_location = "./target/release/rustpython" + import subprocess + result = subprocess.run([rustpython_location, "-m", "test", "-v", test_name], capture_output=True, text=True) + return parse_results(result) + + +if __name__ == "__main__": + args = parse_args() + test_name = args.path.stem + tests = run_test(test_name) + f = open(args.path).read() + for test in tests.tests: + if test.result == "fail" or test.result == "error": + print("Modifying test:", test.name) + f = modify_test_v2(f, path_to_test(test.path), args.platform) + with open(args.path, "w") as file: + # TODO: Find validation method, and make --force override it + file.write(f) From 7aad6e03e31cca9cb7208fa6ddfc30f19c588ed8 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 25 Apr 2025 00:22:26 -0500 Subject: [PATCH 270/317] Update to malachite 0.6 --- Cargo.lock | 59 ++++++++++-------------------------------------------- Cargo.toml | 6 +++--- 2 files changed, 14 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0255535c1b..ffb31bfcbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -725,27 +725,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "derive_more" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", - "unicode-xid", -] - [[package]] name = "digest" version = "0.10.7" @@ -1184,15 +1163,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1457,23 +1427,22 @@ dependencies = [ [[package]] name = "malachite-base" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5063891d2cec8fd20cabccbd3fc277bf8d5666f481fb3f79d999559b39a62713" +checksum = "554bcf7f816ff3c1eae8f2b95c4375156884c79988596a6d01b7b070710fa9e5" dependencies = [ "hashbrown", - "itertools 0.11.0", + "itertools 0.14.0", "libm", "ryu", ] [[package]] name = "malachite-bigint" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1b1fec8b370139968919a5b77071c94b282eaba3da1cf179ae5299060d4e75" +checksum = "df1acde414186498b2a6a1e271f8ce5d65eaa5c492e95271121f30718fe2f925" dependencies = [ - "derive_more", "malachite-base", "malachite-nz", "num-integer", @@ -1483,22 +1452,22 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175263cd5b846c552b9afb9d4b03ca465b4ad10717d789cad7dac24441c4fcbe" +checksum = "f43d406336c42a59e07813b57efd651db00118af84c640a221d666964b2ec71f" dependencies = [ - "itertools 0.11.0", + "itertools 0.14.0", "libm", "malachite-base", ] [[package]] name = "malachite-q" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5261ba8feb1ad20cddab3d625af28206c663c08014b2e5c5f9bd54b0f230234f" +checksum = "25911a58ea0426e0b7bb1dffc8324e82711c82abff868b8523ae69d8a47e8062" dependencies = [ - "itertools 0.11.0", + "itertools 0.14.0", "malachite-base", "malachite-nz", ] @@ -3230,12 +3199,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unicode_names2" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index fb066e5dfd..c59344d2d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,9 +171,9 @@ libc = "0.2.169" libffi = "3.2" log = "0.4.27" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.5" -malachite-q = "0.5" -malachite-base = "0.5" +malachite-bigint = "0.6" +malachite-q = "0.6" +malachite-base = "0.6" memchr = "2.7.4" num-complex = "0.4.6" num-integer = "0.1.46" From ca496fb3b1bc405658c1d9d02124cac3e323cdfb Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 26 Apr 2025 22:59:31 -0700 Subject: [PATCH 271/317] Collect whats_left data with sqlite feature This fixes the output on the website, and correctly shows `_sqlite3` as implemented --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 4e88d749fb..e7d14298cd 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -84,7 +84,7 @@ jobs: - name: Collect what is left data run: | chmod +x ./whats_left.py - ./whats_left.py > whats_left.temp + ./whats_left.py --features "sqlite" > whats_left.temp env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - name: Upload data to the website From d46bcd9291e3e371df6825f6316faac73111e2ec Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 26 Apr 2025 23:26:37 -0700 Subject: [PATCH 272/317] typing upgrade to 3.13.2 (#5590) --- Lib/_py_abc.py | 6 +- Lib/test/support/testcase.py | 57 + Lib/test/test_dataclasses.py | 4 + Lib/test/test_exception_group.py | 2 - Lib/test/test_future_stmt/test_future.py | 2 - Lib/test/test_genericalias.py | 10 + Lib/test/test_types.py | 16 + Lib/test/test_typing.py | 6637 +++++++++++++++--- Lib/test/typinganndata/_typed_dict_helper.py | 30 + Lib/test/typinganndata/ann_module695.py | 72 + Lib/test/typinganndata/mod_generics_cache.py | 26 + Lib/typing.py | 2822 +++++--- vm/src/stdlib/typing.rs | 105 +- 13 files changed, 7817 insertions(+), 1972 deletions(-) create mode 100644 Lib/test/typinganndata/_typed_dict_helper.py create mode 100644 Lib/test/typinganndata/ann_module695.py create mode 100644 Lib/test/typinganndata/mod_generics_cache.py diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index c870ae9048..4780f9a619 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -33,6 +33,8 @@ class ABCMeta(type): _abc_invalidation_counter = 0 def __new__(mcls, name, bases, namespace, /, **kwargs): + # TODO: RUSTPYTHON remove this line (prevents duplicate bases) + bases = tuple(dict.fromkeys(bases)) cls = super().__new__(mcls, name, bases, namespace, **kwargs) # Compute set of abstract method names abstracts = {name @@ -98,8 +100,8 @@ def __instancecheck__(cls, instance): subtype = type(instance) if subtype is subclass: if (cls._abc_negative_cache_version == - ABCMeta._abc_invalidation_counter and - subclass in cls._abc_negative_cache): + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): return False # Fall back to the subclass check. return cls.__subclasscheck__(subclass) diff --git a/Lib/test/support/testcase.py b/Lib/test/support/testcase.py index fad1e4cb34..fd32457d14 100644 --- a/Lib/test/support/testcase.py +++ b/Lib/test/support/testcase.py @@ -1,6 +1,63 @@ from math import copysign, isnan +class ExtraAssertions: + + def assertIsSubclass(self, cls, superclass, msg=None): + if issubclass(cls, superclass): + return + standardMsg = f'{cls!r} is not a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotIsSubclass(self, cls, superclass, msg=None): + if not issubclass(cls, superclass): + return + standardMsg = f'{cls!r} is a subclass of {superclass!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertHasAttr(self, obj, name, msg=None): + if not hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has no attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has no attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has no attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotHasAttr(self, obj, name, msg=None): + if hasattr(obj, name): + if isinstance(obj, types.ModuleType): + standardMsg = f'module {obj.__name__!r} has unexpected attribute {name!r}' + elif isinstance(obj, type): + standardMsg = f'type object {obj.__name__!r} has unexpected attribute {name!r}' + else: + standardMsg = f'{type(obj).__name__!r} object has unexpected attribute {name!r}' + self.fail(self._formatMessage(msg, standardMsg)) + + def assertStartsWith(self, s, prefix, msg=None): + if s.startswith(prefix): + return + standardMsg = f"{s!r} doesn't start with {prefix!r}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotStartsWith(self, s, prefix, msg=None): + if not s.startswith(prefix): + return + self.fail(self._formatMessage(msg, f"{s!r} starts with {prefix!r}")) + + def assertEndsWith(self, s, suffix, msg=None): + if s.endswith(suffix): + return + standardMsg = f"{s!r} doesn't end with {suffix!r}" + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNotEndsWith(self, s, suffix, msg=None): + if not s.endswith(suffix): + return + self.fail(self._formatMessage(msg, f"{s!r} ends with {suffix!r}")) + + class ExceptionIsLikeMixin: def assertExceptionIsLike(self, exc, template): """ diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 62ae5622a8..8094962ccf 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -1906,6 +1906,8 @@ def new_method(self): c = Alias(10, 1.0) self.assertEqual(c.new_method(), 1.0) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_generic_dynamic(self): T = TypeVar('T') @@ -3250,6 +3252,8 @@ def test_classvar_module_level_import(self): # won't exist on the instance. self.assertNotIn('not_iv4', c.__dict__) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_text_annotations(self): from test import dataclass_textanno diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index d0d81490df..9d156a160c 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -15,8 +15,6 @@ def test_exception_is_not_generic_type(self): with self.assertRaisesRegex(TypeError, 'Exception'): Exception[OSError] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_group_is_generic_type(self): E = OSError self.assertIsInstance(ExceptionGroup[E], types.GenericAlias) diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 0e08051038..9c30054963 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -442,8 +442,6 @@ def foo(): def bar(arg: (yield)): pass """)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_type_hints_on_func_with_variadic_arg(self): # `typing.get_type_hints` might break on a function with a variadic # annotation (e.g. `f(*args: *Ts)`) if `from __future__ import diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 560b96f7e3..4d630ed166 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -173,6 +173,8 @@ def test_exposed_type(self): self.assertEqual(a.__args__, (int,)) self.assertEqual(a.__parameters__, ()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_parameters(self): from typing import List, Dict, Callable D0 = dict[str, int] @@ -212,6 +214,8 @@ def test_parameters(self): self.assertEqual(L5.__args__, (Callable[[K, V], K],)) self.assertEqual(L5.__parameters__, (K, V)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_parameter_chaining(self): from typing import List, Dict, Union, Callable self.assertEqual(list[T][int], list[int]) @@ -271,6 +275,8 @@ class MyType(type): with self.assertRaises(TypeError): MyType[int] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pickle(self): alias = GenericAlias(list, T) for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -280,6 +286,8 @@ def test_pickle(self): self.assertEqual(loaded.__args__, alias.__args__) self.assertEqual(loaded.__parameters__, alias.__parameters__) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_copy(self): class X(list): def __copy__(self): @@ -303,6 +311,8 @@ def test_union(self): self.assertEqual(a.__args__, (list[int], list[str])) self.assertEqual(a.__parameters__, ()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_generic(self): a = typing.Union[list[T], tuple[T, ...]] self.assertEqual(a.__args__, (list[T], tuple[T, ...])) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 59dc9814fb..c62bf61181 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -742,6 +742,8 @@ def test_instancecheck_and_subclasscheck(self): self.assertTrue(issubclass(dict, x)) self.assertFalse(issubclass(list, x)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_instancecheck_and_subclasscheck_order(self): T = typing.TypeVar('T') @@ -788,6 +790,8 @@ def __subclasscheck__(cls, sub): self.assertTrue(issubclass(int, x)) self.assertRaises(ZeroDivisionError, issubclass, list, x) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_or_type_operator_with_TypeVar(self): TV = typing.TypeVar('T') assert TV | str == typing.Union[TV, str] @@ -795,6 +799,8 @@ def test_or_type_operator_with_TypeVar(self): self.assertIs((int | TV)[int], int) self.assertIs((TV | int)[int], int) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_args(self): def check(arg, expected): clear_typing_caches() @@ -825,6 +831,8 @@ def check(arg, expected): check(x | None, (x, type(None))) check(None | x, (type(None), x)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_parameter_chaining(self): T = typing.TypeVar("T") S = typing.TypeVar("S") @@ -869,6 +877,8 @@ def eq(actual, expected, typed=True): eq(x[NT], int | NT | bytes) eq(x[S], int | S | bytes) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_pickle(self): orig = list[T] | int for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -878,6 +888,8 @@ def test_union_pickle(self): self.assertEqual(loaded.__args__, orig.__args__) self.assertEqual(loaded.__parameters__, orig.__parameters__) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_copy(self): orig = list[T] | int for copied in (copy.copy(orig), copy.deepcopy(orig)): @@ -885,12 +897,16 @@ def test_union_copy(self): self.assertEqual(copied.__args__, orig.__args__) self.assertEqual(copied.__parameters__, orig.__parameters__) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_parameter_substitution_errors(self): T = typing.TypeVar("T") x = int | T with self.assertRaises(TypeError): x[int, str] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_or_type_operator_with_forward(self): T = typing.TypeVar('T') ForwardAfter = T | 'Forward' diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b6a167f998..70397e2649 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,64 +1,80 @@ import contextlib import collections +import collections.abc +from collections import defaultdict +from functools import lru_cache, wraps, reduce +import gc +import inspect +import itertools +import operator import pickle import re import sys -from unittest import TestCase, main, skipUnless, skip -# TODO: RUSTPYTHON -import unittest +from unittest import TestCase, main, skip +from unittest.mock import patch from copy import copy, deepcopy -from typing import Any, NoReturn -from typing import TypeVar, AnyStr +from typing import Any, NoReturn, Never, assert_never +from typing import overload, get_overloads, clear_overloads +from typing import TypeVar, TypeVarTuple, Unpack, AnyStr from typing import T, KT, VT # Not in __all__. from typing import Union, Optional, Literal from typing import Tuple, List, Dict, MutableMapping from typing import Callable from typing import Generic, ClassVar, Final, final, Protocol -from typing import cast, runtime_checkable +from typing import assert_type, cast, runtime_checkable from typing import get_type_hints -from typing import get_origin, get_args -from typing import is_typeddict +from typing import get_origin, get_args, get_protocol_members +from typing import is_typeddict, is_protocol +from typing import reveal_type +from typing import dataclass_transform from typing import no_type_check, no_type_check_decorator from typing import Type -from typing import NewType -from typing import NamedTuple, TypedDict +from typing import NamedTuple, NotRequired, Required, ReadOnly, TypedDict from typing import IO, TextIO, BinaryIO from typing import Pattern, Match from typing import Annotated, ForwardRef +from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard +from typing import TypeGuard, TypeIs, NoDefault import abc +import textwrap import typing import weakref import types -from test import mod_generics_cache -from test import _typed_dict_helper +from test.support import captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper +from test.support.testcase import ExtraAssertions +from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper +# TODO: RUSTPYTHON +import unittest -class BaseTestCase(TestCase): +CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' +NOT_A_BASE_TYPE = "type 'typing.%s' is not an acceptable base type" +CANNOT_SUBCLASS_INSTANCE = 'Cannot subclass an instance of %s' - def assertIsSubclass(self, cls, class_or_tuple, msg=None): - if not issubclass(cls, class_or_tuple): - message = '%r is not a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) - def assertNotIsSubclass(self, cls, class_or_tuple, msg=None): - if issubclass(cls, class_or_tuple): - message = '%r is a subclass of %r' % (cls, class_or_tuple) - if msg is not None: - message += ' : %s' % msg - raise self.failureException(message) +class BaseTestCase(TestCase, ExtraAssertions): def clear_caches(self): for f in typing._cleanups: f() +def all_pickle_protocols(test_func): + """Runs `test_func` with various values for `proto` argument.""" + + @wraps(test_func) + def wrapper(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_proto=proto): + test_func(self, proto=proto) + + return wrapper + + class Employee: pass @@ -81,28 +97,59 @@ def test_any_instance_type_error(self): with self.assertRaises(TypeError): isinstance(42, Any) - def test_any_subclass_type_error(self): - with self.assertRaises(TypeError): - issubclass(Employee, Any) - with self.assertRaises(TypeError): - issubclass(Any, Employee) - def test_repr(self): self.assertEqual(repr(Any), 'typing.Any') + class Sub(Any): pass + self.assertEqual( + repr(Sub), + f".Sub'>", + ) + def test_errors(self): with self.assertRaises(TypeError): - issubclass(42, Any) + isinstance(42, Any) with self.assertRaises(TypeError): Any[int] # Any is not a generic type. - def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class A(Any): - pass - with self.assertRaises(TypeError): - class A(type(Any)): - pass + def test_can_subclass(self): + class Mock(Any): pass + self.assertTrue(issubclass(Mock, Any)) + self.assertIsInstance(Mock(), Mock) + + class Something: pass + self.assertFalse(issubclass(Something, Any)) + self.assertNotIsInstance(Something(), Mock) + + class MockSomething(Something, Mock): pass + self.assertTrue(issubclass(MockSomething, Any)) + self.assertTrue(issubclass(MockSomething, MockSomething)) + self.assertTrue(issubclass(MockSomething, Something)) + self.assertTrue(issubclass(MockSomething, Mock)) + ms = MockSomething() + self.assertIsInstance(ms, MockSomething) + self.assertIsInstance(ms, Something) + self.assertIsInstance(ms, Mock) + + def test_subclassing_with_custom_constructor(self): + class Sub(Any): + def __init__(self, *args, **kwargs): pass + # The instantiation must not fail. + Sub(0, s="") + + def test_multiple_inheritance_with_custom_constructors(self): + class Foo: + def __init__(self, x): + self.x = x + + class Bar(Any, Foo): + def __init__(self, x, y): + self.y = y + super().__init__(x) + + b = Bar(1, 2) + self.assertEqual(b.x, 1) + self.assertEqual(b.y, 2) def test_cannot_instantiate(self): with self.assertRaises(TypeError): @@ -117,48 +164,276 @@ def test_any_works_with_alias(self): typing.IO[Any] -class NoReturnTests(BaseTestCase): +class BottomTypeTestsMixin: + bottom_type: ClassVar[Any] + + def test_equality(self): + self.assertEqual(self.bottom_type, self.bottom_type) + self.assertIs(self.bottom_type, self.bottom_type) + self.assertNotEqual(self.bottom_type, None) + + def test_get_origin(self): + self.assertIs(get_origin(self.bottom_type), None) + + def test_instance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, self.bottom_type) + + def test_subclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(Employee, self.bottom_type) + with self.assertRaises(TypeError): + issubclass(NoReturn, self.bottom_type) - def test_noreturn_instance_type_error(self): + def test_not_generic(self): with self.assertRaises(TypeError): - isinstance(42, NoReturn) + self.bottom_type[int] + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, + 'Cannot subclass ' + re.escape(str(self.bottom_type))): + class A(self.bottom_type): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class B(type(self.bottom_type)): + pass - def test_noreturn_subclass_type_error(self): + def test_cannot_instantiate(self): with self.assertRaises(TypeError): - issubclass(Employee, NoReturn) + self.bottom_type() with self.assertRaises(TypeError): - issubclass(NoReturn, Employee) + type(self.bottom_type)() + + +class NoReturnTests(BottomTypeTestsMixin, BaseTestCase): + bottom_type = NoReturn def test_repr(self): self.assertEqual(repr(NoReturn), 'typing.NoReturn') - def test_not_generic(self): + def test_get_type_hints(self): + def some(arg: NoReturn) -> NoReturn: ... + def some_str(arg: 'NoReturn') -> 'typing.NoReturn': ... + + expected = {'arg': NoReturn, 'return': NoReturn} + for target in [some, some_str]: + with self.subTest(target=target): + self.assertEqual(gth(target), expected) + + def test_not_equality(self): + self.assertNotEqual(NoReturn, Never) + self.assertNotEqual(Never, NoReturn) + + +class NeverTests(BottomTypeTestsMixin, BaseTestCase): + bottom_type = Never + + def test_repr(self): + self.assertEqual(repr(Never), 'typing.Never') + + def test_get_type_hints(self): + def some(arg: Never) -> Never: ... + def some_str(arg: 'Never') -> 'typing.Never': ... + + expected = {'arg': Never, 'return': Never} + for target in [some, some_str]: + with self.subTest(target=target): + self.assertEqual(gth(target), expected) + + +class AssertNeverTests(BaseTestCase): + def test_exception(self): + with self.assertRaises(AssertionError): + assert_never(None) + + value = "some value" + with self.assertRaisesRegex(AssertionError, value): + assert_never(value) + + # Make sure a huge value doesn't get printed in its entirety + huge_value = "a" * 10000 + with self.assertRaises(AssertionError) as cm: + assert_never(huge_value) + self.assertLess( + len(cm.exception.args[0]), + typing._ASSERT_NEVER_REPR_MAX_LENGTH * 2, + ) + + +class SelfTests(BaseTestCase): + def test_equality(self): + self.assertEqual(Self, Self) + self.assertIs(Self, Self) + self.assertNotEqual(Self, None) + + def test_basics(self): + class Foo: + def bar(self) -> Self: ... + class FooStr: + def bar(self) -> 'Self': ... + class FooStrTyping: + def bar(self) -> 'typing.Self': ... + + for target in [Foo, FooStr, FooStrTyping]: + with self.subTest(target=target): + self.assertEqual(gth(target.bar), {'return': Self}) + self.assertIs(get_origin(Self), None) + + def test_repr(self): + self.assertEqual(repr(Self), 'typing.Self') + + def test_cannot_subscript(self): with self.assertRaises(TypeError): - NoReturn[int] + Self[int] def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class A(NoReturn): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Self)): pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Self'): + class D(Self): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + Self() + with self.assertRaises(TypeError): + type(Self)() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Self) + with self.assertRaises(TypeError): + issubclass(int, Self) + + def test_alias(self): + # TypeAliases are not actually part of the spec + alias_1 = Tuple[Self, Self] + alias_2 = List[Self] + alias_3 = ClassVar[Self] + self.assertEqual(get_args(alias_1), (Self, Self)) + self.assertEqual(get_args(alias_2), (Self,)) + self.assertEqual(get_args(alias_3), (Self,)) + + +class LiteralStringTests(BaseTestCase): + def test_equality(self): + self.assertEqual(LiteralString, LiteralString) + self.assertIs(LiteralString, LiteralString) + self.assertNotEqual(LiteralString, None) + + def test_basics(self): + class Foo: + def bar(self) -> LiteralString: ... + class FooStr: + def bar(self) -> 'LiteralString': ... + class FooStrTyping: + def bar(self) -> 'typing.LiteralString': ... + + for target in [Foo, FooStr, FooStrTyping]: + with self.subTest(target=target): + self.assertEqual(gth(target.bar), {'return': LiteralString}) + self.assertIs(get_origin(LiteralString), None) + + def test_repr(self): + self.assertEqual(repr(LiteralString), 'typing.LiteralString') + + def test_cannot_subscript(self): with self.assertRaises(TypeError): - class A(type(NoReturn)): + LiteralString[int] + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(LiteralString)): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.LiteralString'): + class D(LiteralString): pass - def test_cannot_instantiate(self): + def test_cannot_init(self): + with self.assertRaises(TypeError): + LiteralString() + with self.assertRaises(TypeError): + type(LiteralString)() + + def test_no_isinstance(self): with self.assertRaises(TypeError): - NoReturn() + isinstance(1, LiteralString) with self.assertRaises(TypeError): - type(NoReturn)() + issubclass(int, LiteralString) + def test_alias(self): + alias_1 = Tuple[LiteralString, LiteralString] + alias_2 = List[LiteralString] + alias_3 = ClassVar[LiteralString] + self.assertEqual(get_args(alias_1), (LiteralString, LiteralString)) + self.assertEqual(get_args(alias_2), (LiteralString,)) + self.assertEqual(get_args(alias_3), (LiteralString,)) class TypeVarTests(BaseTestCase): - + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_basic_plain(self): T = TypeVar('T') # T equals itself. self.assertEqual(T, T) # T is an instance of TypeVar self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + self.assertEqual(T.__module__, __name__) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_basic_with_exec(self): + ns = {} + exec('from typing import TypeVar; T = TypeVar("T", bound=float)', ns, ns) + T = ns['T'] + self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, float) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + self.assertIs(T.__module__, None) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_attributes(self): + T_bound = TypeVar('T_bound', bound=int) + self.assertEqual(T_bound.__name__, 'T_bound') + self.assertEqual(T_bound.__constraints__, ()) + self.assertIs(T_bound.__bound__, int) + + T_constraints = TypeVar('T_constraints', int, str) + self.assertEqual(T_constraints.__name__, 'T_constraints') + self.assertEqual(T_constraints.__constraints__, (int, str)) + self.assertIs(T_constraints.__bound__, None) + + T_co = TypeVar('T_co', covariant=True) + self.assertEqual(T_co.__name__, 'T_co') + self.assertIs(T_co.__covariant__, True) + self.assertIs(T_co.__contravariant__, False) + self.assertIs(T_co.__infer_variance__, False) + + T_contra = TypeVar('T_contra', contravariant=True) + self.assertEqual(T_contra.__name__, 'T_contra') + self.assertIs(T_contra.__covariant__, False) + self.assertIs(T_contra.__contravariant__, True) + self.assertIs(T_contra.__infer_variance__, False) + + T_infer = TypeVar('T_infer', infer_variance=True) + self.assertEqual(T_infer.__name__, 'T_infer') + self.assertIs(T_infer.__covariant__, False) + self.assertIs(T_infer.__contravariant__, False) + self.assertIs(T_infer.__infer_variance__, True) def test_typevar_instance_type_error(self): T = TypeVar('T') @@ -172,11 +447,15 @@ def test_typevar_subclass_type_error(self): with self.assertRaises(TypeError): issubclass(T, int) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_constrained_error(self): with self.assertRaises(TypeError): X = TypeVar('X', int) X + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_unique(self): X = TypeVar('X') Y = TypeVar('Y') @@ -190,6 +469,8 @@ def test_union_unique(self): self.assertEqual(Union[X, int].__parameters__, (X,)) self.assertIs(Union[X, int].__origin__, Union) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_or(self): X = TypeVar('X') # use a string because str doesn't implement @@ -204,6 +485,8 @@ def test_union_constrained(self): A = TypeVar('A', str, bytes) self.assertNotEqual(Union[A, str], Union[A]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repr(self): self.assertEqual(repr(T), '~T') self.assertEqual(repr(KT), '~KT') @@ -218,25 +501,30 @@ def test_no_redefinition(self): self.assertNotEqual(TypeVar('T'), TypeVar('T')) self.assertNotEqual(TypeVar('T', int, str), TypeVar('T', int, str)) - def test_cannot_subclass_vars(self): - with self.assertRaises(TypeError): - class V(TypeVar('T')): - pass - - def test_cannot_subclass_var_itself(self): - with self.assertRaises(TypeError): - class V(TypeVar): - pass + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'TypeVar'): + class V(TypeVar): pass + T = TypeVar("T") + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'TypeVar'): + class W(T): pass def test_cannot_instantiate_vars(self): with self.assertRaises(TypeError): TypeVar('A')() + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_bound_errors(self): with self.assertRaises(TypeError): - TypeVar('X', bound=42) + TypeVar('X', bound=Union) with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + with self.assertRaisesRegex(TypeError, + r"Bound must be a type\. Got \(1, 2\)\."): + TypeVar('X', bound=(1, 2)) def test_missing__name__(self): # See bpo-39942 @@ -245,116 +533,1789 @@ def test_missing__name__(self): ) exec(code, {}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_no_bivariant(self): with self.assertRaises(ValueError): TypeVar('T', covariant=True, contravariant=True) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cannot_combine_explicit_and_infer(self): + with self.assertRaises(ValueError): + TypeVar('T', covariant=True, infer_variance=True) + with self.assertRaises(ValueError): + TypeVar('T', contravariant=True, infer_variance=True) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_var_substitution(self): + T = TypeVar('T') + subst = T.__typing_subst__ + self.assertIs(subst(int), int) + self.assertEqual(subst(list[int]), list[int]) + self.assertEqual(subst(List[int]), List[int]) + self.assertEqual(subst(List), List) + self.assertIs(subst(Any), Any) + self.assertIs(subst(None), type(None)) + self.assertIs(subst(T), T) + self.assertEqual(subst(int|str), int|str) + self.assertEqual(subst(Union[int, str]), Union[int, str]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_bad_var_substitution(self): T = TypeVar('T') - for arg in (), (int, str): + bad_args = ( + (), (int, str), Union, + Generic, Generic[T], Protocol, Protocol[T], + Final, Final[int], ClassVar, ClassVar[int], + ) + for arg in bad_args: with self.subTest(arg=arg): + with self.assertRaises(TypeError): + T.__typing_subst__(arg) with self.assertRaises(TypeError): List[T][arg] with self.assertRaises(TypeError): list[T][arg] + def test_many_weakrefs(self): + # gh-108295: this used to segfault + for cls in (ParamSpec, TypeVarTuple, TypeVar): + with self.subTest(cls=cls): + vals = weakref.WeakValueDictionary() -class UnionTests(BaseTestCase): + for x in range(10): + vals[x] = cls(str(x)) + del vals - def test_basics(self): - u = Union[int, float] - self.assertNotEqual(u, Union) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_constructor(self): + T = TypeVar(name="T") + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", bound=type) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, type) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", default=()) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, ()) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", covariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, True) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", contravariant=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, True) + self.assertIs(T.__infer_variance__, False) + + T = TypeVar(name="T", infer_variance=True) + self.assertEqual(T.__name__, "T") + self.assertEqual(T.__constraints__, ()) + self.assertIs(T.__bound__, None) + self.assertIs(T.__default__, typing.NoDefault) + self.assertIs(T.__covariant__, False) + self.assertIs(T.__contravariant__, False) + self.assertIs(T.__infer_variance__, True) + +class TypeParameterDefaultsTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertTrue(T.has_default()) + self.assertIsInstance(T, TypeVar) - def test_subclass_error(self): - with self.assertRaises(TypeError): - issubclass(int, Union) - with self.assertRaises(TypeError): - issubclass(Union, int) - with self.assertRaises(TypeError): - issubclass(Union[int, str], int) + class A(Generic[T]): ... + Alias = Optional[T] - def test_union_any(self): - u = Union[Any] - self.assertEqual(u, Any) - u1 = Union[int, Any] - u2 = Union[Any, int] - u3 = Union[Any, object] - self.assertEqual(u1, u2) - self.assertNotEqual(u1, Any) - self.assertNotEqual(u2, Any) - self.assertNotEqual(u3, Any) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) - def test_union_object(self): - u = Union[object] - self.assertEqual(u, object) - u1 = Union[int, object] - u2 = Union[object, int] - self.assertEqual(u1, u2) - self.assertNotEqual(u1, object) - self.assertNotEqual(u2, object) + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertFalse(T.has_default()) - def test_unordered(self): - u1 = Union[int, float] - u2 = Union[float, int] - self.assertEqual(u1, u2) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertTrue(P.has_default()) + self.assertIsInstance(P, ParamSpec) - def test_single_class_disappears(self): - t = Union[Employee] - self.assertIs(t, Employee) + class A(Generic[P]): ... + Alias = typing.Callable[P, None] - def test_base_class_kept(self): - u = Union[Employee, Manager] - self.assertNotEqual(u, Employee) - self.assertIn(Employee, u.__args__) - self.assertIn(Manager, u.__args__) + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) - def test_union_union(self): - u = Union[int, float] - v = Union[u, Employee] - self.assertEqual(v, Union[int, float, Employee]) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertFalse(P.has_default()) - def test_repr(self): - self.assertEqual(repr(Union), 'typing.Union') - u = Union[Employee, int] - self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) - u = Union[int, Employee] - self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) - T = TypeVar('T') - u = Union[T, int][int] - self.assertEqual(repr(u), repr(int)) - u = Union[List[int], int] - self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') - u = Union[list[int], dict[str, float]] - self.assertEqual(repr(u), 'typing.Union[list[int], dict[str, float]]') - u = Union[int | float] - self.assertEqual(repr(u), 'typing.Union[int, float]') + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertTrue(Ts.has_default()) + self.assertIsInstance(Ts, TypeVarTuple) - u = Union[None, str] - self.assertEqual(repr(u), 'typing.Optional[str]') - u = Union[str, None] - self.assertEqual(repr(u), 'typing.Optional[str]') - u = Union[None, str, int] - self.assertEqual(repr(u), 'typing.Union[NoneType, str, int]') - u = Optional[str] - self.assertEqual(repr(u), 'typing.Optional[str]') + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] - def test_cannot_subclass(self): - with self.assertRaises(TypeError): - class C(Union): - pass - with self.assertRaises(TypeError): - class C(type(Union)): - pass - with self.assertRaises(TypeError): - class C(Union[int, str]): - pass + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) + + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) - def test_cannot_instantiate(self): with self.assertRaises(TypeError): - Union() + class X(Generic[*Ts, T]): ... + with self.assertRaises(TypeError): - type(Union)() - u = Union[int, float] + class Y(Generic[*Ts_default, T]): ... + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_allow_default_after_non_default_in_alias(self): + T_default = TypeVar('T_default', default=int) + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + a1 = Callable[[T_default], T] + self.assertEqual(a1.__args__, (T_default, T)) + + a2 = dict[T_default, T] + self.assertEqual(a2.__args__, (T_default, T)) + + a3 = typing.Dict[T_default, T] + self.assertEqual(a3.__args__, (T_default, T)) + + a4 = Callable[*Ts, T] + self.assertEqual(a4.__args__, (*Ts, T)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertFalse(Ts.has_default()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') + + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] + + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) + + + +def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: + """Renders templates with possible combinations of replacements. + + Example 1: Suppose that: + templates = ["dog_breed are awesome", "dog_breed are cool"] + replacements = ["dog_breed": ["Huskies", "Beagles"]] + Then we would return: + [ + ("Huskies are awesome", "Huskies are cool"), + ("Beagles are awesome", "Beagles are cool") + ] + + Example 2: Suppose that: + templates = ["Huskies are word1 but also word2"] + replacements = {"word1": ["playful", "cute"], + "word2": ["feisty", "tiring"]} + Then we would return: + [ + ("Huskies are playful but also feisty"), + ("Huskies are playful but also tiring"), + ("Huskies are cute but also feisty"), + ("Huskies are cute but also tiring") + ] + + Note that if any of the replacements do not occur in any template: + templates = ["Huskies are word1", "Beagles!"] + replacements = {"word1": ["playful", "cute"], + "word2": ["feisty", "tiring"]} + Then we do not generate duplicates, returning: + [ + ("Huskies are playful", "Beagles!"), + ("Huskies are cute", "Beagles!") + ] + """ + # First, build a structure like: + # [ + # [("word1", "playful"), ("word1", "cute")], + # [("word2", "feisty"), ("word2", "tiring")] + # ] + replacement_combos = [] + for original, possible_replacements in replacements.items(): + original_replacement_tuples = [] + for replacement in possible_replacements: + original_replacement_tuples.append((original, replacement)) + replacement_combos.append(original_replacement_tuples) + + # Second, generate rendered templates, including possible duplicates. + rendered_templates = [] + for replacement_combo in itertools.product(*replacement_combos): + # replacement_combo would be e.g. + # [("word1", "playful"), ("word2", "feisty")] + templates_with_replacements = [] + for template in templates: + for original, replacement in replacement_combo: + template = template.replace(original, replacement) + templates_with_replacements.append(template) + rendered_templates.append(tuple(templates_with_replacements)) + + # Finally, remove the duplicates (but keep the order). + rendered_templates_no_duplicates = [] + for x in rendered_templates: + # Inefficient, but should be fine for our purposes. + if x not in rendered_templates_no_duplicates: + rendered_templates_no_duplicates.append(x) + + return rendered_templates_no_duplicates + + +class TemplateReplacementTests(BaseTestCase): + + def test_two_templates_two_replacements_yields_correct_renders(self): + actual = template_replace( + templates=["Cats are word1", "Dogs are word2"], + replacements={ + "word1": ["small", "cute"], + "word2": ["big", "fluffy"], + }, + ) + expected = [ + ("Cats are small", "Dogs are big"), + ("Cats are small", "Dogs are fluffy"), + ("Cats are cute", "Dogs are big"), + ("Cats are cute", "Dogs are fluffy"), + ] + self.assertEqual(actual, expected) + + def test_no_duplicates_if_replacement_not_in_templates(self): + actual = template_replace( + templates=["Cats are word1", "Dogs!"], + replacements={ + "word1": ["small", "cute"], + "word2": ["big", "fluffy"], + }, + ) + expected = [ + ("Cats are small", "Dogs!"), + ("Cats are cute", "Dogs!"), + ] + self.assertEqual(actual, expected) + + + +class GenericAliasSubstitutionTests(BaseTestCase): + """Tests for type variable substitution in generic aliases. + + For variadic cases, these tests should be regarded as the source of truth, + since we hadn't realised the full complexity of variadic substitution + at the time of finalizing PEP 646. For full discussion, see + https://github.com/python/cpython/issues/91162. + """ + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_one_parameter(self): + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + Ts2 = TypeVarTuple('Ts2') + + class C(Generic[T]): pass + + generics = ['C', 'list', 'List'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[T]', '[()]', 'TypeError'), + ('generic[T]', '[int]', 'generic[int]'), + ('generic[T]', '[int, str]', 'TypeError'), + ('generic[T]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[T]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[T]', '[*tuple_type[()]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, str]]', 'TypeError'), + ('generic[T]', '[*tuple_type[int, ...]]', 'TypeError'), + ('generic[T]', '[*Ts]', 'TypeError'), + ('generic[T]', '[T, *Ts]', 'TypeError'), + ('generic[T]', '[*Ts, T]', 'TypeError'), + # Raises TypeError because C is not variadic. + # (If C _were_ variadic, it'd be fine.) + ('C[T, *tuple_type[int, ...]]', '[int]', 'TypeError'), + # Should definitely raise TypeError: list only takes one argument. + ('list[T, *tuple_type[int, ...]]', '[int]', 'list[int, *tuple_type[int, ...]]'), + ('List[T, *tuple_type[int, ...]]', '[int]', 'TypeError'), + # Should raise, because more than one `TypeVarTuple` is not supported. + ('generic[*Ts, *Ts2]', '[int]', 'TypeError'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_two_parameters(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + + class C(Generic[T1, T2]): pass + + generics = ['C', 'dict', 'Dict'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[T1, T2]', '[()]', 'TypeError'), + ('generic[T1, T2]', '[int]', 'TypeError'), + ('generic[T1, T2]', '[int, str]', 'generic[int, str]'), + ('generic[T1, T2]', '[int, str, bool]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int, str, bool]]', 'TypeError'), + + ('generic[T1, T2]', '[int, *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], str]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[()]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int, str]]', 'generic[int, str]'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[()]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[()], *tuple_type[int]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int], *tuple_type[str, float]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, str], *tuple_type[float, bool]]', 'TypeError'), + + ('generic[T1, T2]', '[tuple_type[int, ...]]', 'TypeError'), + ('generic[T1, T2]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + ('generic[T1, T2]', '[*tuple_type[int, ...]]', 'TypeError'), + ('generic[T1, T2]', '[int, *tuple_type[str, ...]]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], str]', 'TypeError'), + ('generic[T1, T2]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), + ('generic[T1, T2]', '[*Ts]', 'TypeError'), + ('generic[T1, T2]', '[T, *Ts]', 'TypeError'), + ('generic[T1, T2]', '[*Ts, T]', 'TypeError'), + # This one isn't technically valid - none of the things that + # `generic` can be (defined in `generics` above) are variadic, so we + # shouldn't really be able to do `generic[T1, *tuple_type[int, ...]]`. + # So even if type checkers shouldn't allow it, we allow it at + # runtime, in accordance with a general philosophy of "Keep the + # runtime lenient so people can experiment with typing constructs". + ('generic[T1, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_three_parameters(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + + class C(Generic[T1, T2, T3]): pass + + generics = ['C'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[T1, bool, T2]', '[int, str]', 'generic[int, bool, str]'), + ('generic[T1, bool, T2]', '[*tuple_type[int, str]]', 'generic[int, bool, str]'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_parameters(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + + generics = ['C', 'tuple', 'Tuple'] + tuple_types = ['tuple', 'Tuple'] + + tests = [ + # Alias # Args # Expected result + ('generic[*Ts]', '[()]', 'generic[()]'), + ('generic[*Ts]', '[int]', 'generic[int]'), + ('generic[*Ts]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'), + ('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'), + ('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'), + ('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'), + ('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'), + ('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'), + ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'), + + ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), + ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), + ('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'), + ('generic[T, *Ts]', '[()]', 'TypeError'), + ('generic[T, *Ts]', '[int]', 'generic[int]'), + ('generic[T, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[list[T], *Ts]', '[()]', 'TypeError'), + ('generic[list[T], *Ts]', '[int]', 'generic[list[int]]'), + ('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'), + ('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'), + + ('generic[*Ts, T]', '[()]', 'TypeError'), + ('generic[*Ts, T]', '[int]', 'generic[int]'), + ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, list[T]]', '[()]', 'TypeError'), + ('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'), + ('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'), + ('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'), + + ('generic[T1, T2, *Ts]', '[()]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T1, T2, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, T2, *Ts]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[*Ts, T1, T2]', '[()]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T1, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, T1, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[T1, *Ts, T2]', '[()]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int, str]', 'generic[int, str]'), + ('generic[T1, *Ts, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, *Ts, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'), + ('generic[T, *Ts]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...]]'), + ('generic[T, *Ts]', '[*tuple_type[int, ...], str]', 'generic[int, *tuple_type[int, ...], str]'), + ('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'), + ('generic[*Ts, T]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...], int]'), + ('generic[*Ts, T]', '[*tuple_type[int, ...], str]', 'generic[*tuple_type[int, ...], str]'), + ('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'), + ('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'), + ('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'), + ('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'), + ('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'), + + ('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'), + ('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'), + ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'), + ('generic[T1, *tuple_type[int, ...], T2]', '[str, bool, float]', 'TypeError'), + + ('generic[T1, *tuple_type[T2, ...]]', '[int, str]', 'generic[int, *tuple_type[str, ...]]'), + ('generic[*tuple_type[T1, ...], T2]', '[int, str]', 'generic[*tuple_type[int, ...], str]'), + ('generic[T1, *tuple_type[generic[*Ts], ...]]', '[int, str, bool]', 'generic[int, *tuple_type[generic[str, bool], ...]]'), + ('generic[*tuple_type[generic[*Ts], ...], T1]', '[int, str, bool]', 'generic[*tuple_type[generic[int, str], ...], bool]'), + ] + + for alias_template, args_template, expected_template in tests: + rendered_templates = template_replace( + templates=[alias_template, args_template, expected_template], + replacements={'generic': generics, 'tuple_type': tuple_types} + ) + for alias_str, args_str, expected_str in rendered_templates: + with self.subTest(alias=alias_str, args=args_str, expected=expected_str): + if expected_str == 'TypeError': + with self.assertRaises(TypeError): + eval(alias_str + args_str) + else: + self.assertEqual( + eval(alias_str + args_str), + eval(expected_str) + ) + + + + + +class UnpackTests(BaseTestCase): + + def test_accepts_single_type(self): + # (*tuple[int],) + Unpack[Tuple[int]] + + def test_dir(self): + dir_items = set(dir(Unpack[Tuple[int]])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + + def test_rejects_multiple_types(self): + with self.assertRaises(TypeError): + Unpack[Tuple[int], Tuple[str]] + # We can't do the equivalent for `*` here - + # *(Tuple[int], Tuple[str]) is just plain tuple unpacking, + # which is valid. + + def test_rejects_multiple_parameterization(self): + with self.assertRaises(TypeError): + (*tuple[int],)[0][tuple[int]] + with self.assertRaises(TypeError): + Unpack[Tuple[int]][Tuple[int]] + + def test_cannot_be_called(self): + with self.assertRaises(TypeError): + Unpack() + + def test_usage_with_kwargs(self): + Movie = TypedDict('Movie', {'name': str, 'year': int}) + def foo(**kwargs: Unpack[Movie]): ... + self.assertEqual(repr(foo.__annotations__['kwargs']), + f"typing.Unpack[{__name__}.Movie]") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_builtin_tuple(self): + Ts = TypeVarTuple("Ts") + + # TODO: RUSTPYTHON + # class Old(Generic[*Ts]): ... + # class New[*Ts]: ... + + PartOld = Old[int, *Ts] + self.assertEqual(PartOld[str].__args__, (int, str)) + # self.assertEqual(PartOld[*tuple[str]].__args__, (int, str)) + # self.assertEqual(PartOld[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartOld[Unpack[Tuple[str]]].__args__, (int, str)) + + PartNew = New[int, *Ts] + self.assertEqual(PartNew[str].__args__, (int, str)) + # self.assertEqual(PartNew[*tuple[str]].__args__, (int, str)) + # self.assertEqual(PartNew[*Tuple[str]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[tuple[str]]].__args__, (int, str)) + self.assertEqual(PartNew[Unpack[Tuple[str]]].__args__, (int, str)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_unpack_wrong_type(self): + Ts = TypeVarTuple("Ts") + class Gen[*Ts]: ... + # PartGen = Gen[int, *Ts] + + bad_unpack_param = re.escape("Unpack[...] must be used with a tuple type") + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[list[int]]] + with self.assertRaisesRegex(TypeError, bad_unpack_param): + PartGen[Unpack[List[int]]] + +class TypeVarTupleTests(BaseTestCase): + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(Ts.__name__, 'Ts') + Ts2 = TypeVarTuple('Ts2') + self.assertEqual(Ts2.__name__, 'Ts2') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_module(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(Ts.__module__, __name__) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exec(self): + ns = {} + exec('from typing import TypeVarTuple; Ts = TypeVarTuple("Ts")', ns) + Ts = ns['Ts'] + self.assertEqual(Ts.__name__, 'Ts') + self.assertIs(Ts.__module__, None) + + def test_instance_is_equal_to_itself(self): + Ts = TypeVarTuple('Ts') + self.assertEqual(Ts, Ts) + + def test_different_instances_are_different(self): + self.assertNotEqual(TypeVarTuple('Ts'), TypeVarTuple('Ts')) + + def test_instance_isinstance_of_typevartuple(self): + Ts = TypeVarTuple('Ts') + self.assertIsInstance(Ts, TypeVarTuple) + + def test_cannot_call_instance(self): + Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + Ts() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_unpacked_typevartuple_is_equal_to_itself(self): + Ts = TypeVarTuple('Ts') + self.assertEqual((*Ts,)[0], (*Ts,)[0]) + self.assertEqual(Unpack[Ts], Unpack[Ts]) + + def test_parameterised_tuple_is_equal_to_itself(self): + Ts = TypeVarTuple('Ts') + # self.assertEqual(tuple[*Ts], tuple[*Ts]) + self.assertEqual(Tuple[Unpack[Ts]], Tuple[Unpack[Ts]]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def tests_tuple_arg_ordering_matters(self): + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + self.assertNotEqual( + tuple[*Ts1, *Ts2], + tuple[*Ts2, *Ts1], + ) + self.assertNotEqual( + Tuple[Unpack[Ts1], Unpack[Ts2]], + Tuple[Unpack[Ts2], Unpack[Ts1]], + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_tuple_args_and_parameters_are_correct(self): + Ts = TypeVarTuple('Ts') + # t1 = tuple[*Ts] + self.assertEqual(t1.__args__, (*Ts,)) + self.assertEqual(t1.__parameters__, (Ts,)) + t2 = Tuple[Unpack[Ts]] + self.assertEqual(t2.__args__, (Unpack[Ts],)) + self.assertEqual(t2.__parameters__, (Ts,)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_var_substitution(self): + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + # class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + + for A in G1, G2, Tuple, tuple: + # B = A[*Ts] + self.assertEqual(B[()], A[()]) + self.assertEqual(B[float], A[float]) + self.assertEqual(B[float, str], A[float, str]) + + C = A[Unpack[Ts]] + self.assertEqual(C[()], A[()]) + self.assertEqual(C[float], A[float]) + self.assertEqual(C[float, str], A[float, str]) + + # D = list[A[*Ts]] + self.assertEqual(D[()], list[A[()]]) + self.assertEqual(D[float], list[A[float]]) + self.assertEqual(D[float, str], list[A[float, str]]) + + E = List[A[Unpack[Ts]]] + self.assertEqual(E[()], List[A[()]]) + self.assertEqual(E[float], List[A[float]]) + self.assertEqual(E[float, str], List[A[float, str]]) + + F = A[T, *Ts, T2] + with self.assertRaises(TypeError): + F[()] + with self.assertRaises(TypeError): + F[float] + self.assertEqual(F[float, str], A[float, str]) + self.assertEqual(F[float, str, int], A[float, str, int]) + self.assertEqual(F[float, str, int, bytes], A[float, str, int, bytes]) + + G = A[T, Unpack[Ts], T2] + with self.assertRaises(TypeError): + G[()] + with self.assertRaises(TypeError): + G[float] + self.assertEqual(G[float, str], A[float, str]) + self.assertEqual(G[float, str, int], A[float, str, int]) + self.assertEqual(G[float, str, int, bytes], A[float, str, int, bytes]) + + # H = tuple[list[T], A[*Ts], list[T2]] + with self.assertRaises(TypeError): + H[()] + with self.assertRaises(TypeError): + H[float] + if A != Tuple: + self.assertEqual(H[float, str], + tuple[list[float], A[()], list[str]]) + self.assertEqual(H[float, str, int], + tuple[list[float], A[str], list[int]]) + self.assertEqual(H[float, str, int, bytes], + tuple[list[float], A[str, int], list[bytes]]) + + I = Tuple[List[T], A[Unpack[Ts]], List[T2]] + with self.assertRaises(TypeError): + I[()] + with self.assertRaises(TypeError): + I[float] + if A != Tuple: + self.assertEqual(I[float, str], + Tuple[List[float], A[()], List[str]]) + self.assertEqual(I[float, str, int], + Tuple[List[float], A[str], List[int]]) + self.assertEqual(I[float, str, int, bytes], + Tuple[List[float], A[str, int], List[bytes]]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_bad_var_substitution(self): + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T2 = TypeVar('T2') + # class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + + for A in G1, G2, Tuple, tuple: + B = A[Ts] + with self.assertRaises(TypeError): + B[int, str] + + C = A[T, T2] + with self.assertRaises(TypeError): + # C[*Ts] + pass + with self.assertRaises(TypeError): + C[Unpack[Ts]] + + B = A[T, *Ts, str, T2] + with self.assertRaises(TypeError): + B[int, *Ts] + with self.assertRaises(TypeError): + B[int, *Ts, *Ts] + + C = A[T, Unpack[Ts], str, T2] + with self.assertRaises(TypeError): + C[int, Unpack[Ts]] + with self.assertRaises(TypeError): + C[int, Unpack[Ts], Unpack[Ts]] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + + # class G1(Generic[*Ts]): pass + class G2(Generic[Unpack[Ts]]): pass + + self.assertEqual(repr(Ts), 'Ts') + + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') + + # self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') + + # self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + # class A(Generic[*Ts]): pass + class B(Generic[Unpack[Ts]]): pass + + self.assertEndsWith(repr(A[()]), 'A[()]') + self.assertEndsWith(repr(B[()]), 'B[()]') + self.assertEndsWith(repr(A[float]), 'A[float]') + self.assertEndsWith(repr(B[float]), 'B[float]') + self.assertEndsWith(repr(A[float, str]), 'A[float, str]') + self.assertEndsWith(repr(B[float, str]), 'B[float, str]') + + # self.assertEndsWith(repr(A[*tuple[int, ...]]), + # 'A[*tuple[int, ...]]') + self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]]]), + 'B[typing.Unpack[typing.Tuple[int, ...]]]') + + self.assertEndsWith(repr(A[float, *tuple[int, ...]]), + 'A[float, *tuple[int, ...]]') + self.assertEndsWith(repr(A[float, Unpack[Tuple[int, ...]]]), + 'A[float, typing.Unpack[typing.Tuple[int, ...]]]') + + self.assertEndsWith(repr(A[*tuple[int, ...], str]), + 'A[*tuple[int, ...], str]') + self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]], str]), + 'B[typing.Unpack[typing.Tuple[int, ...]], str]') + + self.assertEndsWith(repr(A[float, *tuple[int, ...], str]), + 'A[float, *tuple[int, ...], str]') + self.assertEndsWith(repr(B[float, Unpack[Tuple[int, ...]], str]), + 'B[float, typing.Unpack[typing.Tuple[int, ...]], str]') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_alias_repr_is_correct(self): + Ts = TypeVarTuple('Ts') + class A(Generic[Unpack[Ts]]): pass + + # B = A[*Ts] + # self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') + # self.assertEndsWith(repr(B[()]), 'A[()]') + # self.assertEndsWith(repr(B[float]), 'A[float]') + # self.assertEndsWith(repr(B[float, str]), 'A[float, str]') + + C = A[Unpack[Ts]] + self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') + self.assertEndsWith(repr(C[()]), 'A[()]') + self.assertEndsWith(repr(C[float]), 'A[float]') + self.assertEndsWith(repr(C[float, str]), 'A[float, str]') + + D = A[*Ts, int] + self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(D[()]), 'A[int]') + self.assertEndsWith(repr(D[float]), 'A[float, int]') + self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') + + E = A[Unpack[Ts], int] + self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') + self.assertEndsWith(repr(E[()]), 'A[int]') + self.assertEndsWith(repr(E[float]), 'A[float, int]') + self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') + + F = A[int, *Ts] + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(F[()]), 'A[int]') + self.assertEndsWith(repr(F[float]), 'A[int, float]') + self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') + + G = A[int, Unpack[Ts]] + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') + self.assertEndsWith(repr(G[()]), 'A[int]') + self.assertEndsWith(repr(G[float]), 'A[int, float]') + self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') + + H = A[int, *Ts, str] + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(H[()]), 'A[int, str]') + self.assertEndsWith(repr(H[float]), 'A[int, float, str]') + self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') + + I = A[int, Unpack[Ts], str] + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') + self.assertEndsWith(repr(I[()]), 'A[int, str]') + self.assertEndsWith(repr(I[float]), 'A[int, float, str]') + self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') + + J = A[*Ts, *tuple[str, ...]] + self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') + self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') + self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') + self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') + + K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] + self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'TypeVarTuple'): + class C(TypeVarTuple): pass + Ts = TypeVarTuple('Ts') + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'TypeVarTuple'): + class D(Ts): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class E(type(Unpack)): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class F(type(*Ts)): pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class G(type(Unpack[Ts])): pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Unpack'): + class H(Unpack): pass + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + class I(*Ts): pass + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): + class J(Unpack[Ts]): pass + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_args_are_correct(self): + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + # class A(Generic[*Ts]): pass + class B(Generic[Unpack[Ts]]): pass + + C = A[()] + D = B[()] + self.assertEqual(C.__args__, ()) + self.assertEqual(D.__args__, ()) + + E = A[int] + F = B[int] + self.assertEqual(E.__args__, (int,)) + self.assertEqual(F.__args__, (int,)) + + G = A[int, str] + H = B[int, str] + self.assertEqual(G.__args__, (int, str)) + self.assertEqual(H.__args__, (int, str)) + + I = A[T] + J = B[T] + self.assertEqual(I.__args__, (T,)) + self.assertEqual(J.__args__, (T,)) + + # K = A[*Ts] + # L = B[Unpack[Ts]] + # self.assertEqual(K.__args__, (*Ts,)) + # self.assertEqual(L.__args__, (Unpack[Ts],)) + + M = A[T, *Ts] + N = B[T, Unpack[Ts]] + self.assertEqual(M.__args__, (T, *Ts)) + self.assertEqual(N.__args__, (T, Unpack[Ts])) + + O = A[*Ts, T] + P = B[Unpack[Ts], T] + self.assertEqual(O.__args__, (*Ts, T)) + self.assertEqual(P.__args__, (Unpack[Ts], T)) + + def test_variadic_class_origin_is_correct(self): + Ts = TypeVarTuple('Ts') + + # class C(Generic[*Ts]): pass + self.assertIs(C[int].__origin__, C) + self.assertIs(C[T].__origin__, C) + self.assertIs(C[Unpack[Ts]].__origin__, C) + + class D(Generic[Unpack[Ts]]): pass + self.assertIs(D[int].__origin__, D) + self.assertIs(D[T].__origin__, D) + self.assertIs(D[Unpack[Ts]].__origin__, D) + + def test_get_type_hints_on_unpack_args(self): + Ts = TypeVarTuple('Ts') + + # def func1(*args: *Ts): pass + # self.assertEqual(gth(func1), {'args': Unpack[Ts]}) + + # def func2(*args: *tuple[int, str]): pass + # self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + + # class CustomVariadic(Generic[*Ts]): pass + + # def func3(*args: *CustomVariadic[int, str]): pass + # self.assertEqual(gth(func3), {'args': Unpack[CustomVariadic[int, str]]}) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_get_type_hints_on_unpack_args_string(self): + Ts = TypeVarTuple('Ts') + + def func1(*args: '*Ts'): pass + self.assertEqual(gth(func1, localns={'Ts': Ts}), + {'args': Unpack[Ts]}) + + def func2(*args: '*tuple[int, str]'): pass + self.assertEqual(gth(func2), {'args': Unpack[tuple[int, str]]}) + + # class CustomVariadic(Generic[*Ts]): pass + + # def func3(*args: '*CustomVariadic[int, str]'): pass + # self.assertEqual(gth(func3, localns={'CustomVariadic': CustomVariadic}), + # {'args': Unpack[CustomVariadic[int, str]]}) + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_tuple_args_are_correct(self): + Ts = TypeVarTuple('Ts') + + # self.assertEqual(tuple[*Ts].__args__, (*Ts,)) + self.assertEqual(Tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) + + self.assertEqual(tuple[*Ts, int].__args__, (*Ts, int)) + self.assertEqual(Tuple[Unpack[Ts], int].__args__, (Unpack[Ts], int)) + + self.assertEqual(tuple[int, *Ts].__args__, (int, *Ts)) + self.assertEqual(Tuple[int, Unpack[Ts]].__args__, (int, Unpack[Ts])) + + self.assertEqual(tuple[int, *Ts, str].__args__, + (int, *Ts, str)) + self.assertEqual(Tuple[int, Unpack[Ts], str].__args__, + (int, Unpack[Ts], str)) + + self.assertEqual(tuple[*Ts, int].__args__, (*Ts, int)) + self.assertEqual(Tuple[Unpack[Ts]].__args__, (Unpack[Ts],)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_callable_args_are_correct(self): + Ts = TypeVarTuple('Ts') + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + # TypeVarTuple in the arguments + + a = Callable[[*Ts], None] + b = Callable[[Unpack[Ts]], None] + self.assertEqual(a.__args__, (*Ts, type(None))) + self.assertEqual(b.__args__, (Unpack[Ts], type(None))) + + c = Callable[[int, *Ts], None] + d = Callable[[int, Unpack[Ts]], None] + self.assertEqual(c.__args__, (int, *Ts, type(None))) + self.assertEqual(d.__args__, (int, Unpack[Ts], type(None))) + + e = Callable[[*Ts, int], None] + f = Callable[[Unpack[Ts], int], None] + self.assertEqual(e.__args__, (*Ts, int, type(None))) + self.assertEqual(f.__args__, (Unpack[Ts], int, type(None))) + + g = Callable[[str, *Ts, int], None] + h = Callable[[str, Unpack[Ts], int], None] + self.assertEqual(g.__args__, (str, *Ts, int, type(None))) + self.assertEqual(h.__args__, (str, Unpack[Ts], int, type(None))) + + # TypeVarTuple as the return + + i = Callable[[None], *Ts] + j = Callable[[None], Unpack[Ts]] + self.assertEqual(i.__args__, (type(None), *Ts)) + self.assertEqual(j.__args__, (type(None), Unpack[Ts])) + + k = Callable[[None], tuple[int, *Ts]] + l = Callable[[None], Tuple[int, Unpack[Ts]]] + self.assertEqual(k.__args__, (type(None), tuple[int, *Ts])) + self.assertEqual(l.__args__, (type(None), Tuple[int, Unpack[Ts]])) + + m = Callable[[None], tuple[*Ts, int]] + n = Callable[[None], Tuple[Unpack[Ts], int]] + self.assertEqual(m.__args__, (type(None), tuple[*Ts, int])) + self.assertEqual(n.__args__, (type(None), Tuple[Unpack[Ts], int])) + + o = Callable[[None], tuple[str, *Ts, int]] + p = Callable[[None], Tuple[str, Unpack[Ts], int]] + self.assertEqual(o.__args__, (type(None), tuple[str, *Ts, int])) + self.assertEqual(p.__args__, (type(None), Tuple[str, Unpack[Ts], int])) + + # TypeVarTuple in both + + q = Callable[[*Ts], *Ts] + r = Callable[[Unpack[Ts]], Unpack[Ts]] + self.assertEqual(q.__args__, (*Ts, *Ts)) + self.assertEqual(r.__args__, (Unpack[Ts], Unpack[Ts])) + + s = Callable[[*Ts1], *Ts2] + u = Callable[[Unpack[Ts1]], Unpack[Ts2]] + self.assertEqual(s.__args__, (*Ts1, *Ts2)) + self.assertEqual(u.__args__, (Unpack[Ts1], Unpack[Ts2])) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_with_duplicate_typevartuples_fails(self): + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + with self.assertRaises(TypeError): + class C(Generic[*Ts1, *Ts1]): pass + with self.assertRaises(TypeError): + class D(Generic[Unpack[Ts1], Unpack[Ts1]]): pass + + with self.assertRaises(TypeError): + class E(Generic[*Ts1, *Ts2, *Ts1]): pass + with self.assertRaises(TypeError): + class F(Generic[Unpack[Ts1], Unpack[Ts2], Unpack[Ts1]]): pass + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_type_concatenation_in_variadic_class_argument_list_succeeds(self): + Ts = TypeVarTuple('Ts') + class C(Generic[Unpack[Ts]]): pass + + C[int, *Ts] + C[int, Unpack[Ts]] + + C[*Ts, int] + C[Unpack[Ts], int] + + C[int, *Ts, str] + C[int, Unpack[Ts], str] + + C[int, bool, *Ts, float, str] + C[int, bool, Unpack[Ts], float, str] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_type_concatenation_in_tuple_argument_list_succeeds(self): + Ts = TypeVarTuple('Ts') + + tuple[int, *Ts] + tuple[*Ts, int] + tuple[int, *Ts, str] + tuple[int, bool, *Ts, float, str] + + Tuple[int, Unpack[Ts]] + Tuple[Unpack[Ts], int] + Tuple[int, Unpack[Ts], str] + Tuple[int, bool, Unpack[Ts], float, str] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_definition_using_packed_typevartuple_fails(self): + Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + class C(Generic[Ts]): pass + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_definition_using_concrete_types_fails(self): + Ts = TypeVarTuple('Ts') + with self.assertRaises(TypeError): + class F(Generic[*Ts, int]): pass + with self.assertRaises(TypeError): + class E(Generic[Unpack[Ts], int]): pass + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_with_2_typevars_accepts_2_or_more_args(self): + Ts = TypeVarTuple('Ts') + T1 = TypeVar('T1') + T2 = TypeVar('T2') + + class A(Generic[T1, T2, *Ts]): pass + A[int, str] + A[int, str, float] + A[int, str, float, bool] + + class B(Generic[T1, T2, Unpack[Ts]]): pass + B[int, str] + B[int, str, float] + B[int, str, float, bool] + + class C(Generic[T1, *Ts, T2]): pass + C[int, str] + C[int, str, float] + C[int, str, float, bool] + + class D(Generic[T1, Unpack[Ts], T2]): pass + D[int, str] + D[int, str, float] + D[int, str, float, bool] + + class E(Generic[*Ts, T1, T2]): pass + E[int, str] + E[int, str, float] + E[int, str, float, bool] + + class F(Generic[Unpack[Ts], T1, T2]): pass + F[int, str] + F[int, str, float] + F[int, str, float, bool] + + + def test_variadic_args_annotations_are_correct(self): + Ts = TypeVarTuple('Ts') + + def f(*args: Unpack[Ts]): pass + # def g(*args: *Ts): pass + self.assertEqual(f.__annotations__, {'args': Unpack[Ts]}) + # self.assertEqual(g.__annotations__, {'args': (*Ts,)[0]}) + + + def test_variadic_args_with_ellipsis_annotations_are_correct(self): + # def a(*args: *tuple[int, ...]): pass + # self.assertEqual(a.__annotations__, + # {'args': (*tuple[int, ...],)[0]}) + + def b(*args: Unpack[Tuple[int, ...]]): pass + self.assertEqual(b.__annotations__, + {'args': Unpack[Tuple[int, ...]]}) + + + def test_concatenation_in_variadic_args_annotations_are_correct(self): + Ts = TypeVarTuple('Ts') + + # Unpacking using `*`, native `tuple` type + + # def a(*args: *tuple[int, *Ts]): pass + # self.assertEqual( + # a.__annotations__, + # {'args': (*tuple[int, *Ts],)[0]}, + # ) + + # def b(*args: *tuple[*Ts, int]): pass + # self.assertEqual( + # b.__annotations__, + # {'args': (*tuple[*Ts, int],)[0]}, + # ) + + # def c(*args: *tuple[str, *Ts, int]): pass + # self.assertEqual( + # c.__annotations__, + # {'args': (*tuple[str, *Ts, int],)[0]}, + # ) + + # def d(*args: *tuple[int, bool, *Ts, float, str]): pass + # self.assertEqual( + # d.__annotations__, + # {'args': (*tuple[int, bool, *Ts, float, str],)[0]}, + # ) + + # Unpacking using `Unpack`, `Tuple` type from typing.py + + def e(*args: Unpack[Tuple[int, Unpack[Ts]]]): pass + self.assertEqual( + e.__annotations__, + {'args': Unpack[Tuple[int, Unpack[Ts]]]}, + ) + + def f(*args: Unpack[Tuple[Unpack[Ts], int]]): pass + self.assertEqual( + f.__annotations__, + {'args': Unpack[Tuple[Unpack[Ts], int]]}, + ) + + def g(*args: Unpack[Tuple[str, Unpack[Ts], int]]): pass + self.assertEqual( + g.__annotations__, + {'args': Unpack[Tuple[str, Unpack[Ts], int]]}, + ) + + def h(*args: Unpack[Tuple[int, bool, Unpack[Ts], float, str]]): pass + self.assertEqual( + h.__annotations__, + {'args': Unpack[Tuple[int, bool, Unpack[Ts], float, str]]}, + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_same_args_results_in_equalty(self): + Ts = TypeVarTuple('Ts') + # class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + self.assertEqual(C[int], C[int]) + self.assertEqual(D[int], D[int]) + + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + self.assertEqual( + # C[*Ts1], + # C[*Ts1], + ) + self.assertEqual( + D[Unpack[Ts1]], + D[Unpack[Ts1]], + ) + + self.assertEqual( + C[*Ts1, *Ts2], + C[*Ts1, *Ts2], + ) + self.assertEqual( + D[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts1], Unpack[Ts2]], + ) + + self.assertEqual( + C[int, *Ts1, *Ts2], + C[int, *Ts1, *Ts2], + ) + self.assertEqual( + D[int, Unpack[Ts1], Unpack[Ts2]], + D[int, Unpack[Ts1], Unpack[Ts2]], + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_variadic_class_arg_ordering_matters(self): + Ts = TypeVarTuple('Ts') + # class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + self.assertNotEqual( + C[int, str], + C[str, int], + ) + self.assertNotEqual( + D[int, str], + D[str, int], + ) + + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + self.assertNotEqual( + C[*Ts1, *Ts2], + C[*Ts2, *Ts1], + ) + self.assertNotEqual( + D[Unpack[Ts1], Unpack[Ts2]], + D[Unpack[Ts2], Unpack[Ts1]], + ) + + def test_variadic_class_arg_typevartuple_identity_matters(self): + Ts = TypeVarTuple('Ts') + Ts1 = TypeVarTuple('Ts1') + Ts2 = TypeVarTuple('Ts2') + + # class C(Generic[*Ts]): pass + class D(Generic[Unpack[Ts]]): pass + + # self.assertNotEqual(C[*Ts1], C[*Ts2]) + self.assertNotEqual(D[Unpack[Ts1]], D[Unpack[Ts2]]) + +class TypeVarTuplePicklingTests(BaseTestCase): + # These are slightly awkward tests to run, because TypeVarTuples are only + # picklable if defined in the global scope. We therefore need to push + # various things defined in these tests into the global scope with `global` + # statements at the start of each test. + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @all_pickle_protocols + def test_pickling_then_unpickling_results_in_same_identity(self, proto): + global global_Ts1 # See explanation at start of class. + global_Ts1 = TypeVarTuple('global_Ts1') + global_Ts2 = pickle.loads(pickle.dumps(global_Ts1, proto)) + self.assertIs(global_Ts1, global_Ts2) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @all_pickle_protocols + def test_pickling_then_unpickling_unpacked_results_in_same_identity(self, proto): + global global_Ts # See explanation at start of class. + global_Ts = TypeVarTuple('global_Ts') + + unpacked1 = (*global_Ts,)[0] + unpacked2 = pickle.loads(pickle.dumps(unpacked1, proto)) + self.assertIs(unpacked1, unpacked2) + + unpacked3 = Unpack[global_Ts] + unpacked4 = pickle.loads(pickle.dumps(unpacked3, proto)) + self.assertIs(unpacked3, unpacked4) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @all_pickle_protocols + def test_pickling_then_unpickling_tuple_with_typevartuple_equality( + self, proto + ): + global global_T, global_Ts # See explanation at start of class. + global_T = TypeVar('global_T') + global_Ts = TypeVarTuple('global_Ts') + + tuples = [ + # TODO: RUSTPYTHON + # tuple[*global_Ts], + Tuple[Unpack[global_Ts]], + + tuple[T, *global_Ts], + Tuple[T, Unpack[global_Ts]], + + tuple[int, *global_Ts], + Tuple[int, Unpack[global_Ts]], + ] + for t in tuples: + t2 = pickle.loads(pickle.dumps(t, proto)) + self.assertEqual(t, t2) + + + +class UnionTests(BaseTestCase): + + def test_basics(self): + u = Union[int, float] + self.assertNotEqual(u, Union) + + def test_union_isinstance(self): + self.assertTrue(isinstance(42, Union[int, str])) + self.assertTrue(isinstance('abc', Union[int, str])) + self.assertFalse(isinstance(3.14, Union[int, str])) + self.assertTrue(isinstance(42, Union[int, list[int]])) + self.assertTrue(isinstance(42, Union[int, Any])) + + def test_union_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Union[str, list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], int]) + with self.assertRaises(TypeError): + isinstance(42, Union[list[int], str]) + with self.assertRaises(TypeError): + isinstance(42, Union[str, Any]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, int]) + with self.assertRaises(TypeError): + isinstance(42, Union[Any, str]) + + def test_optional_isinstance(self): + self.assertTrue(isinstance(42, Optional[int])) + self.assertTrue(isinstance(None, Optional[int])) + self.assertFalse(isinstance('abc', Optional[int])) + + def test_optional_isinstance_type_error(self): + with self.assertRaises(TypeError): + isinstance(42, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(None, Optional[list[int]]) + with self.assertRaises(TypeError): + isinstance(42, Optional[Any]) + with self.assertRaises(TypeError): + isinstance(None, Optional[Any]) + + def test_union_issubclass(self): + self.assertTrue(issubclass(int, Union[int, str])) + self.assertTrue(issubclass(str, Union[int, str])) + self.assertFalse(issubclass(float, Union[int, str])) + self.assertTrue(issubclass(int, Union[int, list[int]])) + self.assertTrue(issubclass(int, Union[int, Any])) + self.assertFalse(issubclass(int, Union[str, Any])) + self.assertTrue(issubclass(int, Union[Any, int])) + self.assertFalse(issubclass(int, Union[Any, str])) + + def test_union_issubclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(int, Union) + with self.assertRaises(TypeError): + issubclass(Union, int) + with self.assertRaises(TypeError): + issubclass(Union[int, str], int) + with self.assertRaises(TypeError): + issubclass(int, Union[str, list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], int]) + with self.assertRaises(TypeError): + issubclass(int, Union[list[int], str]) + + def test_optional_issubclass(self): + self.assertTrue(issubclass(int, Optional[int])) + self.assertTrue(issubclass(type(None), Optional[int])) + self.assertFalse(issubclass(str, Optional[int])) + self.assertTrue(issubclass(Any, Optional[Any])) + self.assertTrue(issubclass(type(None), Optional[Any])) + self.assertFalse(issubclass(int, Optional[Any])) + + def test_optional_issubclass_type_error(self): + with self.assertRaises(TypeError): + issubclass(list[int], Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(type(None), Optional[list[int]]) + with self.assertRaises(TypeError): + issubclass(int, Optional[list[int]]) + + def test_union_any(self): + u = Union[Any] + self.assertEqual(u, Any) + u1 = Union[int, Any] + u2 = Union[Any, int] + u3 = Union[Any, object] + self.assertEqual(u1, u2) + self.assertNotEqual(u1, Any) + self.assertNotEqual(u2, Any) + self.assertNotEqual(u3, Any) + + def test_union_object(self): + u = Union[object] + self.assertEqual(u, object) + u1 = Union[int, object] + u2 = Union[object, int] + self.assertEqual(u1, u2) + self.assertNotEqual(u1, object) + self.assertNotEqual(u2, object) + + def test_unordered(self): + u1 = Union[int, float] + u2 = Union[float, int] + self.assertEqual(u1, u2) + + def test_single_class_disappears(self): + t = Union[Employee] + self.assertIs(t, Employee) + + def test_base_class_kept(self): + u = Union[Employee, Manager] + self.assertNotEqual(u, Employee) + self.assertIn(Employee, u.__args__) + self.assertIn(Manager, u.__args__) + + def test_union_union(self): + u = Union[int, float] + v = Union[u, Employee] + self.assertEqual(v, Union[int, float, Employee]) + + def test_union_of_unhashable(self): + class UnhashableMeta(type): + __hash__ = None + + class A(metaclass=UnhashableMeta): ... + class B(metaclass=UnhashableMeta): ... + + self.assertEqual(Union[A, B].__args__, (A, B)) + union1 = Union[A, B] + with self.assertRaises(TypeError): + hash(union1) + + union2 = Union[int, B] + with self.assertRaises(TypeError): + hash(union2) + + union3 = Union[A, int] + with self.assertRaises(TypeError): + hash(union3) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_repr(self): + self.assertEqual(repr(Union), 'typing.Union') + u = Union[Employee, int] + self.assertEqual(repr(u), 'typing.Union[%s.Employee, int]' % __name__) + u = Union[int, Employee] + self.assertEqual(repr(u), 'typing.Union[int, %s.Employee]' % __name__) + T = TypeVar('T') + u = Union[T, int][int] + self.assertEqual(repr(u), repr(int)) + u = Union[List[int], int] + self.assertEqual(repr(u), 'typing.Union[typing.List[int], int]') + u = Union[list[int], dict[str, float]] + self.assertEqual(repr(u), 'typing.Union[list[int], dict[str, float]]') + u = Union[int | float] + self.assertEqual(repr(u), 'typing.Union[int, float]') + + u = Union[None, str] + self.assertEqual(repr(u), 'typing.Optional[str]') + u = Union[str, None] + self.assertEqual(repr(u), 'typing.Optional[str]') + u = Union[None, str, int] + self.assertEqual(repr(u), 'typing.Union[NoneType, str, int]') + u = Optional[str] + self.assertEqual(repr(u), 'typing.Optional[str]') + + def test_dir(self): + dir_items = set(dir(Union[str, int])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Union'): + class C(Union): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(Union)): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Union\[int, str\]'): + class E(Union[int, str]): + pass + + def test_cannot_instantiate(self): + with self.assertRaises(TypeError): + Union() + with self.assertRaises(TypeError): + type(Union)() + u = Union[int, float] with self.assertRaises(TypeError): u() with self.assertRaises(TypeError): @@ -410,6 +2371,35 @@ def Elem(*args): Union[Elem, str] # Nor should this + def test_union_of_literals(self): + self.assertEqual(Union[Literal[1], Literal[2]].__args__, + (Literal[1], Literal[2])) + self.assertEqual(Union[Literal[1], Literal[1]], + Literal[1]) + + self.assertEqual(Union[Literal[False], Literal[0]].__args__, + (Literal[False], Literal[0])) + self.assertEqual(Union[Literal[True], Literal[1]].__args__, + (Literal[True], Literal[1])) + + import enum + class Ints(enum.IntEnum): + A = 0 + B = 1 + + self.assertEqual(Union[Literal[Ints.A], Literal[Ints.A]], + Literal[Ints.A]) + self.assertEqual(Union[Literal[Ints.B], Literal[Ints.B]], + Literal[Ints.B]) + + self.assertEqual(Union[Literal[Ints.A], Literal[Ints.B]].__args__, + (Literal[Ints.A], Literal[Ints.B])) + + self.assertEqual(Union[Literal[0], Literal[Ints.A], Literal[False]].__args__, + (Literal[0], Literal[Ints.A], Literal[False])) + self.assertEqual(Union[Literal[1], Literal[Ints.B], Literal[True]].__args__, + (Literal[1], Literal[Ints.B], Literal[True])) + class TupleTests(BaseTestCase): @@ -441,6 +2431,8 @@ def test_tuple_instance_type_error(self): isinstance((0, 0), Tuple[int, int]) self.assertIsInstance((0, 0), Tuple) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repr(self): self.assertEqual(repr(Tuple), 'typing.Tuple') self.assertEqual(repr(Tuple[()]), 'typing.Tuple[()]') @@ -476,6 +2468,15 @@ def test_eq_hash(self): self.assertNotEqual(C, Callable[..., int]) self.assertNotEqual(C, Callable) + def test_dir(self): + Callable = self.Callable + dir_items = set(dir(Callable[..., int])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_cannot_instantiate(self): Callable = self.Callable with self.assertRaises(TypeError): @@ -505,14 +2506,16 @@ def test_callable_instance_type_error(self): def f(): pass with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], None]) + isinstance(f, Callable[[], None]) with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], Any]) + isinstance(f, Callable[[], Any]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], None]) + isinstance(None, Callable[[], None]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], Any]) + isinstance(None, Callable[[], Any]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repr(self): Callable = self.Callable fullname = f'{Callable.__module__}.Callable' @@ -525,6 +2528,7 @@ def test_repr(self): ct3 = Callable[[str, float], list[int]] self.assertEqual(repr(ct3), f'{fullname}[[str, float], list[int]]') + @unittest.skip("TODO: RUSTPYTHON") def test_callable_with_ellipsis(self): Callable = self.Callable def foo(a: Callable[..., T]): @@ -557,16 +2561,35 @@ def test_weakref(self): alias = Callable[[int, str], float] self.assertEqual(weakref.ref(alias)(), alias) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pickle(self): + global T_pickle, P_pickle, TS_pickle # needed for pickling Callable = self.Callable - alias = Callable[[int, str], float] - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(alias, proto) - loaded = pickle.loads(s) - self.assertEqual(alias.__origin__, loaded.__origin__) - self.assertEqual(alias.__args__, loaded.__args__) - self.assertEqual(alias.__parameters__, loaded.__parameters__) + T_pickle = TypeVar('T_pickle') + P_pickle = ParamSpec('P_pickle') + TS_pickle = TypeVarTuple('TS_pickle') + + samples = [ + Callable[[int, str], float], + Callable[P_pickle, int], + Callable[P_pickle, T_pickle], + Callable[Concatenate[int, P_pickle], int], + Callable[Concatenate[*TS_pickle, P_pickle], int], + ] + for alias in samples: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(alias=alias, proto=proto): + s = pickle.dumps(alias, proto) + loaded = pickle.loads(s) + self.assertEqual(alias.__origin__, loaded.__origin__) + self.assertEqual(alias.__args__, loaded.__args__) + self.assertEqual(alias.__parameters__, loaded.__parameters__) + del T_pickle, P_pickle, TS_pickle # cleaning up global state + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_var_substitution(self): Callable = self.Callable fullname = f"{Callable.__module__}.Callable" @@ -574,8 +2597,7 @@ def test_var_substitution(self): C2 = Callable[[KT, T], VT] C3 = Callable[..., T] self.assertEqual(C1[str], Callable[[int, str], str]) - if Callable is typing.Callable: - self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) + self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) self.assertEqual(C2[int, float, str], Callable[[int, float], str]) self.assertEqual(C3[int], Callable[..., int]) self.assertEqual(C3[NoReturn], Callable[..., NoReturn]) @@ -592,6 +2614,17 @@ def test_var_substitution(self): self.assertEqual(C5[int, str, float], Callable[[typing.List[int], tuple[str, int], float], int]) + @unittest.skip("TODO: RUSTPYTHON") + def test_type_subst_error(self): + Callable = self.Callable + P = ParamSpec('P') + T = TypeVar('T') + + pat = "Expected a list of types, an ellipsis, ParamSpec, or Concatenate." + + with self.assertRaisesRegex(TypeError, pat): + Callable[P, T][0, int] + def test_type_erasure(self): Callable = self.Callable class C1(Callable): @@ -601,6 +2634,8 @@ def __call__(self): self.assertIs(a().__class__, C1) self.assertEqual(a().__orig_class__, C1[[int], T]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_paramspec(self): Callable = self.Callable fullname = f"{Callable.__module__}.Callable" @@ -635,6 +2670,8 @@ def test_paramspec(self): self.assertEqual(repr(C2), f"{fullname}[~P, int]") self.assertEqual(repr(C2[int, str]), f"{fullname}[[int, str], int]") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_concatenate(self): Callable = self.Callable fullname = f"{Callable.__module__}.Callable" @@ -649,8 +2686,7 @@ def test_concatenate(self): self.assertEqual(C[[], int], Callable[[int], int]) self.assertEqual(C[Concatenate[str, P2], int], Callable[Concatenate[int, str, P2], int]) - with self.assertRaises(TypeError): - C[..., int] + self.assertEqual(C[..., int], Callable[Concatenate[int, ...], int]) C = Callable[Concatenate[int, P], int] self.assertEqual(repr(C), @@ -661,9 +2697,54 @@ def test_concatenate(self): self.assertEqual(C[[]], Callable[[int], int]) self.assertEqual(C[Concatenate[str, P2]], Callable[Concatenate[int, str, P2], int]) - with self.assertRaises(TypeError): - C[...] + self.assertEqual(C[...], Callable[Concatenate[int, ...], int]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_nested_paramspec(self): + # Since Callable has some special treatment, we want to be sure + # that substituion works correctly, see gh-103054 + Callable = self.Callable + P = ParamSpec('P') + P2 = ParamSpec('P2') + T = TypeVar('T') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + class My(Generic[P, T]): + pass + + self.assertEqual(My.__parameters__, (P, T)) + + C1 = My[[int, T2], Callable[P2, T2]] + self.assertEqual(C1.__args__, ((int, T2), Callable[P2, T2])) + self.assertEqual(C1.__parameters__, (T2, P2)) + self.assertEqual(C1[str, [list[int], bytes]], + My[[int, str], Callable[[list[int], bytes], str]]) + + C2 = My[[Callable[[T2], int], list[T2]], str] + self.assertEqual(C2.__args__, ((Callable[[T2], int], list[T2]), str)) + self.assertEqual(C2.__parameters__, (T2,)) + self.assertEqual(C2[list[str]], + My[[Callable[[list[str]], int], list[list[str]]], str]) + + C3 = My[[Callable[P2, T2], T2], T2] + self.assertEqual(C3.__args__, ((Callable[P2, T2], T2), T2)) + self.assertEqual(C3.__parameters__, (P2, T2)) + self.assertEqual(C3[[], int], + My[[Callable[[], int], int], int]) + self.assertEqual(C3[[str, bool], int], + My[[Callable[[str, bool], int], int], int]) + self.assertEqual(C3[[str, bool], T][int], + My[[Callable[[str, bool], int], int], int]) + + C4 = My[[Callable[[int, *Ts, str], T2], T2], T2] + self.assertEqual(C4.__args__, ((Callable[[int, *Ts, str], T2], T2), T2)) + self.assertEqual(C4.__parameters__, (Ts, T2)) + self.assertEqual(C4[bool, bytes, float], + My[[Callable[[int, bool, bytes, str], float], float], float]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_errors(self): Callable = self.Callable alias = Callable[[int, str], float] @@ -676,6 +2757,7 @@ def test_errors(self): with self.assertRaisesRegex(TypeError, "few arguments for"): C1[int] + class TypingCallableTests(BaseCallableTests, BaseTestCase): Callable = typing.Callable @@ -690,25 +2772,6 @@ def test_consistency(self): class CollectionsCallableTests(BaseCallableTests, BaseTestCase): Callable = collections.abc.Callable - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_errors(self): # TODO: RUSTPYTHON, remove when this passes - super().test_errors() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError: 'collections.abc.Callable[__main__.ParamSpec, typing.TypeVar]' != 'collections.abc.Callable[~P, ~T]' - @unittest.expectedFailure - def test_paramspec(self): # TODO: RUSTPYTHON, remove when this passes - super().test_paramspec() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_concatenate(self): # TODO: RUSTPYTHON, remove when this passes - super().test_concatenate() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON might be fixed by updating typing to 3.12 - @unittest.expectedFailure - def test_repr(self): # TODO: RUSTPYTHON, remove when this passes - super().test_repr() # TODO: RUSTPYTHON, remove when this passes class LiteralTests(BaseTestCase): @@ -723,9 +2786,16 @@ def test_basics(self): Literal[Literal[1, 2], Literal[4, 5]] Literal[b"foo", u"bar"] + def test_enum(self): + import enum + class My(enum.Enum): + A = 'A' + + self.assertEqual(Literal[My.A].__args__, (My.A,)) + def test_illegal_parameters_do_not_raise_runtime_errors(self): # Type checkers should reject these types, but we do not - # raise errors at runtime to maintain maximium flexibility. + # raise errors at runtime to maintain maximum flexibility. Literal[int] Literal[3j + 2, ..., ()] Literal[{"foo": 3, "bar": 4}] @@ -743,6 +2813,14 @@ def test_repr(self): self.assertEqual(repr(Literal[None]), "typing.Literal[None]") self.assertEqual(repr(Literal[1, 2, 3, 3]), "typing.Literal[1, 2, 3]") + def test_dir(self): + dir_items = set(dir(Literal[1, 2, 3])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_cannot_init(self): with self.assertRaises(TypeError): Literal() @@ -804,6 +2882,20 @@ def test_flatten(self): self.assertEqual(l, Literal[1, 2, 3]) self.assertEqual(l.__args__, (1, 2, 3)) + def test_does_not_flatten_enum(self): + import enum + class Ints(enum.IntEnum): + A = 1 + B = 2 + + l = Literal[ + Literal[Ints.A], + Literal[Ints.B], + Literal[1], + Literal[2], + ] + self.assertEqual(l.__args__, (Ints.A, Ints.B, 1, 2)) + XK = TypeVar('XK', str, bytes) XV = TypeVar('XV') @@ -888,29 +2980,75 @@ class HasCallProtocol(Protocol): class ProtocolTests(BaseTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_protocol(self): @runtime_checkable class P(Protocol): def meth(self): pass - class C: pass + class C: pass + + class D: + def meth(self): + pass + + def f(): + pass + + self.assertIsSubclass(D, P) + self.assertIsInstance(D(), P) + self.assertNotIsSubclass(C, P) + self.assertNotIsInstance(C(), P) + self.assertNotIsSubclass(types.FunctionType, P) + self.assertNotIsInstance(f, P) + + def test_runtime_checkable_generic_non_protocol(self): + # Make sure this doesn't raise AttributeError + with self.assertRaisesRegex( + TypeError, + "@runtime_checkable can be only applied to protocol classes", + ): + @runtime_checkable + class Foo[T]: ... + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_runtime_checkable_generic(self): + # @runtime_checkable + # class Foo[T](Protocol): + # def meth(self) -> T: ... + # pass + + class Impl: + def meth(self) -> int: ... + + self.assertIsSubclass(Impl, Foo) + + class NotImpl: + def method(self) -> int: ... + + self.assertNotIsSubclass(NotImpl, Foo) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep695_generics_can_be_runtime_checkable(self): + # @runtime_checkable + # class HasX(Protocol): + # x: int + + class Bar[T]: + x: T + def __init__(self, x): + self.x = x - class D: - def meth(self): - pass + class Capybara[T]: + y: str + def __init__(self, y): + self.y = y - def f(): - pass + self.assertIsInstance(Bar(1), HasX) + self.assertNotIsInstance(Capybara('a'), HasX) - self.assertIsSubclass(D, P) - self.assertIsInstance(D(), P) - self.assertNotIsSubclass(C, P) - self.assertNotIsInstance(C(), P) - self.assertNotIsSubclass(types.FunctionType, P) - self.assertNotIsInstance(f, P) # TODO: RUSTPYTHON @unittest.expectedFailure @@ -936,20 +3074,22 @@ def f(): self.assertIsInstance(f, HasCallProtocol) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_no_inheritance_from_nominal(self): class C: pass - class BP(Protocol): pass + # class BP(Protocol): pass - with self.assertRaises(TypeError): - class P(C, Protocol): - pass - with self.assertRaises(TypeError): - class P(Protocol, C): - pass - with self.assertRaises(TypeError): - class P(BP, C, Protocol): - pass + # with self.assertRaises(TypeError): + # class P(C, Protocol): + # pass + # with self.assertRaises(TypeError): + # class Q(Protocol, C): + # pass + # with self.assertRaises(TypeError): + # class R(BP, C, Protocol): + # pass class D(BP, C): pass @@ -961,7 +3101,7 @@ class E(C, BP): pass # TODO: RUSTPYTHON @unittest.expectedFailure def test_no_instantiation(self): - class P(Protocol): pass + # class P(Protocol): pass with self.assertRaises(TypeError): P() @@ -989,6 +3129,35 @@ class CG(PG[T]): pass with self.assertRaises(TypeError): CG[int](42) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_protocol_defining_init_does_not_get_overridden(self): + # check that P.__init__ doesn't get clobbered + # see https://bugs.python.org/issue44807 + + # class P(Protocol): + # x: int + # def __init__(self, x: int) -> None: + # self.x = x + class C: pass + + c = C() + P.__init__(c, 1) + self.assertEqual(c.x, 1) + + + def test_concrete_class_inheriting_init_from_protocol(self): + class P(Protocol): + x: int + def __init__(self, x: int) -> None: + self.x = x + + class C(P): pass + + c = C(1) + self.assertIsInstance(c, C) + self.assertEqual(c.x, 1) + def test_cannot_instantiate_abstract(self): @runtime_checkable class P(Protocol): @@ -1007,8 +3176,6 @@ def ameth(self) -> int: B() self.assertIsInstance(C(), P) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subprotocols_extending(self): class P1(Protocol): def meth1(self): @@ -1041,8 +3208,6 @@ def meth2(self): self.assertIsInstance(C(), P2) self.assertIsSubclass(C, P2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subprotocols_merging(self): class P1(Protocol): def meth1(self): @@ -1104,643 +3269,332 @@ def x(self): ... self.assertIsSubclass(C, PG) self.assertIsSubclass(BadP, PG) - with self.assertRaises(TypeError): + no_subscripted_generics = ( + "Subscripted generics cannot be used with class and instance checks" + ) + + with self.assertRaisesRegex(TypeError, no_subscripted_generics): issubclass(C, PG[T]) - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, no_subscripted_generics): issubclass(C, PG[C]) - with self.assertRaises(TypeError): + + only_runtime_checkable_protocols = ( + "Instance and class checks can only be used with " + "@runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, only_runtime_checkable_protocols): issubclass(C, BadP) - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, only_runtime_checkable_protocols): issubclass(C, BadPG) - with self.assertRaises(TypeError): + + with self.assertRaisesRegex(TypeError, no_subscripted_generics): issubclass(P, PG[T]) - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, no_subscripted_generics): issubclass(PG, PG[int]) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_protocols_issubclass_non_callable(self): - class C: - x = 1 - - @runtime_checkable - class PNonCall(Protocol): - x = 1 - - with self.assertRaises(TypeError): - issubclass(C, PNonCall) - self.assertIsInstance(C(), PNonCall) - PNonCall.register(C) - with self.assertRaises(TypeError): - issubclass(C, PNonCall) - self.assertIsInstance(C(), PNonCall) - - # check that non-protocol subclasses are not affected - class D(PNonCall): ... - - self.assertNotIsSubclass(C, D) - self.assertNotIsInstance(C(), D) - D.register(C) - self.assertIsSubclass(C, D) - self.assertIsInstance(C(), D) - with self.assertRaises(TypeError): - issubclass(D, PNonCall) - - def test_protocols_isinstance(self): - T = TypeVar('T') - - @runtime_checkable - class P(Protocol): - def meth(x): ... - - @runtime_checkable - class PG(Protocol[T]): - def meth(x): ... - - class BadP(Protocol): - def meth(x): ... - - class BadPG(Protocol[T]): - def meth(x): ... - - class C: - def meth(x): ... - - self.assertIsInstance(C(), P) - self.assertIsInstance(C(), PG) - with self.assertRaises(TypeError): - isinstance(C(), PG[T]) - with self.assertRaises(TypeError): - isinstance(C(), PG[C]) - with self.assertRaises(TypeError): - isinstance(C(), BadP) - with self.assertRaises(TypeError): - isinstance(C(), BadPG) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_protocols_isinstance_py36(self): - class APoint: - def __init__(self, x, y, label): - self.x = x - self.y = y - self.label = label - - class BPoint: - label = 'B' - - def __init__(self, x, y): - self.x = x - self.y = y - - class C: - def __init__(self, attr): - self.attr = attr - - def meth(self, arg): - return 0 - - class Bad: pass - - self.assertIsInstance(APoint(1, 2, 'A'), Point) - self.assertIsInstance(BPoint(1, 2), Point) - self.assertNotIsInstance(MyPoint(), Point) - self.assertIsInstance(BPoint(1, 2), Position) - self.assertIsInstance(Other(), Proto) - self.assertIsInstance(Concrete(), Proto) - self.assertIsInstance(C(42), Proto) - self.assertNotIsInstance(Bad(), Proto) - self.assertNotIsInstance(Bad(), Point) - self.assertNotIsInstance(Bad(), Position) - self.assertNotIsInstance(Bad(), Concrete) - self.assertNotIsInstance(Other(), Concrete) - self.assertIsInstance(NT(1, 2), Position) - - def test_protocols_isinstance_init(self): - T = TypeVar('T') - - @runtime_checkable - class P(Protocol): - x = 1 - - @runtime_checkable - class PG(Protocol[T]): - x = 1 - - class C: - def __init__(self, x): - self.x = x - - self.assertIsInstance(C(1), P) - self.assertIsInstance(C(1), PG) - - def test_protocol_checks_after_subscript(self): - class P(Protocol[T]): pass - class C(P[T]): pass - class Other1: pass - class Other2: pass - CA = C[Any] - - self.assertNotIsInstance(Other1(), C) - self.assertNotIsSubclass(Other2, C) - - class D1(C[Any]): pass - class D2(C[Any]): pass - CI = C[int] - - self.assertIsInstance(D1(), C) - self.assertIsSubclass(D2, C) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_protocols_support_register(self): - @runtime_checkable - class P(Protocol): - x = 1 - - class PM(Protocol): - def meth(self): pass - - class D(PM): pass - - class C: pass - - D.register(C) - P.register(C) - self.assertIsInstance(C(), P) - self.assertIsInstance(C(), D) - - def test_none_on_non_callable_doesnt_block_implementation(self): - @runtime_checkable - class P(Protocol): - x = 1 - - class A: - x = 1 - - class B(A): - x = None - - class C: - def __init__(self): - self.x = None - - self.assertIsInstance(B(), P) - self.assertIsInstance(C(), P) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_none_on_callable_blocks_implementation(self): - @runtime_checkable - class P(Protocol): - def x(self): ... - - class A: - def x(self): ... - - class B(A): - x = None - - class C: - def __init__(self): - self.x = None - - self.assertNotIsInstance(B(), P) - self.assertNotIsInstance(C(), P) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_non_protocol_subclasses(self): - class P(Protocol): - x = 1 - - @runtime_checkable - class PR(Protocol): - def meth(self): pass - - class NonP(P): - x = 1 - - class NonPR(PR): pass - - class C: - x = 1 - - class D: - def meth(self): pass - - self.assertNotIsInstance(C(), NonP) - self.assertNotIsInstance(D(), NonPR) - self.assertNotIsSubclass(C, NonP) - self.assertNotIsSubclass(D, NonPR) - self.assertIsInstance(NonPR(), PR) - self.assertIsSubclass(NonPR, PR) - - def test_custom_subclasshook(self): - class P(Protocol): - x = 1 - - class OKClass: pass - - class BadClass: - x = 1 - - class C(P): - @classmethod - def __subclasshook__(cls, other): - return other.__name__.startswith("OK") - - self.assertIsInstance(OKClass(), C) - self.assertNotIsInstance(BadClass(), C) - self.assertIsSubclass(OKClass, C) - self.assertNotIsSubclass(BadClass, C) - - def test_issubclass_fails_correctly(self): - @runtime_checkable - class P(Protocol): - x = 1 - - class C: pass - - with self.assertRaises(TypeError): - issubclass(C(), P) - - def test_defining_generic_protocols(self): - T = TypeVar('T') - S = TypeVar('S') - - @runtime_checkable - class PR(Protocol[T, S]): - def meth(self): pass - - class P(PR[int, T], Protocol[T]): - y = 1 - - with self.assertRaises(TypeError): - PR[int] - with self.assertRaises(TypeError): - P[int, str] - - class C(PR[int, T]): pass - - self.assertIsInstance(C[str](), C) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_defining_generic_protocols_old_style(self): - T = TypeVar('T') - S = TypeVar('S') - - @runtime_checkable - class PR(Protocol, Generic[T, S]): - def meth(self): pass - - class P(PR[int, str], Protocol): - y = 1 - - with self.assertRaises(TypeError): - issubclass(PR[int, str], PR) - self.assertIsSubclass(P, PR) - with self.assertRaises(TypeError): - PR[int] - - class P1(Protocol, Generic[T]): - def bar(self, x: T) -> str: ... - - class P2(Generic[T], Protocol): - def bar(self, x: T) -> str: ... - - @runtime_checkable - class PSub(P1[str], Protocol): - x = 1 - - class Test: - x = 1 - - def bar(self, x: str) -> str: - return x - - self.assertIsInstance(Test(), PSub) - - def test_init_called(self): - T = TypeVar('T') - - class P(Protocol[T]): pass - - class C(P[T]): - def __init__(self): - self.test = 'OK' - - self.assertEqual(C[int]().test, 'OK') - - class B: - def __init__(self): - self.test = 'OK' - - class D1(B, P[T]): - pass + only_classes_allowed = r"issubclass\(\) arg 1 must be a class" - self.assertEqual(D1[int]().test, 'OK') + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, P) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, PG) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, BadP) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, BadPG) - class D2(P[T], B): - pass - self.assertEqual(D2[int]().test, 'OK') + def test_implicit_issubclass_between_two_protocols(self): + @runtime_checkable + class CallableMembersProto(Protocol): + def meth(self): ... - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_new_called(self): - T = TypeVar('T') + # All the below protocols should be considered "subclasses" + # of CallableMembersProto at runtime, + # even though none of them explicitly subclass CallableMembersProto - class P(Protocol[T]): pass + class IdenticalProto(Protocol): + def meth(self): ... - class C(P[T]): - def __new__(cls, *args): - self = super().__new__(cls, *args) - self.test = 'OK' - return self + class SupersetProto(Protocol): + def meth(self): ... + def meth2(self): ... - self.assertEqual(C[int]().test, 'OK') - with self.assertRaises(TypeError): - C[int](42) - with self.assertRaises(TypeError): - C[int](a=42) + class NonCallableMembersProto(Protocol): + meth: Callable[[], None] - # TODO: RUSTPYTHON the last line breaks any tests that use unittest.mock - # See https://github.com/RustPython/RustPython/issues/5190#issuecomment-2010535802 - # It's possible that updating typing to 3.12 will resolve this - @unittest.skip("TODO: RUSTPYTHON this test breaks other tests that use unittest.mock") - def test_protocols_bad_subscripts(self): - T = TypeVar('T') - S = TypeVar('S') - with self.assertRaises(TypeError): - class P(Protocol[T, T]): pass - with self.assertRaises(TypeError): - class P(Protocol[int]): pass - with self.assertRaises(TypeError): - class P(Protocol[T], Protocol[S]): pass - with self.assertRaises(TypeError): - class P(typing.Mapping[T, S], Protocol[T]): pass + class NonCallableMembersSupersetProto(Protocol): + meth: Callable[[], None] + meth2: Callable[[str, int], bool] - def test_generic_protocols_repr(self): - T = TypeVar('T') - S = TypeVar('S') + class MixedMembersProto1(Protocol): + meth: Callable[[], None] + def meth2(self): ... - class P(Protocol[T, S]): pass + class MixedMembersProto2(Protocol): + def meth(self): ... + meth2: Callable[[str, int], bool] - self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]')) - self.assertTrue(repr(P[int, str]).endswith('P[int, str]')) + for proto in ( + IdenticalProto, SupersetProto, NonCallableMembersProto, + NonCallableMembersSupersetProto, MixedMembersProto1, MixedMembersProto2 + ): + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, CallableMembersProto) - def test_generic_protocols_eq(self): - T = TypeVar('T') - S = TypeVar('S') + # These two shouldn't be considered subclasses of CallableMembersProto, however, + # since they don't have the `meth` protocol member - class P(Protocol[T, S]): pass + class EmptyProtocol(Protocol): ... + class UnrelatedProtocol(Protocol): + def wut(self): ... - self.assertEqual(P, P) - self.assertEqual(P[int, T], P[int, T]) - self.assertEqual(P[T, T][Tuple[T, S]][int, str], - P[Tuple[int, str], Tuple[int, str]]) + self.assertNotIsSubclass(EmptyProtocol, CallableMembersProto) + self.assertNotIsSubclass(UnrelatedProtocol, CallableMembersProto) - def test_generic_protocols_special_from_generic(self): - T = TypeVar('T') + # These aren't protocols at all (despite having annotations), + # so they should only be considered subclasses of CallableMembersProto + # if they *actually have an attribute* matching the `meth` member + # (just having an annotation is insufficient) - class P(Protocol[T]): pass + class AnnotatedButNotAProtocol: + meth: Callable[[], None] - self.assertEqual(P.__parameters__, (T,)) - self.assertEqual(P[int].__parameters__, ()) - self.assertEqual(P[int].__args__, (int,)) - self.assertIs(P[int].__origin__, P) + class NotAProtocolButAnImplicitSubclass: + def meth(self): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_generic_protocols_special_from_protocol(self): - @runtime_checkable - class PR(Protocol): - x = 1 + class NotAProtocolButAnImplicitSubclass2: + meth: Callable[[], None] + def meth(self): pass - class P(Protocol): - def meth(self): - pass + class NotAProtocolButAnImplicitSubclass3: + meth: Callable[[], None] + meth2: Callable[[int, str], bool] + def meth(self): pass + def meth2(self, x, y): return True - T = TypeVar('T') + self.assertNotIsSubclass(AnnotatedButNotAProtocol, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass2, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass3, CallableMembersProto) - class PG(Protocol[T]): - x = 1 + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON (no gc)") + def test_isinstance_checks_not_at_whim_of_gc(self): + self.addCleanup(gc.enable) + gc.disable() - def meth(self): + with self.assertRaisesRegex( + TypeError, + "Protocols can only inherit from other protocols" + ): + class Foo(collections.abc.Mapping, Protocol): pass - self.assertTrue(P._is_protocol) - self.assertTrue(PR._is_protocol) - self.assertTrue(PG._is_protocol) - self.assertFalse(P._is_runtime_protocol) - self.assertTrue(PR._is_runtime_protocol) - self.assertTrue(PG[int]._is_protocol) - self.assertEqual(typing._get_protocol_attrs(P), {'meth'}) - self.assertEqual(typing._get_protocol_attrs(PR), {'x'}) - self.assertEqual(frozenset(typing._get_protocol_attrs(PG)), - frozenset({'x', 'meth'})) + self.assertNotIsInstance([], collections.abc.Mapping) - def test_no_runtime_deco_on_nominal(self): - with self.assertRaises(TypeError): - @runtime_checkable - class C: pass + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_issubclass_and_isinstance_on_Protocol_itself(self): + class C: + def x(self): pass - class Proto(Protocol): - x = 1 + self.assertNotIsSubclass(object, Protocol) + self.assertNotIsInstance(object(), Protocol) - with self.assertRaises(TypeError): - @runtime_checkable - class Concrete(Proto): - pass + self.assertNotIsSubclass(str, Protocol) + self.assertNotIsInstance('foo', Protocol) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_none_treated_correctly(self): - @runtime_checkable - class P(Protocol): - x = None # type: int + self.assertNotIsSubclass(C, Protocol) + self.assertNotIsInstance(C(), Protocol) - class B(object): pass + only_classes_allowed = r"issubclass\(\) arg 1 must be a class" - self.assertNotIsInstance(B(), P) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, Protocol) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass('foo', Protocol) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(C(), Protocol) - class C: - x = 1 + T = TypeVar('T') - class D: - x = None + @runtime_checkable + class EmptyProtocol(Protocol): pass - self.assertIsInstance(C(), P) - self.assertIsInstance(D(), P) + @runtime_checkable + class SupportsStartsWith(Protocol): + def startswith(self, x: str) -> bool: ... - class CI: - def __init__(self): - self.x = 1 + @runtime_checkable + class SupportsX(Protocol[T]): + def x(self): ... - class DI: - def __init__(self): - self.x = None + for proto in EmptyProtocol, SupportsStartsWith, SupportsX: + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, Protocol) - self.assertIsInstance(C(), P) - self.assertIsInstance(D(), P) + # gh-105237 / PR #105239: + # check that the presence of Protocol subclasses + # where `issubclass(X, )` evaluates to True + # doesn't influence the result of `issubclass(X, Protocol)` - def test_protocols_in_unions(self): - class P(Protocol): - x = None # type: int + self.assertIsSubclass(object, EmptyProtocol) + self.assertIsInstance(object(), EmptyProtocol) + self.assertNotIsSubclass(object, Protocol) + self.assertNotIsInstance(object(), Protocol) - Alias = typing.Union[typing.Iterable, P] - Alias2 = typing.Union[P, typing.Iterable] - self.assertEqual(Alias, Alias2) + self.assertIsSubclass(str, SupportsStartsWith) + self.assertIsInstance('foo', SupportsStartsWith) + self.assertNotIsSubclass(str, Protocol) + self.assertNotIsInstance('foo', Protocol) - def test_protocols_pickleable(self): - global P, CP # pickle wants to reference the class by name - T = TypeVar('T') + self.assertIsSubclass(C, SupportsX) + self.assertIsInstance(C(), SupportsX) + self.assertNotIsSubclass(C, Protocol) + self.assertNotIsInstance(C(), Protocol) - @runtime_checkable - class P(Protocol[T]): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_protocols_issubclass_non_callable(self): + class C: x = 1 - class CP(P[int]): - pass + @runtime_checkable + class PNonCall(Protocol): + x = 1 - c = CP() - c.foo = 42 - c.bar = 'abc' - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - z = pickle.dumps(c, proto) - x = pickle.loads(z) - self.assertEqual(x.foo, 42) - self.assertEqual(x.bar, 'abc') - self.assertEqual(x.x, 1) - self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) - s = pickle.dumps(P) - D = pickle.loads(s) + non_callable_members_illegal = ( + "Protocols with non-method members don't support issubclass()" + ) - class E: - x = 1 + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(C, PNonCall) - self.assertIsInstance(E(), D) + self.assertIsInstance(C(), PNonCall) + PNonCall.register(C) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_supports_int(self): - self.assertIsSubclass(int, typing.SupportsInt) - self.assertNotIsSubclass(str, typing.SupportsInt) + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(C, PNonCall) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_supports_float(self): - self.assertIsSubclass(float, typing.SupportsFloat) - self.assertNotIsSubclass(str, typing.SupportsFloat) + self.assertIsInstance(C(), PNonCall) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_supports_complex(self): + # check that non-protocol subclasses are not affected + class D(PNonCall): ... - # Note: complex itself doesn't have __complex__. - class C: - def __complex__(self): - return 0j + self.assertNotIsSubclass(C, D) + self.assertNotIsInstance(C(), D) + D.register(C) + self.assertIsSubclass(C, D) + self.assertIsInstance(C(), D) - self.assertIsSubclass(C, typing.SupportsComplex) - self.assertNotIsSubclass(str, typing.SupportsComplex) + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(D, PNonCall) # TODO: RUSTPYTHON @unittest.expectedFailure - def test_supports_bytes(self): + def test_no_weird_caching_with_issubclass_after_isinstance(self): + @runtime_checkable + class Spam(Protocol): + x: int - # Note: bytes itself doesn't have __bytes__. - class B: - def __bytes__(self): - return b'' + class Eggs: + def __init__(self) -> None: + self.x = 42 - self.assertIsSubclass(B, typing.SupportsBytes) - self.assertNotIsSubclass(str, typing.SupportsBytes) + self.assertIsInstance(Eggs(), Spam) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_supports_abs(self): - self.assertIsSubclass(float, typing.SupportsAbs) - self.assertIsSubclass(int, typing.SupportsAbs) - self.assertNotIsSubclass(str, typing.SupportsAbs) + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) # TODO: RUSTPYTHON @unittest.expectedFailure - def test_supports_round(self): - issubclass(float, typing.SupportsRound) - self.assertIsSubclass(float, typing.SupportsRound) - self.assertIsSubclass(int, typing.SupportsRound) - self.assertNotIsSubclass(str, typing.SupportsRound) + def test_no_weird_caching_with_issubclass_after_isinstance_2(self): + @runtime_checkable + class Spam(Protocol): + x: int - def test_reversible(self): - self.assertIsSubclass(list, typing.Reversible) - self.assertNotIsSubclass(int, typing.Reversible) + class Eggs: ... - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_supports_index(self): - self.assertIsSubclass(int, typing.SupportsIndex) - self.assertNotIsSubclass(str, typing.SupportsIndex) + self.assertNotIsInstance(Eggs(), Spam) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bundled_protocol_instance_works(self): - self.assertIsInstance(0, typing.SupportsAbs) - class C1(typing.SupportsInt): - def __int__(self) -> int: - return 42 - class C2(C1): - pass - c = C2() - self.assertIsInstance(c, C1) + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) # TODO: RUSTPYTHON @unittest.expectedFailure - def test_collections_protocols_allowed(self): + def test_no_weird_caching_with_issubclass_after_isinstance_3(self): @runtime_checkable - class Custom(collections.abc.Iterable, Protocol): - def close(self): ... - - class A: pass - class B: - def __iter__(self): - return [] - def close(self): - return 0 - - self.assertIsSubclass(B, Custom) - self.assertNotIsSubclass(A, Custom) + class Spam(Protocol): + x: int - def test_builtin_protocol_allowlist(self): - with self.assertRaises(TypeError): - class CustomProtocol(TestCase, Protocol): - pass + class Eggs: + def __getattr__(self, attr): + if attr == "x": + return 42 + raise AttributeError(attr) - class CustomContextManager(typing.ContextManager, Protocol): - pass + self.assertNotIsInstance(Eggs(), Spam) - def test_non_runtime_protocol_isinstance_check(self): - class P(Protocol): - x: int + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) - with self.assertRaisesRegex(TypeError, "@runtime_checkable"): - isinstance(1, P) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_weird_caching_with_issubclass_after_isinstance_pep695(self): + # @runtime_checkable + # class Spam[T](Protocol): + # x: T - def test_super_call_init(self): - class P(Protocol): - x: int + class Eggs[T]: + def __init__(self, x: T) -> None: + self.x = x - class Foo(P): - def __init__(self): - super().__init__() + self.assertIsInstance(Eggs(42), Spam) - Foo() # Previously triggered RecursionError + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + # FIXME(arihant2math): start more porting from test_protocols_isinstance class GenericTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_basics(self): X = SimpleMapping[str, Any] self.assertEqual(X.__parameters__, ()) @@ -1760,6 +3614,8 @@ def test_basics(self): T = TypeVar("T") self.assertEqual(List[list[T] | float].__parameters__, (T,)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_generic_errors(self): T = TypeVar('T') S = TypeVar('S') @@ -1778,8 +3634,33 @@ class NewGeneric(Generic): ... with self.assertRaises(TypeError): class MyGeneric(Generic[T], Generic[S]): ... with self.assertRaises(TypeError): - class MyGeneric(List[T], Generic[S]): ... + class MyGeneric2(List[T], Generic[S]): ... + with self.assertRaises(TypeError): + Generic[()] + class D(Generic[T]): pass + with self.assertRaises(TypeError): + D[()] + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generic_subclass_checks(self): + for typ in [list[int], List[int], + tuple[int, str], Tuple[int, str], + typing.Callable[..., None], + collections.abc.Callable[..., None]]: + with self.subTest(typ=typ): + self.assertRaises(TypeError, issubclass, typ, object) + self.assertRaises(TypeError, issubclass, typ, type) + self.assertRaises(TypeError, issubclass, typ, typ) + self.assertRaises(TypeError, issubclass, object, typ) + + # isinstance is fine: + self.assertTrue(isinstance(typ, object)) + # but, not when the right arg is also a generic: + self.assertRaises(TypeError, isinstance, typ, typ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_init(self): T = TypeVar('T') S = TypeVar('S') @@ -1814,6 +3695,8 @@ def test_repr(self): self.assertEqual(repr(MySimpleMapping), f"") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_chain_repr(self): T = TypeVar('T') S = TypeVar('S') @@ -1838,6 +3721,8 @@ class C(Generic[T]): self.assertTrue(str(Z).endswith( '.C[typing.Tuple[str, int]]')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_new_repr(self): T = TypeVar('T') U = TypeVar('U', covariant=True) @@ -1849,6 +3734,8 @@ def test_new_repr(self): self.assertEqual(repr(List[S][T][int]), 'typing.List[int]') self.assertEqual(repr(List[int]), 'typing.List[int]') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_new_repr_complex(self): T = TypeVar('T') TS = TypeVar('TS') @@ -1861,6 +3748,8 @@ def test_new_repr_complex(self): 'typing.List[typing.Tuple[typing.List[int], typing.List[int]]]' ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_new_repr_bare(self): T = TypeVar('T') self.assertEqual(repr(Generic[T]), 'typing.Generic[~T]') @@ -1886,6 +3775,20 @@ class C(B[int]): c.bar = 'abc' self.assertEqual(c.__dict__, {'bar': 'abc'}) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_setattr_exceptions(self): + class Immutable[T]: + def __setattr__(self, key, value): + raise RuntimeError("immutable") + + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `Immutable` instance + # returned by the `Immutable[int]()` call + self.assertIsInstance(Immutable[int](), Immutable) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_subscripted_generics_as_proxies(self): T = TypeVar('T') class C(Generic[T]): @@ -1959,6 +3862,8 @@ def test_orig_bases(self): class C(typing.Dict[str, T]): ... self.assertEqual(C.__orig_bases__, (typing.Dict[str, T],)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_naive_runtime_checks(self): def naive_dict_check(obj, tp): # Check if a dictionary conforms to Dict type @@ -1995,6 +3900,8 @@ class C(List[int]): ... self.assertTrue(naive_list_base_check([1, 2, 3], C)) self.assertFalse(naive_list_base_check(['a', 'b'], C)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_multi_subscr_base(self): T = TypeVar('T') U = TypeVar('U') @@ -2012,6 +3919,8 @@ class D(C, List[T][U][V]): ... self.assertEqual(C.__orig_bases__, (List[T][U][V],)) self.assertEqual(D.__orig_bases__, (C, List[T][U][V])) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_subscript_meta(self): T = TypeVar('T') class Meta(type): ... @@ -2019,6 +3928,8 @@ class Meta(type): ... self.assertEqual(Union[T, int][Meta], Union[Meta, int]) self.assertEqual(Callable[..., Meta].__args__, (Ellipsis, Meta)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_generic_hashes(self): class A(Generic[T]): ... @@ -2054,14 +3965,15 @@ class A(Generic[T]): self.assertNotEqual(typing.FrozenSet[A[str]], typing.FrozenSet[mod_generics_cache.B.A[str]]) - if sys.version_info[:2] > (3, 2): - self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) - self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) - .endswith('mod_generics_cache.A[str]]')) - self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) - .endswith('mod_generics_cache.B.A[str]]')) + self.assertTrue(repr(Tuple[A[str]]).endswith('.A[str]]')) + self.assertTrue(repr(Tuple[B.A[str]]).endswith('.B.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.A[str]]) + .endswith('mod_generics_cache.A[str]]')) + self.assertTrue(repr(Tuple[mod_generics_cache.B.A[str]]) + .endswith('mod_generics_cache.B.A[str]]')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_extended_generic_rules_eq(self): T = TypeVar('T') U = TypeVar('U') @@ -2075,12 +3987,11 @@ def test_extended_generic_rules_eq(self): class Base: ... class Derived(Base): ... self.assertEqual(Union[T, Base][Union[Base, Derived]], Union[Base, Derived]) - with self.assertRaises(TypeError): - Union[T, int][1] - self.assertEqual(Callable[[T], T][KT], Callable[[KT], KT]) self.assertEqual(Callable[..., List[T]][int], Callable[..., List[int]]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_extended_generic_rules_repr(self): T = TypeVar('T') self.assertEqual(repr(Union[Tuple, Callable]).replace('typing.', ''), @@ -2092,32 +4003,169 @@ def test_extended_generic_rules_repr(self): self.assertEqual(repr(Callable[[], List[T]][int]).replace('typing.', ''), 'Callable[[], List[int]]') - def test_generic_forward_ref(self): - def foobar(x: List[List['CC']]): ... - def foobar2(x: list[list[ForwardRef('CC')]]): ... - def foobar3(x: list[ForwardRef('CC | int')] | int): ... - class CC: ... + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generic_forward_ref(self): + def foobar(x: List[List['CC']]): ... + def foobar2(x: list[list[ForwardRef('CC')]]): ... + def foobar3(x: list[ForwardRef('CC | int')] | int): ... + class CC: ... + self.assertEqual( + get_type_hints(foobar, globals(), locals()), + {'x': List[List[CC]]} + ) + self.assertEqual( + get_type_hints(foobar2, globals(), locals()), + {'x': list[list[CC]]} + ) + self.assertEqual( + get_type_hints(foobar3, globals(), locals()), + {'x': list[CC | int] | int} + ) + + T = TypeVar('T') + AT = Tuple[T, ...] + def barfoo(x: AT): ... + self.assertIs(get_type_hints(barfoo, globals(), locals())['x'], AT) + CT = Callable[..., List[T]] + def barfoo2(x: CT): ... + self.assertIs(get_type_hints(barfoo2, globals(), locals())['x'], CT) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generic_pep585_forward_ref(self): + # See https://bugs.python.org/issue41370 + + class C1: + a: list['C1'] + self.assertEqual( + get_type_hints(C1, globals(), locals()), + {'a': list[C1]} + ) + + class C2: + a: dict['C1', list[List[list['C2']]]] + self.assertEqual( + get_type_hints(C2, globals(), locals()), + {'a': dict[C1, list[List[list[C2]]]]} + ) + + # Test stringified annotations + scope = {} + exec(textwrap.dedent(''' + from __future__ import annotations + class C3: + a: List[list["C2"]] + '''), scope) + C3 = scope['C3'] + self.assertEqual(C3.__annotations__['a'], "List[list['C2']]") + self.assertEqual( + get_type_hints(C3, globals(), locals()), + {'a': List[list[C2]]} + ) + + # Test recursive types + X = list["X"] + def f(x: X): ... + self.assertEqual( + get_type_hints(f, globals(), locals()), + {'x': list[list[ForwardRef('X')]]} + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep695_generic_class_with_future_annotations(self): + original_globals = dict(ann_module695.__dict__) + + hints_for_A = get_type_hints(ann_module695.A) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(hints_for_A["x"], A_type_params[0]) + self.assertEqual(hints_for_A["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(hints_for_A["z"].__args__[0], A_type_params[2]) + + # should not have changed as a result of the get_type_hints() calls! + self.assertEqual(ann_module695.__dict__, original_globals) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + hints_for_B = get_type_hints(ann_module695.B) + self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes}) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + hints_for_C = get_type_hints(ann_module695.C) + self.assertEqual( + set(hints_for_C.values()), + set(ann_module695.C.__type_params__) + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep_695_generic_function_with_future_annotations(self): + hints_for_generic_function = get_type_hints(ann_module695.generic_function) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + hints_for_generic_function.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(hints_for_generic_function["x"], func_t_params[0]) + self.assertEqual(hints_for_generic_function["y"], Unpack[func_t_params[1]]) + self.assertIs(hints_for_generic_function["z"].__origin__, func_t_params[2]) + self.assertIs(hints_for_generic_function["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.generic_function_2).values()), + set(ann_module695.generic_function_2.__type_params__) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + hints_for_generic_method = get_type_hints(ann_module695.D.generic_method) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + hints_for_generic_method, + {"x": params["Foo"], "y": params["Bar"], "return": types.NoneType} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set(get_type_hints(ann_module695.D.generic_method_2).values()), + set(ann_module695.D.generic_method_2.__type_params__) + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = ann_module695.nested() + self.assertEqual( - get_type_hints(foobar, globals(), locals()), - {'x': List[List[CC]]} + set(results.hints_for_E.values()), + set(results.E.__type_params__) ) self.assertEqual( - get_type_hints(foobar2, globals(), locals()), - {'x': list[list[CC]]} + set(results.hints_for_E_meth.values()), + set(results.E.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.hints_for_E_meth.values()), + set(results.E.__type_params__) ) self.assertEqual( - get_type_hints(foobar3, globals(), locals()), - {'x': list[CC | int] | int} + set(results.hints_for_E_meth.values()).intersection(results.E.__type_params__), + set() ) - T = TypeVar('T') - AT = Tuple[T, ...] - def barfoo(x: AT): ... - self.assertIs(get_type_hints(barfoo, globals(), locals())['x'], AT) - CT = Callable[..., List[T]] - def barfoo2(x: CT): ... - self.assertIs(get_type_hints(barfoo2, globals(), locals())['x'], CT) + self.assertEqual( + set(results.hints_for_generic_func.values()), + set(results.generic_func.__type_params__) + ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_extended_generic_rules_subclassing(self): class T1(Tuple[T, KT]): ... class T2(Tuple[T, ...]): ... @@ -2152,11 +4200,11 @@ def test_fail_with_bare_union(self): List[Union] with self.assertRaises(TypeError): Tuple[Optional] - with self.assertRaises(TypeError): - ClassVar[ClassVar] with self.assertRaises(TypeError): List[ClassVar[int]] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fail_with_bare_generic(self): T = TypeVar('T') with self.assertRaises(TypeError): @@ -2179,24 +4227,29 @@ class MyDict(typing.Dict[T, T]): ... class MyDef(typing.DefaultDict[str, T]): ... self.assertIs(MyDef[int]().__class__, MyDef) self.assertEqual(MyDef[int]().__orig_class__, MyDef[int]) - # ChainMap was added in 3.3 - if sys.version_info >= (3, 3): - class MyChain(typing.ChainMap[str, T]): ... - self.assertIs(MyChain[int]().__class__, MyChain) - self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) + class MyChain(typing.ChainMap[str, T]): ... + self.assertIs(MyChain[int]().__class__, MyChain) + self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_all_repr_eq_any(self): objs = (getattr(typing, el) for el in typing.__all__) for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) - if getattr(obj, '__parameters__', None) and len(obj.__parameters__) == 1: + if (getattr(obj, '__parameters__', None) + and not isinstance(obj, typing.TypeVar) + and isinstance(obj.__parameters__, tuple) + and len(obj.__parameters__) == 1): self.assertEqual(obj[Any].__args__, (Any,)) if isinstance(obj, type): for base in obj.__mro__: self.assertNotEqual(repr(base), '') self.assertEqual(base, base) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pickle(self): global C # pickle wants to reference the class by name T = TypeVar('T') @@ -2217,7 +4270,8 @@ class C(B[int]): self.assertEqual(x.bar, 'abc') self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) samples = [Any, Union, Tuple, Callable, ClassVar, - Union[int, str], ClassVar[List], Tuple[int, ...], Callable[[str], bytes], + Union[int, str], ClassVar[List], Tuple[int, ...], Tuple[()], + Callable[[str], bytes], typing.DefaultDict, typing.FrozenSet[int]] for s in samples: for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -2232,10 +4286,25 @@ class C(B[int]): x = pickle.loads(z) self.assertEqual(s, x) + # Test ParamSpec args and kwargs + global PP + PP = ParamSpec('PP') + for thing in [PP.args, PP.kwargs]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(thing=thing, proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(thing, proto)), + thing, + ) + del PP + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_copy_and_deepcopy(self): T = TypeVar('T') class Node(Generic[T]): ... - things = [Union[T, int], Tuple[T, int], Callable[..., T], Callable[[int], int], + things = [Union[T, int], Tuple[T, int], Tuple[()], + Callable[..., T], Callable[[int], int], Tuple[Any, Any], Node[T], Node[int], Node[Any], typing.Iterable[T], typing.Iterable[Any], typing.Iterable[int], typing.Dict[int, str], typing.Dict[T, Any], ClassVar[int], ClassVar[List[T]], Tuple['T', 'T'], @@ -2244,25 +4313,35 @@ class Node(Generic[T]): ... self.assertEqual(t, copy(t)) self.assertEqual(t, deepcopy(t)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_immutability_by_copy_and_pickle(self): # Special forms like Union, Any, etc., generic aliases to containers like List, # Mapping, etc., and type variabcles are considered immutable by copy and pickle. - global TP, TPB, TPV # for pickle + global TP, TPB, TPV, PP # for pickle TP = TypeVar('TP') TPB = TypeVar('TPB', bound=int) TPV = TypeVar('TPV', bytes, str) - for X in [TP, TPB, TPV, List, typing.Mapping, ClassVar, typing.Iterable, + PP = ParamSpec('PP') + for X in [TP, TPB, TPV, PP, + List, typing.Mapping, ClassVar, typing.Iterable, Union, Any, Tuple, Callable]: - self.assertIs(copy(X), X) - self.assertIs(deepcopy(X), X) - self.assertIs(pickle.loads(pickle.dumps(X)), X) + with self.subTest(thing=X): + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertIs(pickle.loads(pickle.dumps(X, proto)), X) + del TP, TPB, TPV, PP + # Check that local type variables are copyable. TL = TypeVar('TL') TLB = TypeVar('TLB', bound=int) TLV = TypeVar('TLV', bytes, str) - for X in [TL, TLB, TLV]: - self.assertIs(copy(X), X) - self.assertIs(deepcopy(X), X) + PL = ParamSpec('PL') + for X in [TL, TLB, TLV, PL]: + with self.subTest(thing=X): + self.assertIs(copy(X), X) + self.assertIs(deepcopy(X), X) def test_copy_generic_instances(self): T = TypeVar('T') @@ -2292,7 +4371,7 @@ def test_weakref_all(self): T = TypeVar('T') things = [Any, Union[T, int], Callable[..., T], Tuple[Any, Any], Optional[List[int]], typing.Mapping[int, str], - typing.re.Match[bytes], typing.Iterable['whatever']] + typing.Match[bytes], typing.Iterable['whatever']] for t in things: self.assertEqual(weakref.ref(t)(), t) @@ -2334,6 +4413,8 @@ class D(Generic[T]): with self.assertRaises(AttributeError): d_int.foobar = 'no' + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_errors(self): with self.assertRaises(TypeError): B = SimpleMapping[XK, Any] @@ -2359,6 +4440,53 @@ class Y(C[int]): self.assertEqual(Y.__qualname__, 'GenericTests.test_repr_2..Y') + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_repr_3(self): + T = TypeVar('T') + T1 = TypeVar('T1') + P = ParamSpec('P') + P2 = ParamSpec('P2') + Ts = TypeVarTuple('Ts') + + class MyCallable(Generic[P, T]): + pass + + class DoubleSpec(Generic[P, P2, T]): + pass + + class TsP(Generic[*Ts, P]): + pass + + object_to_expected_repr = { + MyCallable[P, T]: "MyCallable[~P, ~T]", + MyCallable[Concatenate[T1, P], T]: "MyCallable[typing.Concatenate[~T1, ~P], ~T]", + MyCallable[[], bool]: "MyCallable[[], bool]", + MyCallable[[int], bool]: "MyCallable[[int], bool]", + MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", + MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", + + DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", + DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", + DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", + + TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", + TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", + TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", + + # These lines are just too long to fit: + MyCallable[Concatenate[*Ts, P], int][int, str, [bool, float]]: + "MyCallable[[int, str, bool, float], int]", + } + + for obj, expected_repr in object_to_expected_repr.items(): + with self.subTest(obj=obj, expected_repr=expected_repr): + self.assertRegex( + repr(obj), + fr"^{re.escape(MyCallable.__module__)}.*\.{re.escape(expected_repr)}$", + ) + def test_eq_1(self): self.assertEqual(Generic, Generic) self.assertEqual(Generic[T], Generic[T]) @@ -2377,6 +4505,8 @@ class B(Generic[T]): self.assertEqual(A[T], A[T]) self.assertNotEqual(A[T], B[T]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_multiple_inheritance(self): class A(Generic[T, VT]): @@ -2396,6 +4526,77 @@ class B(Generic[S]): ... class C(List[int], B): ... self.assertEqual(C.__mro__, (C, list, B, Generic, object)) + def test_multiple_inheritance_non_type_with___mro_entries__(self): + class GoodEntries: + def __mro_entries__(self, bases): + return (object,) + + class A(List[int], GoodEntries()): ... + + self.assertEqual(A.__mro__, (A, list, Generic, object)) + + def test_multiple_inheritance_non_type_without___mro_entries__(self): + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex(TypeError, r"^bases must be types"): + class A(List[int], object()): ... + + def test_multiple_inheritance_non_type_bad___mro_entries__(self): + class BadEntries: + def __mro_entries__(self, bases): + return None + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^__mro_entries__ must return a tuple", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance___mro_entries___returns_non_type(self): + class BadEntries: + def __mro_entries__(self, bases): + return (object(),) + + # Error should be from the type machinery, not from typing.py + with self.assertRaisesRegex( + TypeError, + r"^bases must be types", + ): + class A(List[int], BadEntries()): ... + + def test_multiple_inheritance_with_genericalias(self): + class A(typing.Sized, list[int]): ... + + self.assertEqual( + A.__mro__, + (A, collections.abc.Sized, Generic, list, object), + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_multiple_inheritance_with_genericalias_2(self): + T = TypeVar("T") + + class BaseSeq(typing.Sequence[T]): ... + class MySeq(List[T], BaseSeq[T]): ... + + self.assertEqual( + MySeq.__mro__, + ( + MySeq, + list, + BaseSeq, + collections.abc.Sequence, + collections.abc.Reversible, + collections.abc.Collection, + collections.abc.Sized, + collections.abc.Iterable, + collections.abc.Container, + Generic, + object, + ), + ) + def test_init_subclass_super_called(self): class FinalException(Exception): pass @@ -2412,7 +4613,7 @@ class Test(Generic[T], Final): class Subclass(Test): pass with self.assertRaises(FinalException): - class Subclass(Test[int]): + class Subclass2(Test[int]): pass def test_nested(self): @@ -2469,6 +4670,8 @@ def foo(x: T): foo(42) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_implicit_any(self): T = TypeVar('T') @@ -2480,11 +4683,11 @@ class D(C): self.assertEqual(D.__parameters__, ()) - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[int] - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[Any] - with self.assertRaises(Exception): + with self.assertRaises(TypeError): D[T] def test_new_with_args(self): @@ -2567,6 +4770,7 @@ def test_subclass_special_form(self): Literal[1, 2], Concatenate[int, ParamSpec("P")], TypeGuard[int], + TypeIs[range], ): with self.subTest(msg=obj): with self.assertRaisesRegex( @@ -2575,11 +4779,70 @@ def test_subclass_special_form(self): class Foo(obj): pass + def test_complex_subclasses(self): + T_co = TypeVar("T_co", covariant=True) + + class Base(Generic[T_co]): + ... + + T = TypeVar("T") + + # see gh-94607: this fails in that bug + class Sub(Base, Generic[T]): + ... + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parameter_detection(self): + self.assertEqual(List[T].__parameters__, (T,)) + self.assertEqual(List[List[T]].__parameters__, (T,)) + class A: + __parameters__ = (T,) + # Bare classes should be skipped + for a in (List, list): + for b in (A, int, TypeVar, TypeVarTuple, ParamSpec, types.GenericAlias, types.UnionType): + with self.subTest(generic=a, sub=b): + with self.assertRaisesRegex(TypeError, '.* is not a generic class'): + a[b][str] + # Duck-typing anything that looks like it has __parameters__. + # These tests are optional and failure is okay. + self.assertEqual(List[A()].__parameters__, (T,)) + # C version of GenericAlias + self.assertEqual(list[A()].__parameters__, (T,)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_non_generic_subscript(self): + T = TypeVar('T') + class G(Generic[T]): + pass + class A: + __parameters__ = (T,) + + for s in (int, G, A, List, list, + TypeVar, TypeVarTuple, ParamSpec, + types.GenericAlias, types.UnionType): + + for t in Tuple, tuple: + with self.subTest(tuple=t, sub=s): + self.assertEqual(t[s, T][int], t[s, int]) + self.assertEqual(t[T, s][int], t[int, s]) + a = t[s] + with self.assertRaises(TypeError): + a[int] + + for c in Callable, collections.abc.Callable: + with self.subTest(callable=c, sub=s): + self.assertEqual(c[[s], T][int], c[[s], int]) + self.assertEqual(c[[T], s][int], c[[int], s]) + a = c[[s], s] + with self.assertRaises(TypeError): + a[int] + + class ClassVarTests(BaseTestCase): def test_basics(self): - with self.assertRaises(TypeError): - ClassVar[1] with self.assertRaises(TypeError): ClassVar[int, str] with self.assertRaises(TypeError): @@ -2593,11 +4856,19 @@ def test_repr(self): self.assertEqual(repr(cv), 'typing.ClassVar[%s.Employee]' % __name__) def test_cannot_subclass(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): class C(type(ClassVar)): pass - with self.assertRaises(TypeError): - class C(type(ClassVar[int])): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(ClassVar[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.ClassVar'): + class E(ClassVar): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.ClassVar\[int\]'): + class F(ClassVar[int]): pass def test_cannot_init(self): @@ -2614,12 +4885,11 @@ def test_no_isinstance(self): with self.assertRaises(TypeError): issubclass(int, ClassVar) + class FinalTests(BaseTestCase): def test_basics(self): Final[int] # OK - with self.assertRaises(TypeError): - Final[1] with self.assertRaises(TypeError): Final[int, str] with self.assertRaises(TypeError): @@ -2627,6 +4897,8 @@ def test_basics(self): with self.assertRaises(TypeError): Optional[Final[int]] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repr(self): self.assertEqual(repr(Final), 'typing.Final') cv = Final[int] @@ -2637,11 +4909,19 @@ def test_repr(self): self.assertEqual(repr(cv), 'typing.Final[tuple[int]]') def test_cannot_subclass(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): class C(type(Final)): pass - with self.assertRaises(TypeError): - class C(type(Final[int])): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(Final[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Final'): + class E(Final): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Final\[int\]'): + class F(Final[int]): pass def test_cannot_init(self): @@ -2658,10 +4938,216 @@ def test_no_isinstance(self): with self.assertRaises(TypeError): issubclass(int, Final) + +class FinalDecoratorTests(BaseTestCase): def test_final_unmodified(self): def func(x): ... self.assertIs(func, final(func)) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_dunder_final(self): + @final + def func(): ... + @final + class Cls: ... + self.assertIs(True, func.__final__) + self.assertIs(True, Cls.__final__) + + class Wrapper: + __slots__ = ("func",) + def __init__(self, func): + self.func = func + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + # Check that no error is thrown if the attribute + # is not writable. + @final + @Wrapper + def wrapped(): ... + self.assertIsInstance(wrapped, Wrapper) + self.assertIs(False, hasattr(wrapped, "__final__")) + + class Meta(type): + @property + def __final__(self): return "can't set me" + @final + class WithMeta(metaclass=Meta): ... + self.assertEqual(WithMeta.__final__, "can't set me") + + # Builtin classes throw TypeError if you try to set an + # attribute. + final(int) + self.assertIs(False, hasattr(int, "__final__")) + + # Make sure it works with common builtin decorators + class Methods: + @final + @classmethod + def clsmethod(cls): ... + + @final + @staticmethod + def stmethod(): ... + + # The other order doesn't work because property objects + # don't allow attribute assignment. + @property + @final + def prop(self): ... + + @final + @lru_cache() + def cached(self): ... + + # Use getattr_static because the descriptor returns the + # underlying function, which doesn't have __final__. + self.assertIs( + True, + inspect.getattr_static(Methods, "clsmethod").__final__ + ) + self.assertIs( + True, + inspect.getattr_static(Methods, "stmethod").__final__ + ) + self.assertIs(True, Methods.prop.fget.__final__) + self.assertIs(True, Methods.cached.__final__) + + +class OverrideDecoratorTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_override(self): + class Base: + def normal_method(self): ... + @classmethod + def class_method_good_order(cls): ... + @classmethod + def class_method_bad_order(cls): ... + @staticmethod + def static_method_good_order(): ... + @staticmethod + def static_method_bad_order(): ... + + class Derived(Base): + @override + def normal_method(self): + return 42 + + @classmethod + @override + def class_method_good_order(cls): + return 42 + @override + @classmethod + def class_method_bad_order(cls): + return 42 + + @staticmethod + @override + def static_method_good_order(): + return 42 + @override + @staticmethod + def static_method_bad_order(): + return 42 + + self.assertIsSubclass(Derived, Base) + instance = Derived() + self.assertEqual(instance.normal_method(), 42) + self.assertIs(True, Derived.normal_method.__override__) + self.assertIs(True, instance.normal_method.__override__) + + self.assertEqual(Derived.class_method_good_order(), 42) + self.assertIs(True, Derived.class_method_good_order.__override__) + self.assertEqual(Derived.class_method_bad_order(), 42) + self.assertIs(False, hasattr(Derived.class_method_bad_order, "__override__")) + + self.assertEqual(Derived.static_method_good_order(), 42) + self.assertIs(True, Derived.static_method_good_order.__override__) + self.assertEqual(Derived.static_method_bad_order(), 42) + self.assertIs(False, hasattr(Derived.static_method_bad_order, "__override__")) + + # Base object is not changed: + self.assertIs(False, hasattr(Base.normal_method, "__override__")) + self.assertIs(False, hasattr(Base.class_method_good_order, "__override__")) + self.assertIs(False, hasattr(Base.class_method_bad_order, "__override__")) + self.assertIs(False, hasattr(Base.static_method_good_order, "__override__")) + self.assertIs(False, hasattr(Base.static_method_bad_order, "__override__")) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_property(self): + class Base: + @property + def correct(self) -> int: + return 1 + @property + def wrong(self) -> int: + return 1 + + class Child(Base): + @property + @override + def correct(self) -> int: + return 2 + @override + @property + def wrong(self) -> int: + return 2 + + instance = Child() + self.assertEqual(instance.correct, 2) + self.assertTrue(Child.correct.fget.__override__) + self.assertEqual(instance.wrong, 2) + self.assertFalse(hasattr(Child.wrong, "__override__")) + self.assertFalse(hasattr(Child.wrong.fset, "__override__")) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_silent_failure(self): + class CustomProp: + __slots__ = ('fget',) + def __init__(self, fget): + self.fget = fget + def __get__(self, obj, objtype=None): + return self.fget(obj) + + class WithOverride: + @override # must not fail on object with `__slots__` + @CustomProp + def some(self): + return 1 + + self.assertEqual(WithOverride.some, 1) + self.assertFalse(hasattr(WithOverride.some, "__override__")) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_multiple_decorators(self): + def with_wraps(f): # similar to `lru_cache` definition + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + + class WithOverride: + @override + @with_wraps + def on_top(self, a: int) -> int: + return a + 1 + @with_wraps + @override + def on_bottom(self, a: int) -> int: + return a + 2 + + instance = WithOverride() + self.assertEqual(instance.on_top(1), 2) + self.assertTrue(instance.on_top.__override__) + self.assertEqual(instance.on_bottom(1), 3) + self.assertTrue(instance.on_bottom.__override__) + class CastTests(BaseTestCase): @@ -2681,8 +5167,38 @@ def test_errors(self): cast('hello', 42) -class ForwardRefTests(BaseTestCase): +class AssertTypeTests(BaseTestCase): + + def test_basics(self): + arg = 42 + self.assertIs(assert_type(arg, int), arg) + self.assertIs(assert_type(arg, str | float), arg) + self.assertIs(assert_type(arg, AnyStr), arg) + self.assertIs(assert_type(arg, None), arg) + + def test_errors(self): + # Bogus calls are not expected to fail. + arg = 42 + self.assertIs(assert_type(arg, 42), arg) + self.assertIs(assert_type(arg, 'hello'), arg) + + +# We need this to make sure that `@no_type_check` respects `__module__` attr: +from test.typinganndata import ann_module8 + +@no_type_check +class NoTypeCheck_Outer: + Inner = ann_module8.NoTypeCheck_Outer.Inner + +@no_type_check +class NoTypeCheck_WithFunction: + NoTypeCheck_function = ann_module8.NoTypeCheck_function + + +class ForwardRefTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_basics(self): class Node(Generic[T]): @@ -2708,16 +5224,15 @@ def add_right(self, node: 'Node[T]' = None): t = Node[int] both_hints = get_type_hints(t.add_both, globals(), locals()) self.assertEqual(both_hints['left'], Optional[Node[T]]) - self.assertEqual(both_hints['right'], Optional[Node[T]]) - self.assertEqual(both_hints['left'], both_hints['right']) - self.assertEqual(both_hints['stuff'], Optional[int]) + self.assertEqual(both_hints['right'], Node[T]) + self.assertEqual(both_hints['stuff'], int) self.assertNotIn('blah', both_hints) left_hints = get_type_hints(t.add_left, globals(), locals()) self.assertEqual(left_hints['node'], Optional[Node[T]]) right_hints = get_type_hints(t.add_right, globals(), locals()) - self.assertEqual(right_hints['node'], Optional[Node[T]]) + self.assertEqual(right_hints['node'], Node[T]) def test_forwardref_instance_type_error(self): fr = typing.ForwardRef('int') @@ -2811,7 +5326,11 @@ def fun(x: a): def test_forward_repr(self): self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]") + self.assertEqual(repr(List[ForwardRef('int', module='mod')]), + "typing.List[ForwardRef('int', module='mod')]") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_forward(self): def foo(a: Union['T']): @@ -2826,6 +5345,8 @@ def foo(a: tuple[ForwardRef('T')] | int): self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': tuple[T] | int}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_tuple_forward(self): def foo(a: Tuple['T']): @@ -2866,11 +5387,14 @@ def fun(x: a): pass def cmp(o1, o2): return o1 == o2 - r1 = namespace1() - r2 = namespace2() - self.assertIsNot(r1, r2) - self.assertRaises(RecursionError, cmp, r1, r2) + with infinite_recursion(25): + r1 = namespace1() + r2 = namespace2() + self.assertIsNot(r1, r2) + self.assertRaises(RecursionError, cmp, r1, r2) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_union_forward_recursion(self): ValueList = List['Value'] Value = Union[str, ValueList] @@ -2919,20 +5443,28 @@ def foo(a: 'Callable[..., T]'): self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': Callable[..., T]}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_special_forms_forward(self): class C: a: Annotated['ClassVar[int]', (3, 5)] = 4 b: Annotated['Final[int]', "const"] = 4 + x: 'ClassVar' = 4 + y: 'Final' = 4 class CF: b: List['Final[int]'] = 4 self.assertEqual(get_type_hints(C, globals())['a'], ClassVar[int]) self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) + self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) + self.assertEqual(get_type_hints(C, globals())['y'], Final) with self.assertRaises(TypeError): get_type_hints(CF, globals()), + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_syntax_error(self): with self.assertRaises(SyntaxError): @@ -2946,13 +5478,11 @@ def foo(a: 'Node[T'): with self.assertRaises(SyntaxError): get_type_hints(foo) - def test_type_error(self): - - def foo(a: Tuple['42']): - pass - - with self.assertRaises(TypeError): - get_type_hints(foo) + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[''] def test_name_error(self): @@ -2989,9 +5519,104 @@ def meth(self, x: int): ... @no_type_check class D(C): c = C + # verify that @no_type_check never affects bases self.assertEqual(get_type_hints(C.meth), {'x': int}) + # and never child classes: + class Child(D): + def foo(self, x: int): ... + + self.assertEqual(get_type_hints(Child.foo), {'x': int}) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_type_check_nested_types(self): + # See https://bugs.python.org/issue46571 + class Other: + o: int + class B: # Has the same `__name__`` as `A.B` and different `__qualname__` + o: int + @no_type_check + class A: + a: int + class B: + b: int + class C: + c: int + class D: + d: int + + Other = Other + + for klass in [A, A.B, A.B.C, A.D]: + with self.subTest(klass=klass): + self.assertTrue(klass.__no_type_check__) + self.assertEqual(get_type_hints(klass), {}) + + for not_modified in [Other, B]: + with self.subTest(not_modified=not_modified): + with self.assertRaises(AttributeError): + not_modified.__no_type_check__ + self.assertNotEqual(get_type_hints(not_modified), {}) + + def test_no_type_check_class_and_static_methods(self): + @no_type_check + class Some: + @staticmethod + def st(x: int) -> int: ... + @classmethod + def cl(cls, y: int) -> int: ... + + self.assertTrue(Some.st.__no_type_check__) + self.assertEqual(get_type_hints(Some.st), {}) + self.assertTrue(Some.cl.__no_type_check__) + self.assertEqual(get_type_hints(Some.cl), {}) + + def test_no_type_check_other_module(self): + self.assertTrue(NoTypeCheck_Outer.__no_type_check__) + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_Outer.__no_type_check__ + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_Outer.Inner.__no_type_check__ + + self.assertTrue(NoTypeCheck_WithFunction.__no_type_check__) + with self.assertRaises(AttributeError): + ann_module8.NoTypeCheck_function.__no_type_check__ + + def test_no_type_check_foreign_functions(self): + # We should not modify this function: + def some(*args: int) -> int: + ... + + @no_type_check + class A: + some_alias = some + some_class = classmethod(some) + some_static = staticmethod(some) + + with self.assertRaises(AttributeError): + some.__no_type_check__ + self.assertEqual(get_type_hints(some), {'args': int, 'return': int}) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_type_check_lambda(self): + @no_type_check + class A: + # Corner case: `lambda` is both an assignment and a function: + bar: Callable[[int], int] = lambda arg: arg + + self.assertTrue(A.bar.__no_type_check__) + self.assertEqual(get_type_hints(A.bar), {}) + + def test_no_type_check_TypeError(self): + # This simply should not fail with + # `TypeError: can't set attributes of built-in/extension type 'dict'` + no_type_check(dict) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_no_type_check_forward_ref_as_string(self): class C: foo: typing.ClassVar[int] = 7 @@ -3006,21 +5631,15 @@ class F: for clazz in [C, D, E, F]: self.assertEqual(get_type_hints(clazz), expected_result) - def test_nested_classvar_fails_forward_ref_check(self): - class E: - foo: 'typing.ClassVar[typing.ClassVar[int]]' = 7 - class F: - foo: ClassVar['ClassVar[int]'] = 7 - - for clazz in [E, F]: - with self.assertRaises(TypeError): - get_type_hints(clazz) - def test_meta_no_type_check(self): - - @no_type_check_decorator - def magic_decorator(func): - return func + depr_msg = ( + "'typing.no_type_check_decorator' is deprecated " + "and slated for removal in Python 3.15" + ) + with self.assertWarnsRegex(DeprecationWarning, depr_msg): + @no_type_check_decorator + def magic_decorator(func): + return func self.assertEqual(magic_decorator.__name__, 'magic_decorator') @@ -3052,18 +5671,75 @@ def test_default_globals(self): hints = get_type_hints(ns['C'].foo) self.assertEqual(hints, {'a': ns['C'], 'return': ns['D']}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_final_forward_ref(self): self.assertEqual(gth(Loop, globals())['attr'], Final[Loop]) self.assertNotEqual(gth(Loop, globals())['attr'], Final[int]) self.assertNotEqual(gth(Loop, globals())['attr'], Final) + def test_or(self): + X = ForwardRef('X') + # __or__/__ror__ itself + self.assertEqual(X | "x", Union[X, "x"]) + self.assertEqual("x" | X, Union["x", X]) + + +class InternalsTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + + def test_collect_parameters(self): + typing = import_helper.import_fresh_module("typing") + with self.assertWarnsRegex( + DeprecationWarning, + "The private _collect_parameters function is deprecated" + ) as cm: + typing._collect_parameters + self.assertEqual(cm.filename, __file__) + + +@lru_cache() +def cached_func(x, y): + return 3 * x + y + + +class MethodHolder: + @classmethod + def clsmethod(cls): ... + @staticmethod + def stmethod(): ... + def method(self): ... + class OverloadTests(BaseTestCase): def test_overload_fails(self): - from typing import overload - - with self.assertRaises(RuntimeError): + with self.assertRaises(NotImplementedError): @overload def blah(): @@ -3072,8 +5748,6 @@ def blah(): blah() def test_overload_succeeds(self): - from typing import overload - @overload def blah(): pass @@ -3083,9 +5757,80 @@ def blah(): blah() + @cpython_only # gh-98713 + def test_overload_on_compiled_functions(self): + with patch("typing._overload_registry", + defaultdict(lambda: defaultdict(dict))): + # The registry starts out empty: + self.assertEqual(typing._overload_registry, {}) + + # This should just not fail: + overload(sum) + overload(print) + + # No overloads are recorded (but, it still has a side-effect): + self.assertEqual(typing.get_overloads(sum), []) + self.assertEqual(typing.get_overloads(print), []) + + def set_up_overloads(self): + def blah(): + pass + + overload1 = blah + overload(blah) + + def blah(): + pass + + overload2 = blah + overload(blah) + + def blah(): + pass + + return blah, [overload1, overload2] + + # Make sure we don't clear the global overload registry + @patch("typing._overload_registry", + defaultdict(lambda: defaultdict(dict))) + def test_overload_registry(self): + # The registry starts out empty + self.assertEqual(typing._overload_registry, {}) + + impl, overloads = self.set_up_overloads() + self.assertNotEqual(typing._overload_registry, {}) + self.assertEqual(list(get_overloads(impl)), overloads) + + def some_other_func(): pass + overload(some_other_func) + other_overload = some_other_func + def some_other_func(): pass + self.assertEqual(list(get_overloads(some_other_func)), [other_overload]) + # Unrelated function still has no overloads: + def not_overloaded(): pass + self.assertEqual(list(get_overloads(not_overloaded)), []) + + # Make sure that after we clear all overloads, the registry is + # completely empty. + clear_overloads() + self.assertEqual(typing._overload_registry, {}) + self.assertEqual(get_overloads(impl), []) + + # Querying a function with no overloads shouldn't change the registry. + def the_only_one(): pass + self.assertEqual(get_overloads(the_only_one), []) + self.assertEqual(typing._overload_registry, {}) + + def test_overload_registry_repeated(self): + for _ in range(2): + impl, overloads = self.set_up_overloads() + + self.assertEqual(list(get_overloads(impl)), overloads) + +from test.typinganndata import ( + ann_module, ann_module2, ann_module3, ann_module5, ann_module6, +) -ASYNCIO_TESTS = """ -import asyncio T_a = TypeVar('T_a') @@ -3118,19 +5863,7 @@ async def __aenter__(self) -> int: return 42 async def __aexit__(self, etype, eval, tb): return None -""" - -try: - exec(ASYNCIO_TESTS) -except ImportError: - ASYNCIO = False # multithreading is not enabled -else: - ASYNCIO = True - -# Definitions needed for features introduced in Python 3.6 -from test import ann_module, ann_module2, ann_module3, ann_module5, ann_module6 -from typing import AsyncContextManager class A: y: float @@ -3177,20 +5910,60 @@ class Point2D(TypedDict): x: int y: int +class Point2DGeneric(Generic[T], TypedDict): + a: T + b: T + class Bar(_typed_dict_helper.Foo, total=False): b: int +class BarGeneric(_typed_dict_helper.FooGeneric[T], total=False): + b: int + class LabelPoint2D(Point2D, Label): ... class Options(TypedDict, total=False): log_level: int log_path: str -class HasForeignBaseClass(mod_generics_cache.A): - some_xrepr: 'XRepr' - other_a: 'mod_generics_cache.A' +class TotalMovie(TypedDict): + title: str + year: NotRequired[int] + +class NontotalMovie(TypedDict, total=False): + title: Required[str] + year: int + +class ParentNontotalMovie(TypedDict, total=False): + title: Required[str] + +class ChildTotalMovie(ParentNontotalMovie): + year: NotRequired[int] + +class ParentDeeplyAnnotatedMovie(TypedDict): + title: Annotated[Annotated[Required[str], "foobar"], "another level"] + +class ChildDeeplyAnnotatedMovie(ParentDeeplyAnnotatedMovie): + year: NotRequired[Annotated[int, 2000]] -async def g_with(am: AsyncContextManager[int]): +class AnnotatedMovie(TypedDict): + title: Annotated[Required[str], "foobar"] + year: NotRequired[Annotated[int, 2000]] + +class DeeplyAnnotatedMovie(TypedDict): + title: Annotated[Annotated[Required[str], "foobar"], "another level"] + year: NotRequired[Annotated[int, 2000]] + +class WeirdlyQuotedMovie(TypedDict): + title: Annotated['Annotated[Required[str], "foobar"]', "another level"] + year: NotRequired['Annotated[int, 2000]'] + +# TODO: RUSTPYTHON +# class HasForeignBaseClass(mod_generics_cache.A): +# some_xrepr: 'XRepr' +# other_a: 'mod_generics_cache.A' + +async def g_with(am: typing.AsyncContextManager[int]): x: int async with am as x: return x @@ -3202,6 +5975,7 @@ async def g_with(am: AsyncContextManager[int]): gth = get_type_hints + class ForRefExample: @ann_module.dec def func(self: 'ForRefExample'): @@ -3240,6 +6014,8 @@ def test_get_type_hints_modules_forwardref(self): 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) @@ -3264,6 +6040,8 @@ def test_get_type_hints_classes(self): 'my_inner_a2': mod_generics_cache.B.A, 'my_outer_a': mod_generics_cache.A}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_classes_no_implicit_optional(self): class WithNoneDefault: field: int = None # most type-checkers won't be happy with it @@ -3308,6 +6086,8 @@ class B: ... b.__annotations__ = {'x': 'A'} self.assertEqual(gth(b, locals()), {'x': A}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_ClassVar(self): self.assertEqual(gth(ann_module2.CV, ann_module2.__dict__), {'var': typing.ClassVar[ann_module2.CV]}) @@ -3323,6 +6103,8 @@ def test_get_type_hints_wrapped_decoratored_func(self): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_annotated(self): def foobar(x: List['X']): ... X = Annotated[int, (1, 10)] @@ -3349,7 +6131,7 @@ def foobar(x: list[ForwardRef('X')]): ... BA = Tuple[Annotated[T, (1, 0)], ...] def barfoo(x: BA): ... self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], Tuple[T, ...]) - self.assertIs( + self.assertEqual( get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], BA ) @@ -3357,7 +6139,7 @@ def barfoo(x: BA): ... BA = tuple[Annotated[T, (1, 0)], ...] def barfoo(x: BA): ... self.assertEqual(get_type_hints(barfoo, globals(), locals())['x'], tuple[T, ...]) - self.assertIs( + self.assertEqual( get_type_hints(barfoo, globals(), locals(), include_extras=True)['x'], BA ) @@ -3386,6 +6168,8 @@ def barfoo4(x: BA3): ... {"x": typing.Annotated[int | float, "const"]} ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_annotated_in_union(self): # bpo-46603 def with_union(x: int | list[Annotated[str, 'meta']]): ... @@ -3395,6 +6179,8 @@ def with_union(x: int | list[Annotated[str, 'meta']]): ... {'x': int | list[Annotated[str, 'meta']]}, ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_annotated_refs(self): Const = Annotated[T, "Const"] @@ -3422,6 +6208,20 @@ def __iand__(self, other: Const["MySet[T]"]) -> "MySet[T]": {'other': MySet[T], 'return': MySet[T]} ) + def test_get_type_hints_annotated_with_none_default(self): + # See: https://bugs.python.org/issue46195 + def annotated_with_none_default(x: Annotated[int, 'data'] = None): ... + self.assertEqual( + get_type_hints(annotated_with_none_default), + {'x': int}, + ) + self.assertEqual( + get_type_hints(annotated_with_none_default, include_extras=True), + {'x': Annotated[int, 'data']}, + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_classes_str_annotations(self): class Foo: y = str @@ -3437,6 +6237,8 @@ class BadModule: self.assertNotIn('bad', sys.modules) self.assertEqual(get_type_hints(BadModule), {}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints_annotated_bad_module(self): # See https://bugs.python.org/issue44468 class BadBase: @@ -3447,10 +6249,88 @@ class BadType(BadBase): self.assertNotIn('bad', sys.modules) self.assertEqual(get_type_hints(BadType), {'foo': tuple, 'bar': list}) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_forward_ref_and_final(self): + # https://bugs.python.org/issue45166 + hints = get_type_hints(ann_module5) + self.assertEqual(hints, {'name': Final[str]}) + + hints = get_type_hints(ann_module5.MyClass) + self.assertEqual(hints, {'value': Final}) + + def test_top_level_class_var(self): + # https://bugs.python.org/issue45166 + with self.assertRaisesRegex( + TypeError, + r'typing.ClassVar\[int\] is not valid as type argument', + ): + get_type_hints(ann_module6) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_get_type_hints_typeddict(self): + self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { + 'title': str, + 'year': NotRequired[int], + }) + + self.assertEqual(get_type_hints(AnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(AnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(DeeplyAnnotatedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(WeirdlyQuotedMovie), {'title': str, 'year': int}) + self.assertEqual(get_type_hints(WeirdlyQuotedMovie, include_extras=True), { + 'title': Annotated[Required[str], "foobar", "another level"], + 'year': NotRequired[Annotated[int, 2000]], + }) + + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated), {'a': int}) + self.assertEqual(get_type_hints(_typed_dict_helper.VeryAnnotated, include_extras=True), { + 'a': Annotated[Required[int], "a", "b", "c"] + }) + + self.assertEqual(get_type_hints(ChildTotalMovie), {"title": str, "year": int}) + self.assertEqual(get_type_hints(ChildTotalMovie, include_extras=True), { + "title": Required[str], "year": NotRequired[int] + }) + + self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie), {"title": str, "year": int}) + self.assertEqual(get_type_hints(ChildDeeplyAnnotatedMovie, include_extras=True), { + "title": Annotated[Required[str], "foobar", "another level"], + "year": NotRequired[Annotated[int, 2000]] + }) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_get_type_hints_collections_abc_callable(self): + # https://github.com/python/cpython/issues/91621 + P = ParamSpec('P') + def f(x: collections.abc.Callable[[int], int]): ... + def g(x: collections.abc.Callable[..., int]): ... + def h(x: collections.abc.Callable[P, int]): ... + + self.assertEqual(get_type_hints(f), {'x': collections.abc.Callable[[int], int]}) + self.assertEqual(get_type_hints(g), {'x': collections.abc.Callable[..., int]}) + self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) + + class GetUtilitiesTestCase(TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_origin(self): T = TypeVar('T') + Ts = TypeVarTuple('Ts') P = ParamSpec('P') class C(Generic[T]): pass self.assertIs(get_origin(C[int]), C) @@ -3472,27 +6352,46 @@ class C(Generic[T]): pass self.assertIs(get_origin(list | str), types.UnionType) self.assertIs(get_origin(P.args), P) self.assertIs(get_origin(P.kwargs), P) + self.assertIs(get_origin(Required[int]), Required) + self.assertIs(get_origin(NotRequired[int]), NotRequired) + self.assertIs(get_origin((*Ts,)[0]), Unpack) + self.assertIs(get_origin(Unpack[Ts]), Unpack) + # self.assertIs(get_origin((*tuple[*Ts],)[0]), tuple) + self.assertIs(get_origin(Unpack[Tuple[Unpack[Ts]]]), Unpack) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_args(self): T = TypeVar('T') class C(Generic[T]): pass self.assertEqual(get_args(C[int]), (int,)) self.assertEqual(get_args(C[T]), (T,)) + self.assertEqual(get_args(typing.SupportsAbs[int]), (int,)) # Protocol + self.assertEqual(get_args(typing.SupportsAbs[T]), (T,)) + self.assertEqual(get_args(Point2DGeneric[int]), (int,)) # TypedDict + self.assertEqual(get_args(Point2DGeneric[T]), (T,)) + self.assertEqual(get_args(T), ()) self.assertEqual(get_args(int), ()) + self.assertEqual(get_args(Any), ()) + self.assertEqual(get_args(Self), ()) + self.assertEqual(get_args(LiteralString), ()) self.assertEqual(get_args(ClassVar[int]), (int,)) self.assertEqual(get_args(Union[int, str]), (int, str)) self.assertEqual(get_args(Literal[42, 43]), (42, 43)) self.assertEqual(get_args(Final[List[int]]), (List[int],)) + self.assertEqual(get_args(Optional[int]), (int, type(None))) + self.assertEqual(get_args(Union[int, None]), (int, type(None))) self.assertEqual(get_args(Union[int, Tuple[T, int]][str]), (int, Tuple[str, int])) self.assertEqual(get_args(typing.Dict[int, Tuple[T, T]][Optional[int]]), (int, Tuple[Optional[int], Optional[int]])) self.assertEqual(get_args(Callable[[], T][int]), ([], int)) self.assertEqual(get_args(Callable[..., int]), (..., int)) + self.assertEqual(get_args(Callable[[int], str]), ([int], str)) self.assertEqual(get_args(Union[int, Callable[[Tuple[T, ...]], str]]), (int, Callable[[Tuple[T, ...]], str])) self.assertEqual(get_args(Tuple[int, ...]), (int, ...)) - self.assertEqual(get_args(Tuple[()]), ((),)) + self.assertEqual(get_args(Tuple[()]), ()) self.assertEqual(get_args(Annotated[T, 'one', 2, ['three']]), (T, 'one', 2, ['three'])) self.assertEqual(get_args(List), ()) self.assertEqual(get_args(Tuple), ()) @@ -3505,26 +6404,31 @@ class C(Generic[T]): pass self.assertEqual(get_args(collections.abc.Callable[[int], str]), get_args(Callable[[int], str])) P = ParamSpec('P') + self.assertEqual(get_args(P), ()) + self.assertEqual(get_args(P.args), ()) + self.assertEqual(get_args(P.kwargs), ()) self.assertEqual(get_args(Callable[P, int]), (P, int)) + self.assertEqual(get_args(collections.abc.Callable[P, int]), (P, int)) self.assertEqual(get_args(Callable[Concatenate[int, P], int]), (Concatenate[int, P], int)) + self.assertEqual(get_args(collections.abc.Callable[Concatenate[int, P], int]), + (Concatenate[int, P], int)) + self.assertEqual(get_args(Concatenate[int, str, P]), (int, str, P)) self.assertEqual(get_args(list | str), (list, str)) + self.assertEqual(get_args(Required[int]), (int,)) + self.assertEqual(get_args(NotRequired[int]), (int,)) + self.assertEqual(get_args(TypeAlias), ()) + self.assertEqual(get_args(TypeGuard[int]), (int,)) + self.assertEqual(get_args(TypeIs[range]), (range,)) + Ts = TypeVarTuple('Ts') + self.assertEqual(get_args(Ts), ()) + self.assertEqual(get_args((*Ts,)[0]), (Ts,)) + self.assertEqual(get_args(Unpack[Ts]), (Ts,)) + # self.assertEqual(get_args(tuple[*Ts]), (*Ts,)) + self.assertEqual(get_args(tuple[Unpack[Ts]]), (Unpack[Ts],)) + # self.assertEqual(get_args((*tuple[*Ts],)[0]), (*Ts,)) + self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) - def test_forward_ref_and_final(self): - # https://bugs.python.org/issue45166 - hints = get_type_hints(ann_module5) - self.assertEqual(hints, {'name': Final[str]}) - - hints = get_type_hints(ann_module5.MyClass) - self.assertEqual(hints, {'value': Final}) - - def test_top_level_class_var(self): - # https://bugs.python.org/issue45166 - with self.assertRaisesRegex( - TypeError, - r'typing.ClassVar\[int\] is not valid as type argument', - ): - get_type_hints(ann_module6) class CollectionsAbcTests(BaseTestCase): @@ -3549,27 +6453,17 @@ def test_iterator(self): self.assertIsInstance(it, typing.Iterator) self.assertNotIsInstance(42, typing.Iterator) - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_awaitable(self): - ns = {} - exec( - "async def foo() -> typing.Awaitable[int]:\n" - " return await AwaitableWrapper(42)\n", - globals(), ns) - foo = ns['foo'] + async def foo() -> typing.Awaitable[int]: + return await AwaitableWrapper(42) g = foo() self.assertIsInstance(g, typing.Awaitable) self.assertNotIsInstance(foo, typing.Awaitable) g.send(None) # Run foo() till completion, to avoid warning. - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_coroutine(self): - ns = {} - exec( - "async def foo():\n" - " return\n", - globals(), ns) - foo = ns['foo'] + async def foo(): + return g = foo() self.assertIsInstance(g, typing.Coroutine) with self.assertRaises(TypeError): @@ -3580,7 +6474,6 @@ def test_coroutine(self): except StopIteration: pass - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_async_iterable(self): base_it = range(10) # type: Iterator[int] it = AsyncIteratorWrapper(base_it) @@ -3588,7 +6481,6 @@ def test_async_iterable(self): self.assertIsInstance(it, typing.AsyncIterable) self.assertNotIsInstance(42, typing.AsyncIterable) - @skipUnless(ASYNCIO, 'Python 3.5 and multithreading required') def test_async_iterator(self): base_it = range(10) # type: Iterator[int] it = AsyncIteratorWrapper(base_it) @@ -3634,8 +6526,14 @@ def test_mutablesequence(self): self.assertNotIsInstance((), typing.MutableSequence) def test_bytestring(self): - self.assertIsInstance(b'', typing.ByteString) - self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(b'', typing.ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + class Foo(typing.ByteString): ... + with self.assertWarns(DeprecationWarning): + class Bar(typing.ByteString, typing.Awaitable): ... def test_list(self): self.assertIsSubclass(list, typing.List) @@ -3659,6 +6557,8 @@ def test_frozenset(self): def test_dict(self): self.assertIsSubclass(dict, typing.Dict) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dict_subscribe(self): K = TypeVar('K') V = TypeVar('V') @@ -3742,7 +6642,6 @@ class MyOrdDict(typing.OrderedDict[str, int]): self.assertIsSubclass(MyOrdDict, collections.OrderedDict) self.assertNotIsSubclass(collections.OrderedDict, MyOrdDict) - @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') def test_chainmap_instantiation(self): self.assertIs(type(typing.ChainMap()), collections.ChainMap) self.assertIs(type(typing.ChainMap[KT, VT]()), collections.ChainMap) @@ -3750,7 +6649,6 @@ def test_chainmap_instantiation(self): class CM(typing.ChainMap[KT, VT]): ... self.assertIs(type(CM[int, str]()), CM) - @skipUnless(sys.version_info >= (3, 3), 'ChainMap was added in 3.3') def test_chainmap_subclass(self): class MyChainMap(typing.ChainMap[str, int]): @@ -3832,6 +6730,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() @@ -3841,10 +6750,9 @@ def test_no_generator_instantiation(self): typing.Generator[int, int, int]() def test_async_generator(self): - ns = {} - exec("async def f():\n" - " yield 42\n", globals(), ns) - g = ns['f']() + async def f(): + yield 42 + g = f() self.assertIsSubclass(type(g), typing.AsyncGenerator) def test_no_async_generator_instantiation(self): @@ -3878,7 +6786,7 @@ def __len__(self): return 0 self.assertEqual(len(MMC()), 0) - assert callable(MMC.update) + self.assertTrue(callable(MMC.update)) self.assertIsInstance(MMC(), typing.Mapping) class MMB(typing.MutableMapping[KT, VT]): @@ -3933,9 +6841,8 @@ def asend(self, value): def athrow(self, typ, val=None, tb=None): pass - ns = {} - exec('async def g(): yield 0', globals(), ns) - g = ns['g'] + async def g(): yield 0 + self.assertIsSubclass(G, typing.AsyncGenerator) self.assertIsSubclass(G, typing.AsyncIterable) self.assertIsSubclass(G, collections.abc.AsyncGenerator) @@ -4020,7 +6927,17 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) - @skipUnless(ASYNCIO, 'Python 3.5 required') + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -4032,11 +6949,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -4074,16 +6997,24 @@ def foo(a: A) -> Optional[BaseException]: else: return a() - assert isinstance(foo(KeyboardInterrupt), KeyboardInterrupt) - assert foo(None) is None + self.assertIsInstance(foo(KeyboardInterrupt), KeyboardInterrupt) + self.assertIsNone(foo(None)) + + +class TestModules(TestCase): + func_names = ['_idfunc'] + + def test_c_functions(self): + for fname in self.func_names: + self.assertEqual(getattr(typing, fname).__module__, '_typing') class NewTypeTests(BaseTestCase): @classmethod def setUpClass(cls): global UserId - UserId = NewType('UserId', int) - cls.UserName = NewType(cls.__qualname__ + '.UserName', str) + UserId = typing.NewType('UserId', int) + cls.UserName = typing.NewType(cls.__qualname__ + '.UserName', str) @classmethod def tearDownClass(cls): @@ -4091,9 +7022,6 @@ def tearDownClass(cls): del UserId del cls.UserName - def tearDown(self): - self.clear_caches() - def test_basic(self): self.assertIsInstance(UserId(5), int) self.assertIsInstance(self.UserName('Joe'), str) @@ -4109,11 +7037,11 @@ class D(UserId): def test_or(self): for cls in (int, self.UserName): with self.subTest(cls=cls): - self.assertEqual(UserId | cls, Union[UserId, cls]) - self.assertEqual(cls | UserId, Union[cls, UserId]) + self.assertEqual(UserId | cls, typing.Union[UserId, cls]) + self.assertEqual(cls | UserId, typing.Union[cls, UserId]) - self.assertEqual(get_args(UserId | cls), (UserId, cls)) - self.assertEqual(get_args(cls | UserId), (cls, UserId)) + self.assertEqual(typing.get_args(UserId | cls), (UserId, cls)) + self.assertEqual(typing.get_args(cls | UserId), (cls, UserId)) def test_special_attrs(self): self.assertEqual(UserId.__name__, 'UserId') @@ -4134,7 +7062,7 @@ def test_repr(self): f'{__name__}.{self.__class__.__qualname__}.UserName') def test_pickle(self): - UserAge = NewType('UserAge', float) + UserAge = typing.NewType('UserAge', float) for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): pickled = pickle.dumps(UserId, proto) @@ -4154,6 +7082,18 @@ def test_missing__name__(self): ) exec(code, {}) + def test_error_message_when_subclassing(self): + with self.assertRaisesRegex( + TypeError, + re.escape( + "Cannot subclass an instance of NewType. Perhaps you were looking for: " + "`ProUserId = NewType('ProUserId', UserId)`" + ) + ): + class ProUserId(UserId): + ... + + class NamedTupleTests(BaseTestCase): class NestedEmployee(NamedTuple): @@ -4176,14 +7116,6 @@ def test_basics(self): self.assertEqual(Emp.__annotations__, collections.OrderedDict([('name', str), ('id', int)])) - def test_namedtuple_pyversion(self): - if sys.version_info[:2] < (3, 6): - with self.assertRaises(TypeError): - NamedTuple('Name', one=int, other=str) - with self.assertRaises(TypeError): - class NotYet(NamedTuple): - whatever = 0 - def test_annotation_usage(self): tim = CoolEmployee('Tim', 9000) self.assertIsInstance(tim, CoolEmployee) @@ -4239,22 +7171,122 @@ class A: with self.assertRaises(TypeError): class X(NamedTuple, A): x: int + with self.assertRaises(TypeError): + class Y(NamedTuple, tuple): + x: int + with self.assertRaises(TypeError): + class Z(NamedTuple, NamedTuple): + x: int + class B(NamedTuple): + x: int + with self.assertRaises(TypeError): + class C(NamedTuple, B): + y: str + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generic(self): + class X(NamedTuple, Generic[T]): + x: T + self.assertEqual(X.__bases__, (tuple, Generic)) + self.assertEqual(X.__orig_bases__, (NamedTuple, Generic[T])) + self.assertEqual(X.__mro__, (X, tuple, Generic, object)) + + class Y(Generic[T], NamedTuple): + x: T + self.assertEqual(Y.__bases__, (Generic, tuple)) + self.assertEqual(Y.__orig_bases__, (Generic[T], NamedTuple)) + self.assertEqual(Y.__mro__, (Y, Generic, tuple, object)) + + for G in X, Y: + with self.subTest(type=G): + self.assertEqual(G.__parameters__, (T,)) + self.assertEqual(G[T].__args__, (T,)) + self.assertEqual(get_args(G[T]), (T,)) + A = G[int] + self.assertIs(A.__origin__, G) + self.assertEqual(A.__args__, (int,)) + self.assertEqual(get_args(A), (int,)) + self.assertEqual(A.__parameters__, ()) + + a = A(3) + self.assertIs(type(a), G) + self.assertEqual(a.x, 3) + + with self.assertRaises(TypeError): + G[int, str] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generic_pep695(self): + class X[T](NamedTuple): + x: T + T, = X.__type_params__ + self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(X.__bases__, (tuple, Generic)) + self.assertEqual(X.__orig_bases__, (NamedTuple, Generic[T])) + self.assertEqual(X.__mro__, (X, tuple, Generic, object)) + self.assertEqual(X.__parameters__, (T,)) + self.assertEqual(X[str].__args__, (str,)) + self.assertEqual(X[str].__parameters__, ()) + + def test_non_generic_subscript(self): + # For backward compatibility, subscription works + # on arbitrary NamedTuple types. + class Group(NamedTuple): + key: T + group: list[T] + A = Group[int] + self.assertEqual(A.__origin__, Group) + self.assertEqual(A.__parameters__, ()) + self.assertEqual(A.__args__, (int,)) + a = A(1, [2]) + self.assertIs(type(a), Group) + self.assertEqual(a, (1, [2])) def test_namedtuple_keyword_usage(self): - LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) + with self.assertWarnsRegex( + DeprecationWarning, + "Creating NamedTuple classes using keyword arguments is deprecated" + ): + LocalEmployee = NamedTuple("LocalEmployee", name=str, age=int) + nick = LocalEmployee('Nick', 25) self.assertIsInstance(nick, tuple) self.assertEqual(nick.name, 'Nick') self.assertEqual(LocalEmployee.__name__, 'LocalEmployee') self.assertEqual(LocalEmployee._fields, ('name', 'age')) self.assertEqual(LocalEmployee.__annotations__, dict(name=str, age=int)) - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "Either list of fields or keywords can be provided to NamedTuple, not both" + ): NamedTuple('Name', [('x', int)], y=str) - with self.assertRaises(TypeError): - NamedTuple('Name', x=1, y='a') + + with self.assertRaisesRegex( + TypeError, + "Either list of fields or keywords can be provided to NamedTuple, not both" + ): + NamedTuple('Name', [], y=str) + + with self.assertRaisesRegex( + TypeError, + ( + r"Cannot pass `None` as the 'fields' parameter " + r"and also specify fields using keyword arguments" + ) + ): + NamedTuple('Name', None, x=int) def test_namedtuple_special_keyword_names(self): - NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list) + with self.assertWarnsRegex( + DeprecationWarning, + "Creating NamedTuple classes using keyword arguments is deprecated" + ): + NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list) + self.assertEqual(NT.__name__, 'NT') self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields')) a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)]) @@ -4264,51 +7296,182 @@ def test_namedtuple_special_keyword_names(self): self.assertEqual(a.fields, [('bar', tuple)]) def test_empty_namedtuple(self): - NT = NamedTuple('NT') + expected_warning = re.escape( + "Failing to pass a value for the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. `NT1 = NamedTuple('NT1', [])`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + NT1 = NamedTuple('NT1') + + expected_warning = re.escape( + "Passing `None` as the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. `NT2 = NamedTuple('NT2', [])`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + NT2 = NamedTuple('NT2', None) + + NT3 = NamedTuple('NT2', []) class CNT(NamedTuple): pass # empty body - for struct in [NT, CNT]: + for struct in NT1, NT2, NT3, CNT: with self.subTest(struct=struct): self.assertEqual(struct._fields, ()) self.assertEqual(struct._field_defaults, {}) self.assertEqual(struct.__annotations__, {}) self.assertIsInstance(struct(), struct) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_namedtuple_errors(self): with self.assertRaises(TypeError): NamedTuple.__new__() - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "missing 1 required positional argument" + ): NamedTuple() - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "takes from 1 to 2 positional arguments but 3 were given" + ): NamedTuple('Emp', [('name', str)], None) - with self.assertRaises(ValueError): + + with self.assertRaisesRegex( + ValueError, + "Field names cannot start with an underscore" + ): NamedTuple('Emp', [('_name', str)]) - with self.assertRaises(TypeError): + + with self.assertRaisesRegex( + TypeError, + "missing 1 required positional argument: 'typename'" + ): NamedTuple(typename='Emp', name=str, id=int) - with self.assertRaises(TypeError): - NamedTuple('Emp', fields=[('name', str), ('id', int)]) - def test_copy_and_pickle(self): - global Emp # pickle wants to reference the class by name - Emp = NamedTuple('Emp', [('name', str), ('cool', int)]) - for cls in Emp, CoolEmployee, self.NestedEmployee: - with self.subTest(cls=cls): - jane = cls('jane', 37) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - z = pickle.dumps(jane, proto) - jane2 = pickle.loads(z) - self.assertEqual(jane2, jane) - self.assertIsInstance(jane2, cls) + def test_copy_and_pickle(self): + global Emp # pickle wants to reference the class by name + Emp = NamedTuple('Emp', [('name', str), ('cool', int)]) + for cls in Emp, CoolEmployee, self.NestedEmployee: + with self.subTest(cls=cls): + jane = cls('jane', 37) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(jane, proto) + jane2 = pickle.loads(z) + self.assertEqual(jane2, jane) + self.assertIsInstance(jane2, cls) + + jane2 = copy(jane) + self.assertEqual(jane2, jane) + self.assertIsInstance(jane2, cls) + + jane2 = deepcopy(jane) + self.assertEqual(jane2, jane) + self.assertIsInstance(jane2, cls) + + def test_orig_bases(self): + T = TypeVar('T') + + class SimpleNamedTuple(NamedTuple): + pass + + class GenericNamedTuple(NamedTuple, Generic[T]): + pass + + self.assertEqual(SimpleNamedTuple.__orig_bases__, (NamedTuple,)) + self.assertEqual(GenericNamedTuple.__orig_bases__, (NamedTuple, Generic[T])) + + CallNamedTuple = NamedTuple('CallNamedTuple', []) + + self.assertEqual(CallNamedTuple.__orig_bases__, (NamedTuple,)) + + def test_setname_called_on_values_in_class_dictionary(self): + class Vanilla: + def __set_name__(self, owner, name): + self.name = name + + class Foo(NamedTuple): + attr = Vanilla() + + foo = Foo() + self.assertEqual(len(foo), 0) + self.assertNotIn('attr', Foo._fields) + self.assertIsInstance(foo.attr, Vanilla) + self.assertEqual(foo.attr.name, "attr") + + class Bar(NamedTuple): + attr: Vanilla = Vanilla() + + bar = Bar() + self.assertEqual(len(bar), 1) + self.assertIn('attr', Bar._fields) + self.assertIsInstance(bar.attr, Vanilla) + self.assertEqual(bar.attr.name, "attr") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_setname_raises_the_same_as_on_other_classes(self): + class CustomException(BaseException): pass + + class Annoying: + def __set_name__(self, owner, name): + raise CustomException + + annoying = Annoying() + + with self.assertRaises(CustomException) as cm: + class NormalClass: + attr = annoying + normal_exception = cm.exception + + with self.assertRaises(CustomException) as cm: + class NamedTupleClass(NamedTuple): + attr = annoying + namedtuple_exception = cm.exception - jane2 = copy(jane) - self.assertEqual(jane2, jane) - self.assertIsInstance(jane2, cls) + self.assertIs(type(namedtuple_exception), CustomException) + self.assertIs(type(namedtuple_exception), type(normal_exception)) - jane2 = deepcopy(jane) - self.assertEqual(jane2, jane) - self.assertIsInstance(jane2, cls) + self.assertEqual(len(namedtuple_exception.__notes__), 1) + self.assertEqual( + len(namedtuple_exception.__notes__), len(normal_exception.__notes__) + ) + + expected_note = ( + "Error calling __set_name__ on 'Annoying' instance " + "'attr' in 'NamedTupleClass'" + ) + self.assertEqual(namedtuple_exception.__notes__[0], expected_note) + self.assertEqual( + namedtuple_exception.__notes__[0], + normal_exception.__notes__[0].replace("NormalClass", "NamedTupleClass") + ) + + def test_strange_errors_when_accessing_set_name_itself(self): + class CustomException(Exception): pass + + class Meta(type): + def __getattribute__(self, attr): + if attr == "__set_name__": + raise CustomException + return object.__getattribute__(self, attr) + + class VeryAnnoying(metaclass=Meta): pass + + very_annoying = VeryAnnoying() + + with self.assertRaises(CustomException): + class Foo(NamedTuple): + attr = very_annoying class TypedDictTests(BaseTestCase): @@ -4326,33 +7489,10 @@ def test_basics_functional_syntax(self): self.assertEqual(Emp.__bases__, (dict,)) self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) self.assertEqual(Emp.__total__, True) - - def test_basics_keywords_syntax(self): - Emp = TypedDict('Emp', name=str, id=int) - self.assertIsSubclass(Emp, dict) - self.assertIsSubclass(Emp, typing.MutableMapping) - self.assertNotIsSubclass(Emp, collections.abc.Sequence) - jim = Emp(name='Jim', id=1) - self.assertIs(type(jim), dict) - self.assertEqual(jim['name'], 'Jim') - self.assertEqual(jim['id'], 1) - self.assertEqual(Emp.__name__, 'Emp') - self.assertEqual(Emp.__module__, __name__) - self.assertEqual(Emp.__bases__, (dict,)) - self.assertEqual(Emp.__annotations__, {'name': str, 'id': int}) - self.assertEqual(Emp.__total__, True) - - def test_typeddict_special_keyword_names(self): - TD = TypedDict("TD", cls=type, self=object, typename=str, _typename=int, fields=list, _fields=dict) - self.assertEqual(TD.__name__, 'TD') - self.assertEqual(TD.__annotations__, {'cls': type, 'self': object, 'typename': str, '_typename': int, 'fields': list, '_fields': dict}) - a = TD(cls=str, self=42, typename='foo', _typename=53, fields=[('bar', tuple)], _fields={'baz', set}) - self.assertEqual(a['cls'], str) - self.assertEqual(a['self'], 42) - self.assertEqual(a['typename'], 'foo') - self.assertEqual(a['_typename'], 53) - self.assertEqual(a['fields'], [('bar', tuple)]) - self.assertEqual(a['_fields'], {'baz', set}) + self.assertEqual(Emp.__required_keys__, {'name', 'id'}) + self.assertIsInstance(Emp.__required_keys__, frozenset) + self.assertEqual(Emp.__optional_keys__, set()) + self.assertIsInstance(Emp.__optional_keys__, frozenset) def test_typeddict_create_errors(self): with self.assertRaises(TypeError): @@ -4361,11 +7501,10 @@ def test_typeddict_create_errors(self): TypedDict() with self.assertRaises(TypeError): TypedDict('Emp', [('name', str)], None) - with self.assertRaises(TypeError): - TypedDict(_typename='Emp', name=str, id=int) + TypedDict(_typename='Emp') with self.assertRaises(TypeError): - TypedDict('Emp', _fields={'name': str, 'id': int}) + TypedDict('Emp', name=str, id=int) def test_typeddict_errors(self): Emp = TypedDict('Emp', {'name': str, 'id': int}) @@ -4377,10 +7516,6 @@ def test_typeddict_errors(self): isinstance(jim, Emp) with self.assertRaises(TypeError): issubclass(dict, Emp) - with self.assertRaises(TypeError): - TypedDict('Hi', x=1) - with self.assertRaises(TypeError): - TypedDict('Hi', [('x', int), ('y', 1)]) with self.assertRaises(TypeError): TypedDict('Hi', [('x', int)], y=int) @@ -4399,7 +7534,7 @@ def test_py36_class_syntax_usage(self): def test_pickle(self): global EmpD # pickle wants to reference the class by name - EmpD = TypedDict('EmpD', name=str, id=int) + EmpD = TypedDict('EmpD', {'name': str, 'id': int}) jane = EmpD({'name': 'jane', 'id': 37}) for proto in range(pickle.HIGHEST_PROTOCOL + 1): z = pickle.dumps(jane, proto) @@ -4410,8 +7545,19 @@ def test_pickle(self): EmpDnew = pickle.loads(ZZ) self.assertEqual(EmpDnew({'name': 'jane', 'id': 37}), jane) + def test_pickle_generic(self): + point = Point2DGeneric(a=5.0, b=3.0) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(point, proto) + point2 = pickle.loads(z) + self.assertEqual(point2, point) + self.assertEqual(point2, {'a': 5.0, 'b': 3.0}) + ZZ = pickle.dumps(Point2DGeneric, proto) + Point2DGenericNew = pickle.loads(ZZ) + self.assertEqual(Point2DGenericNew({'a': 5.0, 'b': 3.0}), point) + def test_optional(self): - EmpD = TypedDict('EmpD', name=str, id=int) + EmpD = TypedDict('EmpD', {'name': str, 'id': int}) self.assertEqual(typing.Optional[EmpD], typing.Union[None, EmpD]) self.assertNotEqual(typing.List[EmpD], typing.Tuple[EmpD]) @@ -4422,7 +7568,9 @@ def test_total(self): self.assertEqual(D(x=1), {'x': 1}) self.assertEqual(D.__total__, False) self.assertEqual(D.__required_keys__, frozenset()) + self.assertIsInstance(D.__required_keys__, frozenset) self.assertEqual(D.__optional_keys__, {'x'}) + self.assertIsInstance(D.__optional_keys__, frozenset) self.assertEqual(Options(), {}) self.assertEqual(Options(log_level=2), {'log_level': 2}) @@ -4430,12 +7578,25 @@ def test_total(self): self.assertEqual(Options.__required_keys__, frozenset()) self.assertEqual(Options.__optional_keys__, {'log_level', 'log_path'}) + def test_total_inherits_non_total(self): + class TD1(TypedDict, total=False): + a: int + + self.assertIs(TD1.__total__, False) + + class TD2(TD1): + b: str + + self.assertIs(TD2.__total__, True) + def test_optional_keys(self): class Point2Dor3D(Point2D, total=False): z: int - assert Point2Dor3D.__required_keys__ == frozenset(['x', 'y']) - assert Point2Dor3D.__optional_keys__ == frozenset(['z']) + self.assertEqual(Point2Dor3D.__required_keys__, frozenset(['x', 'y'])) + self.assertIsInstance(Point2Dor3D.__required_keys__, frozenset) + self.assertEqual(Point2Dor3D.__optional_keys__, frozenset(['z'])) + self.assertIsInstance(Point2Dor3D.__optional_keys__, frozenset) def test_keys_inheritance(self): class BaseAnimal(TypedDict): @@ -4448,26 +7609,102 @@ class Animal(BaseAnimal, total=False): class Cat(Animal): fur_color: str - assert BaseAnimal.__required_keys__ == frozenset(['name']) - assert BaseAnimal.__optional_keys__ == frozenset([]) - assert BaseAnimal.__annotations__ == {'name': str} + self.assertEqual(BaseAnimal.__required_keys__, frozenset(['name'])) + self.assertEqual(BaseAnimal.__optional_keys__, frozenset([])) + self.assertEqual(BaseAnimal.__annotations__, {'name': str}) - assert Animal.__required_keys__ == frozenset(['name']) - assert Animal.__optional_keys__ == frozenset(['tail', 'voice']) - assert Animal.__annotations__ == { + self.assertEqual(Animal.__required_keys__, frozenset(['name'])) + self.assertEqual(Animal.__optional_keys__, frozenset(['tail', 'voice'])) + self.assertEqual(Animal.__annotations__, { 'name': str, 'tail': bool, 'voice': str, - } + }) - assert Cat.__required_keys__ == frozenset(['name', 'fur_color']) - assert Cat.__optional_keys__ == frozenset(['tail', 'voice']) - assert Cat.__annotations__ == { + self.assertEqual(Cat.__required_keys__, frozenset(['name', 'fur_color'])) + self.assertEqual(Cat.__optional_keys__, frozenset(['tail', 'voice'])) + self.assertEqual(Cat.__annotations__, { 'fur_color': str, 'name': str, 'tail': bool, 'voice': str, - } + }) + + def test_keys_inheritance_with_same_name(self): + class NotTotal(TypedDict, total=False): + a: int + + class Total(NotTotal): + a: int + + self.assertEqual(NotTotal.__required_keys__, frozenset()) + self.assertEqual(NotTotal.__optional_keys__, frozenset(['a'])) + self.assertEqual(Total.__required_keys__, frozenset(['a'])) + self.assertEqual(Total.__optional_keys__, frozenset()) + + class Base(TypedDict): + a: NotRequired[int] + b: Required[int] + + class Child(Base): + a: Required[int] + b: NotRequired[int] + + self.assertEqual(Base.__required_keys__, frozenset(['b'])) + self.assertEqual(Base.__optional_keys__, frozenset(['a'])) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset(['b'])) + + def test_multiple_inheritance_with_same_key(self): + class Base1(TypedDict): + a: NotRequired[int] + + class Base2(TypedDict): + a: Required[str] + + class Child(Base1, Base2): + pass + + # Last base wins + self.assertEqual(Child.__annotations__, {'a': Required[str]}) + self.assertEqual(Child.__required_keys__, frozenset(['a'])) + self.assertEqual(Child.__optional_keys__, frozenset()) + + def test_required_notrequired_keys(self): + self.assertEqual(NontotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(NontotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(TotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(TotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(_typed_dict_helper.VeryAnnotated.__required_keys__, + frozenset()) + self.assertEqual(_typed_dict_helper.VeryAnnotated.__optional_keys__, + frozenset({"a"})) + + self.assertEqual(AnnotatedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(AnnotatedMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(WeirdlyQuotedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(WeirdlyQuotedMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(ChildTotalMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(ChildTotalMovie.__optional_keys__, + frozenset({"year"})) + + self.assertEqual(ChildDeeplyAnnotatedMovie.__required_keys__, + frozenset({"title"})) + self.assertEqual(ChildDeeplyAnnotatedMovie.__optional_keys__, + frozenset({"year"})) def test_multiple_inheritance(self): class One(TypedDict): @@ -4556,18 +7793,181 @@ class ChildWithInlineAndOptional(Untotal, Inline): class Wrong(*bases): pass + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_is_typeddict(self): - assert is_typeddict(Point2D) is True - assert is_typeddict(Union[str, int]) is False + self.assertIs(is_typeddict(Point2D), True) + self.assertIs(is_typeddict(Union[str, int]), False) # classes, not instances - assert is_typeddict(Point2D()) is False + self.assertIs(is_typeddict(Point2D()), False) + call_based = TypedDict('call_based', {'a': int}) + self.assertIs(is_typeddict(call_based), True) + self.assertIs(is_typeddict(call_based()), False) + + T = TypeVar("T") + class BarGeneric(TypedDict, Generic[T]): + a: T + self.assertIs(is_typeddict(BarGeneric), True) + self.assertIs(is_typeddict(BarGeneric[int]), False) + self.assertIs(is_typeddict(BarGeneric()), False) + + class NewGeneric[T](TypedDict): + a: T + self.assertIs(is_typeddict(NewGeneric), True) + self.assertIs(is_typeddict(NewGeneric[int]), False) + self.assertIs(is_typeddict(NewGeneric()), False) + + # The TypedDict constructor is not itself a TypedDict + self.assertIs(is_typeddict(TypedDict), False) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_get_type_hints(self): self.assertEqual( get_type_hints(Bar), {'a': typing.Optional[int], 'b': int} ) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_get_type_hints_generic(self): + self.assertEqual( + get_type_hints(BarGeneric), + {'a': typing.Optional[T], 'b': int} + ) + + class FooBarGeneric(BarGeneric[int]): + c: str + + self.assertEqual( + get_type_hints(FooBarGeneric), + {'a': typing.Optional[T], 'b': int, 'c': str} + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_pep695_generic_typeddict(self): + class A[T](TypedDict): + a: T + + T, = A.__type_params__ + self.assertIsInstance(T, TypeVar) + self.assertEqual(T.__name__, 'T') + self.assertEqual(A.__bases__, (Generic, dict)) + self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) + self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__parameters__, (T,)) + self.assertEqual(A[str].__parameters__, ()) + self.assertEqual(A[str].__args__, (str,)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generic_inheritance(self): + class A(TypedDict, Generic[T]): + a: T + + self.assertEqual(A.__bases__, (Generic, dict)) + self.assertEqual(A.__orig_bases__, (TypedDict, Generic[T])) + self.assertEqual(A.__mro__, (A, Generic, dict, object)) + self.assertEqual(A.__parameters__, (T,)) + self.assertEqual(A[str].__parameters__, ()) + self.assertEqual(A[str].__args__, (str,)) + + class A2(Generic[T], TypedDict): + a: T + + self.assertEqual(A2.__bases__, (Generic, dict)) + self.assertEqual(A2.__orig_bases__, (Generic[T], TypedDict)) + self.assertEqual(A2.__mro__, (A2, Generic, dict, object)) + self.assertEqual(A2.__parameters__, (T,)) + self.assertEqual(A2[str].__parameters__, ()) + self.assertEqual(A2[str].__args__, (str,)) + + class B(A[KT], total=False): + b: KT + + self.assertEqual(B.__bases__, (Generic, dict)) + self.assertEqual(B.__orig_bases__, (A[KT],)) + self.assertEqual(B.__mro__, (B, Generic, dict, object)) + self.assertEqual(B.__parameters__, (KT,)) + self.assertEqual(B.__total__, False) + self.assertEqual(B.__optional_keys__, frozenset(['b'])) + self.assertEqual(B.__required_keys__, frozenset(['a'])) + + self.assertEqual(B[str].__parameters__, ()) + self.assertEqual(B[str].__args__, (str,)) + self.assertEqual(B[str].__origin__, B) + + class C(B[int]): + c: int + + self.assertEqual(C.__bases__, (Generic, dict)) + self.assertEqual(C.__orig_bases__, (B[int],)) + self.assertEqual(C.__mro__, (C, Generic, dict, object)) + self.assertEqual(C.__parameters__, ()) + self.assertEqual(C.__total__, True) + self.assertEqual(C.__optional_keys__, frozenset(['b'])) + self.assertEqual(C.__required_keys__, frozenset(['a', 'c'])) + self.assertEqual(C.__annotations__, { + 'a': T, + 'b': KT, + 'c': int, + }) + with self.assertRaises(TypeError): + C[str] + + + class Point3D(Point2DGeneric[T], Generic[T, KT]): + c: KT + + self.assertEqual(Point3D.__bases__, (Generic, dict)) + self.assertEqual(Point3D.__orig_bases__, (Point2DGeneric[T], Generic[T, KT])) + self.assertEqual(Point3D.__mro__, (Point3D, Generic, dict, object)) + self.assertEqual(Point3D.__parameters__, (T, KT)) + self.assertEqual(Point3D.__total__, True) + self.assertEqual(Point3D.__optional_keys__, frozenset()) + self.assertEqual(Point3D.__required_keys__, frozenset(['a', 'b', 'c'])) + self.assertEqual(Point3D.__annotations__, { + 'a': T, + 'b': T, + 'c': KT, + }) + self.assertEqual(Point3D[int, str].__origin__, Point3D) + + with self.assertRaises(TypeError): + Point3D[int] + + with self.assertRaises(TypeError): + class Point3D(Point2DGeneric[T], Generic[KT]): + c: KT + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_implicit_any_inheritance(self): + class A(TypedDict, Generic[T]): + a: T + + class B(A[KT], total=False): + b: KT + + class WithImplicitAny(B): + c: int + + self.assertEqual(WithImplicitAny.__bases__, (Generic, dict,)) + self.assertEqual(WithImplicitAny.__mro__, (WithImplicitAny, Generic, dict, object)) + # Consistent with GenericTests.test_implicit_any + self.assertEqual(WithImplicitAny.__parameters__, ()) + self.assertEqual(WithImplicitAny.__total__, True) + self.assertEqual(WithImplicitAny.__optional_keys__, frozenset(['b'])) + self.assertEqual(WithImplicitAny.__required_keys__, frozenset(['a', 'c'])) + self.assertEqual(WithImplicitAny.__annotations__, { + 'a': T, + 'b': KT, + 'c': int, + }) + with self.assertRaises(TypeError): + WithImplicitAny[str] + # TODO: RUSTPYTHON @unittest.expectedFailure def test_non_generic_subscript(self): @@ -4583,9 +7983,249 @@ class TD(TypedDict): self.assertIs(type(a), dict) self.assertEqual(a, {'a': 1}) + def test_orig_bases(self): + T = TypeVar('T') + + class Parent(TypedDict): + pass + + class Child(Parent): + pass + + class OtherChild(Parent): + pass + + class MixedChild(Child, OtherChild, Parent): + pass + + class GenericParent(TypedDict, Generic[T]): + pass + + class GenericChild(GenericParent[int]): + pass + + class OtherGenericChild(GenericParent[str]): + pass + + class MixedGenericChild(GenericChild, OtherGenericChild, GenericParent[float]): + pass + + class MultipleGenericBases(GenericParent[int], GenericParent[float]): + pass + + CallTypedDict = TypedDict('CallTypedDict', {}) + + self.assertEqual(Parent.__orig_bases__, (TypedDict,)) + self.assertEqual(Child.__orig_bases__, (Parent,)) + self.assertEqual(OtherChild.__orig_bases__, (Parent,)) + self.assertEqual(MixedChild.__orig_bases__, (Child, OtherChild, Parent,)) + self.assertEqual(GenericParent.__orig_bases__, (TypedDict, Generic[T])) + self.assertEqual(GenericChild.__orig_bases__, (GenericParent[int],)) + self.assertEqual(OtherGenericChild.__orig_bases__, (GenericParent[str],)) + self.assertEqual(MixedGenericChild.__orig_bases__, (GenericChild, OtherGenericChild, GenericParent[float])) + self.assertEqual(MultipleGenericBases.__orig_bases__, (GenericParent[int], GenericParent[float])) + self.assertEqual(CallTypedDict.__orig_bases__, (TypedDict,)) + + def test_zero_fields_typeddicts(self): + T1 = TypedDict("T1", {}) + class T2(TypedDict): pass + class T3[tvar](TypedDict): pass + S = TypeVar("S") + class T4(TypedDict, Generic[S]): pass + + expected_warning = re.escape( + "Failing to pass a value for the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a TypedDict class with 0 fields " + "using the functional syntax, " + "pass an empty dictionary, e.g. `T5 = TypedDict('T5', {})`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + T5 = TypedDict('T5') + + expected_warning = re.escape( + "Passing `None` as the 'fields' parameter is deprecated " + "and will be disallowed in Python 3.15. " + "To create a TypedDict class with 0 fields " + "using the functional syntax, " + "pass an empty dictionary, e.g. `T6 = TypedDict('T6', {})`." + ) + with self.assertWarnsRegex(DeprecationWarning, fr"^{expected_warning}$"): + T6 = TypedDict('T6', None) + + for klass in T1, T2, T3, T4, T5, T6: + with self.subTest(klass=klass.__name__): + self.assertEqual(klass.__annotations__, {}) + self.assertEqual(klass.__required_keys__, set()) + self.assertEqual(klass.__optional_keys__, set()) + self.assertIsInstance(klass(), dict) + + def test_readonly_inheritance(self): + class Base1(TypedDict): + a: ReadOnly[int] + + class Child1(Base1): + b: str + + self.assertEqual(Child1.__readonly_keys__, frozenset({'a'})) + self.assertEqual(Child1.__mutable_keys__, frozenset({'b'})) + + class Base2(TypedDict): + a: int + + class Child2(Base2): + b: ReadOnly[str] + + self.assertEqual(Child2.__readonly_keys__, frozenset({'b'})) + self.assertEqual(Child2.__mutable_keys__, frozenset({'a'})) + + def test_cannot_make_mutable_key_readonly(self): + class Base(TypedDict): + a: int + + with self.assertRaises(TypeError): + class Child(Base): + a: ReadOnly[int] + + def test_can_make_readonly_key_mutable(self): + class Base(TypedDict): + a: ReadOnly[int] + + class Child(Base): + a: int + + self.assertEqual(Child.__readonly_keys__, frozenset()) + self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_combine_qualifiers(self): + class AllTheThings(TypedDict): + a: Annotated[Required[ReadOnly[int]], "why not"] + b: Required[Annotated[ReadOnly[int], "why not"]] + c: ReadOnly[NotRequired[Annotated[int, "why not"]]] + d: NotRequired[Annotated[int, "why not"]] + + self.assertEqual(AllTheThings.__required_keys__, frozenset({'a', 'b'})) + self.assertEqual(AllTheThings.__optional_keys__, frozenset({'c', 'd'})) + self.assertEqual(AllTheThings.__readonly_keys__, frozenset({'a', 'b', 'c'})) + self.assertEqual(AllTheThings.__mutable_keys__, frozenset({'d'})) + + self.assertEqual( + get_type_hints(AllTheThings, include_extras=False), + {'a': int, 'b': int, 'c': int, 'd': int}, + ) + self.assertEqual( + get_type_hints(AllTheThings, include_extras=True), + { + 'a': Annotated[Required[ReadOnly[int]], 'why not'], + 'b': Required[Annotated[ReadOnly[int], 'why not']], + 'c': ReadOnly[NotRequired[Annotated[int, 'why not']]], + 'd': NotRequired[Annotated[int, 'why not']], + }, + ) + + +class RequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + Required[NotRequired] + with self.assertRaises(TypeError): + Required[int, str] + with self.assertRaises(TypeError): + Required[int][str] + + def test_repr(self): + self.assertEqual(repr(Required), 'typing.Required') + cv = Required[int] + self.assertEqual(repr(cv), 'typing.Required[int]') + cv = Required[Employee] + self.assertEqual(repr(cv), f'typing.Required[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(Required)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(Required[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Required'): + class E(Required): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.Required\[int\]'): + class F(Required[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + Required() + with self.assertRaises(TypeError): + type(Required)() + with self.assertRaises(TypeError): + type(Required[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, Required[int]) + with self.assertRaises(TypeError): + issubclass(int, Required) + + +class NotRequiredTests(BaseTestCase): + + def test_basics(self): + with self.assertRaises(TypeError): + NotRequired[Required] + with self.assertRaises(TypeError): + NotRequired[int, str] + with self.assertRaises(TypeError): + NotRequired[int][str] + + def test_repr(self): + self.assertEqual(repr(NotRequired), 'typing.NotRequired') + cv = NotRequired[int] + self.assertEqual(repr(cv), 'typing.NotRequired[int]') + cv = NotRequired[Employee] + self.assertEqual(repr(cv), f'typing.NotRequired[{__name__}.Employee]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(NotRequired)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(NotRequired[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.NotRequired'): + class E(NotRequired): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.NotRequired\[int\]'): + class F(NotRequired[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + NotRequired() + with self.assertRaises(TypeError): + type(NotRequired)() + with self.assertRaises(TypeError): + type(NotRequired[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, NotRequired[int]) + with self.assertRaises(TypeError): + issubclass(int, NotRequired) + class IOTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_io(self): def stuff(a: IO) -> AnyStr: @@ -4594,6 +8234,8 @@ def stuff(a: IO) -> AnyStr: a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, (AnyStr,)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_textio(self): def stuff(a: TextIO) -> str: @@ -4602,6 +8244,8 @@ def stuff(a: TextIO) -> str: a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, ()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_binaryio(self): def stuff(a: BinaryIO) -> bytes: @@ -4610,14 +8254,6 @@ def stuff(a: BinaryIO) -> bytes: a = stuff.__annotations__['a'] self.assertEqual(a.__parameters__, ()) - def test_io_submodule(self): - from typing.io import IO, TextIO, BinaryIO, __all__, __name__ - self.assertIs(IO, typing.IO) - self.assertIs(TextIO, typing.TextIO) - self.assertIs(BinaryIO, typing.BinaryIO) - self.assertEqual(set(__all__), set(['IO', 'TextIO', 'BinaryIO'])) - self.assertEqual(__name__, 'typing.io') - class RETests(BaseTestCase): # Much of this is really testing _TypeAlias. @@ -4662,31 +8298,29 @@ def test_repr(self): self.assertEqual(repr(Match[str]), 'typing.Match[str]') self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') - def test_re_submodule(self): - from typing.re import Match, Pattern, __all__, __name__ - self.assertIs(Match, typing.Match) - self.assertIs(Pattern, typing.Pattern) - self.assertEqual(set(__all__), set(['Match', 'Pattern'])) - self.assertEqual(__name__, 'typing.re') - # TODO: RUSTPYTHON @unittest.expectedFailure def test_cannot_subclass(self): - with self.assertRaises(TypeError) as ex: - + with self.assertRaisesRegex( + TypeError, + r"type 're\.Match' is not an acceptable base type", + ): class A(typing.Match): pass + with self.assertRaisesRegex( + TypeError, + r"type 're\.Pattern' is not an acceptable base type", + ): + class B(typing.Pattern): + pass - self.assertEqual(str(ex.exception), - "type 're.Match' is not an acceptable base type") class AnnotatedTests(BaseTestCase): def test_new(self): with self.assertRaisesRegex( - TypeError, - 'Type Annotated cannot be instantiated', + TypeError, 'Cannot instantiate typing.Annotated', ): Annotated() @@ -4700,12 +8334,93 @@ def test_repr(self): "typing.Annotated[typing.List[int], 4, 5]" ) + def test_dir(self): + dir_items = set(dir(Annotated[int, 4])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + '__metadata__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + def test_flatten(self): A = Annotated[Annotated[int, 4], 5] self.assertEqual(A, Annotated[int, 4, 5]) self.assertEqual(A.__metadata__, (4, 5)) self.assertEqual(A.__origin__, int) + def test_deduplicate_from_union(self): + # Regular: + self.assertEqual(get_args(Annotated[int, 1] | int), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], int]), + (Annotated[int, 1], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[int, 2] | int), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[int, 2], int]), + (Annotated[int, 1], Annotated[int, 2], int)) + self.assertEqual(get_args(Annotated[int, 1] | Annotated[str, 1] | int), + (Annotated[int, 1], Annotated[str, 1], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], Annotated[str, 1], int]), + (Annotated[int, 1], Annotated[str, 1], int)) + + # Duplicates: + self.assertEqual(Annotated[int, 1] | Annotated[int, 1] | int, + Annotated[int, 1] | int) + self.assertEqual(Union[Annotated[int, 1], Annotated[int, 1], int], + Union[Annotated[int, 1], int]) + + # Unhashable metadata: + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[int, set()] | int), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[int, set()], int]), + (str, Annotated[int, {}], Annotated[int, set()], int)) + self.assertEqual(get_args(str | Annotated[int, {}] | Annotated[str, {}] | int), + (str, Annotated[int, {}], Annotated[str, {}], int)) + self.assertEqual(get_args(Union[str, Annotated[int, {}], Annotated[str, {}], int]), + (str, Annotated[int, {}], Annotated[str, {}], int)) + + self.assertEqual(get_args(Annotated[int, 1] | str | Annotated[str, {}] | int), + (Annotated[int, 1], str, Annotated[str, {}], int)) + self.assertEqual(get_args(Union[Annotated[int, 1], str, Annotated[str, {}], int]), + (Annotated[int, 1], str, Annotated[str, {}], int)) + + import dataclasses + @dataclasses.dataclass + class ValueRange: + lo: int + hi: int + v = ValueRange(1, 2) + self.assertEqual(get_args(Annotated[int, v] | None), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Union[Annotated[int, v], None]), + (Annotated[int, v], types.NoneType)) + self.assertEqual(get_args(Optional[Annotated[int, v]]), + (Annotated[int, v], types.NoneType)) + + # Unhashable metadata duplicated: + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + Annotated[int, {}] | int) + self.assertEqual(Annotated[int, {}] | Annotated[int, {}] | int, + int | Annotated[int, {}]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[Annotated[int, {}], int]) + self.assertEqual(Union[Annotated[int, {}], Annotated[int, {}], int], + Union[int, Annotated[int, {}]]) + + def test_order_in_union(self): + expr1 = Annotated[int, 1] | str | Annotated[str, {}] | int + for args in itertools.permutations(get_args(expr1)): + with self.subTest(args=args): + self.assertEqual(expr1, reduce(operator.or_, args)) + + expr2 = Union[Annotated[int, 1], str, Annotated[str, {}], int] + for args in itertools.permutations(get_args(expr2)): + with self.subTest(args=args): + self.assertEqual(expr2, Union[args]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_specialize(self): L = Annotated[List[T], "my decoration"] LI = Annotated[List[int], "my decoration"] @@ -4726,6 +8441,16 @@ def test_hash_eq(self): {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, {Annotated[int, 4, 5], Annotated[T, 4, 5]} ) + # Unhashable `metadata` raises `TypeError`: + a1 = Annotated[int, []] + with self.assertRaises(TypeError): + hash(a1) + + class A: + __hash__ = None + a2 = Annotated[int, A()] + with self.assertRaises(TypeError): + hash(a2) def test_instantiate(self): class C: @@ -4746,11 +8471,24 @@ def __eq__(self, other): self.assertEqual(a.x, c.x) self.assertEqual(a.classvar, c.classvar) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_instantiate_generic(self): MyCount = Annotated[typing.Counter[T], "my decoration"] self.assertEqual(MyCount([4, 4, 5]), {4: 2, 5: 1}) self.assertEqual(MyCount[int]([4, 4, 5]), {4: 2, 5: 1}) + def test_instantiate_immutable(self): + class C: + def __setattr__(self, key, value): + raise Exception("should be ignored") + + A = Annotated[C, "a decoration"] + # gh-115165: This used to cause RuntimeError to be raised + # when we tried to set `__orig_class__` on the `C` instance + # returned by the `A()` call + self.assertIsInstance(A(), C) + def test_cannot_instantiate_forward(self): A = Annotated["int", (5, 6)] with self.assertRaises(TypeError): @@ -4774,23 +8512,45 @@ class C: A.x = 5 self.assertEqual(C.x, 5) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_special_form_containment(self): class C: classvar: Annotated[ClassVar[int], "a decoration"] = 4 const: Annotated[Final[int], "Const"] = 4 - self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) - self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + self.assertEqual(get_type_hints(C, globals())['classvar'], ClassVar[int]) + self.assertEqual(get_type_hints(C, globals())['const'], Final[int]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_special_forms_nesting(self): + # These are uncommon types and are to ensure runtime + # is lax on validation. See gh-89547 for more context. + class CF: + x: ClassVar[Final[int]] + + class FC: + x: Final[ClassVar[int]] + + class ACF: + x: Annotated[ClassVar[Final[int]], "a decoration"] + + class CAF: + x: ClassVar[Annotated[Final[int], "a decoration"]] + + class AFC: + x: Annotated[Final[ClassVar[int]], "a decoration"] + + class FAC: + x: Final[Annotated[ClassVar[int], "a decoration"]] - def test_hash_eq(self): - self.assertEqual(len({Annotated[int, 4, 5], Annotated[int, 4, 5]}), 1) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[int, 5, 4]) - self.assertNotEqual(Annotated[int, 4, 5], Annotated[str, 4, 5]) - self.assertNotEqual(Annotated[int, 4], Annotated[int, 4, 4]) - self.assertEqual( - {Annotated[int, 4, 5], Annotated[int, 4, 5], Annotated[T, 4, 5]}, - {Annotated[int, 4, 5], Annotated[T, 4, 5]} - ) + self.assertEqual(get_type_hints(CF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(FC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(ACF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(CAF, globals())['x'], ClassVar[Final[int]]) + self.assertEqual(get_type_hints(AFC, globals())['x'], Final[ClassVar[int]]) + self.assertEqual(get_type_hints(FAC, globals())['x'], Final[ClassVar[int]]) def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, "Cannot subclass .*Annotated"): @@ -4809,6 +8569,8 @@ def test_too_few_type_args(self): with self.assertRaisesRegex(TypeError, 'at least two arguments'): Annotated[int] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pickle(self): samples = [typing.Any, typing.Union[int, str], typing.Optional[str], Tuple[int, ...], @@ -4839,6 +8601,8 @@ class _Annotated_test_G(Generic[T]): self.assertEqual(x.bar, 'abc') self.assertEqual(x.x, 1) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_subst(self): dec = "a decoration" dec2 = "another decoration" @@ -4868,6 +8632,124 @@ def test_subst(self): with self.assertRaises(TypeError): LI[None] + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevar_subst(self): + dec = "a decoration" + Ts = TypeVarTuple('Ts') + T = TypeVar('T') + T1 = TypeVar('T1') + T2 = TypeVar('T2') + + # A = Annotated[tuple[*Ts], dec] + self.assertEqual(A[int], Annotated[tuple[int], dec]) + self.assertEqual(A[str, int], Annotated[tuple[str, int], dec]) + with self.assertRaises(TypeError): + Annotated[*Ts, dec] + + B = Annotated[Tuple[Unpack[Ts]], dec] + self.assertEqual(B[int], Annotated[Tuple[int], dec]) + self.assertEqual(B[str, int], Annotated[Tuple[str, int], dec]) + with self.assertRaises(TypeError): + Annotated[Unpack[Ts], dec] + + C = Annotated[tuple[T, *Ts], dec] + self.assertEqual(C[int], Annotated[tuple[int], dec]) + self.assertEqual(C[int, str], Annotated[tuple[int, str], dec]) + self.assertEqual( + C[int, str, float], + Annotated[tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + C[()] + + D = Annotated[Tuple[T, Unpack[Ts]], dec] + self.assertEqual(D[int], Annotated[Tuple[int], dec]) + self.assertEqual(D[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + D[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + D[()] + + E = Annotated[tuple[*Ts, T], dec] + self.assertEqual(E[int], Annotated[tuple[int], dec]) + self.assertEqual(E[int, str], Annotated[tuple[int, str], dec]) + self.assertEqual( + E[int, str, float], + Annotated[tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + E[()] + + F = Annotated[Tuple[Unpack[Ts], T], dec] + self.assertEqual(F[int], Annotated[Tuple[int], dec]) + self.assertEqual(F[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + F[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + with self.assertRaises(TypeError): + F[()] + + G = Annotated[tuple[T1, *Ts, T2], dec] + self.assertEqual(G[int, str], Annotated[tuple[int, str], dec]) + self.assertEqual( + G[int, str, float], + Annotated[tuple[int, str, float], dec] + ) + self.assertEqual( + G[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec] + ) + with self.assertRaises(TypeError): + G[int] + + H = Annotated[Tuple[T1, Unpack[Ts], T2], dec] + self.assertEqual(H[int, str], Annotated[Tuple[int, str], dec]) + self.assertEqual( + H[int, str, float], + Annotated[Tuple[int, str, float], dec] + ) + self.assertEqual( + H[int, str, bool, float], + Annotated[Tuple[int, str, bool, float], dec] + ) + with self.assertRaises(TypeError): + H[int] + + # Now let's try creating an alias from an alias. + + Ts2 = TypeVarTuple('Ts2') + T3 = TypeVar('T3') + T4 = TypeVar('T4') + + # G is Annotated[tuple[T1, *Ts, T2], dec]. + I = G[T3, *Ts2, T4] + J = G[T3, Unpack[Ts2], T4] + + for x, y in [ + (I, Annotated[tuple[T3, *Ts2, T4], dec]), + (J, Annotated[tuple[T3, Unpack[Ts2], T4], dec]), + (I[int, str], Annotated[tuple[int, str], dec]), + (J[int, str], Annotated[tuple[int, str], dec]), + (I[int, str, float], Annotated[tuple[int, str, float], dec]), + (J[int, str, float], Annotated[tuple[int, str, float], dec]), + (I[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec]), + (J[int, str, bool, float], + Annotated[tuple[int, str, bool, float], dec]), + ]: + self.assertEqual(x, y) + + with self.assertRaises(TypeError): + I[int] + with self.assertRaises(TypeError): + J[int] + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_annotated_in_other_types(self): X = List[Annotated[T, 5]] self.assertEqual(X[int], List[Annotated[int, 5]]) @@ -4877,6 +8759,40 @@ class X(Annotated[int, (1, 10)]): ... self.assertEqual(X.__mro__, (X, int, object), "Annotated should be transparent.") + def test_annotated_cached_with_types(self): + class A(str): ... + class B(str): ... + + field_a1 = Annotated[str, A("X")] + field_a2 = Annotated[str, B("X")] + a1_metadata = field_a1.__metadata__[0] + a2_metadata = field_a2.__metadata__[0] + + self.assertIs(type(a1_metadata), A) + self.assertEqual(a1_metadata, A("X")) + self.assertIs(type(a2_metadata), B) + self.assertEqual(a2_metadata, B("X")) + self.assertIsNot(type(a1_metadata), type(a2_metadata)) + + field_b1 = Annotated[str, A("Y")] + field_b2 = Annotated[str, B("Y")] + b1_metadata = field_b1.__metadata__[0] + b2_metadata = field_b2.__metadata__[0] + + self.assertIs(type(b1_metadata), A) + self.assertEqual(b1_metadata, A("Y")) + self.assertIs(type(b2_metadata), B) + self.assertEqual(b2_metadata, B("Y")) + self.assertIsNot(type(b1_metadata), type(b2_metadata)) + + field_c1 = Annotated[int, 1] + field_c2 = Annotated[int, 1.0] + field_c3 = Annotated[int, True] + + self.assertIs(type(field_c1.__metadata__[0]), int) + self.assertIs(type(field_c2.__metadata__[0]), float) + self.assertIs(type(field_c3.__metadata__[0]), bool) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): @@ -4893,6 +8809,8 @@ def test_no_isinstance(self): with self.assertRaises(TypeError): isinstance(42, TypeAlias) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_stringized_usage(self): class A: a: "TypeAlias" @@ -4906,12 +8824,13 @@ def test_no_issubclass(self): issubclass(TypeAlias, Employee) def test_cannot_subclass(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeAlias'): class C(TypeAlias): pass with self.assertRaises(TypeError): - class C(type(TypeAlias)): + class D(type(TypeAlias)): pass def test_repr(self): @@ -4922,12 +8841,27 @@ def test_cannot_subscript(self): TypeAlias[int] + class ParamSpecTests(BaseTestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_basic_plain(self): P = ParamSpec('P') self.assertEqual(P, P) self.assertIsInstance(P, ParamSpec) + self.assertEqual(P.__name__, 'P') + self.assertEqual(P.__module__, __name__) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_basic_with_exec(self): + ns = {} + exec('from typing import ParamSpec; P = ParamSpec("P")', ns, ns) + P = ns['P'] + self.assertIsInstance(P, ParamSpec) + self.assertEqual(P.__name__, 'P') + self.assertIs(P.__module__, None) # TODO: RUSTPYTHON @unittest.expectedFailure @@ -4948,6 +8882,8 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (P, T)) self.assertEqual(C4.__parameters__, (P, T)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_args_kwargs(self): P = ParamSpec('P') P_2 = ParamSpec('P_2') @@ -4967,6 +8903,9 @@ def test_args_kwargs(self): self.assertEqual(repr(P.args), "P.args") self.assertEqual(repr(P.kwargs), "P.kwargs") + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_stringized(self): P = ParamSpec('P') class C(Generic[P]): @@ -4979,6 +8918,8 @@ def foo(self, *args: "P.args", **kwargs: "P.kwargs"): gth(C.foo, globals(), locals()), {"args": P.args, "kwargs": P.kwargs} ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_user_generics(self): T = TypeVar("T") P = ParamSpec("P") @@ -5033,6 +8974,8 @@ class Z(Generic[P]): with self.assertRaisesRegex(TypeError, "many arguments for"): Z[P_2, bool] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_multiple_paramspecs_in_user_generics(self): P = ParamSpec("P") P2 = ParamSpec("P2") @@ -5047,33 +8990,221 @@ class X(Generic[P, P2]): self.assertEqual(G1.__args__, ((int, str), (bytes,))) self.assertEqual(G2.__args__, ((int,), (str, bytes))) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevartuple_and_paramspecs_in_user_generics(self): + Ts = TypeVarTuple("Ts") + P = ParamSpec("P") + + # class X(Generic[*Ts, P]): + # f: Callable[P, int] + # g: Tuple[*Ts] + + G1 = X[int, [bytes]] + self.assertEqual(G1.__args__, (int, (bytes,))) + G2 = X[int, str, [bytes]] + self.assertEqual(G2.__args__, (int, str, (bytes,))) + G3 = X[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = X[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + X[()] + + # class Y(Generic[P, *Ts]): + # f: Callable[P, int] + # g: Tuple[*Ts] + + G1 = Y[[bytes], int] + self.assertEqual(G1.__args__, ((bytes,), int)) + G2 = Y[[bytes], int, str] + self.assertEqual(G2.__args__, ((bytes,), int, str)) + G3 = Y[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = Y[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + Y[()] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_typevartuple_and_paramspecs_in_generic_aliases(self): + P = ParamSpec('P') + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + for C in Callable, collections.abc.Callable: + with self.subTest(generic=C): + # A = C[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, (int, str, Tuple[bytes, float])) + + class X(Generic[T, P]): + pass + + # A = X[Tuple[*Ts], P] + B = A[bytes, float, [int, str]] + self.assertEqual(B.__args__, (Tuple[bytes, float], (int, str,))) + + class Y(Generic[P, T]): + pass + + # A = Y[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float])) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_var_substitution(self): + P = ParamSpec("P") + subst = P.__typing_subst__ + self.assertEqual(subst((int, str)), (int, str)) + self.assertEqual(subst([int, str]), (int, str)) + self.assertEqual(subst([None]), (type(None),)) + self.assertIs(subst(...), ...) + self.assertIs(subst(P), P) + self.assertEqual(subst(Concatenate[int, P]), Concatenate[int, P]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_bad_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') bad_args = (42, int, None, T, int|str, Union[int, str]) for arg in bad_args: with self.subTest(arg=arg): + with self.assertRaises(TypeError): + P.__typing_subst__(arg) with self.assertRaises(TypeError): typing.Callable[P, T][arg, str] with self.assertRaises(TypeError): collections.abc.Callable[P, T][arg, str] - def test_no_paramspec_in__parameters__(self): - # ParamSpec should not be found in __parameters__ - # of generics. Usages outside Callable, Concatenate - # and Generic are invalid. - T = TypeVar("T") - P = ParamSpec("P") - self.assertNotIn(P, List[P].__parameters__) - self.assertIn(T, Tuple[T, P].__parameters__) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_type_var_subst_for_other_type_vars(self): + T = TypeVar('T') + T2 = TypeVar('T2') + P = ParamSpec('P') + P2 = ParamSpec('P2') + Ts = TypeVarTuple('Ts') + + class Base(Generic[P]): + pass - # Test for consistency with builtin generics. - self.assertNotIn(P, list[P].__parameters__) - self.assertIn(T, tuple[T, P].__parameters__) + A1 = Base[T] + self.assertEqual(A1.__parameters__, (T,)) + self.assertEqual(A1.__args__, ((T,),)) + self.assertEqual(A1[int], Base[int]) + + A2 = Base[[T]] + self.assertEqual(A2.__parameters__, (T,)) + self.assertEqual(A2.__args__, ((T,),)) + self.assertEqual(A2[int], Base[int]) + + A3 = Base[[int, T]] + self.assertEqual(A3.__parameters__, (T,)) + self.assertEqual(A3.__args__, ((int, T),)) + self.assertEqual(A3[str], Base[[int, str]]) + + A4 = Base[[T, int, T2]] + self.assertEqual(A4.__parameters__, (T, T2)) + self.assertEqual(A4.__args__, ((T, int, T2),)) + self.assertEqual(A4[str, bool], Base[[str, int, bool]]) + + A5 = Base[[*Ts, int]] + self.assertEqual(A5.__parameters__, (Ts,)) + self.assertEqual(A5.__args__, ((*Ts, int),)) + self.assertEqual(A5[str, bool], Base[[str, bool, int]]) + + A5_2 = Base[[int, *Ts]] + self.assertEqual(A5_2.__parameters__, (Ts,)) + self.assertEqual(A5_2.__args__, ((int, *Ts),)) + self.assertEqual(A5_2[str, bool], Base[[int, str, bool]]) + + A6 = Base[[T, *Ts]] + self.assertEqual(A6.__parameters__, (T, Ts)) + self.assertEqual(A6.__args__, ((T, *Ts),)) + self.assertEqual(A6[int, str, bool], Base[[int, str, bool]]) + + A7 = Base[[T, T]] + self.assertEqual(A7.__parameters__, (T,)) + self.assertEqual(A7.__args__, ((T, T),)) + self.assertEqual(A7[int], Base[[int, int]]) + + A8 = Base[[T, list[T]]] + self.assertEqual(A8.__parameters__, (T,)) + self.assertEqual(A8.__args__, ((T, list[T]),)) + self.assertEqual(A8[int], Base[[int, list[int]]]) + + # A9 = Base[[Tuple[*Ts], *Ts]] + # self.assertEqual(A9.__parameters__, (Ts,)) + # self.assertEqual(A9.__args__, ((Tuple[*Ts], *Ts),)) + # self.assertEqual(A9[int, str], Base[Tuple[int, str], int, str]) + + A10 = Base[P2] + self.assertEqual(A10.__parameters__, (P2,)) + self.assertEqual(A10.__args__, (P2,)) + self.assertEqual(A10[[int, str]], Base[[int, str]]) + + class DoubleP(Generic[P, P2]): + pass + + B1 = DoubleP[P, P2] + self.assertEqual(B1.__parameters__, (P, P2)) + self.assertEqual(B1.__args__, (P, P2)) + self.assertEqual(B1[[int, str], [bool]], DoubleP[[int, str], [bool]]) + self.assertEqual(B1[[], []], DoubleP[[], []]) + + B2 = DoubleP[[int, str], P2] + self.assertEqual(B2.__parameters__, (P2,)) + self.assertEqual(B2.__args__, ((int, str), P2)) + self.assertEqual(B2[[bool, bool]], DoubleP[[int, str], [bool, bool]]) + self.assertEqual(B2[[]], DoubleP[[int, str], []]) + + B3 = DoubleP[P, [bool, bool]] + self.assertEqual(B3.__parameters__, (P,)) + self.assertEqual(B3.__args__, (P, (bool, bool))) + self.assertEqual(B3[[int, str]], DoubleP[[int, str], [bool, bool]]) + self.assertEqual(B3[[]], DoubleP[[], [bool, bool]]) + + B4 = DoubleP[[T, int], [bool, T2]] + self.assertEqual(B4.__parameters__, (T, T2)) + self.assertEqual(B4.__args__, ((T, int), (bool, T2))) + self.assertEqual(B4[str, float], DoubleP[[str, int], [bool, float]]) + + B5 = DoubleP[[*Ts, int], [bool, T2]] + self.assertEqual(B5.__parameters__, (Ts, T2)) + self.assertEqual(B5.__args__, ((*Ts, int), (bool, T2))) + self.assertEqual(B5[str, bytes, float], + DoubleP[[str, bytes, int], [bool, float]]) + + B6 = DoubleP[[T, int], [bool, *Ts]] + self.assertEqual(B6.__parameters__, (T, Ts)) + self.assertEqual(B6.__args__, ((T, int), (bool, *Ts))) + self.assertEqual(B6[str, bytes, float], + DoubleP[[str, int], [bool, bytes, float]]) + + class PandT(Generic[P, T]): + pass + + C1 = PandT[P, T] + self.assertEqual(C1.__parameters__, (P, T)) + self.assertEqual(C1.__args__, (P, T)) + self.assertEqual(C1[[int, str], bool], PandT[[int, str], bool]) - self.assertNotIn(P, (list[P] | int).__parameters__) - self.assertIn(T, (tuple[T, P] | int).__parameters__) + C2 = PandT[[int, T], T] + self.assertEqual(C2.__parameters__, (T,)) + self.assertEqual(C2.__args__, ((int, T), T)) + self.assertEqual(C2[str], PandT[[int, str], str]) + C3 = PandT[[int, *Ts], T] + self.assertEqual(C3.__parameters__, (Ts, T)) + self.assertEqual(C3.__args__, ((int, *Ts), T)) + self.assertEqual(C3[str, bool, bytes], PandT[[int, str, bool], bytes]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_paramspec_in_nested_generics(self): # Although ParamSpec should not be found in __parameters__ of most # generics, they probably should be found when nested in @@ -5092,6 +9223,8 @@ def test_paramspec_in_nested_generics(self): self.assertEqual(G2[[int, str], float], list[C]) self.assertEqual(G3[[int, str], float], list[C] | int) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_paramspec_gets_copied(self): # bpo-46581 P = ParamSpec('P') @@ -5113,6 +9246,27 @@ def test_paramspec_gets_copied(self): self.assertEqual(C2[Concatenate[str, P2]].__parameters__, (P2,)) self.assertEqual(C2[Concatenate[T, P2]].__parameters__, (T, P2)) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpec'): + class C(ParamSpec): pass + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpecArgs'): + class D(ParamSpecArgs): pass + with self.assertRaisesRegex(TypeError, NOT_A_BASE_TYPE % 'ParamSpecKwargs'): + class E(ParamSpecKwargs): pass + P = ParamSpec('P') + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'ParamSpec'): + class F(P): pass + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'ParamSpecArgs'): + class G(P.args): pass + with self.assertRaisesRegex(TypeError, + CANNOT_SUBCLASS_INSTANCE % 'ParamSpecKwargs'): + class H(P.kwargs): pass + + class ConcatenateTests(BaseTestCase): def test_basics(self): @@ -5121,6 +9275,17 @@ class MyClass: ... c = Concatenate[MyClass, P] self.assertNotEqual(c, Concatenate) + def test_dir(self): + P = ParamSpec('P') + dir_items = set(dir(Concatenate[int, P])) + for required_item in [ + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_valid_uses(self): P = ParamSpec('P') T = TypeVar('T') @@ -5139,6 +9304,20 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (Concatenate[int, T, P], T)) self.assertEqual(C4.__parameters__, (T, P)) + def test_invalid_uses(self): + with self.assertRaisesRegex(TypeError, 'Concatenate of no types'): + Concatenate[()] + with self.assertRaisesRegex( + TypeError, + ( + 'The last parameter to Concatenate should be a ' + 'ParamSpec variable or ellipsis' + ), + ): + Concatenate[int] + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') @@ -5149,8 +9328,7 @@ def test_var_substitution(self): self.assertEqual(C[int, []], (int,)) self.assertEqual(C[int, Concatenate[str, P2]], Concatenate[int, str, P2]) - with self.assertRaises(TypeError): - C[int, ...] + self.assertEqual(C[int, ...], Concatenate[int, ...]) C = Concatenate[int, P] self.assertEqual(C[P2], Concatenate[int, P2]) @@ -5158,8 +9336,8 @@ def test_var_substitution(self): self.assertEqual(C[str, float], (int, str, float)) self.assertEqual(C[[]], (int,)) self.assertEqual(C[Concatenate[str, P2]], Concatenate[int, str, P2]) - with self.assertRaises(TypeError): - C[...] + self.assertEqual(C[...], Concatenate[int, ...]) + class TypeGuardTests(BaseTestCase): def test_basics(self): @@ -5168,6 +9346,11 @@ def test_basics(self): def foo(arg) -> TypeGuard[int]: ... self.assertEqual(gth(foo), {'return': TypeGuard[int]}) + with self.assertRaises(TypeError): + TypeGuard[int, str] + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_repr(self): self.assertEqual(repr(TypeGuard), 'typing.TypeGuard') cv = TypeGuard[int] @@ -5178,11 +9361,19 @@ def test_repr(self): self.assertEqual(repr(cv), 'typing.TypeGuard[tuple[int]]') def test_cannot_subclass(self): - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): class C(type(TypeGuard)): pass - with self.assertRaises(TypeError): - class C(type(TypeGuard[int])): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeGuard[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeGuard'): + class E(TypeGuard): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeGuard\[int\]'): + class F(TypeGuard[int]): pass def test_cannot_init(self): @@ -5200,12 +9391,63 @@ def test_no_isinstance(self): issubclass(int, TypeGuard) +class TypeIsTests(BaseTestCase): + def test_basics(self): + TypeIs[int] # OK + + def foo(arg) -> TypeIs[int]: ... + self.assertEqual(gth(foo), {'return': TypeIs[int]}) + + with self.assertRaises(TypeError): + TypeIs[int, str] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_repr(self): + self.assertEqual(repr(TypeIs), 'typing.TypeIs') + cv = TypeIs[int] + self.assertEqual(repr(cv), 'typing.TypeIs[int]') + cv = TypeIs[Employee] + self.assertEqual(repr(cv), 'typing.TypeIs[%s.Employee]' % __name__) + cv = TypeIs[tuple[int]] + self.assertEqual(repr(cv), 'typing.TypeIs[tuple[int]]') + + def test_cannot_subclass(self): + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class C(type(TypeIs)): + pass + with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): + class D(type(TypeIs[int])): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs'): + class E(TypeIs): + pass + with self.assertRaisesRegex(TypeError, + r'Cannot subclass typing\.TypeIs\[int\]'): + class F(TypeIs[int]): + pass + + def test_cannot_init(self): + with self.assertRaises(TypeError): + TypeIs() + with self.assertRaises(TypeError): + type(TypeIs)() + with self.assertRaises(TypeError): + type(TypeIs[Optional[int]])() + + def test_no_isinstance(self): + with self.assertRaises(TypeError): + isinstance(1, TypeIs[int]) + with self.assertRaises(TypeError): + issubclass(int, TypeIs) + + SpecialAttrsP = typing.ParamSpec('SpecialAttrsP') SpecialAttrsT = typing.TypeVar('SpecialAttrsT', int, float, complex) class SpecialAttrsTests(BaseTestCase): - # TODO: RUSTPYTHON @unittest.expectedFailure def test_special_attrs(self): @@ -5251,7 +9493,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -5261,7 +9503,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', @@ -5297,13 +9539,17 @@ def test_special_attrs(self): typing.Literal: 'Literal', typing.NewType: 'NewType', typing.NoReturn: 'NoReturn', + typing.Never: 'Never', typing.Optional: 'Optional', typing.TypeAlias: 'TypeAlias', typing.TypeGuard: 'TypeGuard', + typing.TypeIs: 'TypeIs', typing.TypeVar: 'TypeVar', typing.Union: 'Union', + typing.Self: 'Self', # Subscribed special forms typing.Annotated[Any, "Annotation"]: 'Annotated', + typing.Annotated[int, 'Annotation']: 'Annotated', typing.ClassVar[Any]: 'ClassVar', typing.Concatenate[Any, SpecialAttrsP]: 'Concatenate', typing.Final[Any]: 'Final', @@ -5312,6 +9558,7 @@ def test_special_attrs(self): typing.Literal[True, 2]: 'Literal', typing.Optional[Any]: 'Optional', typing.TypeGuard[Any]: 'TypeGuard', + typing.TypeIs[Any]: 'TypeIs', typing.Union[Any]: 'Any', typing.Union[int, float]: 'Union', # Incompatible special forms (tested in test_special_attrs2) @@ -5345,7 +9592,7 @@ def test_special_attrs2(self): self.assertEqual(fr.__module__, 'typing') # Forward refs are currently unpicklable. for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(TypeError) as exc: + with self.assertRaises(TypeError): pickle.dumps(fr, proto) self.assertEqual(SpecialAttrsTests.TypeName.__name__, 'TypeName') @@ -5385,15 +9632,168 @@ def test_special_attrs2(self): loaded = pickle.loads(s) self.assertIs(SpecialAttrsP, loaded) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_genericalias_dir(self): class Foo(Generic[T]): def bar(self): pass baz = 3 + __magic__ = 4 + # The class attributes of the original class should be visible even # in dir() of the GenericAlias. See bpo-45755. - self.assertIn('bar', dir(Foo[int])) - self.assertIn('baz', dir(Foo[int])) + dir_items = set(dir(Foo[int])) + for required_item in [ + 'bar', 'baz', + '__args__', '__parameters__', '__origin__', + ]: + with self.subTest(required_item=required_item): + self.assertIn(required_item, dir_items) + self.assertNotIn('__magic__', dir_items) + + +class RevealTypeTests(BaseTestCase): + def test_reveal_type(self): + obj = object() + with captured_stderr() as stderr: + self.assertIs(obj, reveal_type(obj)) + self.assertEqual(stderr.getvalue(), "Runtime type is 'object'\n") + + +class DataclassTransformTests(BaseTestCase): + def test_decorator(self): + def create_model(*, frozen: bool = False, kw_only: bool = True): + return lambda cls: cls + + decorated = dataclass_transform(kw_only_default=True, order_default=False)(create_model) + + class CustomerModel: + id: int + + self.assertIs(decorated, create_model) + self.assertEqual( + decorated.__dataclass_transform__, + { + "eq_default": True, + "order_default": False, + "kw_only_default": True, + "frozen_default": False, + "field_specifiers": (), + "kwargs": {}, + } + ) + self.assertIs( + decorated(frozen=True, kw_only=False)(CustomerModel), + CustomerModel + ) + + def test_base_class(self): + class ModelBase: + def __init_subclass__(cls, *, frozen: bool = False): ... + + Decorated = dataclass_transform( + eq_default=True, + order_default=True, + # Arbitrary unrecognized kwargs are accepted at runtime. + make_everything_awesome=True, + )(ModelBase) + + class CustomerModel(Decorated, frozen=True): + id: int + + self.assertIs(Decorated, ModelBase) + self.assertEqual( + Decorated.__dataclass_transform__, + { + "eq_default": True, + "order_default": True, + "kw_only_default": False, + "frozen_default": False, + "field_specifiers": (), + "kwargs": {"make_everything_awesome": True}, + } + ) + self.assertIsSubclass(CustomerModel, Decorated) + + def test_metaclass(self): + class Field: ... + + class ModelMeta(type): + def __new__( + cls, name, bases, namespace, *, init: bool = True, + ): + return super().__new__(cls, name, bases, namespace) + + Decorated = dataclass_transform( + order_default=True, frozen_default=True, field_specifiers=(Field,) + )(ModelMeta) + + class ModelBase(metaclass=Decorated): ... + + class CustomerModel(ModelBase, init=False): + id: int + + self.assertIs(Decorated, ModelMeta) + self.assertEqual( + Decorated.__dataclass_transform__, + { + "eq_default": True, + "order_default": True, + "kw_only_default": False, + "frozen_default": True, + "field_specifiers": (Field,), + "kwargs": {}, + } + ) + self.assertIsInstance(CustomerModel, Decorated) + + +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + type(NoDefault)(1) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + @requires_docstrings + def test_doc(self): + self.assertIsInstance(NoDefault.__doc__, str) + + def test_class(self): + self.assertIs(NoDefault.__class__, type(NoDefault)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_attributes(self): + with self.assertRaises(AttributeError): + NoDefault.foo = 3 + with self.assertRaises(AttributeError): + NoDefault.foo + + # TypeError is consistent with the behavior of NoneType + with self.assertRaises(TypeError): + type(NoDefault).foo = 3 + with self.assertRaises(AttributeError): + type(NoDefault).foo class AllTests(BaseTestCase): @@ -5409,7 +9809,7 @@ def test_all(self): # Context managers. self.assertIn('ContextManager', a) self.assertIn('AsyncContextManager', a) - # Check that io and re are not exported. + # Check that former namespaces io and re are not exported. self.assertNotIn('io', a) self.assertNotIn('re', a) # Spot-check that stdlib modules aren't exported. @@ -5421,7 +9821,13 @@ def test_all(self): self.assertIn('SupportsBytes', a) self.assertIn('SupportsComplex', a) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_all_exported_names(self): + # ensure all dynamically created objects are actualised + for name in typing.__all__: + getattr(typing, name) + actual_all = set(typing.__all__) computed_all = { k for k, v in vars(typing).items() @@ -5429,10 +9835,6 @@ def test_all_exported_names(self): if k in actual_all or ( # avoid private names not k.startswith('_') and - # avoid things in the io / re typing submodules - k not in typing.io.__all__ and - k not in typing.re.__all__ and - k not in {'io', 're'} and # there's a few types and metaclasses that aren't exported not k.endswith(('Meta', '_contra', '_co')) and not k.upper() == k and @@ -5443,6 +9845,45 @@ def test_all_exported_names(self): self.assertSetEqual(computed_all, actual_all) +class TypeIterationTests(BaseTestCase): + _UNITERABLE_TYPES = ( + Any, + Union, + Union[str, int], + Union[str, T], + List, + Tuple, + Callable, + Callable[..., T], + Callable[[T], str], + Annotated, + Annotated[T, ''], + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cannot_iterate(self): + expected_error_regex = "object is not iterable" + for test_type in self._UNITERABLE_TYPES: + with self.subTest(type=test_type): + with self.assertRaisesRegex(TypeError, expected_error_regex): + iter(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + list(test_type) + with self.assertRaisesRegex(TypeError, expected_error_regex): + for _ in test_type: + pass + + def test_is_not_instance_of_iterable(self): + for type_to_test in self._UNITERABLE_TYPES: + self.assertNotIsInstance(type_to_test, collections.abc.Iterable) + + +def load_tests(loader, tests, pattern): + import doctest + tests.addTests(doctest.DocTestSuite(typing)) + return tests + if __name__ == '__main__': main() diff --git a/Lib/test/typinganndata/_typed_dict_helper.py b/Lib/test/typinganndata/_typed_dict_helper.py new file mode 100644 index 0000000000..9df0ede7d4 --- /dev/null +++ b/Lib/test/typinganndata/_typed_dict_helper.py @@ -0,0 +1,30 @@ +"""Used to test `get_type_hints()` on a cross-module inherited `TypedDict` class + +This script uses future annotations to postpone a type that won't be available +on the module inheriting from to `Foo`. The subclass in the other module should +look something like this: + + class Bar(_typed_dict_helper.Foo, total=False): + b: int + +In addition, it uses multiple levels of Annotated to test the interaction +between the __future__ import, Annotated, and Required. +""" + +from __future__ import annotations + +from typing import Annotated, Generic, Optional, Required, TypedDict, TypeVar + + +OptionalIntType = Optional[int] + +class Foo(TypedDict): + a: OptionalIntType + +T = TypeVar("T") + +class FooGeneric(TypedDict, Generic[T]): + a: Optional[T] + +class VeryAnnotated(TypedDict, total=False): + a: Annotated[Annotated[Annotated[Required[int], "a"], "b"], "c"] diff --git a/Lib/test/typinganndata/ann_module695.py b/Lib/test/typinganndata/ann_module695.py new file mode 100644 index 0000000000..b6f3b06bd5 --- /dev/null +++ b/Lib/test/typinganndata/ann_module695.py @@ -0,0 +1,72 @@ +from __future__ import annotations +from typing import Callable + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: *Ts, z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +def nested(): + from types import SimpleNamespace + from typing import get_type_hints + + Eggs = bytes + Spam = memoryview + + + class E[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + return SimpleNamespace( + E=E, + hints_for_E=get_type_hints(E), + hints_for_E_meth=get_type_hints(E.generic_method), + generic_func=generic_function, + hints_for_generic_func=get_type_hints(generic_function) + ) diff --git a/Lib/test/typinganndata/mod_generics_cache.py b/Lib/test/typinganndata/mod_generics_cache.py new file mode 100644 index 0000000000..62deea9859 --- /dev/null +++ b/Lib/test/typinganndata/mod_generics_cache.py @@ -0,0 +1,26 @@ +"""Module for testing the behavior of generics across different modules.""" + +from typing import TypeVar, Generic, Optional, TypeAliasType + +# TODO: RUSTPYTHON + +# default_a: Optional['A'] = None +# default_b: Optional['B'] = None + +# T = TypeVar('T') + + +# class A(Generic[T]): +# some_b: 'B' + + +# class B(Generic[T]): +# class A(Generic[T]): +# pass + +# my_inner_a1: 'B.A' +# my_inner_a2: A +# my_outer_a: 'A' # unless somebody calls get_type_hints with localns=B.__dict__ + +# type Alias = int +# OldStyle = TypeAliasType("OldStyle", int) diff --git a/Lib/typing.py b/Lib/typing.py index 75ec2a6a2e..b64a6b6714 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,34 +1,46 @@ """ -The typing module: Support for gradual typing as defined by PEP 484. - -At large scale, the structure of the module is following: -* Imports and exports, all public names should be explicitly added to __all__. -* Internal helper functions: these should never be used in code outside this module. -* _SpecialForm and its instances (special forms): - Any, NoReturn, ClassVar, Union, Optional, Concatenate -* Classes whose instances can be type arguments in addition to types: - ForwardRef, TypeVar and ParamSpec -* The core of internal generics API: _GenericAlias and _VariadicGenericAlias, the latter is - currently only used by Tuple and Callable. All subscripted types like X[int], Union[int, str], - etc., are instances of either of these classes. -* The public counterpart of the generics API consists of two classes: Generic and Protocol. -* Public helper functions: get_type_hints, overload, cast, no_type_check, - no_type_check_decorator. -* Generic aliases for collections.abc ABCs and few additional protocols. +The typing module: Support for gradual typing as defined by PEP 484 and subsequent PEPs. + +Among other things, the module includes the following: +* Generic, Protocol, and internal machinery to support generic aliases. + All subscripted types like X[int], Union[int, str] are generic aliases. +* Various "special forms" that have unique meanings in type annotations: + NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. +* Classes whose instances can be type arguments to generic classes and functions: + TypeVar, ParamSpec, TypeVarTuple. +* Public helper functions: get_type_hints, overload, cast, final, and others. +* Several protocols to support duck-typing: + SupportsFloat, SupportsIndex, SupportsAbs, and others. * Special types: NewType, NamedTuple, TypedDict. -* Wrapper submodules for re and io related types. +* Deprecated aliases for builtin types and collections.abc ABCs. + +Any name not present in __all__ is an implementation detail +that may be changed without notice. Use at your own risk! """ from abc import abstractmethod, ABCMeta import collections +from collections import defaultdict import collections.abc +import copyreg import functools import operator -import re as stdlib_re # Avoid confusion with the re we export. import sys import types from types import WrapperDescriptorType, MethodWrapperType, MethodDescriptorType, GenericAlias +from _typing import ( + _idfunc, + TypeVar, + ParamSpec, + TypeVarTuple, + ParamSpecArgs, + ParamSpecKwargs, + TypeAliasType, + Generic, + NoDefault, +) + # Please keep __all__ alphabetized within each category. __all__ = [ # Super-special typing primitives. @@ -47,6 +59,7 @@ 'Tuple', 'Type', 'TypeVar', + 'TypeVarTuple', 'Union', # ABCs (from collections.abc). @@ -108,30 +121,45 @@ # One-off things. 'AnyStr', + 'assert_type', + 'assert_never', 'cast', + 'clear_overloads', + 'dataclass_transform', 'final', 'get_args', 'get_origin', + 'get_overloads', + 'get_protocol_members', 'get_type_hints', + 'is_protocol', 'is_typeddict', + 'LiteralString', + 'Never', 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', + 'NotRequired', 'overload', + 'override', 'ParamSpecArgs', 'ParamSpecKwargs', + 'ReadOnly', + 'Required', + 'reveal_type', 'runtime_checkable', + 'Self', 'Text', 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TypeIs', + 'TypeAliasType', + 'Unpack', ] -# The pseudo-submodules 're' and 'io' are part of the public -# namespace, but excluded from __all__ because they might stomp on -# legitimate imports of those modules. - def _type_convert(arg, module=None, *, allow_special_forms=False): """For converting None to type(None), and strings to ForwardRef.""" @@ -148,7 +176,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= As a special case, accept None and return type(None) instead. Also wrap strings into ForwardRef instances. Consider several corner cases, for example plain special forms like Union are not valid, while Union[int, str] is OK, etc. - The msg argument is a human-readable error message, e.g:: + The msg argument is a human-readable error message, e.g.:: "Union[arg, ...]: arg should be a type." @@ -164,14 +192,13 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") - if arg in (Any, NoReturn, Final, TypeAlias): + if arg in (Any, LiteralString, NoReturn, Never, Self, TypeAlias): + return arg + if allow_special_forms and arg in (ClassVar, Final): return arg if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol): raise TypeError(f"Plain {arg} is not valid as type argument") - if isinstance(arg, (type, TypeVar, ForwardRef, types.UnionType, ParamSpec, - ParamSpecArgs, ParamSpecKwargs)): - return arg - if not callable(arg): + if type(arg) is tuple: raise TypeError(f"{msg} Got {arg!r:.100}.") return arg @@ -181,6 +208,28 @@ def _is_param_expr(arg): (tuple, list, ParamSpec, _ConcatenateGenericAlias)) +def _should_unflatten_callable_args(typ, args): + """Internal helper for munging collections.abc.Callable's __args__. + + The canonical representation for a Callable's __args__ flattens the + argument types, see https://github.com/python/cpython/issues/86361. + + For example:: + + >>> import collections.abc + >>> P = ParamSpec('P') + >>> collections.abc.Callable[[int, int], str].__args__ == (int, int, str) + True + + As a result, if we need to reconstruct the Callable from its __args__, + we need to unflatten it. + """ + return ( + typ.__origin__ is collections.abc.Callable + and not (len(args) == 2 and _is_param_expr(args[0])) + ) + + def _type_repr(obj): """Return the repr() of an object, special-casing types (internal helper). @@ -189,99 +238,158 @@ def _type_repr(obj): typically enough to uniquely identify a type. For everything else, we fall back on repr(obj). """ - if isinstance(obj, types.GenericAlias): - return repr(obj) + # When changing this function, don't forget about + # `_collections_abc._type_repr`, which does the same thing + # and must be consistent with this one. if isinstance(obj, type): if obj.__module__ == 'builtins': return obj.__qualname__ return f'{obj.__module__}.{obj.__qualname__}' if obj is ...: - return('...') + return '...' if isinstance(obj, types.FunctionType): return obj.__name__ + if isinstance(obj, tuple): + # Special case for `repr` of types with `ParamSpec`: + return '[' + ', '.join(_type_repr(t) for t in obj) + ']' return repr(obj) -def _collect_type_vars(types_, typevar_types=None): - """Collect all type variable contained - in types in order of first appearance (lexicographic order). For example:: +def _collect_type_parameters(args, *, enforce_default_ordering: bool = True): + """Collect all type parameters in args + in order of first appearance (lexicographic order). + + For example:: - _collect_type_vars((T, List[S, T])) == (T, S) + >>> P = ParamSpec('P') + >>> T = TypeVar('T') """ - if typevar_types is None: - typevar_types = TypeVar - tvars = [] - for t in types_: - if isinstance(t, typevar_types) and t not in tvars: - tvars.append(t) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - return tuple(tvars) + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False + parameters = [] + for t in args: + if isinstance(t, type): + # We don't want __parameters__ descriptor of a bare Python class. + pass + elif isinstance(t, tuple): + # `t` might be a tuple, when `ParamSpec` is substituted with + # `[T, int]`, or `[int, *Ts]`, etc. + for x in t: + for collected in _collect_type_parameters([x]): + if collected not in parameters: + parameters.append(collected) + elif hasattr(t, '__typing_subst__'): + if t not in parameters: + if enforce_default_ordering: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + + parameters.append(t) + else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True + for x in getattr(t, '__parameters__', ()): + if x not in parameters: + parameters.append(x) + return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). + This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") - -def _prepare_paramspec_params(cls, params): - """Prepares the parameters for a Generic containing ParamSpec - variables (internal helper). - """ - # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. - if (len(cls.__parameters__) == 1 - and params and not _is_param_expr(params[0])): - assert isinstance(cls.__parameters__[0], ParamSpec) - return (params,) - else: - _check_generic(cls, params, len(cls.__parameters__)) - _params = [] - # Convert lists to tuples to help other libraries cache the results. - for p, tvar in zip(params, cls.__parameters__): - if isinstance(tvar, ParamSpec) and isinstance(p, list): - p = tuple(p) - _params.append(p) - return tuple(_params) - -def _deduplicate(params): - # Weed out strict duplicates, preserving the first of each occurrence. - all_params = set(params) - if len(all_params) < len(params): - new_params = [] - for t in params: - if t in all_params: - new_params.append(t) - all_params.remove(t) - params = new_params - assert not all_params, all_params - return params + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_type_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") +def _unpack_args(*args): + newargs = [] + for arg in args: + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs is not None and not (subargs and subargs[-1] is ...): + newargs.extend(subargs) + else: + newargs.append(arg) + return newargs + +def _deduplicate(params, *, unhashable_fallback=False): + # Weed out strict duplicates, preserving the first of each occurrence. + try: + return dict.fromkeys(params) + except TypeError: + if not unhashable_fallback: + raise + # Happens for cases like `Annotated[dict, {'x': IntValidator()}]` + return _deduplicate_unhashable(params) + +def _deduplicate_unhashable(unhashable_params): + new_unhashable = [] + for t in unhashable_params: + if t not in new_unhashable: + new_unhashable.append(t) + return new_unhashable + +def _compare_args_orderless(first_args, second_args): + first_unhashable = _deduplicate_unhashable(first_args) + second_unhashable = _deduplicate_unhashable(second_args) + t = list(second_unhashable) + try: + for elem in first_unhashable: + t.remove(elem) + except ValueError: + return False + return not t + def _remove_dups_flatten(parameters): - """An internal helper for Union creation and substitution: flatten Unions - among parameters, then remove duplicates. + """Internal helper for Union creation and substitution. + + Flatten Unions among parameters, then remove duplicates. """ # Flatten out Union[Union[...], ...]. params = [] for p in parameters: if isinstance(p, (_UnionGenericAlias, types.UnionType)): params.extend(p.__args__) - elif isinstance(p, tuple) and len(p) > 0 and p[0] is Union: - params.extend(p[1:]) else: params.append(p) - return tuple(_deduplicate(params)) + return tuple(_deduplicate(params, unhashable_fallback=True)) def _flatten_literal_params(parameters): - """An internal helper for Literal creation: flatten Literals among parameters""" + """Internal helper for Literal creation: flatten Literals among parameters.""" params = [] for p in parameters: if isinstance(p, _LiteralGenericAlias): @@ -292,20 +400,29 @@ def _flatten_literal_params(parameters): _cleanups = [] +_caches = {} def _tp_cache(func=None, /, *, typed=False): - """Internal wrapper caching __getitem__ of generic types with a fallback to - original function for non-hashable arguments. + """Internal wrapper caching __getitem__ of generic types. + + For non-hashable arguments, the original function is used as a fallback. """ def decorator(func): - cached = functools.lru_cache(typed=typed)(func) - _cleanups.append(cached.cache_clear) + # The callback 'inner' references the newly created lru_cache + # indirectly by performing a lookup in the global '_caches' dictionary. + # This breaks a reference that can be problematic when combined with + # C API extensions that leak references to types. See GH-98253. + + cache = functools.lru_cache(typed=typed)(func) + _caches[func] = cache + _cleanups.append(cache.cache_clear) + del cache @functools.wraps(func) def inner(*args, **kwds): try: - return cached(*args, **kwds) + return _caches[func](*args, **kwds) except TypeError: pass # All real errors (not unhashable args) are raised below. return func(*args, **kwds) @@ -316,16 +433,61 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, recursive_guard=frozenset()): + +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset()): """Evaluate all forward references in the given type t. + For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () if isinstance(t, ForwardRef): - return t._evaluate(globalns, localns, recursive_guard) + return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): - ev_args = tuple(_eval_type(a, globalns, localns, recursive_guard) for a in t.__args__) + if isinstance(t, GenericAlias): + args = tuple( + ForwardRef(arg) if isinstance(arg, str) else arg + for arg in t.__args__ + ) + is_unpacked = t.__unpacked__ + if _should_unflatten_callable_args(t, args): + t = t.__origin__[(args[:-1], args[-1])] + else: + t = t.__origin__[args] + if is_unpacked: + t = Unpack[t] + + ev_args = tuple( + _eval_type( + a, globalns, localns, type_params, recursive_guard=recursive_guard + ) + for a in t.__args__ + ) if ev_args == t.__args__: return t if isinstance(t, GenericAlias): @@ -338,28 +500,36 @@ def _eval_type(t, globalns, localns, recursive_guard=frozenset()): class _Final: - """Mixin to prohibit subclassing""" + """Mixin to prohibit subclassing.""" __slots__ = ('__weakref__',) - def __init_subclass__(self, /, *args, **kwds): + def __init_subclass__(cls, /, *args, **kwds): if '_root' not in kwds: raise TypeError("Cannot subclass special typing classes") -class _Immutable: - """Mixin to indicate that object should not be copied.""" - __slots__ = () - def __copy__(self): - return self +class _NotIterable: + """Mixin to prevent iteration, without being compatible with Iterable. + + That is, we could do:: + + def __iter__(self): raise TypeError() + + But this would make users of this mixin duck type-compatible with + collections.abc.Iterable - isinstance(foo, Iterable) would be True. - def __deepcopy__(self, memo): - return self + Luckily, we can instead prevent iteration by setting __iter__ to None, which + is treated specially. + """ + + __slots__ = () + __iter__ = None # Internal indicator of special typing constructs. # See __doc__ instance attribute for specific docs. -class _SpecialForm(_Final, _root=True): +class _SpecialForm(_Final, _NotIterable, _root=True): __slots__ = ('_name', '__doc__', '_getitem') def __init__(self, getitem): @@ -402,15 +572,26 @@ def __getitem__(self, parameters): return self._getitem(self, parameters) -class _LiteralSpecialForm(_SpecialForm, _root=True): +class _TypedCacheSpecialForm(_SpecialForm, _root=True): def __getitem__(self, parameters): if not isinstance(parameters, tuple): parameters = (parameters,) return self._getitem(self, *parameters) -@_SpecialForm -def Any(self, parameters): +class _AnyMeta(type): + def __instancecheck__(self, obj): + if self is Any: + raise TypeError("typing.Any cannot be used with isinstance()") + return super().__instancecheck__(obj) + + def __repr__(self): + if self is Any: + return "typing.Any" + return super().__repr__() # respect to subclasses + + +class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - Any is compatible with every type. @@ -419,43 +600,128 @@ def Any(self, parameters): Note that all the above statements are true from the point of view of static type checkers. At runtime, Any should not be used with instance - or class checks. + checks. """ - raise TypeError(f"{self} is not subscriptable") + + def __new__(cls, *args, **kwargs): + if cls is Any: + raise TypeError("Any cannot be instantiated") + return super().__new__(cls) + @_SpecialForm def NoReturn(self, parameters): """Special type indicating functions that never return. + + Example:: + + from typing import NoReturn + + def stop() -> NoReturn: + raise Exception('no way') + + NoReturn can also be used as a bottom type, a type that + has no values. Starting in Python 3.11, the Never type should + be used for this concept instead. Type checkers should treat the two + equivalently. + """ + raise TypeError(f"{self} is not subscriptable") + +# This is semantically identical to NoReturn, but it is implemented +# separately so that type checkers can distinguish between the two +# if they want. +@_SpecialForm +def Never(self, parameters): + """The bottom type, a type that has no members. + + This can be used to define a function that should never be + called, or a function that never returns:: + + from typing import Never + + def never_call_me(arg: Never) -> None: + pass + + def int_or_str(arg: int | str) -> None: + never_call_me(arg) # type checker error + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _: + never_call_me(arg) # OK, arg is of type Never + """ + raise TypeError(f"{self} is not subscriptable") + + +@_SpecialForm +def Self(self, parameters): + """Used to spell the type of "self" in classes. + + Example:: + + from typing import Self + + class Foo: + def return_self(self) -> Self: + ... + return self + + This is especially useful for: + - classmethods that are used as alternative constructors + - annotating an `__enter__` method which returns self + """ + raise TypeError(f"{self} is not subscriptable") + + +@_SpecialForm +def LiteralString(self, parameters): + """Represents an arbitrary literal string. + Example:: - from typing import NoReturn + from typing import LiteralString + + def run_query(sql: LiteralString) -> None: + ... - def stop() -> NoReturn: - raise Exception('no way') + def caller(arbitrary_string: str, literal_string: LiteralString) -> None: + run_query("SELECT * FROM students") # OK + run_query(literal_string) # OK + run_query("SELECT * FROM " + literal_string) # OK + run_query(arbitrary_string) # type checker error + run_query( # type checker error + f"SELECT * FROM students WHERE name = {arbitrary_string}" + ) - This type is invalid in other positions, e.g., ``List[NoReturn]`` - will fail in static type checkers. + Only string literals and other LiteralStrings are compatible + with LiteralString. This provides a tool to help prevent + security issues such as SQL injection. """ raise TypeError(f"{self} is not subscriptable") + @_SpecialForm def ClassVar(self, parameters): """Special type construct to mark class variables. An annotation wrapped in ClassVar indicates that a given attribute is intended to be used as a class variable and - should not be set on instances of that class. Usage:: + should not be set on instances of that class. + + Usage:: - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable + class Starship: + stats: ClassVar[dict[str, int]] = {} # class variable + damage: int = 10 # instance variable ClassVar accepts only types and cannot be further subscribed. Note that ClassVar is not a class itself, and should not be used with isinstance() or issubclass(). """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm @@ -463,45 +729,50 @@ def Final(self, parameters): """Special typing construct to indicate final names to type checkers. A final name cannot be re-assigned or overridden in a subclass. - For example: - MAX_SIZE: Final = 9000 - MAX_SIZE += 1 # Error reported by type checker + For example:: + + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker - class Connection: - TIMEOUT: Final[int] = 10 + class Connection: + TIMEOUT: Final[int] = 10 - class FastConnector(Connection): - TIMEOUT = 1 # Error reported by type checker + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker There is no runtime checking of these properties. """ - item = _type_check(parameters, f'{self} accepts only single type.') + item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) return _GenericAlias(self, (item,)) @_SpecialForm def Union(self, parameters): """Union type; Union[X, Y] means either X or Y. - To define a union, use e.g. Union[int, str]. Details: + On Python 3.10 and higher, the | operator + can also be used to denote unions; + X | Y means the same thing to the type checker as Union[X, Y]. + + To define a union, use e.g. Union[int, str]. Details: - The arguments must be types and there must be at least one. - None as an argument is a special case and is replaced by type(None). - Unions of unions are flattened, e.g.:: - Union[Union[int, str], float] == Union[int, str, float] + assert Union[Union[int, str], float] == Union[int, str, float] - Unions of a single argument vanish, e.g.:: - Union[int] == int # The constructor actually returns int + assert Union[int] == int # The constructor actually returns int - Redundant arguments are skipped, e.g.:: - Union[int, str, int] == Union[int, str] + assert Union[int, str, int] == Union[int, str] - When comparing unions, the argument order is ignored, e.g.:: - Union[int, str] == Union[str, int] + assert Union[int, str] == Union[str, int] - You cannot subclass or instantiate a union. - You can use Optional[X] as a shorthand for Union[X, None]. @@ -519,33 +790,39 @@ def Union(self, parameters): return _UnionGenericAlias(self, parameters, name="Optional") return _UnionGenericAlias(self, parameters) -@_SpecialForm -def Optional(self, parameters): - """Optional type. +def _make_union(left, right): + """Used from the C implementation of TypeVar. - Optional[X] is equivalent to Union[X, None]. + TypeVar.__or__ calls this instead of returning types.UnionType + because we want to allow unions between TypeVars and strings + (forward references). """ + return Union[left, right] + +@_SpecialForm +def Optional(self, parameters): + """Optional[X] is equivalent to Union[X, None].""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] -@_LiteralSpecialForm +@_TypedCacheSpecialForm @_tp_cache(typed=True) def Literal(self, *parameters): """Special typing form to define literal types (a.k.a. value types). This form can be used to indicate to type checkers that the corresponding variable or function parameter has a value equivalent to the provided - literal (or one of several literals): + literal (or one of several literals):: - def validate_simple(data: Any) -> Literal[True]: # always returns True - ... + def validate_simple(data: Any) -> Literal[True]: # always returns True + ... - MODE = Literal['r', 'rb', 'w', 'wb'] - def open_helper(file: str, mode: MODE) -> str: - ... + MODE = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: MODE) -> str: + ... - open_helper('/some/path', 'r') # Passes type check - open_helper('/other/path', 'typo') # Error in type checker + open_helper('/some/path', 'r') # Passes type check + open_helper('/other/path', 'typo') # Error in type checker Literal[...] cannot be subclassed. At runtime, an arbitrary value is allowed as type argument to Literal[...], but type checkers may @@ -565,7 +842,9 @@ def open_helper(file: str, mode: MODE) -> str: @_SpecialForm def TypeAlias(self, parameters): - """Special marker indicating that an assignment should + """Special form for marking type aliases. + + Use TypeAlias to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -580,13 +859,15 @@ def TypeAlias(self, parameters): @_SpecialForm def Concatenate(self, parameters): - """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a - higher order function which adds, removes or transforms parameters of a - callable. + """Special form for annotating higher-order functions. + + ``Concatenate`` can be used in conjunction with ``ParamSpec`` and + ``Callable`` to represent a higher-order function which adds, removes or + transforms the parameters of a callable. For example:: - Callable[Concatenate[int, P], int] + Callable[Concatenate[int, P], int] See PEP 612 for detailed information. """ @@ -594,56 +875,62 @@ def Concatenate(self, parameters): raise TypeError("Cannot take a Concatenate of no types.") if not isinstance(parameters, tuple): parameters = (parameters,) - if not isinstance(parameters[-1], ParamSpec): + if not (parameters[-1] is ... or isinstance(parameters[-1], ParamSpec)): raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable.") + "ParamSpec variable or ellipsis.") msg = "Concatenate[arg, ...]: each arg must be a type." parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) - return _ConcatenateGenericAlias(self, parameters, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) + return _ConcatenateGenericAlias(self, parameters) @_SpecialForm def TypeGuard(self, parameters): - """Special typing form used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. + """Special typing construct for marking user-defined type predicate functions. + + ``TypeGuard`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeGuard`` only accepts a single type argument. At runtime, functions marked this way should return a boolean. ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static type checkers to determine a more precise type of an expression within a program's code flow. Usually type narrowing is done by analyzing conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". + conditional expression here is sometimes referred to as a "type predicate". Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. + as a type predicate. Such a function should use ``TypeGuard[...]`` or + ``TypeIs[...]`` as its return type to alert static type checkers to + this intention. ``TypeGuard`` should be used over ``TypeIs`` when narrowing + from an incompatible type (e.g., ``list[object]`` to ``list[int]``) or when + the function does not return ``True`` for all instances of the narrowed type. - Using ``-> TypeGuard`` tells the static type checker that for a given - function: + Using ``-> TypeGuard[NarrowedType]`` tells the static type checker that + for a given function: 1. The return value is a boolean. 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. + is ``NarrowedType``. - For example:: + For example:: + + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... + def func1(val: list[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``list[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``list[object]``. + print("Not a list of strings!") Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower form of ``TypeA`` (it can even be a wider form) and this may lead to type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of - writing type-safe type guards is left to the user. + narrowing ``list[object]`` to ``list[str]`` even though the latter is not + a subtype of the former, since ``list`` is invariant. The responsibility of + writing type-safe type predicates is left to the user. ``TypeGuard`` also works with type variables. For more information, see PEP 647 (User-Defined Type Guards). @@ -652,6 +939,75 @@ def is_str(val: Union[str, float]): return _GenericAlias(self, (item,)) +@_SpecialForm +def TypeIs(self, parameters): + """Special typing construct for marking user-defined type predicate functions. + + ``TypeIs`` can be used to annotate the return type of a user-defined + type predicate function. ``TypeIs`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean and accept + at least one argument. + + ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type predicate". + + Sometimes it would be convenient to use a user-defined boolean function + as a type predicate. Such a function should use ``TypeIs[...]`` or + ``TypeGuard[...]`` as its return type to alert static type checkers to + this intention. ``TypeIs`` usually has more intuitive behavior than + ``TypeGuard``, but it cannot be used when the input and output types + are incompatible (e.g., ``list[object]`` to ``list[int]``) or when the + function does not return ``True`` for all instances of the narrowed type. + + Using ``-> TypeIs[NarrowedType]`` tells the static type checker that for + a given function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the intersection of the argument's original type and + ``NarrowedType``. + 3. If the return value is ``False``, the type of its argument + is narrowed to exclude ``NarrowedType``. + + For example:: + + from typing import assert_type, final, TypeIs + + class Parent: pass + class Child(Parent): pass + @final + class Unrelated: pass + + def is_parent(val: object) -> TypeIs[Parent]: + return isinstance(val, Parent) + + def run(arg: Child | Unrelated): + if is_parent(arg): + # Type of ``arg`` is narrowed to the intersection + # of ``Parent`` and ``Child``, which is equivalent to + # ``Child``. + assert_type(arg, Child) + else: + # Type of ``arg`` is narrowed to exclude ``Parent``, + # so only ``Unrelated`` is left. + assert_type(arg, Unrelated) + + The type inside ``TypeIs`` must be consistent with the type of the + function's argument; if it is not, static type checkers will raise + an error. An incorrectly written ``TypeIs`` function can lead to + unsound behavior in the type system; it is the user's responsibility + to write such functions in a type-safe manner. + + ``TypeIs`` also works with type variables. For more information, see + PEP 742 (Narrowing types with ``TypeIs``). + """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _GenericAlias(self, (item,)) + + class ForwardRef(_Final, _root=True): """Internal wrapper to hold a forward reference.""" @@ -663,10 +1019,19 @@ class ForwardRef(_Final, _root=True): def __init__(self, arg, is_argument=True, module=None, *, is_class=False): if not isinstance(arg, str): raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. + # Unfortunately, this isn't a valid expression on its own, so we + # do the unpacking manually. + if arg.startswith('*'): + arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + arg_to_compile = arg try: - code = compile(arg, '', 'eval') + code = compile(arg_to_compile, '', 'eval') except SyntaxError: raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + self.__forward_arg__ = arg self.__forward_code__ = code self.__forward_evaluated__ = False @@ -675,7 +1040,10 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): self.__forward_is_class__ = is_class self.__forward_module__ = module - def _evaluate(self, globalns, localns, recursive_guard): + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.ForwardRef._evaluate") + type_params = () if self.__forward_arg__ in recursive_guard: return self if not self.__forward_evaluated__ or localns is not globalns: @@ -689,6 +1057,22 @@ def _evaluate(self, globalns, localns, recursive_guard): globalns = getattr( sys.modules.get(self.__forward_module__, None), '__dict__', globalns ) + + # type parameters require some special handling, + # as they exist in their own scope + # but `eval()` does not have a dedicated parameter for that scope. + # For classes, names in type parameter scopes should override + # names in the global scope (which here are called `localns`!), + # but should in turn be overridden by names in the class scope + # (which here are called `globalns`!) + if type_params: + globalns, localns = dict(globalns), dict(localns) + for param in type_params: + param_name = param.__name__ + if not self.__forward_is_class__ or param_name not in globalns: + globalns[param_name] = param + localns.pop(param_name, None) + type_ = _type_check( eval(self.__forward_code__, globalns, localns), "Forward references must evaluate to types.", @@ -696,7 +1080,11 @@ def _evaluate(self, globalns, localns, recursive_guard): allow_special_forms=self.__forward_is_class__, ) self.__forward_value__ = _eval_type( - type_, globalns, localns, recursive_guard | {self.__forward_arg__} + type_, + globalns, + localns, + type_params, + recursive_guard=(recursive_guard | {self.__forward_arg__}), ) self.__forward_evaluated__ = True return self.__forward_value__ @@ -713,236 +1101,205 @@ def __eq__(self, other): def __hash__(self): return hash((self.__forward_arg__, self.__forward_module__)) - def __repr__(self): - return f'ForwardRef({self.__forward_arg__!r})' - -class _TypeVarLike: - """Mixin for TypeVar-like types (TypeVar and ParamSpec).""" - def __init__(self, bound, covariant, contravariant): - """Used to setup TypeVars and ParamSpec's bound, covariant and - contravariant attributes. - """ - if covariant and contravariant: - raise ValueError("Bivariant types are not supported.") - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) - if bound: - self.__bound__ = _type_check(bound, "Bound must be a type.") - else: - self.__bound__ = None - - def __or__(self, right): - return Union[self, right] + def __or__(self, other): + return Union[self, other] - def __ror__(self, left): - return Union[left, self] + def __ror__(self, other): + return Union[other, self] def __repr__(self): - if self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' + if self.__forward_module__ is None: + module_repr = '' else: - prefix = '~' - return prefix + self.__name__ - - def __reduce__(self): - return self.__name__ - - -class TypeVar( _Final, _Immutable, _TypeVarLike, _root=True): - """Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows: - - def repeat(x: T, n: int) -> List[T]: - '''Return a list containing n references to x.''' - return [x]*n - - def longest(x: A, y: A) -> A: - '''Return the longest of two strings.''' - return x if len(x) >= len(y) else y - - The latter example's signature is essentially the overloading - of (str, str) -> str and (bytes, bytes) -> bytes. Also note - that if the arguments are instances of some subclass of str, - the return type is still plain str. - - At runtime, isinstance(x, T) and issubclass(C, T) will raise TypeError. - - Type variables defined with covariant=True or contravariant=True - can be used to declare covariant or contravariant generic types. - See PEP 484 for more details. By default generic types are invariant - in all type variables. - - Type variables can be introspected. e.g.: - - T.__name__ == 'T' - T.__constraints__ == () - T.__covariant__ == False - T.__contravariant__ = False - A.__constraints__ == (str, bytes) - - Note that only type variables defined in global scope can be pickled. - """ - - __slots__ = ('__name__', '__bound__', '__constraints__', - '__covariant__', '__contravariant__', '__dict__') - - def __init__(self, name, *constraints, bound=None, - covariant=False, contravariant=False): - self.__name__ = name - super().__init__(bound, covariant, contravariant) - if constraints and bound is not None: - raise TypeError("Constraints cannot be combined with bound=...") - if constraints and len(constraints) == 1: - raise TypeError("A single constraint is not allowed") - msg = "TypeVar(name, constraint, ...): constraints must be types." - self.__constraints__ = tuple(_type_check(t, msg) for t in constraints) - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing': - self.__module__ = def_mod + module_repr = f', module={self.__forward_module__!r}' + return f'ForwardRef({self.__forward_arg__!r}{module_repr})' -class ParamSpecArgs(_Final, _Immutable, _root=True): - """The args for a ParamSpec object. +def _is_unpacked_typevartuple(x: Any) -> bool: + return ((not isinstance(x, type)) and + getattr(x, '__typing_is_unpacked_typevartuple__', False)) - Given a ParamSpec object P, P.args is an instance of ParamSpecArgs. - ParamSpecArgs objects have a reference back to their ParamSpec: +def _is_typevar_like(x: Any) -> bool: + return isinstance(x, (TypeVar, ParamSpec)) or _is_unpacked_typevartuple(x) - P.args.__origin__ is P - This type is meant for runtime introspection and has no special meaning to - static type checkers. - """ - def __init__(self, origin): - self.__origin__ = origin +def _typevar_subst(self, arg): + msg = "Parameters to generic types must be types." + arg = _type_check(arg, msg, is_argument=True) + if ((isinstance(arg, _GenericAlias) and arg.__origin__ is Unpack) or + (isinstance(arg, GenericAlias) and getattr(arg, '__unpacked__', False))): + raise TypeError(f"{arg} is not valid as type argument") + return arg - def __repr__(self): - return f"{self.__origin__.__name__}.args" - def __eq__(self, other): - if not isinstance(other, ParamSpecArgs): - return NotImplemented - return self.__origin__ == other.__origin__ +def _typevartuple_prepare_subst(self, alias, args): + params = alias.__parameters__ + typevartuple_index = params.index(self) + for param in params[typevartuple_index + 1:]: + if isinstance(param, TypeVarTuple): + raise TypeError(f"More than one TypeVarTuple parameter in {alias}") + + alen = len(args) + plen = len(params) + left = typevartuple_index + right = plen - typevartuple_index - 1 + var_tuple_index = None + fillarg = None + for k, arg in enumerate(args): + if not isinstance(arg, type): + subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) + if subargs and len(subargs) == 2 and subargs[-1] is ...: + if var_tuple_index is not None: + raise TypeError("More than one unpacked arbitrary-length tuple argument") + var_tuple_index = k + fillarg = subargs[0] + if var_tuple_index is not None: + left = min(left, var_tuple_index) + right = min(right, alen - var_tuple_index - 1) + elif left + right > alen: + raise TypeError(f"Too few arguments for {alias};" + f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] + + return ( + *args[:left], + *([fillarg]*(typevartuple_index - left)), + replacement, + *([fillarg]*(plen - right - left - typevartuple_index - 1)), + *args[alen - right:], + ) + + +def _paramspec_subst(self, arg): + if isinstance(arg, (list, tuple)): + arg = tuple(_type_check(a, "Expected a type.") for a in arg) + elif not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + return arg -class ParamSpecKwargs(_Final, _Immutable, _root=True): - """The kwargs for a ParamSpec object. +def _paramspec_prepare_subst(self, alias, args): + params = alias.__parameters__ + i = params.index(self) + if i == len(args) and self.has_default(): + args = [*args, self.__default__] + if i >= len(args): + raise TypeError(f"Too few arguments for {alias}") + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if len(params) == 1 and not _is_param_expr(args[0]): + assert i == 0 + args = (args,) + # Convert lists to tuples to help other libraries cache the results. + elif isinstance(args[i], list): + args = (*args[:i], tuple(args[i]), *args[i+1:]) + return args - Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs. - ParamSpecKwargs objects have a reference back to their ParamSpec: +@_tp_cache +def _generic_class_getitem(cls, args): + """Parameterizes a generic class. - P.kwargs.__origin__ is P + At least, parameterizing a generic class is the *main* thing this method + does. For example, for some generic class `Foo`, this is called when we + do `Foo[int]` - there, with `cls=Foo` and `args=int`. - This type is meant for runtime introspection and has no special meaning to - static type checkers. + However, note that this method is also called when defining generic + classes in the first place with `class Foo(Generic[T]): ...`. """ - def __init__(self, origin): - self.__origin__ = origin - - def __repr__(self): - return f"{self.__origin__.__name__}.kwargs" + if not isinstance(args, tuple): + args = (args,) - def __eq__(self, other): - if not isinstance(other, ParamSpecKwargs): - return NotImplemented - return self.__origin__ == other.__origin__ - - -class ParamSpec(_Final, _Immutable, _TypeVarLike, _root=True): - """Parameter specification variable. - - Usage:: + args = tuple(_type_convert(p) for p in args) + is_generic_or_protocol = cls in (Generic, Protocol) - P = ParamSpec('P') - - Parameter specification variables exist primarily for the benefit of static - type checkers. They are used to forward the parameter types of one - callable to another callable, a pattern commonly found in higher order - functions and decorators. They are only valid when used in ``Concatenate``, - or as the first argument to ``Callable``, or as parameters for user-defined - Generics. See class Generic for more information on generic types. An - example for annotating a decorator:: - - T = TypeVar('T') - P = ParamSpec('P') - - def add_logging(f: Callable[P, T]) -> Callable[P, T]: - '''A type-safe decorator to add logging to a function.''' - def inner(*args: P.args, **kwargs: P.kwargs) -> T: - logging.info(f'{f.__name__} was called') - return f(*args, **kwargs) - return inner - - @add_logging - def add_two(x: float, y: float) -> float: - '''Add two numbers together.''' - return x + y - - Parameter specification variables defined with covariant=True or - contravariant=True can be used to declare covariant or contravariant - generic types. These keyword arguments are valid, but their actual semantics - are yet to be decided. See PEP 612 for details. - - Parameter specification variables can be introspected. e.g.: - - P.__name__ == 'T' - P.__bound__ == None - P.__covariant__ == False - P.__contravariant__ == False - - Note that only parameter specification variables defined in global scope can - be pickled. - """ + if is_generic_or_protocol: + # Generic and Protocol can only be subscripted with unique type variables. + if not args: + raise TypeError( + f"Parameter list to {cls.__qualname__}[...] cannot be empty" + ) + if not all(_is_typevar_like(p) for p in args): + raise TypeError( + f"Parameters to {cls.__name__}[...] must all be type variables " + f"or parameter specification variables.") + if len(set(args)) != len(args): + raise TypeError( + f"Parameters to {cls.__name__}[...] must all be unique") + else: + # Subscripting a regular Generic subclass. + for param in cls.__parameters__: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + args = prepare(cls, args) + _check_generic_specialization(cls, args) - __slots__ = ('__name__', '__bound__', '__covariant__', '__contravariant__', - '__dict__') + new_args = [] + for param, new_arg in zip(cls.__parameters__, args): + if isinstance(param, TypeVarTuple): + new_args.extend(new_arg) + else: + new_args.append(new_arg) + args = tuple(new_args) - @property - def args(self): - return ParamSpecArgs(self) + return _GenericAlias(cls, args) - @property - def kwargs(self): - return ParamSpecKwargs(self) - def __init__(self, name, *, bound=None, covariant=False, contravariant=False): - self.__name__ = name - super().__init__(bound, covariant, contravariant) - try: - def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - def_mod = None - if def_mod != 'typing': - self.__module__ = def_mod +def _generic_init_subclass(cls, *args, **kwargs): + super(Generic, cls).__init_subclass__(*args, **kwargs) + tvars = [] + if '__orig_bases__' in cls.__dict__: + error = Generic in cls.__orig_bases__ + else: + error = (Generic in cls.__bases__ and + cls.__name__ != 'Protocol' and + type(cls) != _TypedDictMeta) + if error: + raise TypeError("Cannot inherit from plain Generic") + if '__orig_bases__' in cls.__dict__: + tvars = _collect_type_parameters(cls.__orig_bases__) + # Look for Generic[T1, ..., Tn]. + # If found, tvars must be a subset of it. + # If not found, tvars is it. + # Also check for and reject plain Generic, + # and reject multiple Generic[...]. + gvars = None + for base in cls.__orig_bases__: + if (isinstance(base, _GenericAlias) and + base.__origin__ is Generic): + if gvars is not None: + raise TypeError( + "Cannot inherit from Generic[...] multiple times.") + gvars = base.__parameters__ + if gvars is not None: + tvarset = set(tvars) + gvarset = set(gvars) + if not tvarset <= gvarset: + s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) + s_args = ', '.join(str(g) for g in gvars) + raise TypeError(f"Some type variables ({s_vars}) are" + f" not listed in Generic[{s_args}]") + tvars = gvars + cls.__parameters__ = tuple(tvars) def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') class _BaseGenericAlias(_Final, _root=True): - """The central part of internal API. + """The central part of the internal API. This represents a generic version of type 'origin' with type arguments 'params'. There are two kind of these aliases: user defined and special. The special ones are wrappers around builtin collections and ABCs in collections.abc. These must - have 'name' always set. If 'inst' is False, then the alias can't be instantiated, + have 'name' always set. If 'inst' is False, then the alias can't be instantiated; this is used by e.g. typing.List and typing.Dict. """ + def __init__(self, origin, *, inst=True, name=None): self._inst = inst self._name = name @@ -956,7 +1313,9 @@ def __call__(self, *args, **kwargs): result = self.__origin__(*args, **kwargs) try: result.__orig_class__ = self - except AttributeError: + # Some objects raise TypeError (or something even more exotic) + # if you try to set attributes on them; we guard against that here + except Exception: pass return result @@ -964,9 +1323,29 @@ def __mro_entries__(self, bases): res = [] if self.__origin__ not in bases: res.append(self.__origin__) + + # Check if any base that occurs after us in `bases` is either itself a + # subclass of Generic, or something which will add a subclass of Generic + # to `__bases__` via its `__mro_entries__`. If not, add Generic + # ourselves. The goal is to ensure that Generic (or a subclass) will + # appear exactly once in the final bases tuple. If we let it appear + # multiple times, we risk "can't form a consistent MRO" errors. i = bases.index(self) for b in bases[i+1:]: - if isinstance(b, _BaseGenericAlias) or issubclass(b, Generic): + if isinstance(b, _BaseGenericAlias): + break + if not isinstance(b, type): + meth = getattr(b, "__mro_entries__", None) + new_bases = meth(bases) if meth else None + if ( + isinstance(new_bases, tuple) and + any( + isinstance(b2, type) and issubclass(b2, Generic) + for b2 in new_bases + ) + ): + break + elif issubclass(b, Generic): break else: res.append(Generic) @@ -983,8 +1362,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_typevar_types', '_paramspec_tvars'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1000,6 +1378,7 @@ def __dir__(self): return list(set(super().__dir__() + [attr for attr in dir(self.__origin__) if not _is_dunder(attr)])) + # Special typing constructs Union, Optional, Generic, Callable and Tuple # use three special attributes for internal bookkeeping of generic types: # * __parameters__ is a tuple of unique free type parameters of a generic @@ -1012,18 +1391,42 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): - def __init__(self, origin, params, *, inst=True, name=None, - _typevar_types=TypeVar, - _paramspec_tvars=False): + # The type of parameterized generics. + # + # That is, for example, `type(List[int])` is `_GenericAlias`. + # + # Objects which are instances of this class include: + # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. + # * Note that native container types, e.g. `tuple`, `list`, use + # `types.GenericAlias` instead. + # * Parameterized classes: + # class C[T]: pass + # # C[int] is a _GenericAlias + # * `Callable` aliases, generic `Callable` aliases, and + # parameterized `Callable` aliases: + # T = TypeVar('T') + # # _CallableGenericAlias inherits from _GenericAlias. + # A = Callable[[], None] # _CallableGenericAlias + # B = Callable[[T], None] # _CallableGenericAlias + # C = B[int] # _CallableGenericAlias + # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: + # # All _GenericAlias + # Final[int] + # ClassVar[float] + # TypeGuard[bool] + # TypeIs[range] + + def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else - () if a is _TypingEmpty else - a for a in params) - self.__parameters__ = _collect_type_vars(params, typevar_types=_typevar_types) - self._typevar_types = _typevar_types - self._paramspec_tvars = _paramspec_tvars + a for a in args) + enforce_default_ordering = origin in (Generic, Protocol) + self.__parameters__ = _collect_type_parameters( + args, + enforce_default_ordering=enforce_default_ordering, + ) if not name: self.__module__ = origin.__module__ @@ -1043,53 +1446,140 @@ def __ror__(self, left): return Union[left, self] @_tp_cache - def __getitem__(self, params): + def __getitem__(self, args): + # Parameterizes an already-parameterized object. + # + # For example, we arrive here doing something like: + # T1 = TypeVar('T1') + # T2 = TypeVar('T2') + # T3 = TypeVar('T3') + # class A(Generic[T1]): pass + # B = A[T2] # B is a _GenericAlias + # C = B[T3] # Invokes _GenericAlias.__getitem__ + # + # We also arrive here when parameterizing a generic `Callable` alias: + # T = TypeVar('T') + # C = Callable[[T], None] + # C[int] # Invokes _GenericAlias.__getitem__ + if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. raise TypeError(f"Cannot subscript already-subscripted {self}") - if not isinstance(params, tuple): - params = (params,) - params = tuple(_type_convert(p) for p in params) - if (self._paramspec_tvars - and any(isinstance(t, ParamSpec) for t in self.__parameters__)): - params = _prepare_paramspec_params(self, params) - else: - _check_generic(self, params, len(self.__parameters__)) + if not self.__parameters__: + raise TypeError(f"{self} is not a generic class") - subst = dict(zip(self.__parameters__, params)) + # Preprocess `args`. + if not isinstance(args, tuple): + args = (args,) + args = _unpack_args(*(_type_convert(p) for p in args)) + new_args = self._determine_new_args(args) + r = self.copy_with(new_args) + return r + + def _determine_new_args(self, args): + # Determines new __args__ for __getitem__. + # + # For example, suppose we had: + # T1 = TypeVar('T1') + # T2 = TypeVar('T2') + # class A(Generic[T1, T2]): pass + # T3 = TypeVar('T3') + # B = A[int, T3] + # C = B[str] + # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. + # Unfortunately, this is harder than it looks, because if `T3` is + # anything more exotic than a plain `TypeVar`, we need to consider + # edge cases. + + params = self.__parameters__ + # In the example above, this would be {T3: str} + for param in params: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + args = prepare(self, args) + alen = len(args) + plen = len(params) + if alen != plen: + raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};" + f" actual {alen}, expected {plen}") + new_arg_by_param = dict(zip(params, args)) + return tuple(self._make_substitution(self.__args__, new_arg_by_param)) + + def _make_substitution(self, args, new_arg_by_param): + """Create a list of new type arguments.""" new_args = [] - for arg in self.__args__: - if isinstance(arg, self._typevar_types): - if isinstance(arg, ParamSpec): - arg = subst[arg] - if not _is_param_expr(arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {arg}") + for old_arg in args: + if isinstance(old_arg, type): + new_args.append(old_arg) + continue + + substfunc = getattr(old_arg, '__typing_subst__', None) + if substfunc: + new_arg = substfunc(new_arg_by_param[old_arg]) + else: + subparams = getattr(old_arg, '__parameters__', ()) + if not subparams: + new_arg = old_arg else: - arg = subst[arg] - elif isinstance(arg, (_GenericAlias, GenericAlias, types.UnionType)): - subparams = arg.__parameters__ - if subparams: - subargs = tuple(subst[x] for x in subparams) - arg = arg[subargs] - # Required to flatten out the args for CallableGenericAlias - if self.__origin__ == collections.abc.Callable and isinstance(arg, tuple): - new_args.extend(arg) + subargs = [] + for x in subparams: + if isinstance(x, TypeVarTuple): + subargs.extend(new_arg_by_param[x]) + else: + subargs.append(new_arg_by_param[x]) + new_arg = old_arg[tuple(subargs)] + + if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple): + # Consider the following `Callable`. + # C = Callable[[int], str] + # Here, `C.__args__` should be (int, str) - NOT ([int], str). + # That means that if we had something like... + # P = ParamSpec('P') + # T = TypeVar('T') + # C = Callable[P, T] + # D = C[[int, str], float] + # ...we need to be careful; `new_args` should end up as + # `(int, str, float)` rather than `([int, str], float)`. + new_args.extend(new_arg) + elif _is_unpacked_typevartuple(old_arg): + # Consider the following `_GenericAlias`, `B`: + # class A(Generic[*Ts]): ... + # B = A[T, *Ts] + # If we then do: + # B[float, int, str] + # The `new_arg` corresponding to `T` will be `float`, and the + # `new_arg` corresponding to `*Ts` will be `(int, str)`. We + # should join all these types together in a flat list + # `(float, int, str)` - so again, we should `extend`. + new_args.extend(new_arg) + elif isinstance(old_arg, tuple): + # Corner case: + # P = ParamSpec('P') + # T = TypeVar('T') + # class Base(Generic[P]): ... + # Can be substituted like this: + # X = Base[[int, T]] + # In this case, `old_arg` will be a tuple: + new_args.append( + tuple(self._make_substitution(old_arg, new_arg_by_param)), + ) else: - new_args.append(arg) - return self.copy_with(tuple(new_args)) + new_args.append(new_arg) + return new_args - def copy_with(self, params): - return self.__class__(self.__origin__, params, name=self._name, inst=self._inst, - _typevar_types=self._typevar_types, - _paramspec_tvars=self._paramspec_tvars) + def copy_with(self, args): + return self.__class__(self.__origin__, args, name=self._name, inst=self._inst) def __repr__(self): if self._name: name = 'typing.' + self._name else: name = _type_repr(self.__origin__) - args = ", ".join([_type_repr(a) for a in self.__args__]) + if self.__args__: + args = ", ".join([_type_repr(a) for a in self.__args__]) + else: + # To ensure the repr is eval-able. + args = "()" return f'{name}[{args}]' def __reduce__(self): @@ -1117,17 +1607,21 @@ def __mro_entries__(self, bases): return () return (self.__origin__,) + def __iter__(self): + yield Unpack[self] + # _nparams is the number of accepted parameters, e.g. 0 for Hashable, # 1 for List and 2 for Dict. It may be -1 if variable number of # parameters are accepted (needs custom __getitem__). -class _SpecialGenericAlias(_BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): +class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': self.__doc__ = f'A generic version of {origin.__qualname__}.' else: @@ -1139,7 +1633,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) + actual_len = len(params) + + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -1165,7 +1674,23 @@ def __or__(self, right): def __ror__(self, left): return Union[left, self] -class _CallableGenericAlias(_GenericAlias, _root=True): + +class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + +class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' args = self.__args__ @@ -1185,9 +1710,7 @@ def __reduce__(self): class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, - name=self._name, inst=self._inst, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) + name=self._name, inst=self._inst) def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: @@ -1220,27 +1743,28 @@ def __getitem_inner__(self, params): class _TupleType(_SpecialGenericAlias, _root=True): @_tp_cache def __getitem__(self, params): - if params == (): - return self.copy_with((_TypingEmpty,)) if not isinstance(params, tuple): params = (params,) - if len(params) == 2 and params[1] is ...: + if len(params) >= 2 and params[-1] is ...: msg = "Tuple[t, ...]: t must be a type." - p = _type_check(params[0], msg) - return self.copy_with((p, _TypingEllipsis)) + params = tuple(_type_check(p, msg) for p in params[:-1]) + return self.copy_with((*params, _TypingEllipsis)) msg = "Tuple[t0, t1, ...]: each t must be a type." params = tuple(_type_check(p, msg) for p in params) return self.copy_with(params) -class _UnionGenericAlias(_GenericAlias, _root=True): +class _UnionGenericAlias(_NotIterable, _GenericAlias, _root=True): def copy_with(self, params): return Union[params] def __eq__(self, other): if not isinstance(other, (_UnionGenericAlias, types.UnionType)): return NotImplemented - return set(self.__args__) == set(other.__args__) + try: # fast path + return set(self.__args__) == set(other.__args__) + except TypeError: # not hashable, slow path + return _compare_args_orderless(self.__args__, other.__args__) def __hash__(self): return hash(frozenset(self.__args__)) @@ -1255,12 +1779,16 @@ def __repr__(self): return super().__repr__() def __instancecheck__(self, obj): - return self.__subclasscheck__(type(obj)) + for arg in self.__args__: + if isinstance(obj, arg): + return True + return False def __subclasscheck__(self, cls): for arg in self.__args__: if issubclass(cls, arg): return True + return False def __reduce__(self): func, (origin, args) = super().__reduce__() @@ -1272,7 +1800,6 @@ def _value_and_type_iter(parameters): class _LiteralGenericAlias(_GenericAlias, _root=True): - def __eq__(self, other): if not isinstance(other, _LiteralGenericAlias): return NotImplemented @@ -1289,118 +1816,108 @@ def copy_with(self, params): return (*params[:-1], *params[-1]) if isinstance(params[-1], _ConcatenateGenericAlias): params = (*params[:-1], *params[-1].__args__) - elif not isinstance(params[-1], ParamSpec): - raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable.") return super().copy_with(params) -class Generic: - """Abstract base class for generic types. +@_SpecialForm +def Unpack(self, parameters): + """Type unpack operator. - A generic type is typically declared by inheriting from - this class parameterized with one or more type variables. - For example, a generic mapping type might be defined as:: + The type unpack operator takes the child types from some container type, + such as `tuple[int, str]` or a `TypeVarTuple`, and 'pulls them out'. - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. + For example:: - This class can then be used as follows:: + # For some generic class `Foo`: + Foo[Unpack[tuple[int, str]]] # Equivalent to Foo[int, str] - def lookup_name(mapping: Mapping[KT, VT], key: KT, default: VT) -> VT: - try: - return mapping[key] - except KeyError: - return default - """ - __slots__ = () - _is_protocol = False + Ts = TypeVarTuple('Ts') + # Specifies that `Bar` is generic in an arbitrary number of types. + # (Think of `Ts` as a tuple of an arbitrary number of individual + # `TypeVar`s, which the `Unpack` is 'pulling out' directly into the + # `Generic[]`.) + class Bar(Generic[Unpack[Ts]]): ... + Bar[int] # Valid + Bar[int, str] # Also valid - @_tp_cache - def __class_getitem__(cls, params): - if not isinstance(params, tuple): - params = (params,) - if not params and cls is not Tuple: - raise TypeError( - f"Parameter list to {cls.__qualname__}[...] cannot be empty") - params = tuple(_type_convert(p) for p in params) - if cls in (Generic, Protocol): - # Generic and Protocol can only be subscripted with unique type variables. - if not all(isinstance(p, (TypeVar, ParamSpec)) for p in params): - raise TypeError( - f"Parameters to {cls.__name__}[...] must all be type variables " - f"or parameter specification variables.") - if len(set(params)) != len(params): - raise TypeError( - f"Parameters to {cls.__name__}[...] must all be unique") - else: - # Subscripting a regular Generic subclass. - if any(isinstance(t, ParamSpec) for t in cls.__parameters__): - params = _prepare_paramspec_params(cls, params) - else: - _check_generic(cls, params, len(cls.__parameters__)) - return _GenericAlias(cls, params, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) + From Python 3.11, this can also be done using the `*` operator:: - def __init_subclass__(cls, *args, **kwargs): - super().__init_subclass__(*args, **kwargs) - tvars = [] - if '__orig_bases__' in cls.__dict__: - error = Generic in cls.__orig_bases__ - else: - error = Generic in cls.__bases__ and cls.__name__ != 'Protocol' - if error: - raise TypeError("Cannot inherit from plain Generic") - if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_vars(cls.__orig_bases__, (TypeVar, ParamSpec)) - # Look for Generic[T1, ..., Tn]. - # If found, tvars must be a subset of it. - # If not found, tvars is it. - # Also check for and reject plain Generic, - # and reject multiple Generic[...]. - gvars = None - for base in cls.__orig_bases__: - if (isinstance(base, _GenericAlias) and - base.__origin__ is Generic): - if gvars is not None: - raise TypeError( - "Cannot inherit from Generic[...] multiple types.") - gvars = base.__parameters__ - if gvars is not None: - tvarset = set(tvars) - gvarset = set(gvars) - if not tvarset <= gvarset: - s_vars = ', '.join(str(t) for t in tvars if t not in gvarset) - s_args = ', '.join(str(g) for g in gvars) - raise TypeError(f"Some type variables ({s_vars}) are" - f" not listed in Generic[{s_args}]") - tvars = gvars - cls.__parameters__ = tuple(tvars) - - -class _TypingEmpty: - """Internal placeholder for () or []. Used by TupleMeta and CallableMeta - to allow empty list/tuple in specific places, without allowing them - to sneak in where prohibited. + Foo[*tuple[int, str]] + class Bar(Generic[*Ts]): ... + + And from Python 3.12, it can be done using built-in syntax for generics:: + + Foo[*tuple[int, str]] + class Bar[*Ts]: ... + + The operator can also be used along with a `TypedDict` to annotate + `**kwargs` in a function signature:: + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - *name* of type `str` and + # *year* of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + + Note that there is only some runtime checking of this operator. Not + everything the runtime allows may be accepted by static type checkers. + + For more information, see PEPs 646 and 692. """ + item = _type_check(parameters, f'{self} accepts only single type.') + return _UnpackGenericAlias(origin=self, args=(item,)) + + +class _UnpackGenericAlias(_GenericAlias, _root=True): + def __repr__(self): + # `Unpack` only takes one argument, so __args__ should contain only + # a single item. + return f'typing.Unpack[{_type_repr(self.__args__[0])}]' + + def __getitem__(self, args): + if self.__typing_is_unpacked_typevartuple__: + return args + return super().__getitem__(args) + + @property + def __typing_unpacked_tuple_args__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + arg, = self.__args__ + if isinstance(arg, (_GenericAlias, types.GenericAlias)): + if arg.__origin__ is not tuple: + raise TypeError("Unpack[...] must be used with a tuple type") + return arg.__args__ + return None + + @property + def __typing_is_unpacked_typevartuple__(self): + assert self.__origin__ is Unpack + assert len(self.__args__) == 1 + return isinstance(self.__args__[0], TypeVarTuple) class _TypingEllipsis: """Internal placeholder for ... (ellipsis).""" -_TYPING_INTERNALS = ['__parameters__', '__orig_bases__', '__orig_class__', - '_is_protocol', '_is_runtime_protocol'] +_TYPING_INTERNALS = frozenset({ + '__parameters__', '__orig_bases__', '__orig_class__', + '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', + '__non_callable_proto_members__', '__type_params__', +}) -_SPECIAL_NAMES = ['__abstractmethods__', '__annotations__', '__dict__', '__doc__', - '__init__', '__module__', '__new__', '__slots__', - '__subclasshook__', '__weakref__', '__class_getitem__'] +_SPECIAL_NAMES = frozenset({ + '__abstractmethods__', '__annotations__', '__dict__', '__doc__', + '__init__', '__module__', '__new__', '__slots__', + '__subclasshook__', '__weakref__', '__class_getitem__', + '__match_args__', '__static_attributes__', '__firstlineno__', +}) # These special attributes will be not collected as protocol members. -EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS + _SPECIAL_NAMES + ['_MutableMapping__marker'] +EXCLUDED_ATTRIBUTES = _TYPING_INTERNALS | _SPECIAL_NAMES | {'_MutableMapping__marker'} def _get_protocol_attrs(cls): @@ -1411,20 +1928,15 @@ def _get_protocol_attrs(cls): """ attrs = set() for base in cls.__mro__[:-1]: # without object - if base.__name__ in ('Protocol', 'Generic'): + if base.__name__ in {'Protocol', 'Generic'}: continue annotations = getattr(base, '__annotations__', {}) - for attr in list(base.__dict__.keys()) + list(annotations.keys()): + for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: attrs.add(attr) return attrs -def _is_callable_members_only(cls): - # PEP 544 prohibits using issubclass() with protocols that have non-method members. - return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls)) - - def _no_init_or_replace_init(self, *args, **kwargs): cls = type(self) @@ -1455,59 +1967,192 @@ def _no_init_or_replace_init(self, *args, **kwargs): def _caller(depth=1, default='__main__'): + try: + return sys._getframemodulename(depth + 1) or default + except AttributeError: # For platforms without _getframemodulename() + pass try: return sys._getframe(depth + 1).f_globals.get('__name__', default) except (AttributeError, ValueError): # For platforms without _getframe() - return None - + pass + return None -def _allow_reckless_class_checks(depth=3): +def _allow_reckless_class_checks(depth=2): """Allow instance and class checks for special stdlib modules. The abc and functools modules indiscriminately call isinstance() and issubclass() on the whole MRO of a user class, which may contain protocols. """ - try: - return sys._getframe(depth).f_globals['__name__'] in ['abc', 'functools'] - except (AttributeError, ValueError): # For platforms without _getframe(). - return True + return _caller(depth) in {'abc', 'functools', None} + + +_PROTO_ALLOWLIST = { + 'collections.abc': [ + 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', + 'AsyncIterator', 'Hashable', 'Sized', 'Container', 'Collection', + 'Reversible', 'Buffer', + ], + 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], +} + + +@functools.cache +def _lazy_load_getattr_static(): + # Import getattr_static lazily so as not to slow down the import of typing.py + # Cache the result so we don't slow down _ProtocolMeta.__instancecheck__ unnecessarily + from inspect import getattr_static + return getattr_static + + +_cleanups.append(_lazy_load_getattr_static.cache_clear) + +def _pickle_psargs(psargs): + return ParamSpecArgs, (psargs.__origin__,) + +copyreg.pickle(ParamSpecArgs, _pickle_psargs) + +def _pickle_pskwargs(pskwargs): + return ParamSpecKwargs, (pskwargs.__origin__,) + +copyreg.pickle(ParamSpecKwargs, _pickle_pskwargs) + +del _pickle_psargs, _pickle_pskwargs + + +# Preload these once, as globals, as a micro-optimisation. +# This makes a significant difference to the time it takes +# to do `isinstance()`/`issubclass()` checks +# against runtime-checkable protocols with only one callable member. +_abc_instancecheck = ABCMeta.__instancecheck__ +_abc_subclasscheck = ABCMeta.__subclasscheck__ + +def _type_check_issubclass_arg_1(arg): + """Raise TypeError if `arg` is not an instance of `type` + in `issubclass(arg, )`. -_PROTO_ALLOWLIST = { - 'collections.abc': [ - 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', - ], - 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], -} + In most cases, this is verified by type.__subclasscheck__. + Checking it again unnecessarily would slow down issubclass() checks, + so, we don't perform this check unless we absolutely have to. + + For various error paths, however, + we want to ensure that *this* error message is shown to the user + where relevant, rather than a typing.py-specific error message. + """ + if not isinstance(arg, type): + # Same error message as for issubclass(1, int). + raise TypeError('issubclass() arg 1 must be a class') class _ProtocolMeta(ABCMeta): - # This metaclass is really unfortunate and exists only because of - # the lack of __instancehook__. + # This metaclass is somewhat unfortunate, + # but is necessary for several reasons... + def __new__(mcls, name, bases, namespace, /, **kwargs): + if name == "Protocol" and bases == (Generic,): + pass + elif Protocol in bases: + for base in bases: + if not ( + base in {object, Generic} + or base.__name__ in _PROTO_ALLOWLIST.get(base.__module__, []) + or ( + issubclass(base, Generic) + and getattr(base, "_is_protocol", False) + ) + ): + raise TypeError( + f"Protocols can only inherit from other protocols, " + f"got {base!r}" + ) + return super().__new__(mcls, name, bases, namespace, **kwargs) + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + if getattr(cls, "_is_protocol", False): + cls.__protocol_attrs__ = _get_protocol_attrs(cls) + + def __subclasscheck__(cls, other): + if cls is Protocol: + return type.__subclasscheck__(cls, other) + if ( + getattr(cls, '_is_protocol', False) + and not _allow_reckless_class_checks() + ): + if not getattr(cls, '_is_runtime_protocol', False): + _type_check_issubclass_arg_1(other) + raise TypeError( + "Instance and class checks can only be used with " + "@runtime_checkable protocols" + ) + if ( + # this attribute is set by @runtime_checkable: + cls.__non_callable_proto_members__ + and cls.__dict__.get("__subclasshook__") is _proto_hook + ): + _type_check_issubclass_arg_1(other) + # non_method_attrs = sorted(cls.__non_callable_proto_members__) + # raise TypeError( + # "Protocols with non-method members don't support issubclass()." + # f" Non-method members: {str(non_method_attrs)[1:-1]}." + # ) + return _abc_subclasscheck(cls, other) + def __instancecheck__(cls, instance): # We need this method for situations where attributes are # assigned in __init__. + if cls is Protocol: + return type.__instancecheck__(cls, instance) + if not getattr(cls, "_is_protocol", False): + # i.e., it's a concrete subclass of a protocol + return _abc_instancecheck(cls, instance) + if ( - getattr(cls, '_is_protocol', False) and not getattr(cls, '_is_runtime_protocol', False) and - not _allow_reckless_class_checks(depth=2) + not _allow_reckless_class_checks() ): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - if ((not getattr(cls, '_is_protocol', False) or - _is_callable_members_only(cls)) and - issubclass(instance.__class__, cls)): + if _abc_instancecheck(cls, instance): return True - if cls._is_protocol: - if all(hasattr(instance, attr) and - # All *methods* can be blocked by setting them to None. - (not callable(getattr(cls, attr, None)) or - getattr(instance, attr) is not None) - for attr in _get_protocol_attrs(cls)): - return True - return super().__instancecheck__(instance) + + getattr_static = _lazy_load_getattr_static() + for attr in cls.__protocol_attrs__: + try: + val = getattr_static(instance, attr) + except AttributeError: + break + # this attribute is set by @runtime_checkable: + if val is None and attr not in cls.__non_callable_proto_members__: + break + else: + return True + + return False + + +@classmethod +def _proto_hook(cls, other): + if not cls.__dict__.get('_is_protocol', False): + return NotImplemented + + for attr in cls.__protocol_attrs__: + for base in other.__mro__: + # Check if the members appears in the class dictionary... + if attr in base.__dict__: + if base.__dict__[attr] is None: + return NotImplemented + break + + # ...or in annotations, if it is a sub-protocol. + annotations = getattr(base, '__annotations__', {}) + if (isinstance(annotations, collections.abc.Mapping) and + attr in annotations and + issubclass(other, Generic) and getattr(other, '_is_protocol', False)): + break + else: + return NotImplemented + return True class Protocol(Generic, metaclass=_ProtocolMeta): @@ -1520,7 +2165,9 @@ def meth(self) -> int: ... Such classes are primarily used with static type checkers that recognize - structural subtyping (static duck-typing), for example:: + structural subtyping (static duck-typing). + + For example:: class C: def meth(self) -> int: @@ -1536,10 +2183,11 @@ def func(x: Proto) -> int: only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as:: - class GenProto(Protocol[T]): + class GenProto[T](Protocol): def meth(self) -> T: ... """ + __slots__ = () _is_protocol = True _is_runtime_protocol = False @@ -1552,75 +2200,30 @@ def __init_subclass__(cls, *args, **kwargs): cls._is_protocol = any(b is Protocol for b in cls.__bases__) # Set (or override) the protocol subclass hook. - def _proto_hook(other): - if not cls.__dict__.get('_is_protocol', False): - return NotImplemented - - # First, perform various sanity checks. - if not getattr(cls, '_is_runtime_protocol', False): - if _allow_reckless_class_checks(): - return NotImplemented - raise TypeError("Instance and class checks can only be used with" - " @runtime_checkable protocols") - if not _is_callable_members_only(cls): - if _allow_reckless_class_checks(): - return NotImplemented - raise TypeError("Protocols with non-method members" - " don't support issubclass()") - if not isinstance(other, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') - - # Second, perform the actual structural compatibility check. - for attr in _get_protocol_attrs(cls): - for base in other.__mro__: - # Check if the members appears in the class dictionary... - if attr in base.__dict__: - if base.__dict__[attr] is None: - return NotImplemented - break - - # ...or in annotations, if it is a sub-protocol. - annotations = getattr(base, '__annotations__', {}) - if (isinstance(annotations, collections.abc.Mapping) and - attr in annotations and - issubclass(other, Generic) and other._is_protocol): - break - else: - return NotImplemented - return True - if '__subclasshook__' not in cls.__dict__: cls.__subclasshook__ = _proto_hook - # We have nothing more to do for non-protocols... - if not cls._is_protocol: - return + # Prohibit instantiation for protocol classes + if cls._is_protocol and cls.__init__ is Protocol.__init__: + cls.__init__ = _no_init_or_replace_init - # ... otherwise check consistency of bases, and prohibit instantiation. - for base in cls.__bases__: - if not (base in (object, Generic) or - base.__module__ in _PROTO_ALLOWLIST and - base.__name__ in _PROTO_ALLOWLIST[base.__module__] or - issubclass(base, Generic) and base._is_protocol): - raise TypeError('Protocols can only inherit from other' - ' protocols, got %r' % base) - cls.__init__ = _no_init_or_replace_init - -class _AnnotatedAlias(_GenericAlias, _root=True): +class _AnnotatedAlias(_NotIterable, _GenericAlias, _root=True): """Runtime representation of an annotated type. At its core 'Annotated[t, dec1, dec2, ...]' is an alias for the type 't' - with extra annotations. The alias behaves like a normal typing alias, - instantiating is the same as instantiating the underlying type, binding + with extra annotations. The alias behaves like a normal typing alias. + Instantiating is the same as instantiating the underlying type; binding it to types is also the same. + + The metadata itself is stored in a '__metadata__' attribute as a tuple. """ + def __init__(self, origin, metadata): if isinstance(origin, _AnnotatedAlias): metadata = origin.__metadata__ + metadata origin = origin.__origin__ - super().__init__(origin, origin) + super().__init__(origin, origin, name='Annotated') self.__metadata__ = metadata def copy_with(self, params): @@ -1653,9 +2256,14 @@ def __getattr__(self, attr): return 'Annotated' return super().__getattr__(attr) + def __mro_entries__(self, bases): + return (self.__origin__,) + -class Annotated: - """Add context specific metadata to a type. +@_TypedCacheSpecialForm +@_tp_cache(typed=True) +def Annotated(self, *params): + """Add context-specific metadata to a type. Example: Annotated[int, runtime_check.Unsigned] indicates to the hypothetical runtime_check module that this type is an unsigned int. @@ -1667,44 +2275,51 @@ class Annotated: Details: - It's an error to call `Annotated` with less than two arguments. - - Nested Annotated are flattened:: + - Access the metadata via the ``__metadata__`` attribute:: + + assert Annotated[int, '$'].__metadata__ == ('$',) - Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] + - Nested Annotated types are flattened:: + + assert Annotated[Annotated[T, Ann1, Ann2], Ann3] == Annotated[T, Ann1, Ann2, Ann3] - Instantiating an annotated type is equivalent to instantiating the underlying type:: - Annotated[C, Ann1](5) == C(5) + assert Annotated[C, Ann1](5) == C(5) - Annotated can be used as a generic type alias:: - Optimized = Annotated[T, runtime.Optimize()] - Optimized[int] == Annotated[int, runtime.Optimize()] + type Optimized[T] = Annotated[T, runtime.Optimize()] + # type checker will treat Optimized[int] + # as equivalent to Annotated[int, runtime.Optimize()] - OptimizedList = Annotated[List[T], runtime.Optimize()] - OptimizedList[int] == Annotated[List[int], runtime.Optimize()] - """ + type OptimizedList[T] = Annotated[list[T], runtime.Optimize()] + # type checker will treat OptimizedList[int] + # as equivalent to Annotated[list[int], runtime.Optimize()] - __slots__ = () + - Annotated cannot be used with an unpacked TypeVarTuple:: - def __new__(cls, *args, **kwargs): - raise TypeError("Type Annotated cannot be instantiated.") + type Variadic[*Ts] = Annotated[*Ts, Ann1] # NOT valid - @_tp_cache - def __class_getitem__(cls, params): - if not isinstance(params, tuple) or len(params) < 2: - raise TypeError("Annotated[...] should be used " - "with at least two arguments (a type and an " - "annotation).") - msg = "Annotated[t, ...]: t must be a type." - origin = _type_check(params[0], msg, allow_special_forms=True) - metadata = tuple(params[1:]) - return _AnnotatedAlias(origin, metadata) + This would be equivalent to:: - def __init_subclass__(cls, *args, **kwargs): - raise TypeError( - "Cannot subclass {}.Annotated".format(cls.__module__) - ) + Annotated[T1, T2, T3, ..., Ann1] + + where T1, T2 etc. are TypeVars, which would be invalid, because + only one type should be passed to Annotated. + """ + if len(params) < 2: + raise TypeError("Annotated[...] should be used " + "with at least two arguments (a type and an " + "annotation).") + if _is_unpacked_typevartuple(params[0]): + raise TypeError("Annotated[...] should not be used with an " + "unpacked TypeVarTuple") + msg = "Annotated[t, ...]: t must be a type." + origin = _type_check(params[0], msg, allow_special_forms=True) + metadata = tuple(params[1:]) + return _AnnotatedAlias(origin, metadata) def runtime_checkable(cls): @@ -1714,6 +2329,7 @@ def runtime_checkable(cls): Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to one trick ponies in collections.abc such as Iterable. + For example:: @runtime_checkable @@ -1725,10 +2341,26 @@ def close(self): ... Warning: this will check only the presence of the required methods, not their type signatures! """ - if not issubclass(cls, Generic) or not cls._is_protocol: + if not issubclass(cls, Generic) or not getattr(cls, '_is_protocol', False): raise TypeError('@runtime_checkable can be only applied to protocol classes,' ' got %r' % cls) cls._is_runtime_protocol = True + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + # See gh-113320 for why we compute this attribute here, + # rather than in `_ProtocolMeta.__init__` + cls.__non_callable_proto_members__ = set() + for attr in cls.__protocol_attrs__: + try: + is_callable = callable(getattr(cls, attr, None)) + except Exception as e: + raise TypeError( + f"Failed to determine whether protocol member {attr!r} " + "is a method member" + ) from e + else: + if not is_callable: + cls.__non_callable_proto_members__.add(attr) return cls @@ -1743,24 +2375,20 @@ def cast(typ, val): return val -def _get_defaults(func): - """Internal helper to extract the default arguments, by name.""" - try: - code = func.__code__ - except AttributeError: - # Some built-in functions don't have __code__, __defaults__, etc. - return {} - pos_count = code.co_argcount - arg_names = code.co_varnames - arg_names = arg_names[:pos_count] - defaults = func.__defaults__ or () - kwdefaults = func.__kwdefaults__ - res = dict(kwdefaults) if kwdefaults else {} - pos_offset = pos_count - len(defaults) - for name, value in zip(arg_names[pos_offset:], defaults): - assert name not in res - res[name] = value - return res +def assert_type(val, typ, /): + """Ask a static type checker to confirm that the value is of the given type. + + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to assert_type(), it + emits an error if the value is not of the specified type:: + + def greet(name: str) -> None: + assert_type(name, str) # OK + assert_type(name, int) # type checker error + """ + return val _allowed_types = (types.FunctionType, types.BuiltinFunctionType, @@ -1772,8 +2400,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): """Return type hints for an object. This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, adds Optional[t] if a - default value equal to None is set and recursively replaces all + forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -1800,7 +2427,6 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): - If two dict arguments are passed, they specify globals and locals, respectively. """ - if getattr(obj, '__no_type_check__', None): return {} # Classes require a special treatment. @@ -1828,7 +2454,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): value = type(None) if isinstance(value, str): value = ForwardRef(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals) + value = _eval_type(value, base_globals, base_locals, base.__type_params__) hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} @@ -1853,8 +2479,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): else: raise TypeError('{!r} is not a module, class, method, ' 'or function.'.format(obj)) - defaults = _get_defaults(obj) hints = dict(hints) + type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): if value is None: value = type(None) @@ -1866,18 +2492,16 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - value = _eval_type(value, globalns, localns) - if name in defaults and defaults[name] is None: - value = Optional[value] - hints[name] = value + hints[name] = _eval_type(value, globalns, localns, type_params) return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} def _strip_annotations(t): - """Strips the annotations from a given type. - """ + """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): return _strip_annotations(t.__origin__) + if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): + return _strip_annotations(t.__args__[0]) if isinstance(t, _GenericAlias): stripped_args = tuple(_strip_annotations(a) for a in t.__args__) if stripped_args == t.__args__: @@ -1900,17 +2524,8 @@ def _strip_annotations(t): def get_origin(tp): """Get the unsubscripted version of a type. - This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar - and Annotated. Return None for unsupported types. Examples:: - - get_origin(Literal[42]) is Literal - get_origin(int) is None - get_origin(ClassVar[int]) is ClassVar - get_origin(Generic) is Generic - get_origin(Generic[T]) is Generic - get_origin(Union[T, int]) is Union - get_origin(List[Tuple[T, T]][int]) == list - get_origin(P.args) is P + This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar, + Annotated, and others. Return None for unsupported types. """ if isinstance(tp, _AnnotatedAlias): return Annotated @@ -1928,19 +2543,17 @@ def get_args(tp): """Get type arguments with all substitutions performed. For unions, basic simplifications used by Union constructor are performed. + Examples:: - get_args(Dict[str, int]) == (str, int) - get_args(int) == () - get_args(Union[int, Union[T, int], str][int]) == (int, str) - get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - get_args(Callable[[], T][int]) == ([], int) + + >>> T = TypeVar('T') + >>> assert get_args(Dict[str, int]) == (str, int) """ if isinstance(tp, _AnnotatedAlias): return (tp.__origin__,) + tp.__metadata__ if isinstance(tp, (_GenericAlias, GenericAlias)): res = tp.__args__ - if (tp.__origin__ is collections.abc.Callable - and not (len(res) == 2 and _is_param_expr(res[0]))): + if _should_unflatten_callable_args(tp, res): res = (list(res[:-1]), res[-1]) return res if isinstance(tp, types.UnionType): @@ -1949,19 +2562,51 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class + """Check if an annotation is a TypedDict class. For example:: - class Film(TypedDict): - title: str - year: int - is_typeddict(Film) # => True - is_typeddict(Union[list, str]) # => False + >>> from typing import TypedDict + >>> class Film(TypedDict): + ... title: str + ... year: int + ... + >>> is_typeddict(Film) + True + >>> is_typeddict(dict) + False """ return isinstance(tp, _TypedDictMeta) +_ASSERT_NEVER_REPR_MAX_LENGTH = 100 + + +def assert_never(arg: Never, /) -> Never: + """Statically assert that a line of code is unreachable. + + Example:: + + def int_or_str(arg: int | str) -> None: + match arg: + case int(): + print("It's an int") + case str(): + print("It's a str") + case _: + assert_never(arg) + + If a type checker finds that a call to assert_never() is + reachable, it will emit an error. + + At runtime, this throws an exception when called. + """ + value = repr(arg) + if len(value) > _ASSERT_NEVER_REPR_MAX_LENGTH: + value = value[:_ASSERT_NEVER_REPR_MAX_LENGTH] + '...' + raise AssertionError(f"Expected code to be unreachable, but got: {value}") + + def no_type_check(arg): """Decorator to indicate that annotations are not type hints. @@ -1972,13 +2617,23 @@ def no_type_check(arg): This mutates the function(s) or class(es) in place. """ if isinstance(arg, type): - arg_attrs = arg.__dict__.copy() - for attr, val in arg.__dict__.items(): - if val in arg.__bases__ + (arg,): - arg_attrs.pop(attr) - for obj in arg_attrs.values(): + for key in dir(arg): + obj = getattr(arg, key) + if ( + not hasattr(obj, '__qualname__') + or obj.__qualname__ != f'{arg.__qualname__}.{obj.__name__}' + or getattr(obj, '__module__', None) != arg.__module__ + ): + # We only modify objects that are defined in this type directly. + # If classes / methods are nested in multiple layers, + # we will modify them when processing their direct holders. + continue + # Instance, class, and static methods: if isinstance(obj, types.FunctionType): obj.__no_type_check__ = True + if isinstance(obj, types.MethodType): + obj.__func__.__no_type_check__ = True + # Nested types: if isinstance(obj, type): no_type_check(obj) try: @@ -1994,7 +2649,8 @@ def no_type_check_decorator(decorator): This wraps the decorator with something that wraps the decorated function in @no_type_check. """ - + import warnings + warnings._deprecated("typing.no_type_check_decorator", remove=(3, 15)) @functools.wraps(decorator) def wrapped_decorator(*args, **kwds): func = decorator(*args, **kwds) @@ -2013,63 +2669,107 @@ def _overload_dummy(*args, **kwds): "by an implementation that is not @overload-ed.") +# {module: {qualname: {firstlineno: func}}} +_overload_registry = defaultdict(functools.partial(defaultdict, dict)) + + def overload(func): """Decorator for overloaded functions/methods. In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. For example: + function in a row, each decorated with @overload. + + For example:: - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... In a non-stub file (i.e. a regular .py file), do the same but follow it with an implementation. The implementation should *not* - be decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - def utf8(value): - # implementation goes here + be decorated with @overload:: + + @overload + def utf8(value: None) -> None: ... + @overload + def utf8(value: bytes) -> bytes: ... + @overload + def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here + + The overloads for a function can be retrieved at runtime using the + get_overloads() function. """ + # classmethod and staticmethod + f = getattr(func, "__func__", func) + try: + _overload_registry[f.__module__][f.__qualname__][f.__code__.co_firstlineno] = func + except AttributeError: + # Not a normal function; ignore. + pass return _overload_dummy +def get_overloads(func): + """Return all defined overloads for *func* as a sequence.""" + # classmethod and staticmethod + f = getattr(func, "__func__", func) + if f.__module__ not in _overload_registry: + return [] + mod_dict = _overload_registry[f.__module__] + if f.__qualname__ not in mod_dict: + return [] + return list(mod_dict[f.__qualname__].values()) + + +def clear_overloads(): + """Clear all overloads in the registry.""" + _overload_registry.clear() + + def final(f): - """A decorator to indicate final methods and final classes. + """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated method cannot be overridden, and decorated class cannot be subclassed. - For example: - - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker + + For example:: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker ... - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... - There is no runtime checking of these properties. + There is no runtime checking of these properties. The decorator + attempts to set the ``__final__`` attribute to ``True`` on the decorated + object to allow runtime introspection. """ + try: + f.__final__ = True + except (AttributeError, TypeError): + # Skip the attribute silently if it is not writable. + # AttributeError happens if the object has __slots__ or a + # read-only property, TypeError if it's a builtin class. + pass return f -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) +# Some unconstrained type variables. These were initially used by the container types. +# They were never meant for export and are now unused, but we keep them around to +# avoid breaking compatibility with users who import them. T = TypeVar('T') # Any type. KT = TypeVar('KT') # Key type. VT = TypeVar('VT') # Value type. @@ -2080,6 +2780,7 @@ class Other(Leaf): # Error reported by type checker # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) + # A useful type variable with constraints. This represents string types. # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) @@ -2101,13 +2802,17 @@ class Other(Leaf): # Error reported by type checker Collection = _alias(collections.abc.Collection, 1) Callable = _CallableType(collections.abc.Callable, 2) Callable.__doc__ = \ - """Callable type; Callable[[int], str] is a function of (int) -> str. + """Deprecated alias to collections.abc.Callable. + + Callable[[int], str] signifies a function that takes a single + parameter of type int and returns a str. The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or ellipsis; the return type must be a single type. + values: the argument list and the return type. + The argument list must be a list of types, a ParamSpec, + Concatenate or ellipsis. The return type must be a single type. - There is no syntax to indicate optional or keyword arguments, + There is no syntax to indicate optional or keyword arguments; such function types are rarely used as callback types. """ AbstractSet = _alias(collections.abc.Set, 1, name='AbstractSet') @@ -2117,11 +2822,15 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _alias(collections.abc.ByteString, 0) # Not generic +ByteString = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 14) # Not generic. +) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ - """Tuple type; Tuple[X, Y] is the cross-product type of X and Y. + """Deprecated alias to builtins.tuple. + + Tuple[X, Y] is the cross-product type of X and Y. Example: Tuple[T1, T2] is a tuple of two elements corresponding to type variables T1 and T2. Tuple[int, float, str] is a tuple @@ -2137,41 +2846,34 @@ class Other(Leaf): # Error reported by type checker KeysView = _alias(collections.abc.KeysView, 1) ItemsView = _alias(collections.abc.ItemsView, 2) ValuesView = _alias(collections.abc.ValuesView, 1) -try: - # XXX: RUSTPYTHON; contextlib support for wasm - import contextlib - ContextManager = _alias(contextlib.AbstractContextManager, 1, name='ContextManager') - AsyncContextManager = _alias(contextlib.AbstractAsyncContextManager, 1, name='AsyncContextManager') -except ImportError: - pass Dict = _alias(dict, 2, inst=False, name='Dict') DefaultDict = _alias(collections.defaultdict, 2, name='DefaultDict') OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ - """A special construct usable to annotate class objects. + """Deprecated alias to builtins.type. + builtins.type or typing.Type can be used to annotate class objects. For example, suppose we have the following classes:: - class User: ... # Abstract base for User classes - class BasicUser(User): ... - class ProUser(User): ... - class TeamUser(User): ... + class User: ... # Abstract base for User classes + class BasicUser(User): ... + class ProUser(User): ... + class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - U = TypeVar('U', bound=User) - def new_user(user_class: Type[U]) -> U: - user = user_class() - # (Here we could write the user object to a database) - return user + def new_user[U](user_class: Type[U]) -> U: + user = user_class() + # (Here we could write the user object to a database) + return user - joe = new_user(BasicUser) + joe = new_user(BasicUser) At this point the type checker knows that joe has type BasicUser. """ @@ -2180,6 +2882,7 @@ def new_user(user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): """An ABC with one abstract method __int__.""" + __slots__ = () @abstractmethod @@ -2190,6 +2893,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): """An ABC with one abstract method __float__.""" + __slots__ = () @abstractmethod @@ -2200,6 +2904,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): """An ABC with one abstract method __complex__.""" + __slots__ = () @abstractmethod @@ -2210,6 +2915,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): """An ABC with one abstract method __bytes__.""" + __slots__ = () @abstractmethod @@ -2220,6 +2926,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): """An ABC with one abstract method __index__.""" + __slots__ = () @abstractmethod @@ -2228,22 +2935,24 @@ def __index__(self) -> int: @runtime_checkable -class SupportsAbs(Protocol[T_co]): +class SupportsAbs[T](Protocol): """An ABC with one abstract method __abs__ that is covariant in its return type.""" + __slots__ = () @abstractmethod - def __abs__(self) -> T_co: + def __abs__(self) -> T: pass @runtime_checkable -class SupportsRound(Protocol[T_co]): +class SupportsRound[T](Protocol): """An ABC with one abstract method __round__ that is covariant in its return type.""" + __slots__ = () @abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: + def __round__(self, ndigits: int = 0) -> T: pass @@ -2266,9 +2975,13 @@ def _make_nmtuple(name, types, module, defaults = ()): class NamedTupleMeta(type): - def __new__(cls, typename, bases, ns): - assert bases[0] is _NamedTuple + assert _NamedTuple in bases + for base in bases: + if base is not _NamedTuple and base is not Generic: + raise TypeError( + 'can only inherit from a NamedTuple type and Generic') + bases = tuple(tuple if base is _NamedTuple else base for base in bases) types = ns.get('__annotations__', {}) default_names = [] for field_name in types: @@ -2282,19 +2995,40 @@ def __new__(cls, typename, bases, ns): nm_tpl = _make_nmtuple(typename, types.items(), defaults=[ns[n] for n in default_names], module=ns['__module__']) + nm_tpl.__bases__ = bases + if Generic in bases: + class_getitem = _generic_class_getitem + nm_tpl.__class_getitem__ = classmethod(class_getitem) # update from user namespace without overriding special namedtuple attributes - for key in ns: + for key, val in ns.items(): if key in _prohibited: raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special and key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) + elif key not in _special: + if key not in nm_tpl._fields: + setattr(nm_tpl, key, val) + try: + set_name = type(val).__set_name__ + except AttributeError: + pass + else: + try: + set_name(val, nm_tpl, key) + except BaseException as e: + e.add_note( + f"Error calling __set_name__ on {type(val).__name__!r} " + f"instance {key!r} in {typename!r}" + ) + raise + + if Generic in bases: + nm_tpl.__init_subclass__() return nm_tpl -def NamedTuple(typename, fields=None, /, **kwargs): +def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. - Usage in Python versions >= 3.6:: + Usage:: class Employee(NamedTuple): name: str @@ -2307,39 +3041,86 @@ class Employee(NamedTuple): The resulting class has an extra __annotations__ attribute, giving a dict that maps field names to types. (The field names are also in the _fields attribute, which is part of the namedtuple API.) - Alternative equivalent keyword syntax is also accepted:: - - Employee = NamedTuple('Employee', name=str, id=int) - - In Python versions <= 3.5 use:: + An alternative equivalent functional syntax is also accepted:: Employee = NamedTuple('Employee', [('name', str), ('id', int)]) """ - if fields is None: - fields = kwargs.items() + if fields is _sentinel: + if kwargs: + deprecated_thing = "Creating NamedTuple classes using keyword arguments" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "Use the class-based or functional syntax instead." + ) + else: + deprecated_thing = "Failing to pass a value for the 'fields' parameter" + example = f"`{typename} = NamedTuple({typename!r}, [])`" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. " + ) + example + "." + elif fields is None: + if kwargs: + raise TypeError( + "Cannot pass `None` as the 'fields' parameter " + "and also specify fields using keyword arguments" + ) + else: + deprecated_thing = "Passing `None` as the 'fields' parameter" + example = f"`{typename} = NamedTuple({typename!r}, [])`" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "To create a NamedTuple class with 0 fields " + "using the functional syntax, " + "pass an empty list, e.g. " + ) + example + "." elif kwargs: raise TypeError("Either list of fields or keywords" " can be provided to NamedTuple, not both") - try: - module = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - module = None - return _make_nmtuple(typename, fields, module=module) + if fields is _sentinel or fields is None: + import warnings + warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) + fields = kwargs.items() + nt = _make_nmtuple(typename, fields, module=_caller()) + nt.__orig_bases__ = (NamedTuple,) + return nt _NamedTuple = type.__new__(NamedTupleMeta, 'NamedTuple', (), {}) def _namedtuple_mro_entries(bases): - if len(bases) > 1: - raise TypeError("Multiple inheritance with NamedTuple is not supported") - assert bases[0] is NamedTuple + assert NamedTuple in bases return (_NamedTuple,) NamedTuple.__mro_entries__ = _namedtuple_mro_entries +def _get_typeddict_qualifiers(annotation_type): + while True: + annotation_origin = get_origin(annotation_type) + if annotation_origin is Annotated: + annotation_args = get_args(annotation_type) + if annotation_args: + annotation_type = annotation_args[0] + else: + break + elif annotation_origin is Required: + yield Required + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is NotRequired: + yield NotRequired + (annotation_type,) = get_args(annotation_type) + elif annotation_origin is ReadOnly: + yield ReadOnly + (annotation_type,) = get_args(annotation_type) + else: + break + + class _TypedDictMeta(type): def __new__(cls, name, bases, ns, total=True): - """Create new typed dict class object. + """Create a new typed dict class object. This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way @@ -2347,14 +3128,22 @@ def __new__(cls, name, bases, ns, total=True): Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: - if type(base) is not _TypedDictMeta: + if type(base) is not _TypedDictMeta and base is not Generic: raise TypeError('cannot inherit from both a TypedDict type ' 'and a non-TypedDict base class') - tp_dict = type.__new__(_TypedDictMeta, name, (dict,), ns) + + if any(issubclass(b, Generic) for b in bases): + generic_base = (Generic,) + else: + generic_base = () + + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) + + if not hasattr(tp_dict, '__orig_bases__'): + tp_dict.__orig_bases__ = bases annotations = {} own_annotations = ns.get('__annotations__', {}) - own_annotation_keys = set(own_annotations.keys()) msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) @@ -2362,23 +3151,61 @@ def __new__(cls, name, bases, ns, total=True): } required_keys = set() optional_keys = set() + readonly_keys = set() + mutable_keys = set() for base in bases: annotations.update(base.__dict__.get('__annotations__', {})) - required_keys.update(base.__dict__.get('__required_keys__', ())) - optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + base_required = base.__dict__.get('__required_keys__', set()) + required_keys |= base_required + optional_keys -= base_required + + base_optional = base.__dict__.get('__optional_keys__', set()) + required_keys -= base_optional + optional_keys |= base_optional + + readonly_keys.update(base.__dict__.get('__readonly_keys__', ())) + mutable_keys.update(base.__dict__.get('__mutable_keys__', ())) annotations.update(own_annotations) - if total: - required_keys.update(own_annotation_keys) - else: - optional_keys.update(own_annotation_keys) + for annotation_key, annotation_type in own_annotations.items(): + qualifiers = set(_get_typeddict_qualifiers(annotation_type)) + if Required in qualifiers: + is_required = True + elif NotRequired in qualifiers: + is_required = False + else: + is_required = total + if is_required: + required_keys.add(annotation_key) + optional_keys.discard(annotation_key) + else: + optional_keys.add(annotation_key) + required_keys.discard(annotation_key) + + if ReadOnly in qualifiers: + if annotation_key in mutable_keys: + raise TypeError( + f"Cannot override mutable key {annotation_key!r}" + " with read-only key" + ) + readonly_keys.add(annotation_key) + else: + mutable_keys.add(annotation_key) + readonly_keys.discard(annotation_key) + + assert required_keys.isdisjoint(optional_keys), ( + f"Required keys overlap with optional keys in {name}:" + f" {required_keys=}, {optional_keys=}" + ) tp_dict.__annotations__ = annotations tp_dict.__required_keys__ = frozenset(required_keys) tp_dict.__optional_keys__ = frozenset(optional_keys) - if not hasattr(tp_dict, '__total__'): - tp_dict.__total__ = total + tp_dict.__readonly_keys__ = frozenset(readonly_keys) + tp_dict.__mutable_keys__ = frozenset(mutable_keys) + tp_dict.__total__ = total return tp_dict __call__ = dict # static method @@ -2390,72 +3217,152 @@ def __subclasscheck__(cls, other): __instancecheck__ = __subclasscheck__ -def TypedDict(typename, fields=None, /, *, total=True, **kwargs): +def TypedDict(typename, fields=_sentinel, /, *, total=True): """A simple typed namespace. At runtime it is equivalent to a plain dict. - TypedDict creates a dictionary type that expects all of its + TypedDict creates a dictionary type such that a type checker will expect all instances to have a certain set of keys, where each key is associated with a value of a consistent type. This expectation - is not checked at runtime but is only enforced by type checkers. - Usage:: - - class Point2D(TypedDict): - x: int - y: int - label: str - - a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK - b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check - - assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') + is not checked at runtime. The type info can be accessed via the Point2D.__annotations__ dict, and the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. - TypedDict supports two additional equivalent forms:: + TypedDict supports an additional equivalent form:: - Point2D = TypedDict('Point2D', x=int, y=int, label=str) Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality. - Usage:: + to override this by specifying totality:: - class point2D(TypedDict, total=False): + class Point2D(TypedDict, total=False): x: int y: int - This means that a point2D TypedDict can have any of the keys omitted.A type + This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the class body be required. - The class syntax is only supported in Python 3.6+, while two other - syntax forms work for Python 2.7 and 3.2+ + The Required and NotRequired special forms can also be used to mark + individual keys as being required or not required:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + See PEP 655 for more details on Required and NotRequired. + + The ReadOnly special form can be used + to mark individual keys as immutable for type checkers:: + + class DatabaseUser(TypedDict): + id: ReadOnly[int] # the "id" key must not be modified + username: str # the "username" key can be changed + """ - if fields is None: - fields = kwargs - elif kwargs: - raise TypeError("TypedDict takes either a dict or keyword arguments," - " but not both") + if fields is _sentinel or fields is None: + import warnings + + if fields is _sentinel: + deprecated_thing = "Failing to pass a value for the 'fields' parameter" + else: + deprecated_thing = "Passing `None` as the 'fields' parameter" + + example = f"`{typename} = TypedDict({typename!r}, {{{{}}}})`" + deprecation_msg = ( + "{name} is deprecated and will be disallowed in Python {remove}. " + "To create a TypedDict class with 0 fields " + "using the functional syntax, " + "pass an empty dictionary, e.g. " + ) + example + "." + warnings._deprecated(deprecated_thing, message=deprecation_msg, remove=(3, 15)) + fields = {} ns = {'__annotations__': dict(fields)} - try: + module = _caller() + if module is not None: # Setting correct module is necessary to make typed dict classes pickleable. - ns['__module__'] = sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass + ns['__module__'] = module - return _TypedDictMeta(typename, (), ns, total=total) + td = _TypedDictMeta(typename, (), ns, total=total) + td.__orig_bases__ = (TypedDict,) + return td _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {}) TypedDict.__mro_entries__ = lambda bases: (_TypedDict,) +@_SpecialForm +def Required(self, parameters): + """Special typing construct to mark a TypedDict key as required. + + This is mainly useful for total=False TypedDicts. + + For example:: + + class Movie(TypedDict, total=False): + title: Required[str] + year: int + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + + There is no runtime checking that a required key is actually provided + when instantiating a related TypedDict. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + +@_SpecialForm +def NotRequired(self, parameters): + """Special typing construct to mark a TypedDict key as potentially missing. + + For example:: + + class Movie(TypedDict): + title: str + year: NotRequired[int] + + m = Movie( + title='The Matrix', # typechecker error if key is omitted + year=1999, + ) + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + +@_SpecialForm +def ReadOnly(self, parameters): + """A special typing construct to mark an item of a TypedDict as read-only. + + For example:: + + class Movie(TypedDict): + title: ReadOnly[str] + year: int + + def mutate_movie(m: Movie) -> None: + m["year"] = 1992 # allowed + m["title"] = "The Matrix" # typechecker error + + There is no runtime checking for this property. + """ + item = _type_check(parameters, f'{self._name} accepts only a single type.') + return _GenericAlias(self, (item,)) + + class NewType: - """NewType creates simple unique types with almost zero - runtime overhead. NewType(name, tp) is considered a subtype of tp + """NewType creates simple unique types with almost zero runtime overhead. + + NewType(name, tp) is considered a subtype of tp by static type checkers. At runtime, NewType(name, tp) returns - a dummy callable that simply returns its argument. Usage:: + a dummy callable that simply returns its argument. + + Usage:: UserId = NewType('UserId', int) @@ -2470,6 +3377,8 @@ def name_by_id(user_id: UserId) -> str: num = UserId(5) + 1 # type: int """ + __call__ = _idfunc + def __init__(self, name, tp): self.__qualname__ = name if '.' in name: @@ -2480,12 +3389,24 @@ def __init__(self, name, tp): if def_mod != 'typing': self.__module__ = def_mod + def __mro_entries__(self, bases): + # We defined __mro_entries__ to get a better error message + # if a user attempts to subclass a NewType instance. bpo-46170 + superclass_name = self.__name__ + + class Dummy: + def __init_subclass__(cls): + subclass_name = cls.__name__ + raise TypeError( + f"Cannot subclass an instance of NewType. Perhaps you were looking for: " + f"`{subclass_name} = NewType({subclass_name!r}, {superclass_name})`" + ) + + return (Dummy,) + def __repr__(self): return f'{self.__module__}.{self.__qualname__}' - def __call__(self, x): - return x - def __reduce__(self): return self.__qualname__ @@ -2652,28 +3573,205 @@ def __enter__(self) -> 'TextIO': pass -class io: - """Wrapper namespace for IO generic classes.""" +def reveal_type[T](obj: T, /) -> T: + """Ask a static type checker to reveal the inferred type of an expression. + + When a static type checker encounters a call to ``reveal_type()``, + it will emit the inferred type of the argument:: + + x: int = 1 + reveal_type(x) + + Running a static type checker (e.g., mypy) on this example + will produce output similar to 'Revealed type is "builtins.int"'. - __all__ = ['IO', 'TextIO', 'BinaryIO'] - IO = IO - TextIO = TextIO - BinaryIO = BinaryIO + At runtime, the function prints the runtime type of the + argument and returns the argument unchanged. + """ + print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr) + return obj + + +class _IdentityCallable(Protocol): + def __call__[T](self, arg: T, /) -> T: + ... + + +def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + frozen_default: bool = False, + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), + **kwargs: Any, +) -> _IdentityCallable: + """Decorator to mark an object as providing dataclass-like behaviour. + + The decorator can be applied to a function, class, or metaclass. + + Example usage with a decorator function:: + + @dataclass_transform() + def create_model[T](cls: type[T]) -> type[T]: + ... + return cls + + @create_model + class CustomerModel: + id: int + name: str + + On a base class:: + + @dataclass_transform() + class ModelBase: ... + + class CustomerModel(ModelBase): + id: int + name: str + + On a metaclass:: + + @dataclass_transform() + class ModelMeta(type): ... + + class ModelBase(metaclass=ModelMeta): ... + class CustomerModel(ModelBase): + id: int + name: str + + The ``CustomerModel`` classes defined above will + be treated by type checkers similarly to classes created with + ``@dataclasses.dataclass``. + For example, type checkers will assume these classes have + ``__init__`` methods that accept ``id`` and ``name``. + + The arguments to this decorator can be used to customize this behavior: + - ``eq_default`` indicates whether the ``eq`` parameter is assumed to be + ``True`` or ``False`` if it is omitted by the caller. + - ``order_default`` indicates whether the ``order`` parameter is + assumed to be True or False if it is omitted by the caller. + - ``kw_only_default`` indicates whether the ``kw_only`` parameter is + assumed to be True or False if it is omitted by the caller. + - ``frozen_default`` indicates whether the ``frozen`` parameter is + assumed to be True or False if it is omitted by the caller. + - ``field_specifiers`` specifies a static list of supported classes + or functions that describe fields, similar to ``dataclasses.field()``. + - Arbitrary other keyword arguments are accepted in order to allow for + possible future extensions. + + At runtime, this decorator records its arguments in the + ``__dataclass_transform__`` attribute on the decorated object. + It has no other runtime effect. + + See PEP 681 for more details. + """ + def decorator(cls_or_fn): + cls_or_fn.__dataclass_transform__ = { + "eq_default": eq_default, + "order_default": order_default, + "kw_only_default": kw_only_default, + "frozen_default": frozen_default, + "field_specifiers": field_specifiers, + "kwargs": kwargs, + } + return cls_or_fn + return decorator -io.__name__ = __name__ + '.io' -sys.modules[io.__name__] = io +# TODO: RUSTPYTHON + +# type _Func = Callable[..., Any] + + +# def override[F: _Func](method: F, /) -> F: +# """Indicate that a method is intended to override a method in a base class. +# +# Usage:: +# +# class Base: +# def method(self) -> None: +# pass +# +# class Child(Base): +# @override +# def method(self) -> None: +# super().method() +# +# When this decorator is applied to a method, the type checker will +# validate that it overrides a method or attribute with the same name on a +# base class. This helps prevent bugs that may occur when a base class is +# changed without an equivalent change to a child class. +# +# There is no runtime checking of this property. The decorator attempts to +# set the ``__override__`` attribute to ``True`` on the decorated object to +# allow runtime introspection. +# +# See PEP 698 for details. +# """ +# try: +# method.__override__ = True +# except (AttributeError, TypeError): +# # Skip the attribute silently if it is not writable. +# # AttributeError happens if the object has __slots__ or a +# # read-only property, TypeError if it's a builtin class. +# pass +# return method + + +def is_protocol(tp: type, /) -> bool: + """Return True if the given type is a Protocol. -Pattern = _alias(stdlib_re.Pattern, 1) -Match = _alias(stdlib_re.Match, 1) + Example:: -class re: - """Wrapper namespace for re type aliases.""" + >>> from typing import Protocol, is_protocol + >>> class P(Protocol): + ... def a(self) -> str: ... + ... b: int + >>> is_protocol(P) + True + >>> is_protocol(int) + False + """ + return ( + isinstance(tp, type) + and getattr(tp, '_is_protocol', False) + and tp != Protocol + ) + +def get_protocol_members(tp: type, /) -> frozenset[str]: + """Return the set of members defined in a Protocol. + Raise a TypeError for arguments that are not Protocols. + """ + if not is_protocol(tp): + raise TypeError(f'{tp!r} is not a Protocol') + return frozenset(tp.__protocol_attrs__) - __all__ = ['Pattern', 'Match'] - Pattern = Pattern - Match = Match +def __getattr__(attr): + """Improve the import time of the typing module. -re.__name__ = __name__ + '.re' -sys.modules[re.__name__] = re + Soft-deprecated objects which are costly to create + are only created on-demand here. + """ + if attr in {"Pattern", "Match"}: + import re + obj = _alias(getattr(re, attr), 1) + elif attr in {"ContextManager", "AsyncContextManager"}: + import contextlib + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) + elif attr == "_collect_parameters": + import warnings + + depr_message = ( + "The private _collect_parameters function is deprecated and will be" + " removed in a future version of Python. Any use of private functions" + " is discouraged and may break in the future." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=2) + obj = _collect_type_parameters + else: + raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") + globals()[attr] = obj + return obj diff --git a/vm/src/stdlib/typing.rs b/vm/src/stdlib/typing.rs index c266e811ca..d7008ee739 100644 --- a/vm/src/stdlib/typing.rs +++ b/vm/src/stdlib/typing.rs @@ -5,7 +5,7 @@ pub(crate) mod _typing { use crate::{ PyObjectRef, PyPayload, PyResult, VirtualMachine, builtins::{PyGenericAlias, PyTupleRef, PyTypeRef, pystr::AsPyStr}, - function::IntoFuncArgs, + function::{FuncArgs, IntoFuncArgs}, }; pub(crate) fn _call_typing_func_object<'a>( @@ -20,8 +20,10 @@ pub(crate) mod _typing { // func.call(args, vm) } - #[pyattr] - pub(crate) fn _idfunc(_vm: &VirtualMachine) {} + #[pyfunction] + pub(crate) fn _idfunc(args: FuncArgs, _vm: &VirtualMachine) -> PyObjectRef { + args.args[0].clone() + } #[pyattr] #[pyclass(name = "TypeVar")] @@ -51,6 +53,11 @@ pub(crate) mod _typing { Ok(vm.ctx.none()) } } + + #[pygetset(magic)] + fn name(&self) -> PyObjectRef { + self.name.clone() + } } pub(crate) fn make_typevar( @@ -77,15 +84,102 @@ pub(crate) mod _typing { #[allow(dead_code)] pub(crate) struct ParamSpec { name: PyObjectRef, + bound: Option, + default_value: Option, + evaluate_default: Option, + covariant: bool, + contravariant: bool, + infer_variance: bool, } #[pyclass(flags(BASETYPE))] - impl ParamSpec {} + impl ParamSpec { + #[pygetset(magic)] + fn name(&self) -> PyObjectRef { + self.name.clone() + } + + #[pygetset(magic)] + fn bound(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(bound) = self.bound.clone() { + return bound; + } + vm.ctx.none() + } + + #[pygetset(magic)] + fn covariant(&self) -> bool { + self.covariant + } + + #[pygetset(magic)] + fn contravariant(&self) -> bool { + self.contravariant + } + + #[pygetset(magic)] + fn infer_variance(&self) -> bool { + self.infer_variance + } + + #[pygetset(magic)] + fn default(&self, vm: &VirtualMachine) -> PyResult { + if let Some(default_value) = self.default_value.clone() { + return Ok(default_value); + } + // handle evaluate_default + if let Some(evaluate_default) = self.evaluate_default.clone() { + let default_value = vm.call_method(evaluate_default.as_ref(), "__call__", ())?; + return Ok(default_value); + } + // TODO: this isn't up to spec + Ok(vm.ctx.none()) + } + + #[pygetset] + fn evaluate_default(&self, vm: &VirtualMachine) -> PyObjectRef { + if let Some(evaluate_default) = self.evaluate_default.clone() { + return evaluate_default; + } + // TODO: default_value case + vm.ctx.none() + } + + #[pymethod(magic)] + fn reduce(&self) -> PyResult { + Ok(self.name.clone()) + } + + #[pymethod] + fn has_default(&self) -> PyResult { + // TODO: fix + Ok(self.evaluate_default.is_some() || self.default_value.is_some()) + } + } pub(crate) fn make_paramspec(name: PyObjectRef) -> ParamSpec { - ParamSpec { name } + ParamSpec { + name, + bound: None, + default_value: None, + evaluate_default: None, + covariant: false, + contravariant: false, + infer_variance: false, + } + } + + #[pyattr] + #[pyclass(name = "NoDefault")] + #[derive(Debug, PyPayload)] + #[allow(dead_code)] + pub(crate) struct NoDefault { + name: PyObjectRef, } + #[pyclass(flags(BASETYPE))] + impl NoDefault {} + #[pyattr] #[pyclass(name = "TypeVarTuple")] #[derive(Debug, PyPayload)] @@ -161,7 +255,6 @@ pub(crate) mod _typing { // fn as_mapping() -> &'static PyMappingMethods { // static AS_MAPPING: Lazy = Lazy::new(|| PyMappingMethods { // subscript: atomic_func!(|mapping, needle, vm| { - // println!("gigity"); // call_typing_func_object(vm, "_GenericAlias", (mapping.obj, needle)) // }), // ..PyMappingMethods::NOT_IMPLEMENTED From 392d1c04f63677ccbe93b88f919c3797fc71f38f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 27 Apr 2025 03:48:06 -0700 Subject: [PATCH 273/317] More overlapped implementation (#5748) --- stdlib/src/overlapped.rs | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/stdlib/src/overlapped.rs b/stdlib/src/overlapped.rs index 6fd8a1516d..85a391c753 100644 --- a/stdlib/src/overlapped.rs +++ b/stdlib/src/overlapped.rs @@ -297,6 +297,10 @@ mod _overlapped { } } + unsafe fn u64_to_handle(raw_ptr_value: u64) -> HANDLE { + raw_ptr_value as HANDLE + } + #[pyfunction] fn CreateIoCompletionPort( handle: isize, @@ -354,4 +358,56 @@ mod _overlapped { ]); Ok(value.into()) } + + #[pyfunction] + fn CreateEvent( + event_attributes: PyObjectRef, + manual_reset: bool, + initial_state: bool, + name: Option, + vm: &VirtualMachine, + ) -> PyResult { + if !vm.is_none(&event_attributes) { + return Err(vm.new_value_error("EventAttributes must be None".to_owned())); + } + + let name = match name { + Some(name) => { + let name = widestring::WideCString::from_str(&name).unwrap(); + name.as_ptr() + } + None => std::ptr::null(), + }; + let event = unsafe { + windows_sys::Win32::System::Threading::CreateEventW( + std::ptr::null(), + manual_reset as _, + initial_state as _, + name, + ) as isize + }; + if event == NULL { + return Err(errno_err(vm)); + } + Ok(event) + } + + #[pyfunction] + fn SetEvent(handle: u64, vm: &VirtualMachine) -> PyResult<()> { + let ret = unsafe { windows_sys::Win32::System::Threading::SetEvent(u64_to_handle(handle)) }; + if ret == 0 { + return Err(errno_err(vm)); + } + Ok(()) + } + + #[pyfunction] + fn ResetEvent(handle: u64, vm: &VirtualMachine) -> PyResult<()> { + let ret = + unsafe { windows_sys::Win32::System::Threading::ResetEvent(u64_to_handle(handle)) }; + if ret == 0 { + return Err(errno_err(vm)); + } + Ok(()) + } } From 5561b6ead4cd2f761541f85124821751435a0e2a Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 27 Apr 2025 20:20:42 -0700 Subject: [PATCH 274/317] Re-add ssl feature to cron-ci.yaml for whats-left (#5750) --- .github/workflows/cron-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index e7d14298cd..6389fee1cb 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -84,7 +84,7 @@ jobs: - name: Collect what is left data run: | chmod +x ./whats_left.py - ./whats_left.py --features "sqlite" > whats_left.temp + ./whats_left.py --features "ssl,sqlite" > whats_left.temp env: RUSTPYTHONPATH: ${{ github.workspace }}/Lib - name: Upload data to the website From ff10a647274ab621df5f014345d4416186116db0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Mon, 28 Apr 2025 12:29:21 +0900 Subject: [PATCH 275/317] Fix test_poll::test_poll3 (#5718) * test_poll3 * Refactor EventMask --- Lib/test/test_poll.py | 2 -- stdlib/src/select.rs | 60 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 338eb00f0c..a9bfb755c3 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -152,8 +152,6 @@ def test_poll2(self): else: self.fail('Unexpected return value from select.poll: %s' % fdlist) - # TODO: RUSTPYTHON int overflow - @unittest.expectedFailure def test_poll3(self): # test int overflow pollster = select.poll() diff --git a/stdlib/src/select.rs b/stdlib/src/select.rs index f600d60ede..1119f0cd9d 100644 --- a/stdlib/src/select.rs +++ b/stdlib/src/select.rs @@ -338,7 +338,10 @@ mod decl { }; use libc::pollfd; use num_traits::{Signed, ToPrimitive}; - use std::time::{Duration, Instant}; + use std::{ + convert::TryFrom, + time::{Duration, Instant}, + }; #[derive(Default)] pub(super) struct TimeoutArg(pub Option); @@ -417,25 +420,62 @@ mod decl { search(fds, fd).ok().map(|i| fds.remove(i)) } + // new EventMask type + #[derive(Copy, Clone)] + #[repr(transparent)] + pub struct EventMask(pub i16); + + impl TryFromObject for EventMask { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + use crate::builtins::PyInt; + let int = obj + .downcast::() + .map_err(|_| vm.new_type_error("argument must be an integer".to_owned()))?; + + let val = int.as_bigint(); + if val.is_negative() { + return Err(vm.new_value_error("negative event mask".to_owned())); + } + + // Try converting to i16, should raise OverflowError if too large + let mask = i16::try_from(val).map_err(|_| { + vm.new_overflow_error("event mask value out of range".to_owned()) + })?; + + Ok(EventMask(mask)) + } + } + const DEFAULT_EVENTS: i16 = libc::POLLIN | libc::POLLPRI | libc::POLLOUT; #[pyclass] impl PyPoll { #[pymethod] - fn register(&self, Fildes(fd): Fildes, eventmask: OptionalArg) { - insert_fd( - &mut self.fds.lock(), - fd, - eventmask.map_or(DEFAULT_EVENTS, |e| e as i16), - ) + fn register( + &self, + Fildes(fd): Fildes, + eventmask: OptionalArg, + ) -> PyResult<()> { + let mask = match eventmask { + OptionalArg::Present(event_mask) => event_mask.0, + OptionalArg::Missing => DEFAULT_EVENTS, + }; + insert_fd(&mut self.fds.lock(), fd, mask); + Ok(()) } #[pymethod] - fn modify(&self, Fildes(fd): Fildes, eventmask: u16) -> io::Result<()> { + fn modify( + &self, + Fildes(fd): Fildes, + eventmask: EventMask, + vm: &VirtualMachine, + ) -> PyResult<()> { let mut fds = self.fds.lock(); + // CPython raises KeyError if fd is not registered, match that behavior let pfd = get_fd_mut(&mut fds, fd) - .ok_or_else(|| io::Error::from_raw_os_error(libc::ENOENT))?; - pfd.events = eventmask as i16; + .ok_or_else(|| vm.new_key_error(vm.ctx.new_int(fd).into()))?; + pfd.events = eventmask.0; Ok(()) } From f55bf8f83bbe5e28091032014b77b2fa79115880 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 27 Apr 2025 20:30:14 -0700 Subject: [PATCH 276/317] fix openssl error reasons (#5739) --- Lib/test/test_httplib.py | 4 ---- stdlib/src/ssl.rs | 8 ++++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8f095d52ac..b73f081bb8 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1695,8 +1695,6 @@ def test_attributes(self): h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) self.assertEqual(h.timeout, 30) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_networked(self): # Default settings: requires a valid cert from a trusted CA import ssl @@ -1769,8 +1767,6 @@ def test_networked_good_cert(self): h.close() self.assertIn('nginx', server_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_networked_bad_cert(self): # We feed a "CA" cert that is unrelated to the server's cert import ssl diff --git a/stdlib/src/ssl.rs b/stdlib/src/ssl.rs index 6cc7f3ed32..16e6cf5b34 100644 --- a/stdlib/src/ssl.rs +++ b/stdlib/src/ssl.rs @@ -1183,8 +1183,12 @@ mod _ssl { let file = file .rsplit_once(&['/', '\\'][..]) .map_or(file, |(_, basename)| basename); - // TODO: map the error codes to code names, e.g. "CERTIFICATE_VERIFY_FAILED", just requires a big hashmap/dict - let errstr = e.reason().unwrap_or("unknown error"); + // TODO: finish map + let default_errstr = e.reason().unwrap_or("unknown error"); + let errstr = match default_errstr { + "certificate verify failed" => "CERTIFICATE_VERIFY_FAILED", + _ => default_errstr, + }; let msg = if let Some(lib) = e.library() { // add `library` attribute let attr_name = vm.ctx.as_ref().intern_str("library"); From b0991e28a25564d4b92f853dafa85b767ef9a237 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 22 Apr 2025 13:46:12 +0900 Subject: [PATCH 277/317] Replace puruspe to pymath Might be lower quality, but better compatibility --- .cspell.dict/rust-more.txt | 1 - Cargo.lock | 22 ++++++--------------- Cargo.toml | 1 + Lib/test/test_math.py | 3 ++- stdlib/Cargo.toml | 2 +- stdlib/src/math.rs | 39 +++++++++++++++++--------------------- 6 files changed, 27 insertions(+), 41 deletions(-) diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index e5d5ac86c4..6a98daa9db 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -51,7 +51,6 @@ powf powi prepended punct -puruspe replacen rmatch rposition diff --git a/Cargo.lock b/Cargo.lock index ffb31bfcbb..13ad0c4935 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1240,16 +1240,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "lambert_w" -version = "1.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4d9b9fa6582f5d77f954729c91c32a7c85834332e470b014d12e1678fd1793" -dependencies = [ - "num-complex", - "num-traits", -] - [[package]] name = "lazy_static" version = "0.2.11" @@ -1889,13 +1879,13 @@ dependencies = [ ] [[package]] -name = "puruspe" -version = "0.4.0" +name = "pymath" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76c522e44709f541a403db419a7e34d6fbbc8e6b208589ae29a030cddeefd96" +checksum = "c574544c9acbbb7cab157def68e24e7a417c4ff6f7e528a805160c8b6cc3aa9f" dependencies = [ - "lambert_w", - "num-complex", + "errno", + "libc", ] [[package]] @@ -2495,7 +2485,7 @@ dependencies = [ "page_size", "parking_lot", "paste", - "puruspe", + "pymath", "rand_core 0.9.3", "rustix", "rustpython-common", diff --git a/Cargo.toml b/Cargo.toml index c59344d2d5..2be9db7477 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -184,6 +184,7 @@ once_cell = "1.20.3" parking_lot = "0.12.3" paste = "1.0.15" proc-macro2 = "1.0.93" +pymath = "0.0.2" quote = "1.0.38" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index bb02041644..0e14fad774 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1374,6 +1374,7 @@ def test_sumprod_accuracy(self): self.assertEqual(sumprod([True, False] * 10, [0.1] * 20), 1.0) self.assertEqual(sumprod([1.0, 10E100, 1.0, -10E100], [1.0]*4), 2.0) + @unittest.skip("TODO: RUSTPYTHON, Taking a few minutes.") @support.requires_resource('cpu') def test_sumprod_stress(self): sumprod = math.sumprod @@ -2079,7 +2080,7 @@ def test_testfile(self): self.fail('Failures in test_testfile:\n ' + '\n '.join(failures)) - @unittest.skip("TODO: RUSTPYTHON, Currently hangs. Function never finishes.") + @unittest.skip("TODO: RUSTPYTHON, Taking a few minutes.") @requires_IEEE_754 def test_mtestfile(self): fail_fmt = "{}: {}({!r}): {}" diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index d29bd3b21e..7b6aba0a14 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -45,7 +45,7 @@ memchr = { workspace = true } base64 = "0.22" csv-core = "0.1.11" dyn-clone = "1.0.10" -puruspe = "0.4.0" +pymath = { workspace = true } xml-rs = "0.8.14" # random diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 4854bdd75e..9ab4bed5ce 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -1,5 +1,7 @@ pub(crate) use math::make_module; +use crate::{builtins::PyBaseExceptionRef, vm::VirtualMachine}; + #[pymodule] mod math { use crate::vm::{ @@ -17,6 +19,8 @@ mod math { // Constants #[pyattr] use std::f64::consts::{E as e, PI as pi, TAU as tau}; + + use super::pymath_error_to_exception; #[pyattr(name = "inf")] const INF: f64 = f64::INFINITY; #[pyattr(name = "nan")] @@ -475,38 +479,22 @@ mod math { // Special functions: #[pyfunction] fn erf(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_nan() { x } else { puruspe::erf(x) } + pymath::erf(*x) } #[pyfunction] fn erfc(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_nan() { x } else { puruspe::erfc(x) } + pymath::erfc(*x) } #[pyfunction] - fn gamma(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_finite() { - puruspe::gamma(x) - } else if x.is_nan() || x.is_sign_positive() { - x - } else { - f64::NAN - } + fn gamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult { + pymath::gamma(*x).map_err(|err| pymath_error_to_exception(err, vm)) } #[pyfunction] - fn lgamma(x: ArgIntoFloat) -> f64 { - let x = *x; - if x.is_finite() { - puruspe::ln_gamma(x) - } else if x.is_nan() { - x - } else { - f64::INFINITY - } + fn lgamma(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult { + pymath::lgamma(*x).map_err(|err| pymath_error_to_exception(err, vm)) } fn try_magic_method( @@ -1000,3 +988,10 @@ mod math { Ok(result) } } + +fn pymath_error_to_exception(err: pymath::Error, vm: &VirtualMachine) -> PyBaseExceptionRef { + match err { + pymath::Error::EDOM => vm.new_value_error("math domain error".to_owned()), + pymath::Error::ERANGE => vm.new_overflow_error("math range error".to_owned()), + } +} From d1b7dc551cfc07a20b5ed1fa8af2c8d866f01529 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 30 Apr 2025 14:12:25 +0900 Subject: [PATCH 278/317] skip flaky test_weakref --- Lib/test/test_weakref.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 751002bfc9..7d204f3c4c 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1997,6 +1997,7 @@ def test_threaded_weak_key_dict_deepcopy(self): # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) + @unittest.skip("TODO: RUSTPYTHON; occasionally crash (Exit code -6)") def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. From 5e770e9c9eb88e5634ef71139ef587894f66bd37 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 11:27:23 -0700 Subject: [PATCH 279/317] update getopt to 3.13.3 --- Lib/test/support/i18n_helper.py | 63 +++ Lib/test/test_getopt.py | 22 +- Lib/test/test_tools/__init__.py | 43 ++ Lib/test/test_tools/__main__.py | 4 + .../test_tools/i18n_data/ascii-escapes.pot | 45 ++ Lib/test/test_tools/i18n_data/docstrings.pot | 40 ++ Lib/test/test_tools/i18n_data/docstrings.py | 41 ++ Lib/test/test_tools/i18n_data/escapes.pot | 45 ++ Lib/test/test_tools/i18n_data/escapes.py | 23 + Lib/test/test_tools/i18n_data/fileloc.pot | 35 ++ Lib/test/test_tools/i18n_data/fileloc.py | 26 + Lib/test/test_tools/i18n_data/messages.pot | 67 +++ Lib/test/test_tools/i18n_data/messages.py | 64 +++ Lib/test/test_tools/msgfmt_data/fuzzy.json | 1 + Lib/test/test_tools/msgfmt_data/fuzzy.mo | Bin 0 -> 28 bytes Lib/test/test_tools/msgfmt_data/fuzzy.po | 23 + Lib/test/test_tools/msgfmt_data/general.json | 58 +++ Lib/test/test_tools/msgfmt_data/general.mo | Bin 0 -> 671 bytes Lib/test/test_tools/msgfmt_data/general.po | 47 ++ Lib/test/test_tools/test_freeze.py | 37 ++ Lib/test/test_tools/test_i18n.py | 444 ++++++++++++++++++ Lib/test/test_tools/test_makefile.py | 81 ++++ Lib/test/test_tools/test_makeunicodedata.py | 122 +++++ Lib/test/test_tools/test_msgfmt.py | 159 +++++++ Lib/test/test_tools/test_reindent.py | 35 ++ Lib/test/test_tools/test_sundry.py | 30 ++ 26 files changed, 1549 insertions(+), 6 deletions(-) create mode 100644 Lib/test/support/i18n_helper.py create mode 100644 Lib/test/test_tools/__init__.py create mode 100644 Lib/test/test_tools/__main__.py create mode 100644 Lib/test/test_tools/i18n_data/ascii-escapes.pot create mode 100644 Lib/test/test_tools/i18n_data/docstrings.pot create mode 100644 Lib/test/test_tools/i18n_data/docstrings.py create mode 100644 Lib/test/test_tools/i18n_data/escapes.pot create mode 100644 Lib/test/test_tools/i18n_data/escapes.py create mode 100644 Lib/test/test_tools/i18n_data/fileloc.pot create mode 100644 Lib/test/test_tools/i18n_data/fileloc.py create mode 100644 Lib/test/test_tools/i18n_data/messages.pot create mode 100644 Lib/test/test_tools/i18n_data/messages.py create mode 100644 Lib/test/test_tools/msgfmt_data/fuzzy.json create mode 100644 Lib/test/test_tools/msgfmt_data/fuzzy.mo create mode 100644 Lib/test/test_tools/msgfmt_data/fuzzy.po create mode 100644 Lib/test/test_tools/msgfmt_data/general.json create mode 100644 Lib/test/test_tools/msgfmt_data/general.mo create mode 100644 Lib/test/test_tools/msgfmt_data/general.po create mode 100644 Lib/test/test_tools/test_freeze.py create mode 100644 Lib/test/test_tools/test_i18n.py create mode 100644 Lib/test/test_tools/test_makefile.py create mode 100644 Lib/test/test_tools/test_makeunicodedata.py create mode 100644 Lib/test/test_tools/test_msgfmt.py create mode 100644 Lib/test/test_tools/test_reindent.py create mode 100644 Lib/test/test_tools/test_sundry.py diff --git a/Lib/test/support/i18n_helper.py b/Lib/test/support/i18n_helper.py new file mode 100644 index 0000000000..2e304f29e8 --- /dev/null +++ b/Lib/test/support/i18n_helper.py @@ -0,0 +1,63 @@ +import re +import subprocess +import sys +import unittest +from pathlib import Path +from test.support import REPO_ROOT, TEST_HOME_DIR, requires_subprocess +from test.test_tools import skip_if_missing + + +pygettext = Path(REPO_ROOT) / 'Tools' / 'i18n' / 'pygettext.py' + +msgid_pattern = re.compile(r'msgid(.*?)(?:msgid_plural|msgctxt|msgstr)', + re.DOTALL) +msgid_string_pattern = re.compile(r'"((?:\\"|[^"])*)"') + + +def _generate_po_file(path, *, stdout_only=True): + res = subprocess.run([sys.executable, pygettext, + '--no-location', '-o', '-', path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + if stdout_only: + return res.stdout + return res + + +def _extract_msgids(po): + msgids = [] + for msgid in msgid_pattern.findall(po): + msgid_string = ''.join(msgid_string_pattern.findall(msgid)) + msgid_string = msgid_string.replace(r'\"', '"') + if msgid_string: + msgids.append(msgid_string) + return sorted(msgids) + + +def _get_snapshot_path(module_name): + return Path(TEST_HOME_DIR) / 'translationdata' / module_name / 'msgids.txt' + + +@requires_subprocess() +class TestTranslationsBase(unittest.TestCase): + + def assertMsgidsEqual(self, module): + '''Assert that msgids extracted from a given module match a + snapshot. + + ''' + skip_if_missing('i18n') + res = _generate_po_file(module.__file__, stdout_only=False) + self.assertEqual(res.returncode, 0) + self.assertEqual(res.stderr, '') + msgids = _extract_msgids(res.stdout) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot = snapshot_path.read_text().splitlines() + self.assertListEqual(msgids, snapshot) + + +def update_translation_snapshots(module): + contents = _generate_po_file(module.__file__) + msgids = _extract_msgids(contents) + snapshot_path = _get_snapshot_path(module.__name__) + snapshot_path.write_text('\n'.join(msgids)) diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index c8b3442de4..295a2c8136 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -1,19 +1,19 @@ # test_getopt.py # David Goodger 2000-08-19 -from test.support.os_helper import EnvironmentVarGuard import doctest -import unittest - import getopt +import sys +import unittest +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots +from test.support.os_helper import EnvironmentVarGuard sentinel = object() class GetoptTests(unittest.TestCase): def setUp(self): self.env = self.enterContext(EnvironmentVarGuard()) - if "POSIXLY_CORRECT" in self.env: - del self.env["POSIXLY_CORRECT"] + del self.env["POSIXLY_CORRECT"] def assertError(self, *args, **kwargs): self.assertRaises(getopt.GetoptError, *args, **kwargs) @@ -173,10 +173,20 @@ def test_libref_examples(): ['a1', 'a2'] """ + +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(getopt) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite()) return tests -if __name__ == "__main__": +if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(getopt) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py new file mode 100644 index 0000000000..c4395c7c0a --- /dev/null +++ b/Lib/test/test_tools/__init__.py @@ -0,0 +1,43 @@ +"""Support functions for testing scripts in the Tools directory.""" +import contextlib +import importlib +import os.path +import unittest +from test import support +from test.support import import_helper + + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + +basepath = os.path.normpath( + os.path.dirname( # + os.path.dirname( # Lib + os.path.dirname( # test + os.path.dirname(__file__))))) # test_tools + +toolsdir = os.path.join(basepath, 'Tools') +scriptsdir = os.path.join(toolsdir, 'scripts') + +def skip_if_missing(tool=None): + if tool: + tooldir = os.path.join(toolsdir, tool) + else: + tool = 'scripts' + tooldir = scriptsdir + if not os.path.isdir(tooldir): + raise unittest.SkipTest(f'{tool} directory could not be found') + +@contextlib.contextmanager +def imports_under_tool(name, *subdirs): + tooldir = os.path.join(toolsdir, name, *subdirs) + with import_helper.DirsOnSysPath(tooldir) as cm: + yield cm + +def import_tool(toolname): + with import_helper.DirsOnSysPath(scriptsdir): + return importlib.import_module(toolname) + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/__main__.py b/Lib/test/test_tools/__main__.py new file mode 100644 index 0000000000..b6f13e534e --- /dev/null +++ b/Lib/test/test_tools/__main__.py @@ -0,0 +1,4 @@ +from test.test_tools import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_tools/i18n_data/ascii-escapes.pot b/Lib/test/test_tools/i18n_data/ascii-escapes.pot new file mode 100644 index 0000000000..18d868b6a2 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/ascii-escapes.pot @@ -0,0 +1,45 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: escapes.py:5 +msgid "" +"\"\t\n" +"\r\\" +msgstr "" + +#: escapes.py:8 +msgid "" +"\000\001\002\003\004\005\006\007\010\t\n" +"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" +msgstr "" + +#: escapes.py:13 +msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" +msgstr "" + +#: escapes.py:17 +msgid "\177" +msgstr "" + +#: escapes.py:20 +msgid "€   ÿ" +msgstr "" + +#: escapes.py:23 +msgid "α ㄱ 𓂀" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.pot b/Lib/test/test_tools/i18n_data/docstrings.pot new file mode 100644 index 0000000000..5af1d41422 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.pot @@ -0,0 +1,40 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: docstrings.py:7 +#, docstring +msgid "" +msgstr "" + +#: docstrings.py:18 +#, docstring +msgid "" +"multiline\n" +" docstring\n" +" " +msgstr "" + +#: docstrings.py:25 +#, docstring +msgid "docstring1" +msgstr "" + +#: docstrings.py:30 +#, docstring +msgid "Hello, {}!" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/docstrings.py b/Lib/test/test_tools/i18n_data/docstrings.py new file mode 100644 index 0000000000..85d7f159d3 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/docstrings.py @@ -0,0 +1,41 @@ +# Test docstring extraction +from gettext import gettext as _ + + +# Empty docstring +def test(x): + """""" + + +# Leading empty line +def test2(x): + + """docstring""" # XXX This should be extracted but isn't. + + +# XXX Multiline docstrings should be cleaned with `inspect.cleandoc`. +def test3(x): + """multiline + docstring + """ + + +# Multiple docstrings - only the first should be extracted +def test4(x): + """docstring1""" + """docstring2""" + + +def test5(x): + """Hello, {}!""".format("world!") # XXX This should not be extracted. + + +# Nested docstrings +def test6(x): + def inner(y): + """nested docstring""" # XXX This should be extracted but isn't. + + +class Outer: + class Inner: + "nested class docstring" # XXX This should be extracted but isn't. diff --git a/Lib/test/test_tools/i18n_data/escapes.pot b/Lib/test/test_tools/i18n_data/escapes.pot new file mode 100644 index 0000000000..2c7899d59d --- /dev/null +++ b/Lib/test/test_tools/i18n_data/escapes.pot @@ -0,0 +1,45 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: escapes.py:5 +msgid "" +"\"\t\n" +"\r\\" +msgstr "" + +#: escapes.py:8 +msgid "" +"\000\001\002\003\004\005\006\007\010\t\n" +"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037" +msgstr "" + +#: escapes.py:13 +msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" +msgstr "" + +#: escapes.py:17 +msgid "\177" +msgstr "" + +#: escapes.py:20 +msgid "\302\200 \302\240 \303\277" +msgstr "" + +#: escapes.py:23 +msgid "\316\261 \343\204\261 \360\223\202\200" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/escapes.py b/Lib/test/test_tools/i18n_data/escapes.py new file mode 100644 index 0000000000..900bd97a70 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/escapes.py @@ -0,0 +1,23 @@ +import gettext as _ + + +# Special characters that are always escaped in the POT file +_('"\t\n\r\\') + +# All ascii characters 0-31 +_('\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n' + '\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15' + '\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f') + +# All ascii characters 32-126 +_(' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ' + '[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~') + +# ascii char 127 +_('\x7f') + +# some characters in the 128-255 range +_('\x80 \xa0 ÿ') + +# some characters >= 256 encoded as 2, 3 and 4 bytes, respectively +_('α ㄱ 𓂀') diff --git a/Lib/test/test_tools/i18n_data/fileloc.pot b/Lib/test/test_tools/i18n_data/fileloc.pot new file mode 100644 index 0000000000..dbd28687a7 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.pot @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: fileloc.py:5 fileloc.py:6 +msgid "foo" +msgstr "" + +#: fileloc.py:9 +msgid "bar" +msgstr "" + +#: fileloc.py:14 fileloc.py:18 +#, docstring +msgid "docstring" +msgstr "" + +#: fileloc.py:22 fileloc.py:26 +#, docstring +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/fileloc.py b/Lib/test/test_tools/i18n_data/fileloc.py new file mode 100644 index 0000000000..c5d4d0595f --- /dev/null +++ b/Lib/test/test_tools/i18n_data/fileloc.py @@ -0,0 +1,26 @@ +# Test file locations +from gettext import gettext as _ + +# Duplicate strings +_('foo') +_('foo') + +# Duplicate strings on the same line should only add one location to the output +_('bar'), _('bar') + + +# Duplicate docstrings +class A: + """docstring""" + + +def f(): + """docstring""" + + +# Duplicate message and docstring +_('baz') + + +def g(): + """baz""" diff --git a/Lib/test/test_tools/i18n_data/messages.pot b/Lib/test/test_tools/i18n_data/messages.pot new file mode 100644 index 0000000000..ddfbd18349 --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.pot @@ -0,0 +1,67 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2000-01-01 00:00+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: pygettext.py 1.5\n" + + +#: messages.py:5 +msgid "" +msgstr "" + +#: messages.py:8 messages.py:9 +msgid "parentheses" +msgstr "" + +#: messages.py:12 +msgid "Hello, world!" +msgstr "" + +#: messages.py:15 +msgid "" +"Hello,\n" +" multiline!\n" +msgstr "" + +#: messages.py:29 +msgid "Hello, {}!" +msgstr "" + +#: messages.py:33 +msgid "1" +msgstr "" + +#: messages.py:33 +msgid "2" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "A" +msgstr "" + +#: messages.py:34 messages.py:35 +msgid "B" +msgstr "" + +#: messages.py:36 +msgid "set" +msgstr "" + +#: messages.py:42 +msgid "nested string" +msgstr "" + +#: messages.py:47 +msgid "baz" +msgstr "" + diff --git a/Lib/test/test_tools/i18n_data/messages.py b/Lib/test/test_tools/i18n_data/messages.py new file mode 100644 index 0000000000..f220294b8d --- /dev/null +++ b/Lib/test/test_tools/i18n_data/messages.py @@ -0,0 +1,64 @@ +# Test message extraction +from gettext import gettext as _ + +# Empty string +_("") + +# Extra parentheses +(_("parentheses")) +((_("parentheses"))) + +# Multiline strings +_("Hello, " + "world!") + +_("""Hello, + multiline! +""") + +# Invalid arguments +_() +_(None) +_(1) +_(False) +_(x="kwargs are not allowed") +_("foo", "bar") +_("something", x="something else") + +# .format() +_("Hello, {}!").format("world") # valid +_("Hello, {}!".format("world")) # invalid + +# Nested structures +_("1"), _("2") +arr = [_("A"), _("B")] +obj = {'a': _("A"), 'b': _("B")} +{{{_('set')}}} + + +# Nested functions and classes +def test(): + _("nested string") # XXX This should be extracted but isn't. + [_("nested string")] + + +class Foo: + def bar(self): + return _("baz") + + +def bar(x=_('default value')): # XXX This should be extracted but isn't. + pass + + +def baz(x=[_('default value')]): # XXX This should be extracted but isn't. + pass + + +# Shadowing _() +def _(x): + pass + + +def _(x="don't extract me"): + pass diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.json b/Lib/test/test_tools/msgfmt_data/fuzzy.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/fuzzy.json @@ -0,0 +1 @@ +[] diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.mo b/Lib/test/test_tools/msgfmt_data/fuzzy.mo new file mode 100644 index 0000000000000000000000000000000000000000..4b144831cf5f75c62f6e9446b91ce164bb3c8dee GIT binary patch literal 28 Tcmca7#4?ou3S@vZ2*UsXEKdQ# literal 0 HcmV?d00001 diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.po b/Lib/test/test_tools/msgfmt_data/fuzzy.po new file mode 100644 index 0000000000..05e8354948 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/fuzzy.po @@ -0,0 +1,23 @@ +# Fuzzy translations are not written to the .mo file. +#, fuzzy +msgid "foo" +msgstr "bar" + +# comment +#, fuzzy +msgctxt "abc" +msgid "foo" +msgstr "bar" + +#, fuzzy +# comment +msgctxt "xyz" +msgid "foo" +msgstr "bar" + +#, fuzzy +msgctxt "abc" +msgid "One email sent." +msgid_plural "%d emails sent." +msgstr[0] "One email sent." +msgstr[1] "%d emails sent." diff --git a/Lib/test/test_tools/msgfmt_data/general.json b/Lib/test/test_tools/msgfmt_data/general.json new file mode 100644 index 0000000000..0586113985 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/general.json @@ -0,0 +1,58 @@ +[ + [ + "", + "Project-Id-Version: PACKAGE VERSION\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME \nLanguage-Team: LANGUAGE \nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n" + ], + [ + "\n newlines \n", + "\n translated \n" + ], + [ + "\"escapes\"", + "\"translated\"" + ], + [ + "Multilinestring", + "Multilinetranslation" + ], + [ + "abc\u0004foo", + "bar" + ], + [ + "bar", + "baz" + ], + [ + "xyz\u0004foo", + "bar" + ], + [ + [ + "One email sent.", + 0 + ], + "One email sent." + ], + [ + [ + "One email sent.", + 1 + ], + "%d emails sent." + ], + [ + [ + "abc\u0004One email sent.", + 0 + ], + "One email sent." + ], + [ + [ + "abc\u0004One email sent.", + 1 + ], + "%d emails sent." + ] +] diff --git a/Lib/test/test_tools/msgfmt_data/general.mo b/Lib/test/test_tools/msgfmt_data/general.mo new file mode 100644 index 0000000000000000000000000000000000000000..ee905cbb3ec58db04c2f2fca5f67e15ce7a852c9 GIT binary patch literal 671 zcma)&&rjPh6vw@-(zIw|(zL@4ld#G-F?ZvJGN_1%3lT|E^#{^+bCX9g3QlDkX!~zW z(+>O#@DFh2PvE@2V%+ykGqpEn>7$RI^WwaGf4{B$B)DE7ACTM?OY0i~ng=yS*rFd!3YZ;X=BHEEE=e`i%!&mZU5U z=t1fy(Z=_*&AnI{%lcXBShv7g6EO#{Ou>>e=2$x`B2OrqyGyTTCMK@ literal 0 HcmV?d00001 diff --git a/Lib/test/test_tools/msgfmt_data/general.po b/Lib/test/test_tools/msgfmt_data/general.po new file mode 100644 index 0000000000..8f84042682 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/general.po @@ -0,0 +1,47 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"POT-Creation-Date: 2024-10-26 18:06+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "foo" +msgstr "" + +msgid "bar" +msgstr "baz" + +msgctxt "abc" +msgid "foo" +msgstr "bar" + +# comment +msgctxt "xyz" +msgid "foo" +msgstr "bar" + +msgid "Multiline" +"string" +msgstr "Multiline" +"translation" + +msgid "\"escapes\"" +msgstr "\"translated\"" + +msgid "\n newlines \n" +msgstr "\n translated \n" + +msgid "One email sent." +msgid_plural "%d emails sent." +msgstr[0] "One email sent." +msgstr[1] "%d emails sent." + +msgctxt "abc" +msgid "One email sent." +msgid_plural "%d emails sent." +msgstr[0] "One email sent." +msgstr[1] "%d emails sent." diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py new file mode 100644 index 0000000000..0e7ed67de7 --- /dev/null +++ b/Lib/test/test_tools/test_freeze.py @@ -0,0 +1,37 @@ +"""Sanity-check tests for the "freeze" tool.""" + +import sys +import textwrap +import unittest + +from test import support +from test.support import os_helper +from test.test_tools import imports_under_tool, skip_if_missing + +skip_if_missing('freeze') +with imports_under_tool('freeze', 'test'): + import freeze as helper + +@support.requires_zlib() +@unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows') +@unittest.skipIf(sys.platform == 'darwin' and sys._framework, + 'not supported for frameworks builds on macOS') +@support.skip_if_buildbot('not all buildbots have enough space') +# gh-103053: Skip test if Python is built with Profile Guided Optimization +# (PGO), since the test is just too slow in this case. +@unittest.skipIf(support.check_cflags_pgo(), + 'test is too slow with PGO') +class TestFreeze(unittest.TestCase): + + @support.requires_resource('cpu') # Building Python is slow + def test_freeze_simple_script(self): + script = textwrap.dedent(""" + import sys + print('running...') + sys.exit(0) + """) + with os_helper.temp_dir() as outdir: + outdir, scriptfile, python = helper.prepare(script, outdir) + executable = helper.freeze(python, scriptfile, outdir) + text = helper.run(executable) + self.assertEqual(text, 'running...') diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py new file mode 100644 index 0000000000..ffa1b1178e --- /dev/null +++ b/Lib/test/test_tools/test_i18n.py @@ -0,0 +1,444 @@ +"""Tests to cover the Tools/i18n package""" + +import os +import re +import sys +import unittest +from textwrap import dedent +from pathlib import Path + +from test.support.script_helper import assert_python_ok +from test.test_tools import skip_if_missing, toolsdir +from test.support.os_helper import temp_cwd, temp_dir + + +skip_if_missing() + +DATA_DIR = Path(__file__).resolve().parent / 'i18n_data' + + +def normalize_POT_file(pot): + """Normalize the POT creation timestamp, charset and + file locations to make the POT file easier to compare. + + """ + # Normalize the creation date. + date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"') + header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"' + pot = re.sub(date_pattern, header, pot) + + # Normalize charset to UTF-8 (currently there's no way to specify the output charset). + charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"') + charset = r'"Content-Type: text/plain; charset=UTF-8\\n"' + pot = re.sub(charset_pattern, charset, pot) + + # Normalize file location path separators in case this test is + # running on Windows (which uses '\'). + fileloc_pattern = re.compile(r'#:.+') + + def replace(match): + return match[0].replace(os.sep, "/") + pot = re.sub(fileloc_pattern, replace, pot) + return pot + + +class Test_pygettext(unittest.TestCase): + """Tests for the pygettext.py tool""" + + script = Path(toolsdir, 'i18n', 'pygettext.py') + + def get_header(self, data): + """ utility: return the header of a .po file as a dictionary """ + headers = {} + for line in data.split('\n'): + if not line or line.startswith(('#', 'msgid', 'msgstr')): + continue + line = line.strip('"') + key, val = line.split(':', 1) + headers[key] = val.strip() + return headers + + def get_msgids(self, data): + """ utility: return all msgids in .po file as a list of strings """ + msgids = [] + reading_msgid = False + cur_msgid = [] + for line in data.split('\n'): + if reading_msgid: + if line.startswith('"'): + cur_msgid.append(line.strip('"')) + else: + msgids.append('\n'.join(cur_msgid)) + cur_msgid = [] + reading_msgid = False + continue + if line.startswith('msgid '): + line = line[len('msgid '):] + cur_msgid.append(line.strip('"')) + reading_msgid = True + else: + if reading_msgid: + msgids.append('\n'.join(cur_msgid)) + + return msgids + + def assert_POT_equal(self, expected, actual): + """Check if two POT files are equal""" + self.maxDiff = None + self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual)) + + def extract_from_str(self, module_content, *, args=(), strict=True): + """Return all msgids extracted from module_content.""" + filename = 'test.py' + with temp_cwd(None): + with open(filename, 'w', encoding='utf-8') as fp: + fp.write(module_content) + res = assert_python_ok('-Xutf8', self.script, *args, filename) + if strict: + self.assertEqual(res.err, b'') + with open('messages.pot', encoding='utf-8') as fp: + data = fp.read() + return self.get_msgids(data) + + def extract_docstrings_from_str(self, module_content): + """Return all docstrings extracted from module_content.""" + return self.extract_from_str(module_content, args=('--docstrings',), strict=False) + + def test_header(self): + """Make sure the required fields are in the header, according to: + http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry + """ + with temp_cwd(None) as cwd: + assert_python_ok('-Xutf8', self.script) + with open('messages.pot', encoding='utf-8') as fp: + data = fp.read() + header = self.get_header(data) + + self.assertIn("Project-Id-Version", header) + self.assertIn("POT-Creation-Date", header) + self.assertIn("PO-Revision-Date", header) + self.assertIn("Last-Translator", header) + self.assertIn("Language-Team", header) + self.assertIn("MIME-Version", header) + self.assertIn("Content-Type", header) + self.assertIn("Content-Transfer-Encoding", header) + self.assertIn("Generated-By", header) + + # not clear if these should be required in POT (template) files + #self.assertIn("Report-Msgid-Bugs-To", header) + #self.assertIn("Language", header) + + #"Plural-Forms" is optional + + @unittest.skipIf(sys.platform.startswith('aix'), + 'bpo-29972: broken test on AIX') + def test_POT_Creation_Date(self): + """ Match the date format from xgettext for POT-Creation-Date """ + from datetime import datetime + with temp_cwd(None) as cwd: + assert_python_ok('-Xutf8', self.script) + with open('messages.pot', encoding='utf-8') as fp: + data = fp.read() + header = self.get_header(data) + creationDate = header['POT-Creation-Date'] + + # peel off the escaped newline at the end of string + if creationDate.endswith('\\n'): + creationDate = creationDate[:-len('\\n')] + + # This will raise if the date format does not exactly match. + datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + + def test_funcdocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar): + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_funcdocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar): + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_funcdocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar): + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_classdocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + class C: + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_classdocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + class C: + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_classdocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + class C: + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_moduledocstring(self): + for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'): + with self.subTest(doc): + msgids = self.extract_docstrings_from_str(dedent('''\ + %s + ''' % doc)) + self.assertIn('doc', msgids) + + def test_moduledocstring_bytes(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + b"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_moduledocstring_fstring(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""doc""" + ''')) + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_msgid(self): + msgids = self.extract_docstrings_from_str( + '''_("""doc""" r'str' u"ing")''') + self.assertIn('docstring', msgids) + + def test_msgid_bytes(self): + msgids = self.extract_docstrings_from_str('_(b"""doc""")') + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_msgid_fstring(self): + msgids = self.extract_docstrings_from_str('_(f"""doc""")') + self.assertFalse([msgid for msgid in msgids if 'doc' in msgid]) + + def test_funcdocstring_annotated_args(self): + """ Test docstrings for functions with annotated args """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar: str): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_annotated_return(self): + """ Test docstrings for functions with annotated return type """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar) -> str: + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_defvalue_args(self): + """ Test docstring for functions with default arg values """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo(bar=()): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_multiple_funcs(self): + """ Test docstring extraction for multiple functions combining + annotated args, annotated return types and default arg values + """ + msgids = self.extract_docstrings_from_str(dedent('''\ + def foo1(bar: tuple=()) -> str: + """doc1""" + + def foo2(bar: List[1:2]) -> (lambda x: x): + """doc2""" + + def foo3(bar: 'func'=lambda x: x) -> {1: 2}: + """doc3""" + ''')) + self.assertIn('doc1', msgids) + self.assertIn('doc2', msgids) + self.assertIn('doc3', msgids) + + def test_classdocstring_early_colon(self): + """ Test docstring extraction for a class with colons occurring within + the parentheses. + """ + msgids = self.extract_docstrings_from_str(dedent('''\ + class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_calls_in_fstrings(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_raw(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + rf"{_('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_nested(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""{f'{_("foo bar")}'}""" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_attribute(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{obj._('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_with_call_on_call(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{type(str)('foo bar')}" + ''')) + self.assertNotIn('foo bar', msgids) + + def test_calls_in_fstrings_with_format(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo {bar}').format(bar='baz')}" + ''')) + self.assertIn('foo {bar}', msgids) + + def test_calls_in_fstrings_with_wrong_input_1(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(f'foo {bar}')}" + ''')) + self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid]) + + def test_calls_in_fstrings_with_wrong_input_2(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(1)}" + ''')) + self.assertNotIn(1, msgids) + + def test_calls_in_fstring_with_multiple_args(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo', 'bar')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertNotIn('bar', msgids) + + def test_calls_in_fstring_with_keyword_args(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo', bar='baz')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertNotIn('bar', msgids) + self.assertNotIn('baz', msgids) + + def test_calls_in_fstring_with_partially_wrong_expression(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(f'foo') + _('bar')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertIn('bar', msgids) + + def test_function_and_class_names(self): + """Test that function and class names are not mistakenly extracted.""" + msgids = self.extract_from_str(dedent('''\ + def _(x): + pass + + def _(x="foo"): + pass + + async def _(x): + pass + + class _(object): + pass + ''')) + self.assertEqual(msgids, ['']) + + def test_pygettext_output(self): + """Test that the pygettext output exactly matches snapshots.""" + for input_file, output_file, output in extract_from_snapshots(): + with self.subTest(input_file=input_file): + expected = output_file.read_text(encoding='utf-8') + self.assert_POT_equal(expected, output) + + def test_files_list(self): + """Make sure the directories are inspected for source files + bpo-31920 + """ + text1 = 'Text to translate1' + text2 = 'Text to translate2' + text3 = 'Text to ignore' + with temp_cwd(None), temp_dir(None) as sdir: + pymod = Path(sdir, 'pypkg', 'pymod.py') + pymod.parent.mkdir() + pymod.write_text(f'_({text1!r})', encoding='utf-8') + + pymod2 = Path(sdir, 'pkg.py', 'pymod2.py') + pymod2.parent.mkdir() + pymod2.write_text(f'_({text2!r})', encoding='utf-8') + + pymod3 = Path(sdir, 'CVS', 'pymod3.py') + pymod3.parent.mkdir() + pymod3.write_text(f'_({text3!r})', encoding='utf-8') + + assert_python_ok('-Xutf8', self.script, sdir) + data = Path('messages.pot').read_text(encoding='utf-8') + self.assertIn(f'msgid "{text1}"', data) + self.assertIn(f'msgid "{text2}"', data) + self.assertNotIn(text3, data) + + +def extract_from_snapshots(): + snapshots = { + 'messages.py': ('--docstrings',), + 'fileloc.py': ('--docstrings',), + 'docstrings.py': ('--docstrings',), + # == Test character escaping + # Escape ascii and unicode: + 'escapes.py': ('--escape',), + # Escape only ascii and let unicode pass through: + ('escapes.py', 'ascii-escapes.pot'): (), + } + + for filename, args in snapshots.items(): + if isinstance(filename, tuple): + filename, output_file = filename + output_file = DATA_DIR / output_file + input_file = DATA_DIR / filename + else: + input_file = DATA_DIR / filename + output_file = input_file.with_suffix('.pot') + contents = input_file.read_bytes() + with temp_cwd(None): + Path(input_file.name).write_bytes(contents) + assert_python_ok('-Xutf8', Test_pygettext.script, *args, + input_file.name) + yield (input_file, output_file, + Path('messages.pot').read_text(encoding='utf-8')) + + +def update_POT_snapshots(): + for _, output_file, output in extract_from_snapshots(): + output = normalize_POT_file(output) + output_file.write_text(output, encoding='utf-8') + + +if __name__ == '__main__': + # To regenerate POT files + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_POT_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py new file mode 100644 index 0000000000..4c7588d4d9 --- /dev/null +++ b/Lib/test/test_tools/test_makefile.py @@ -0,0 +1,81 @@ +""" +Tests for `Makefile`. +""" + +import os +import unittest +from test import support +import sysconfig + +MAKEFILE = sysconfig.get_makefile_filename() + +if not support.check_impl_detail(cpython=True): + raise unittest.SkipTest('cpython only') +if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE): + raise unittest.SkipTest('Makefile could not be found') + + +class TestMakefile(unittest.TestCase): + def list_test_dirs(self): + result = [] + found_testsubdirs = False + with open(MAKEFILE, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('TESTSUBDIRS='): + found_testsubdirs = True + result.append( + line.removeprefix('TESTSUBDIRS=').replace( + '\\', '', + ).strip(), + ) + continue + if found_testsubdirs: + if '\t' not in line: + break + result.append(line.replace('\\', '').strip()) + return result + + @unittest.skipUnless(support.TEST_MODULES_ENABLED, "requires test modules") + def test_makefile_test_folders(self): + test_dirs = self.list_test_dirs() + idle_test = 'idlelib/idle_test' + self.assertIn(idle_test, test_dirs) + + used = set([idle_test]) + for dirpath, dirs, files in os.walk(support.TEST_HOME_DIR): + dirname = os.path.basename(dirpath) + # Skip temporary dirs: + if dirname == '__pycache__' or dirname.startswith('.'): + dirs.clear() # do not process subfolders + continue + # Skip empty dirs: + if not dirs and not files: + continue + # Skip dirs with hidden-only files: + if files and all( + filename.startswith('.') or filename == '__pycache__' + for filename in files + ): + continue + + relpath = os.path.relpath(dirpath, support.STDLIB_DIR) + with self.subTest(relpath=relpath): + self.assertIn( + relpath, + test_dirs, + msg=( + f"{relpath!r} is not included in the Makefile's list " + "of test directories to install" + ) + ) + used.add(relpath) + + # Don't check the wheel dir when Python is built --with-wheel-pkg-dir + if sysconfig.get_config_var('WHEEL_PKG_DIR'): + test_dirs.remove('test/wheeldata') + used.discard('test/wheeldata') + + # Check that there are no extra entries: + unique_test_dirs = set(test_dirs) + self.assertSetEqual(unique_test_dirs, used) + self.assertEqual(len(test_dirs), len(unique_test_dirs)) diff --git a/Lib/test/test_tools/test_makeunicodedata.py b/Lib/test/test_tools/test_makeunicodedata.py new file mode 100644 index 0000000000..f31375117e --- /dev/null +++ b/Lib/test/test_tools/test_makeunicodedata.py @@ -0,0 +1,122 @@ +import unittest +from test.test_tools import skip_if_missing, imports_under_tool +from test import support +from test.support.hypothesis_helper import hypothesis + +st = hypothesis.strategies +given = hypothesis.given +example = hypothesis.example + + +skip_if_missing("unicode") +with imports_under_tool("unicode"): + from dawg import Dawg, build_compression_dawg, lookup, inverse_lookup + + +@st.composite +def char_name_db(draw, min_length=1, max_length=30): + m = draw(st.integers(min_value=min_length, max_value=max_length)) + names = draw( + st.sets(st.text("abcd", min_size=1, max_size=10), min_size=m, max_size=m) + ) + characters = draw(st.sets(st.characters(), min_size=m, max_size=m)) + return list(zip(names, characters)) + + +class TestDawg(unittest.TestCase): + """Tests for the directed acyclic word graph data structure that is used + to store the unicode character names in unicodedata. Tests ported from PyPy + """ + + def test_dawg_direct_simple(self): + dawg = Dawg() + dawg.insert("a", -4) + dawg.insert("c", -2) + dawg.insert("cat", -1) + dawg.insert("catarr", 0) + dawg.insert("catnip", 1) + dawg.insert("zcatnip", 5) + packed, data, inverse = dawg.finish() + + self.assertEqual(lookup(packed, data, b"a"), -4) + self.assertEqual(lookup(packed, data, b"c"), -2) + self.assertEqual(lookup(packed, data, b"cat"), -1) + self.assertEqual(lookup(packed, data, b"catarr"), 0) + self.assertEqual(lookup(packed, data, b"catnip"), 1) + self.assertEqual(lookup(packed, data, b"zcatnip"), 5) + self.assertRaises(KeyError, lookup, packed, data, b"b") + self.assertRaises(KeyError, lookup, packed, data, b"catni") + self.assertRaises(KeyError, lookup, packed, data, b"catnipp") + + self.assertEqual(inverse_lookup(packed, inverse, -4), b"a") + self.assertEqual(inverse_lookup(packed, inverse, -2), b"c") + self.assertEqual(inverse_lookup(packed, inverse, -1), b"cat") + self.assertEqual(inverse_lookup(packed, inverse, 0), b"catarr") + self.assertEqual(inverse_lookup(packed, inverse, 1), b"catnip") + self.assertEqual(inverse_lookup(packed, inverse, 5), b"zcatnip") + self.assertRaises(KeyError, inverse_lookup, packed, inverse, 12) + + def test_forbid_empty_dawg(self): + dawg = Dawg() + self.assertRaises(ValueError, dawg.finish) + + @given(char_name_db()) + @example([("abc", "a"), ("abd", "b")]) + @example( + [ + ("bab", "1"), + ("a", ":"), + ("ad", "@"), + ("b", "<"), + ("aacc", "?"), + ("dab", "D"), + ("aa", "0"), + ("ab", "F"), + ("aaa", "7"), + ("cbd", "="), + ("abad", ";"), + ("ac", "B"), + ("abb", "4"), + ("bb", "2"), + ("aab", "9"), + ("caaaaba", "E"), + ("ca", ">"), + ("bbaaa", "5"), + ("d", "3"), + ("baac", "8"), + ("c", "6"), + ("ba", "A"), + ] + ) + @example( + [ + ("bcdac", "9"), + ("acc", "g"), + ("d", "d"), + ("daabdda", "0"), + ("aba", ";"), + ("c", "6"), + ("aa", "7"), + ("abbd", "c"), + ("badbd", "?"), + ("bbd", "f"), + ("cc", "@"), + ("bb", "8"), + ("daca", ">"), + ("ba", ":"), + ("baac", "3"), + ("dbdddac", "a"), + ("a", "2"), + ("cabd", "b"), + ("b", "="), + ("abd", "4"), + ("adcbd", "5"), + ("abc", "e"), + ("ab", "1"), + ] + ) + def test_dawg(self, data): + # suppress debug prints + with support.captured_stdout() as output: + # it's enough to build it, building will also check the result + build_compression_dawg(data) diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py new file mode 100644 index 0000000000..8cd31680f7 --- /dev/null +++ b/Lib/test/test_tools/test_msgfmt.py @@ -0,0 +1,159 @@ +"""Tests for the Tools/i18n/msgfmt.py tool.""" + +import json +import sys +import unittest +from gettext import GNUTranslations +from pathlib import Path + +from test.support.os_helper import temp_cwd +from test.support.script_helper import assert_python_failure, assert_python_ok +from test.test_tools import skip_if_missing, toolsdir + + +skip_if_missing('i18n') + +data_dir = (Path(__file__).parent / 'msgfmt_data').resolve() +script_dir = Path(toolsdir) / 'i18n' +msgfmt = script_dir / 'msgfmt.py' + + +def compile_messages(po_file, mo_file): + assert_python_ok(msgfmt, '-o', mo_file, po_file) + + +class CompilationTest(unittest.TestCase): + + def test_compilation(self): + self.maxDiff = None + with temp_cwd(): + for po_file in data_dir.glob('*.po'): + with self.subTest(po_file=po_file): + mo_file = po_file.with_suffix('.mo') + with open(mo_file, 'rb') as f: + expected = GNUTranslations(f) + + tmp_mo_file = mo_file.name + compile_messages(po_file, tmp_mo_file) + with open(tmp_mo_file, 'rb') as f: + actual = GNUTranslations(f) + + self.assertDictEqual(actual._catalog, expected._catalog) + + def test_translations(self): + with open(data_dir / 'general.mo', 'rb') as f: + t = GNUTranslations(f) + + self.assertEqual(t.gettext('foo'), 'foo') + self.assertEqual(t.gettext('bar'), 'baz') + self.assertEqual(t.pgettext('abc', 'foo'), 'bar') + self.assertEqual(t.pgettext('xyz', 'foo'), 'bar') + self.assertEqual(t.gettext('Multilinestring'), 'Multilinetranslation') + self.assertEqual(t.gettext('"escapes"'), '"translated"') + self.assertEqual(t.gettext('\n newlines \n'), '\n translated \n') + self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 1), + 'One email sent.') + self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 2), + '%d emails sent.') + self.assertEqual(t.npgettext('abc', 'One email sent.', + '%d emails sent.', 1), + 'One email sent.') + self.assertEqual(t.npgettext('abc', 'One email sent.', + '%d emails sent.', 2), + '%d emails sent.') + + def test_invalid_msgid_plural(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +msgid_plural "plural" +msgstr[0] "singular" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('msgid_plural not preceded by msgid', err) + + def test_plural_without_msgid_plural(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +msgid "foo" +msgstr[0] "bar" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('plural without msgid_plural', err) + + def test_indexed_msgstr_without_msgid_plural(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +msgid "foo" +msgid_plural "foos" +msgstr "bar" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('indexed msgstr required for plural', err) + + def test_generic_syntax_error(self): + with temp_cwd(): + Path('invalid.po').write_text('''\ +"foo" +''') + + res = assert_python_failure(msgfmt, 'invalid.po') + err = res.err.decode('utf-8') + self.assertIn('Syntax error', err) + +class CLITest(unittest.TestCase): + + def test_help(self): + for option in ('--help', '-h'): + res = assert_python_ok(msgfmt, option) + err = res.err.decode('utf-8') + self.assertIn('Generate binary message catalog from textual translation description.', err) + + def test_version(self): + for option in ('--version', '-V'): + res = assert_python_ok(msgfmt, option) + out = res.out.decode('utf-8').strip() + self.assertEqual('msgfmt.py 1.2', out) + + def test_invalid_option(self): + res = assert_python_failure(msgfmt, '--invalid-option') + err = res.err.decode('utf-8') + self.assertIn('Generate binary message catalog from textual translation description.', err) + self.assertIn('option --invalid-option not recognized', err) + + def test_no_input_file(self): + res = assert_python_ok(msgfmt) + err = res.err.decode('utf-8').replace('\r\n', '\n') + self.assertIn('No input file given\n' + "Try `msgfmt --help' for more information.", err) + + def test_nonexistent_file(self): + assert_python_failure(msgfmt, 'nonexistent.po') + + +def update_catalog_snapshots(): + for po_file in data_dir.glob('*.po'): + mo_file = po_file.with_suffix('.mo') + compile_messages(po_file, mo_file) + # Create a human-readable JSON file which is + # easier to review than the binary .mo file. + with open(mo_file, 'rb') as f: + translations = GNUTranslations(f) + catalog_file = po_file.with_suffix('.json') + with open(catalog_file, 'w') as f: + data = translations._catalog.items() + data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0])) + json.dump(data, f, indent=4) + f.write('\n') + + +if __name__ == '__main__': + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_catalog_snapshots() + sys.exit(0) + unittest.main() diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py new file mode 100644 index 0000000000..64e31c2b77 --- /dev/null +++ b/Lib/test/test_tools/test_reindent.py @@ -0,0 +1,35 @@ +"""Tests for scripts in the Tools directory. + +This file contains regression tests for some of the scripts found in the +Tools directory of a Python checkout or tarball, such as reindent.py. +""" + +import os +import unittest +from test.support.script_helper import assert_python_ok +from test.support import findfile + +from test.test_tools import toolsdir, skip_if_missing + +skip_if_missing() + +class ReindentTests(unittest.TestCase): + script = os.path.join(toolsdir, 'patchcheck', 'reindent.py') + + def test_noargs(self): + assert_python_ok(self.script) + + def test_help(self): + rc, out, err = assert_python_ok(self.script, '-h') + self.assertEqual(out, b'') + self.assertGreater(err, b'') + + def test_reindent_file_with_bad_encoding(self): + bad_coding_path = findfile('bad_coding.py', subdir='tokenizedata') + rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path) + self.assertEqual(out, b'') + self.assertNotEqual(err, b'') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py new file mode 100644 index 0000000000..d0b702d392 --- /dev/null +++ b/Lib/test/test_tools/test_sundry.py @@ -0,0 +1,30 @@ +"""Tests for scripts in the Tools/scripts directory. + +This file contains extremely basic regression tests for the scripts found in +the Tools directory of a Python checkout or tarball which don't have separate +tests of their own. +""" + +import os +import unittest +from test.support import import_helper + +from test.test_tools import scriptsdir, import_tool, skip_if_missing + +skip_if_missing() + +class TestSundryScripts(unittest.TestCase): + # import logging registers "atfork" functions which keep indirectly the + # logging module dictionary alive. Mock the function to be able to unload + # cleanly the logging module. + @import_helper.mock_register_at_fork + def test_sundry(self, mock_os): + for fn in os.listdir(scriptsdir): + if not fn.endswith('.py'): + continue + name = fn[:-3] + import_tool(name) + + +if __name__ == '__main__': + unittest.main() From dd40bf7566742e850e5e08276b05f345744adc5e Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 11:27:53 -0700 Subject: [PATCH 280/317] update getpass to 3.13.3 --- Lib/getpass.py | 15 +++++++++++---- Lib/test/test_getpass.py | 7 +++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/getpass.py b/Lib/getpass.py index 6970d8adfb..bd0097ced9 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -18,7 +18,6 @@ import io import os import sys -import warnings __all__ = ["getpass","getuser","GetPassWarning"] @@ -118,6 +117,7 @@ def win_getpass(prompt='Password: ', stream=None): def fallback_getpass(prompt='Password: ', stream=None): + import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: @@ -156,7 +156,11 @@ def getuser(): First try various environment variables, then the password database. This works on Windows as long as USERNAME is set. + Any failure to find a username raises OSError. + .. versionchanged:: 3.13 + Previously, various exceptions beyond just :exc:`OSError` + were raised. """ for name in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): @@ -164,9 +168,12 @@ def getuser(): if user: return user - # If this fails, the exception will "explain" why - import pwd - return pwd.getpwuid(os.getuid())[0] + try: + import pwd + return pwd.getpwuid(os.getuid())[0] + except (ImportError, KeyError) as e: + raise OSError('No username set in the environment') from e + # Bind the name getpass to the appropriate function try: diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index 3452e46213..80dda2caaa 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -26,7 +26,10 @@ def test_username_priorities_of_env_values(self, environ): environ.get.return_value = None try: getpass.getuser() - except ImportError: # in case there's no pwd module + except OSError: # in case there's no pwd module + pass + except KeyError: + # current user has no pwd entry pass self.assertEqual( environ.get.call_args_list, @@ -44,7 +47,7 @@ def test_username_falls_back_to_pwd(self, environ): getpass.getuser()) getpw.assert_called_once_with(42) else: - self.assertRaises(ImportError, getpass.getuser) + self.assertRaises(OSError, getpass.getuser) class GetpassRawinputTest(unittest.TestCase): From 1a4b035dacaab8dc37be53aace2532bf433ac803 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 11:29:54 -0700 Subject: [PATCH 281/317] update timeit to 3.13.3 --- Lib/timeit.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/Lib/timeit.py b/Lib/timeit.py index f323e65572..258dedccd0 100755 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -50,9 +50,9 @@ """ import gc +import itertools import sys import time -import itertools __all__ = ["Timer", "timeit", "repeat", "default_timer"] @@ -77,9 +77,11 @@ def inner(_it, _timer{init}): return _t1 - _t0 """ + def reindent(src, indent): """Helper to reindent a multi-line statement.""" - return src.replace("\n", "\n" + " "*indent) + return src.replace("\n", "\n" + " " * indent) + class Timer: """Class for timing execution speed of small code snippets. @@ -166,7 +168,7 @@ def timeit(self, number=default_number): To be precise, this executes the setup statement once, and then returns the time it takes to execute the main statement - a number of times, as a float measured in seconds. The + a number of times, as float seconds if using the default timer. The argument is the number of times through the loop, defaulting to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. @@ -230,16 +232,19 @@ def autorange(self, callback=None): return (number, time_taken) i *= 10 + def timeit(stmt="pass", setup="pass", timer=default_timer, number=default_number, globals=None): """Convenience function to create Timer object and call timeit method.""" return Timer(stmt, setup, timer, globals).timeit(number) + def repeat(stmt="pass", setup="pass", timer=default_timer, repeat=default_repeat, number=default_number, globals=None): """Convenience function to create Timer object and call repeat method.""" return Timer(stmt, setup, timer, globals).repeat(repeat, number) + def main(args=None, *, _wrap_timer=None): """Main program, used when run as a script. @@ -261,10 +266,9 @@ def main(args=None, *, _wrap_timer=None): args = sys.argv[1:] import getopt try: - opts, args = getopt.getopt(args, "n:u:s:r:tcpvh", + opts, args = getopt.getopt(args, "n:u:s:r:pvh", ["number=", "setup=", "repeat=", - "time", "clock", "process", - "verbose", "unit=", "help"]) + "process", "verbose", "unit=", "help"]) except getopt.error as err: print(err) print("use -h/--help for command line help") @@ -272,7 +276,7 @@ def main(args=None, *, _wrap_timer=None): timer = default_timer stmt = "\n".join(args) or "pass" - number = 0 # auto-determine + number = 0 # auto-determine setup = [] repeat = default_repeat verbose = 0 @@ -289,7 +293,7 @@ def main(args=None, *, _wrap_timer=None): time_unit = a else: print("Unrecognized unit. Please select nsec, usec, msec, or sec.", - file=sys.stderr) + file=sys.stderr) return 2 if o in ("-r", "--repeat"): repeat = int(a) @@ -323,7 +327,7 @@ def callback(number, time_taken): msg = "{num} loop{s} -> {secs:.{prec}g} secs" plural = (number != 1) print(msg.format(num=number, s='s' if plural else '', - secs=time_taken, prec=precision)) + secs=time_taken, prec=precision)) try: number, _ = t.autorange(callback) except: @@ -374,5 +378,6 @@ def format_time(dt): UserWarning, '', 0) return None + if __name__ == "__main__": sys.exit(main()) From d73f03b9bac9a6fc5763ea8d4a02bf5057411200 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 11:32:20 -0700 Subject: [PATCH 282/317] update reprlib to 3.13.3 --- Lib/reprlib.py | 112 +++++++--- Lib/test/test_reprlib.py | 431 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 514 insertions(+), 29 deletions(-) diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 616b3439b5..19dbe3a07e 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -29,49 +29,100 @@ def wrapper(self): wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__type_params__ = getattr(user_function, '__type_params__', ()) + wrapper.__wrapped__ = user_function return wrapper return decorating_function class Repr: - - def __init__(self): - self.maxlevel = 6 - self.maxtuple = 6 - self.maxlist = 6 - self.maxarray = 5 - self.maxdict = 4 - self.maxset = 6 - self.maxfrozenset = 6 - self.maxdeque = 6 - self.maxstring = 30 - self.maxlong = 40 - self.maxother = 30 + _lookup = { + 'tuple': 'builtins', + 'list': 'builtins', + 'array': 'array', + 'set': 'builtins', + 'frozenset': 'builtins', + 'deque': 'collections', + 'dict': 'builtins', + 'str': 'builtins', + 'int': 'builtins' + } + + def __init__( + self, *, maxlevel=6, maxtuple=6, maxlist=6, maxarray=5, maxdict=4, + maxset=6, maxfrozenset=6, maxdeque=6, maxstring=30, maxlong=40, + maxother=30, fillvalue='...', indent=None, + ): + self.maxlevel = maxlevel + self.maxtuple = maxtuple + self.maxlist = maxlist + self.maxarray = maxarray + self.maxdict = maxdict + self.maxset = maxset + self.maxfrozenset = maxfrozenset + self.maxdeque = maxdeque + self.maxstring = maxstring + self.maxlong = maxlong + self.maxother = maxother + self.fillvalue = fillvalue + self.indent = indent def repr(self, x): return self.repr1(x, self.maxlevel) def repr1(self, x, level): - typename = type(x).__name__ + cls = type(x) + typename = cls.__name__ + if ' ' in typename: parts = typename.split() typename = '_'.join(parts) - if hasattr(self, 'repr_' + typename): - return getattr(self, 'repr_' + typename)(x, level) - else: - return self.repr_instance(x, level) + + method = getattr(self, 'repr_' + typename, None) + if method: + # not defined in this class + if typename not in self._lookup: + return method(x, level) + module = getattr(cls, '__module__', None) + # defined in this class and is the module intended + if module == self._lookup[typename]: + return method(x, level) + + return self.repr_instance(x, level) + + def _join(self, pieces, level): + if self.indent is None: + return ', '.join(pieces) + if not pieces: + return '' + indent = self.indent + if isinstance(indent, int): + if indent < 0: + raise ValueError( + f'Repr.indent cannot be negative int (was {indent!r})' + ) + indent *= ' ' + try: + sep = ',\n' + (self.maxlevel - level + 1) * indent + except TypeError as error: + raise TypeError( + f'Repr.indent must be a str, int or None, not {type(indent)}' + ) from error + return sep.join(('', *pieces, ''))[1:-len(indent) or None] def _repr_iterable(self, x, level, left, right, maxiter, trail=''): n = len(x) if level <= 0 and n: - s = '...' + s = self.fillvalue else: newlevel = level - 1 repr1 = self.repr1 pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] - if n > maxiter: pieces.append('...') - s = ', '.join(pieces) - if n == 1 and trail: right = trail + right + if n > maxiter: + pieces.append(self.fillvalue) + s = self._join(pieces, level) + if n == 1 and trail and self.indent is None: + right = trail + right return '%s%s%s' % (left, s, right) def repr_tuple(self, x, level): @@ -104,8 +155,10 @@ def repr_deque(self, x, level): def repr_dict(self, x, level): n = len(x) - if n == 0: return '{}' - if level <= 0: return '{...}' + if n == 0: + return '{}' + if level <= 0: + return '{' + self.fillvalue + '}' newlevel = level - 1 repr1 = self.repr1 pieces = [] @@ -113,8 +166,9 @@ def repr_dict(self, x, level): keyrepr = repr1(key, newlevel) valrepr = repr1(x[key], newlevel) pieces.append('%s: %s' % (keyrepr, valrepr)) - if n > self.maxdict: pieces.append('...') - s = ', '.join(pieces) + if n > self.maxdict: + pieces.append(self.fillvalue) + s = self._join(pieces, level) return '{%s}' % (s,) def repr_str(self, x, level): @@ -123,7 +177,7 @@ def repr_str(self, x, level): i = max(0, (self.maxstring-3)//2) j = max(0, self.maxstring-3-i) s = builtins.repr(x[:i] + x[len(x)-j:]) - s = s[:i] + '...' + s[len(s)-j:] + s = s[:i] + self.fillvalue + s[len(s)-j:] return s def repr_int(self, x, level): @@ -131,7 +185,7 @@ def repr_int(self, x, level): if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) - s = s[:i] + '...' + s[len(s)-j:] + s = s[:i] + self.fillvalue + s[len(s)-j:] return s def repr_instance(self, x, level): @@ -144,7 +198,7 @@ def repr_instance(self, x, level): if len(s) > self.maxother: i = max(0, (self.maxother-3)//2) j = max(0, self.maxother-3-i) - s = s[:i] + '...' + s[len(s)-j:] + s = s[:i] + self.fillvalue + s[len(s)-j:] return s diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 611fb9d1e4..f84dec1ed9 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -9,6 +9,7 @@ import importlib import importlib.util import unittest +import textwrap from test.support import verbose from test.support.os_helper import create_empty_file @@ -25,6 +26,29 @@ def nestedTuple(nesting): class ReprTests(unittest.TestCase): + def test_init_kwargs(self): + example_kwargs = { + "maxlevel": 101, + "maxtuple": 102, + "maxlist": 103, + "maxarray": 104, + "maxdict": 105, + "maxset": 106, + "maxfrozenset": 107, + "maxdeque": 108, + "maxstring": 109, + "maxlong": 110, + "maxother": 111, + "fillvalue": "x" * 112, + "indent": "x" * 113, + } + r1 = Repr() + for attr, val in example_kwargs.items(): + setattr(r1, attr, val) + r2 = Repr(**example_kwargs) + for attr in example_kwargs: + self.assertEqual(getattr(r1, attr), getattr(r2, attr), msg=attr) + def test_string(self): eq = self.assertEqual eq(r("abc"), "'abc'") @@ -51,6 +75,15 @@ def test_tuple(self): expected = repr(t3)[:-2] + "...)" eq(r2.repr(t3), expected) + # modified fillvalue: + r3 = Repr() + r3.fillvalue = '+++' + r3.maxtuple = 2 + expected = repr(t3)[:-2] + "+++)" + eq(r3.repr(t3), expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_container(self): from array import array from collections import deque @@ -223,6 +256,382 @@ def test_unsortable(self): r(y) r(z) + def test_valid_indent(self): + test_cases = [ + { + 'object': (), + 'tests': ( + (dict(indent=None), '()'), + (dict(indent=False), '()'), + (dict(indent=True), '()'), + (dict(indent=0), '()'), + (dict(indent=1), '()'), + (dict(indent=4), '()'), + (dict(indent=4, maxlevel=2), '()'), + (dict(indent=''), '()'), + (dict(indent='-->'), '()'), + (dict(indent='....'), '()'), + ), + }, + { + 'object': '', + 'tests': ( + (dict(indent=None), "''"), + (dict(indent=False), "''"), + (dict(indent=True), "''"), + (dict(indent=0), "''"), + (dict(indent=1), "''"), + (dict(indent=4), "''"), + (dict(indent=4, maxlevel=2), "''"), + (dict(indent=''), "''"), + (dict(indent='-->'), "''"), + (dict(indent='....'), "''"), + ), + }, + { + 'object': [1, 'spam', {'eggs': True, 'ham': []}], + 'tests': ( + (dict(indent=None), '''\ + [1, 'spam', {'eggs': True, 'ham': []}]'''), + (dict(indent=False), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=True), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=0), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=1), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=4), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=4, maxlevel=2), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent=''), '''\ + [ + 1, + 'spam', + { + 'eggs': True, + 'ham': [], + }, + ]'''), + (dict(indent='-->'), '''\ + [ + -->1, + -->'spam', + -->{ + -->-->'eggs': True, + -->-->'ham': [], + -->}, + ]'''), + (dict(indent='....'), '''\ + [ + ....1, + ....'spam', + ....{ + ........'eggs': True, + ........'ham': [], + ....}, + ]'''), + ), + }, + { + 'object': { + 1: 'two', + b'three': [ + (4.5, 6.7), + [set((8, 9)), frozenset((10, 11))], + ], + }, + 'tests': ( + (dict(indent=None), '''\ + {1: 'two', b'three': [(4.5, 6.7), [{8, 9}, frozenset({10, 11})]]}'''), + (dict(indent=False), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.7, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=True), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.7, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=0), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.7, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=1), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.7, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=4), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.7, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent=4, maxlevel=2), '''\ + { + 1: 'two', + b'three': [ + (...), + [...], + ], + }'''), + (dict(indent=''), '''\ + { + 1: 'two', + b'three': [ + ( + 4.5, + 6.7, + ), + [ + { + 8, + 9, + }, + frozenset({ + 10, + 11, + }), + ], + ], + }'''), + (dict(indent='-->'), '''\ + { + -->1: 'two', + -->b'three': [ + -->-->( + -->-->-->4.5, + -->-->-->6.7, + -->-->), + -->-->[ + -->-->-->{ + -->-->-->-->8, + -->-->-->-->9, + -->-->-->}, + -->-->-->frozenset({ + -->-->-->-->10, + -->-->-->-->11, + -->-->-->}), + -->-->], + -->], + }'''), + (dict(indent='....'), '''\ + { + ....1: 'two', + ....b'three': [ + ........( + ............4.5, + ............6.7, + ........), + ........[ + ............{ + ................8, + ................9, + ............}, + ............frozenset({ + ................10, + ................11, + ............}), + ........], + ....], + }'''), + ), + }, + ] + for test_case in test_cases: + with self.subTest(test_object=test_case['object']): + for repr_settings, expected_repr in test_case['tests']: + with self.subTest(repr_settings=repr_settings): + r = Repr() + for attribute, value in repr_settings.items(): + setattr(r, attribute, value) + resulting_repr = r.repr(test_case['object']) + expected_repr = textwrap.dedent(expected_repr) + self.assertEqual(resulting_repr, expected_repr) + + def test_invalid_indent(self): + test_object = [1, 'spam', {'eggs': True, 'ham': []}] + test_cases = [ + (-1, (ValueError, '[Nn]egative|[Pp]ositive')), + (-4, (ValueError, '[Nn]egative|[Pp]ositive')), + ((), (TypeError, None)), + ([], (TypeError, None)), + ((4,), (TypeError, None)), + ([4,], (TypeError, None)), + (object(), (TypeError, None)), + ] + for indent, (expected_error, expected_msg) in test_cases: + with self.subTest(indent=indent): + r = Repr() + r.indent = indent + expected_msg = expected_msg or f'{type(indent)}' + with self.assertRaisesRegex(expected_error, expected_msg): + r.repr(test_object) + + def test_shadowed_stdlib_array(self): + # Issue #113570: repr() should not be fooled by an array + class array: + def __repr__(self): + return "not array.array" + + self.assertEqual(r(array()), "not array.array") + + def test_shadowed_builtin(self): + # Issue #113570: repr() should not be fooled + # by a shadowed builtin function + class list: + def __repr__(self): + return "not builtins.list" + + self.assertEqual(r(list()), "not builtins.list") + + def test_custom_repr(self): + class MyRepr(Repr): + + def repr_TextIOWrapper(self, obj, level): + if obj.name in {'', '', ''}: + return obj.name + return repr(obj) + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(sys.stdin), "") + + def test_custom_repr_class_with_spaces(self): + class TypeWithSpaces: + pass + + t = TypeWithSpaces() + type(t).__name__ = "type with spaces" + self.assertEqual(type(t).__name__, "type with spaces") + + class MyRepr(Repr): + def repr_type_with_spaces(self, obj, level): + return "Type With Spaces" + + + aRepr = MyRepr() + self.assertEqual(aRepr.repr(t), "Type With Spaces") + def write_file(path, text): with open(path, 'w', encoding='ASCII') as fp: fp.write(text) @@ -408,5 +817,27 @@ def test_assigned_attributes(self): for name in assigned: self.assertIs(getattr(wrapper, name), getattr(wrapped, name)) + def test__wrapped__(self): + class X: + def __repr__(self): + return 'X()' + f = __repr__ # save reference to check it later + __repr__ = recursive_repr()(__repr__) + + self.assertIs(X.f, X.__repr__.__wrapped__) + + # TODO: RUSTPYTHON: AttributeError: 'TypeVar' object has no attribute '__name__' + @unittest.expectedFailure + def test__type_params__(self): + class My: + @recursive_repr() + def __repr__[T: str](self, default: T = '') -> str: + return default + + type_params = My().__repr__.__type_params__ + self.assertEqual(len(type_params), 1) + self.assertEqual(type_params[0].__name__, 'T') + self.assertEqual(type_params[0].__bound__, str) + if __name__ == "__main__": unittest.main() From b84d6a36a6f19e72486dd41235af5a73959bb6a5 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 23 Apr 2025 11:52:00 -0700 Subject: [PATCH 283/317] update fileinput to 3.13.3 --- Lib/fileinput.py | 4 +-- Lib/test/test_fileinput.py | 60 ++++++++++++++++++++++++-------------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/Lib/fileinput.py b/Lib/fileinput.py index e234dc9ea6..3dba3d2fbf 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -53,7 +53,7 @@ sequence must be accessed in strictly sequential order; sequence access and readline() cannot be mixed. -Optional in-place filtering: if the keyword argument inplace=1 is +Optional in-place filtering: if the keyword argument inplace=True is passed to input() or to the FileInput constructor, the file is moved to a backup file and standard output is directed to the input file. This makes it possible to write a filter that rewrites its input file @@ -399,7 +399,7 @@ def isstdin(self): def hook_compressed(filename, mode, *, encoding=None, errors=None): - if encoding is None: # EncodingWarning is emitted in FileInput() already. + if encoding is None and "b" not in mode: # EncodingWarning is emitted in FileInput() already. encoding = "locale" ext = os.path.splitext(filename)[1] if ext == '.gz': diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index bda6dee6bf..1a6ef3cd27 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -23,10 +23,9 @@ from io import BytesIO, StringIO from fileinput import FileInput, hook_encoded -from pathlib import Path from test.support import verbose -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, FakePath from test.support.os_helper import unlink as safe_unlink from test.support import os_helper from test import support @@ -151,7 +150,7 @@ def test_buffer_sizes(self): print('6. Inplace') savestdout = sys.stdout try: - fi = FileInput(files=(t1, t2, t3, t4), inplace=1, encoding="utf-8") + fi = FileInput(files=(t1, t2, t3, t4), inplace=True, encoding="utf-8") for line in fi: line = line[:-1].upper() print(line) @@ -256,7 +255,7 @@ def test_detached_stdin_binary_mode(self): def test_file_opening_hook(self): try: # cannot use openhook and inplace mode - fi = FileInput(inplace=1, openhook=lambda f, m: None) + fi = FileInput(inplace=True, openhook=lambda f, m: None) self.fail("FileInput should raise if both inplace " "and openhook arguments are given") except ValueError: @@ -478,23 +477,23 @@ def test_iteration_buffering(self): self.assertRaises(StopIteration, next, fi) self.assertEqual(src.linesread, []) - def test_pathlib_file(self): - t1 = Path(self.writeTmp("Pathlib file.")) + def test_pathlike_file(self): + t1 = FakePath(self.writeTmp("Path-like file.")) with FileInput(t1, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') self.assertEqual(fi.lineno(), 1) self.assertEqual(fi.filelineno(), 1) self.assertEqual(fi.filename(), os.fspath(t1)) - def test_pathlib_file_inplace(self): - t1 = Path(self.writeTmp('Pathlib file.')) + def test_pathlike_file_inplace(self): + t1 = FakePath(self.writeTmp('Path-like file.')) with FileInput(t1, inplace=True, encoding="utf-8") as fi: line = fi.readline() - self.assertEqual(line, 'Pathlib file.') + self.assertEqual(line, 'Path-like file.') print('Modified %s' % line) with open(t1, encoding="utf-8") as f: - self.assertEqual(f.read(), 'Modified Pathlib file.\n') + self.assertEqual(f.read(), 'Modified Path-like file.\n') class MockFileInput: @@ -855,29 +854,29 @@ def setUp(self): self.fake_open = InvocationRecorder() def test_empty_string(self): - self.do_test_use_builtin_open("", 1) + self.do_test_use_builtin_open_text("", "r") def test_no_ext(self): - self.do_test_use_builtin_open("abcd", 2) + self.do_test_use_builtin_open_text("abcd", "r") @unittest.skipUnless(gzip, "Requires gzip and zlib") def test_gz_ext_fake(self): original_open = gzip.open gzip.open = self.fake_open try: - result = fileinput.hook_compressed("test.gz", "3") + result = fileinput.hook_compressed("test.gz", "r") finally: gzip.open = original_open self.assertEqual(self.fake_open.invocation_count, 1) - self.assertEqual(self.fake_open.last_invocation, (("test.gz", "3"), {})) + self.assertEqual(self.fake_open.last_invocation, (("test.gz", "r"), {})) @unittest.skipUnless(gzip, "Requires gzip and zlib") def test_gz_with_encoding_fake(self): original_open = gzip.open gzip.open = lambda filename, mode: io.BytesIO(b'Ex-binary string') try: - result = fileinput.hook_compressed("test.gz", "3", encoding="utf-8") + result = fileinput.hook_compressed("test.gz", "r", encoding="utf-8") finally: gzip.open = original_open self.assertEqual(list(result), ['Ex-binary string']) @@ -887,23 +886,40 @@ def test_bz2_ext_fake(self): original_open = bz2.BZ2File bz2.BZ2File = self.fake_open try: - result = fileinput.hook_compressed("test.bz2", "4") + result = fileinput.hook_compressed("test.bz2", "r") finally: bz2.BZ2File = original_open self.assertEqual(self.fake_open.invocation_count, 1) - self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "4"), {})) + self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "r"), {})) def test_blah_ext(self): - self.do_test_use_builtin_open("abcd.blah", "5") + self.do_test_use_builtin_open_binary("abcd.blah", "rb") def test_gz_ext_builtin(self): - self.do_test_use_builtin_open("abcd.Gz", "6") + self.do_test_use_builtin_open_binary("abcd.Gz", "rb") def test_bz2_ext_builtin(self): - self.do_test_use_builtin_open("abcd.Bz2", "7") + self.do_test_use_builtin_open_binary("abcd.Bz2", "rb") - def do_test_use_builtin_open(self, filename, mode): + def test_binary_mode_encoding(self): + self.do_test_use_builtin_open_binary("abcd", "rb") + + def test_text_mode_encoding(self): + self.do_test_use_builtin_open_text("abcd", "r") + + def do_test_use_builtin_open_binary(self, filename, mode): + original_open = self.replace_builtin_open(self.fake_open) + try: + result = fileinput.hook_compressed(filename, mode) + finally: + self.replace_builtin_open(original_open) + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, + ((filename, mode), {'encoding': None, 'errors': None})) + + def do_test_use_builtin_open_text(self, filename, mode): original_open = self.replace_builtin_open(self.fake_open) try: result = fileinput.hook_compressed(filename, mode) From 02f120aaf4d6ca5bf8d5c78359eceb3bcdbfaf86 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 20 Apr 2025 18:56:59 +0900 Subject: [PATCH 284/317] copilot-instructions --- .github/copilot-instructions.md | 186 ++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..2991e3c626 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,186 @@ +# GitHub Copilot Instructions for RustPython + +This document provides guidelines for working with GitHub Copilot when contributing to the RustPython project. + +## Project Overview + +RustPython is a Python 3 interpreter written in Rust, implementing Python 3.13.0+ compatibility. The project aims to provide: + +- A complete Python-3 environment entirely in Rust (not CPython bindings) +- A clean implementation without compatibility hacks +- Cross-platform support, including WebAssembly compilation +- The ability to embed Python scripting in Rust applications + +## Repository Structure + +- `src/` - Top-level code for the RustPython binary +- `vm/` - The Python virtual machine implementation + - `builtins/` - Python built-in types and functions + - `stdlib/` - Essential standard library modules implemented in Rust, required to run the Python core +- `compiler/` - Python compiler components + - `parser/` - Parser for converting Python source to AST + - `core/` - Bytecode representation in Rust structures + - `codegen/` - AST to bytecode compiler +- `Lib/` - CPython's standard library in Python (copied from CPython) +- `derive/` - Rust macros for RustPython +- `common/` - Common utilities +- `extra_tests/` - Integration tests and snippets +- `stdlib/` - Non-essential Python standard library modules implemented in Rust (useful but not required for core functionality) +- `wasm/` - WebAssembly support +- `jit/` - Experimental JIT compiler implementation +- `pylib/` - Python standard library packaging (do not modify this directory directly - its contents are generated automatically) + +## Important Development Notes + +### Running Python Code + +When testing Python code, always use RustPython instead of the standard `python` command: + +```bash +# Use this instead of python script.py +cargo run -- script.py + +# For interactive REPL +cargo run + +# With specific features +cargo run --features ssl + +# Release mode (recommended for better performance) +cargo run --release -- script.py +``` + +### Comparing with CPython + +When you need to compare behavior with CPython or run test suites: + +```bash +# Use python command to explicitly run CPython +python my_test_script.py + +# Run RustPython +cargo run -- my_test_script.py +``` + +### Working with the Lib Directory + +The `Lib/` directory contains Python standard library files copied from the CPython repository. Important notes: + +- These files should be edited very conservatively +- Modifications should be minimal and only to work around RustPython limitations +- Tests in `Lib/test` often use one of the following markers: + - Add a `# TODO: RUSTPYTHON` comment when modifications are made + - `unittest.skip("TODO: RustPython ")` + - `unittest.expectedFailure` with `# TODO: RUSTPYTHON ` comment + +### Testing + +```bash +# Run Rust unit tests +cargo test --workspace --exclude rustpython_wasm + +# Run Python snippets tests +cd extra_tests +pytest -v + +# Run the Python test module +cargo run --release -- -m test +``` + +### Determining What to Implement + +Run `./whats_left.py` to get a list of unimplemented methods, which is helpful when looking for contribution opportunities. + +## Coding Guidelines + +### Rust Code + +- Follow the default rustfmt code style (`cargo fmt` to format) +- Use clippy to lint code (`cargo clippy`) +- Follow Rust best practices for error handling and memory management +- Use the macro system (`pyclass`, `pymodule`, `pyfunction`, etc.) when implementing Python functionality in Rust + +### Python Code + +- Follow PEP 8 style for custom Python code +- Use ruff for linting Python code +- Minimize modifications to CPython standard library files + +## Integration Between Rust and Python + +The project provides several mechanisms for integration: + +- `pymodule` macro for creating Python modules in Rust +- `pyclass` macro for implementing Python classes in Rust +- `pyfunction` macro for exposing Rust functions to Python +- `PyObjectRef` and other types for working with Python objects in Rust + +## Common Patterns + +### Implementing a Python Module in Rust + +```rust +#[pymodule] +mod mymodule { + use rustpython_vm::prelude::*; + + #[pyfunction] + fn my_function(value: i32) -> i32 { + value * 2 + } + + #[pyattr] + #[pyclass(name = "MyClass")] + #[derive(Debug, PyPayload)] + struct MyClass { + value: usize, + } + + #[pyclass] + impl MyClass { + #[pymethod] + fn get_value(&self) -> usize { + self.value + } + } +} +``` + +### Adding a Python Module to the Interpreter + +```rust +vm.add_native_module( + "my_module_name".to_owned(), + Box::new(my_module::make_module), +); +``` + +## Building for Different Targets + +### WebAssembly + +```bash +# Build for WASM +cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdlib,stdlib --release +``` + +### JIT Support + +```bash +# Enable JIT support +cargo run --features jit +``` + +### SSL Support + +```bash +# Enable SSL support +cargo run --features ssl +``` + +## Documentation + +- Check the [architecture document](architecture/architecture.md) for a high-level overview +- Read the [development guide](DEVELOPMENT.md) for detailed setup instructions +- Generate documentation with `cargo doc --no-deps --all` +- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) \ No newline at end of file From e8df06582e9886d469d420a7e8cf78a74a604fd6 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Tue, 29 Apr 2025 20:05:51 -0700 Subject: [PATCH 285/317] bump dependencies --- Cargo.lock | 28 ++++++++++++++++++---------- Cargo.toml | 2 +- stdlib/Cargo.toml | 4 ++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13ad0c4935..65fa7952fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1296,9 +1296,9 @@ checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libffi" -version = "3.2.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce826c243048e3d5cec441799724de52e2d42f820468431fc3fceee2341871e2" +checksum = "4a9434b6fc77375fb624698d5f8c49d7e80b10d59eb1219afda27d1f824d4074" dependencies = [ "libc", "libffi-sys", @@ -1306,9 +1306,9 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "2.3.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36115160c57e8529781b4183c2bb51fdc1f6d6d1ed345591d84be7703befb3c" +checksum = "ead36a2496acfc8edd6cc32352110e9478ac5b9b5f5b9856ebd3d28019addb84" dependencies = [ "cc", ] @@ -2758,6 +2758,14 @@ dependencies = [ "keccak", ] +[[package]] +name = "shared-build" +version = "0.2.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" +dependencies = [ + "bindgen", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2891,11 +2899,11 @@ checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tcl-sys" -version = "0.1.0" -source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.1.0#09a4f62e894df64692b34e6c7f81af1e1ae376dd" +version = "0.2.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" dependencies = [ - "bindgen", "pkg-config", + "shared-build", ] [[package]] @@ -3007,11 +3015,11 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tk-sys" -version = "0.1.0" -source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.1.0#09a4f62e894df64692b34e6c7f81af1e1ae376dd" +version = "0.2.0" +source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" dependencies = [ - "bindgen", "pkg-config", + "shared-build", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2be9db7477..b3c5b49e11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,7 +168,7 @@ itertools = "0.14.0" is-macro = "0.3.7" junction = "1.2.0" libc = "0.2.169" -libffi = "3.2" +libffi = "4.0" log = "0.4.27" nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } malachite-bigint = "0.6" diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 7b6aba0a14..8fcaaa24d9 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -82,8 +82,8 @@ libz-sys = { package = "libz-rs-sys", version = "0.5" } bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] } # tkinter -tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true } -tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.1.0", optional = true } +tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } +tcl-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } # uuid [target.'cfg(not(any(target_os = "ios", target_os = "android", target_os = "windows", target_arch = "wasm32", target_os = "redox")))'.dependencies] From e49e7436699f9aa3dd9c3d2deb33de9f8c901a82 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Wed, 30 Apr 2025 00:53:37 -0500 Subject: [PATCH 286/317] Support multiline function/class definitions in REPL and InteractiveConsole (#5743) * Support multiline function/class definitions in REPL and InteractiveConsole --- Lib/codeop.py | 7 ++++++- src/shell.rs | 17 ++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Lib/codeop.py b/Lib/codeop.py index 4dd096574b..96868047cb 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -66,7 +66,12 @@ def _maybe_compile(compiler, source, filename, symbol): compiler(source + "\n", filename, symbol) return None except SyntaxError as e: - if "incomplete input" in str(e): + # XXX: RustPython; support multiline definitions in REPL + # See also: https://github.com/RustPython/RustPython/pull/5743 + strerr = str(e) + if source.endswith(":") and "expected an indented block" in strerr: + return None + elif "incomplete input" in str(e): return None # fallthrough diff --git a/src/shell.rs b/src/shell.rs index f733e03ddc..cbe2c9efe0 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -49,13 +49,16 @@ fn shell_exec( let bad_error = match err { CompileError::Parse(ref p) => { - if matches!( - p.error, - ParseErrorType::Lexical(LexicalErrorType::IndentationError) - ) { - continuing // && p.location.is_some() - } else { - true // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) + match &p.error { + ParseErrorType::Lexical(LexicalErrorType::IndentationError) => continuing, // && p.location.is_some() + ParseErrorType::OtherError(msg) => { + if msg.starts_with("Expected an indented block") { + continuing + } else { + true + } + } + _ => true, // !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) } } _ => true, // It is a bad error for everything else From cd1c9be0e1a3b59c5d49c9edf593d66538c5e530 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 30 Apr 2025 15:03:32 +0900 Subject: [PATCH 287/317] Fix Cargo.lock --- Cargo.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65fa7952fb..38aad89fed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1880,11 +1880,10 @@ dependencies = [ [[package]] name = "pymath" -version = "0.0.1" +version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c574544c9acbbb7cab157def68e24e7a417c4ff6f7e528a805160c8b6cc3aa9f" +checksum = "5b66ab66a8610ce209d8b36cd0fecc3a15c494f715e0cb26f0586057f293abc9" dependencies = [ - "errno", "libc", ] From 301c32dba0902fde35d4482ab550a0f9fea64f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Kr=C3=A1l?= Date: Tue, 29 Apr 2025 22:07:39 +0200 Subject: [PATCH 288/317] Increase numerical precision of log1p Use Rust native ln_1p to increase numerical precision matching what is expected by `test_mtestfile`. --- stdlib/src/math.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/math.rs b/stdlib/src/math.rs index 9ab4bed5ce..524660a434 100644 --- a/stdlib/src/math.rs +++ b/stdlib/src/math.rs @@ -147,7 +147,7 @@ mod math { fn log1p(x: ArgIntoFloat, vm: &VirtualMachine) -> PyResult { let x = *x; if x.is_nan() || x > -1.0_f64 { - Ok((x + 1.0_f64).ln()) + Ok(x.ln_1p()) } else { Err(vm.new_value_error("math domain error".to_owned())) } From 253cc4e8469cbbd158d971fa5965081aa9ad3963 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Thu, 1 May 2025 13:55:19 +0800 Subject: [PATCH 289/317] =?UTF-8?q?Fix=20usize=20not=20using=20the=20same?= =?UTF-8?q?=20hash=20as=20PyInt=20when=20used=20as=20key=20into=20a=20di?= =?UTF-8?q?=E2=80=A6=20(#5756)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix usize not using the same hash as PyInt when used as key into a dictionary * Fix test that unexpectedly succeed! * Update extra_tests/snippets/builtin_object.py --- Lib/test/test_fstring.py | 2 -- common/src/hash.rs | 5 +++++ extra_tests/snippets/builtin_object.py | 7 +++++++ vm/src/dict_inner.rs | 5 +++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index c0c987ca2f..c727b5b22c 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1631,8 +1631,6 @@ def test_empty_format_specifier(self): self.assertEqual(f"{x!s:}", "test") self.assertEqual(f"{x!r:}", "'test'") - # TODO: RUSTPYTHON d[0] error - @unittest.expectedFailure def test_str_format_differences(self): d = { "a": "string", diff --git a/common/src/hash.rs b/common/src/hash.rs index 88af9b6f56..9fea1e717e 100644 --- a/common/src/hash.rs +++ b/common/src/hash.rs @@ -139,6 +139,11 @@ pub fn hash_bigint(value: &BigInt) -> PyHash { fix_sentinel(ret) } +#[inline] +pub fn hash_usize(data: usize) -> PyHash { + fix_sentinel(mod_int(data as i64)) +} + #[inline(always)] pub fn fix_sentinel(x: PyHash) -> PyHash { if x == SENTINEL { -2 } else { x } diff --git a/extra_tests/snippets/builtin_object.py b/extra_tests/snippets/builtin_object.py index 5a12afbf45..0cb6f31d66 100644 --- a/extra_tests/snippets/builtin_object.py +++ b/extra_tests/snippets/builtin_object.py @@ -24,3 +24,10 @@ class MyObject: assert not hasattr(obj, 'a') obj.__dict__ = {'a': 1} assert obj.a == 1 + +# Value inside the formatter goes through a different path of resolution. +# Check that it still works all the same +d = { + 0: "ab", +} +assert "ab ab" == "{k[0]} {vv}".format(k=d, vv=d[0]) diff --git a/vm/src/dict_inner.rs b/vm/src/dict_inner.rs index 56ebc5ebaf..a165c7669a 100644 --- a/vm/src/dict_inner.rs +++ b/vm/src/dict_inner.rs @@ -17,6 +17,7 @@ use crate::{ object::{Traverse, TraverseFn}, }; use num_traits::ToPrimitive; +use rustpython_common::hash::hash_integer; use std::{fmt, mem::size_of, ops::ControlFlow}; // HashIndex is intended to be same size with hash::PyHash @@ -993,8 +994,8 @@ impl DictKey for usize { *self } - fn key_hash(&self, vm: &VirtualMachine) -> PyResult { - Ok(vm.state.hash_secret.hash_value(self)) + fn key_hash(&self, _vm: &VirtualMachine) -> PyResult { + Ok(hash_usize(*self)) } fn key_is(&self, _other: &PyObject) -> bool { From 94b38a51c4322d4e54c3f1fd3c7fe38c1ce88831 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Thu, 1 May 2025 15:38:50 +0900 Subject: [PATCH 290/317] Fix build (#5765) --- vm/src/dict_inner.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vm/src/dict_inner.rs b/vm/src/dict_inner.rs index a165c7669a..1dd701b0b6 100644 --- a/vm/src/dict_inner.rs +++ b/vm/src/dict_inner.rs @@ -17,7 +17,6 @@ use crate::{ object::{Traverse, TraverseFn}, }; use num_traits::ToPrimitive; -use rustpython_common::hash::hash_integer; use std::{fmt, mem::size_of, ops::ControlFlow}; // HashIndex is intended to be same size with hash::PyHash @@ -995,7 +994,7 @@ impl DictKey for usize { } fn key_hash(&self, _vm: &VirtualMachine) -> PyResult { - Ok(hash_usize(*self)) + Ok(hash::hash_usize(*self)) } fn key_is(&self, _other: &PyObject) -> bool { From 85f7ba51f4f9eb8d46f45b00f2ede5705a77ad9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karel=20Kr=C3=A1l?= Date: Thu, 1 May 2025 19:33:28 +0200 Subject: [PATCH 291/317] Enable test_mtestfile --- Lib/test/test_math.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 0e14fad774..16d12f9688 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2080,7 +2080,6 @@ def test_testfile(self): self.fail('Failures in test_testfile:\n ' + '\n '.join(failures)) - @unittest.skip("TODO: RUSTPYTHON, Taking a few minutes.") @requires_IEEE_754 def test_mtestfile(self): fail_fmt = "{}: {}({!r}): {}" From 6aa80aa596064a037dcc7975d4ec099e75817aed Mon Sep 17 00:00:00 2001 From: Undersk0re <44556359+Undersk0re@users.noreply.github.com> Date: Tue, 6 May 2025 07:07:12 +0200 Subject: [PATCH 292/317] Update .gitignore, folder fix (#5762) --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 485272adfb..cb7165aaca 100644 --- a/.gitignore +++ b/.gitignore @@ -2,11 +2,11 @@ /*/target **/*.rs.bk **/*.bytecode -__pycache__ +__pycache__/ **/*.pytest_cache .*sw* .repl_history.txt -.vscode +.vscode/ wasm-pack.log .idea/ .envrc From 9c88475b315541a5a86e5feb59432ab638a93d6d Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 6 May 2025 01:26:54 -0500 Subject: [PATCH 293/317] Split out common compression routines into separate file (#5728) --- stdlib/src/{ => compression}/bz2.rs | 8 +- stdlib/src/compression/generic.rs | 268 +++++++++++++++++++++++++ stdlib/src/compression/mod.rs | 5 + stdlib/src/{ => compression}/zlib.rs | 283 +-------------------------- stdlib/src/lib.rs | 7 +- 5 files changed, 286 insertions(+), 285 deletions(-) rename stdlib/src/{ => compression}/bz2.rs (98%) create mode 100644 stdlib/src/compression/generic.rs create mode 100644 stdlib/src/compression/mod.rs rename stdlib/src/{ => compression}/zlib.rs (71%) diff --git a/stdlib/src/bz2.rs b/stdlib/src/compression/bz2.rs similarity index 98% rename from stdlib/src/bz2.rs rename to stdlib/src/compression/bz2.rs index 6339a44a24..27cfcb14ae 100644 --- a/stdlib/src/bz2.rs +++ b/stdlib/src/compression/bz2.rs @@ -4,6 +4,9 @@ pub(crate) use _bz2::make_module; #[pymodule] mod _bz2 { + use super::super::{ + DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, + }; use crate::common::lock::PyMutex; use crate::vm::{ VirtualMachine, @@ -12,9 +15,6 @@ mod _bz2 { object::{PyPayload, PyResult}, types::Constructor, }; - use crate::zlib::{ - DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, - }; use bzip2::{Decompress, Status, write::BzEncoder}; use rustpython_vm::convert::ToPyException; use std::{fmt, io::Write}; @@ -74,7 +74,7 @@ mod _bz2 { impl BZ2Decompressor { #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length = args.max_length(); + let max_length = args.max_length_negative_is_none(); let data = &*args.data(); let mut state = self.state.lock(); diff --git a/stdlib/src/compression/generic.rs b/stdlib/src/compression/generic.rs new file mode 100644 index 0000000000..64fc687723 --- /dev/null +++ b/stdlib/src/compression/generic.rs @@ -0,0 +1,268 @@ +// cspell:ignore chunker +use crate::vm::{ + VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef}, + convert::ToPyException, + function::{ArgBytesLike, ArgSize, OptionalArg}, +}; + +#[derive(FromArgs)] +pub(super) struct DecompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, optional)] + pub max_length: OptionalArg, +} + +impl DecompressArgs { + pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { + self.data.borrow_buf() + } + pub fn max_length_negative_is_none(&self) -> Option { + self.max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()) + } +} + +pub(super) trait Decompressor { + type Flush: FlushKind; + type Status: DecompressStatus; + type Error; + + fn total_in(&self) -> u64; + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result; + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + Err(err) + } +} + +pub(super) trait DecompressStatus { + fn is_stream_end(&self) -> bool; +} + +pub(super) trait FlushKind: Copy { + const SYNC: Self; +} + +impl FlushKind for () { + const SYNC: Self = (); +} + +pub(super) fn flush_sync(_final_chunk: bool) -> T { + T::SYNC +} + +pub(super) const CHUNKSIZE: usize = u32::MAX as usize; + +#[derive(Clone)] +pub(super) struct Chunker<'a> { + data1: &'a [u8], + data2: &'a [u8], +} +impl<'a> Chunker<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { + data1: data, + data2: &[], + } + } + pub fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { + if data1.is_empty() { + Self { + data1: data2, + data2: &[], + } + } else { + Self { data1, data2 } + } + } + pub fn len(&self) -> usize { + self.data1.len() + self.data2.len() + } + pub fn is_empty(&self) -> bool { + self.data1.is_empty() + } + pub fn to_vec(&self) -> Vec { + [self.data1, self.data2].concat() + } + pub fn chunk(&self) -> &'a [u8] { + self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) + } + pub fn advance(&mut self, consumed: usize) { + self.data1 = &self.data1[consumed..]; + if self.data1.is_empty() { + self.data1 = std::mem::take(&mut self.data2); + } + } +} + +pub(super) fn _decompress( + data: &[u8], + d: &mut D, + bufsize: usize, + max_length: Option, + calc_flush: impl Fn(bool) -> D::Flush, +) -> Result<(Vec, bool), D::Error> { + let mut data = Chunker::new(data); + _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) +} + +pub(super) fn _decompress_chunks( + data: &mut Chunker<'_>, + d: &mut D, + bufsize: usize, + max_length: Option, + calc_flush: impl Fn(bool) -> D::Flush, +) -> Result<(Vec, bool), D::Error> { + if data.is_empty() { + return Ok((Vec::new(), true)); + } + let max_length = max_length.unwrap_or(usize::MAX); + let mut buf = Vec::new(); + + 'outer: loop { + let chunk = data.chunk(); + let flush = calc_flush(chunk.len() == data.len()); + loop { + let additional = std::cmp::min(bufsize, max_length - buf.capacity()); + if additional == 0 { + return Ok((buf, false)); + } + buf.reserve_exact(additional); + + let prev_in = d.total_in(); + let res = d.decompress_vec(chunk, &mut buf, flush); + let consumed = d.total_in() - prev_in; + + data.advance(consumed as usize); + + match res { + Ok(status) => { + let stream_end = status.is_stream_end(); + if stream_end || data.is_empty() { + // we've reached the end of the stream, we're done + buf.shrink_to_fit(); + return Ok((buf, stream_end)); + } else if !chunk.is_empty() && consumed == 0 { + // we're gonna need a bigger buffer + continue; + } else { + // next chunk + continue 'outer; + } + } + Err(e) => { + d.maybe_set_dict(e)?; + // now try the next chunk + continue 'outer; + } + }; + } + } +} + +#[derive(Debug)] +pub(super) struct DecompressState { + decompress: D, + unused_data: PyBytesRef, + input_buffer: Vec, + eof: bool, + needs_input: bool, +} + +impl DecompressState { + pub fn new(decompress: D, vm: &VirtualMachine) -> Self { + Self { + decompress, + unused_data: vm.ctx.empty_bytes.clone(), + input_buffer: Vec::new(), + eof: false, + needs_input: true, + } + } + + pub fn eof(&self) -> bool { + self.eof + } + + pub fn unused_data(&self) -> PyBytesRef { + self.unused_data.clone() + } + + pub fn needs_input(&self) -> bool { + self.needs_input + } + + pub fn decompress( + &mut self, + data: &[u8], + max_length: Option, + bufsize: usize, + vm: &VirtualMachine, + ) -> Result, DecompressError> { + if self.eof { + return Err(DecompressError::Eof(EofError)); + } + + let input_buffer = &mut self.input_buffer; + let d = &mut self.decompress; + + let mut chunks = Chunker::chain(input_buffer, data); + + let prev_len = chunks.len(); + let (ret, stream_end) = + match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = prev_len - chunks.len(); + + self.eof |= stream_end; + + if self.eof { + self.needs_input = false; + if !chunks.is_empty() { + self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); + } + } else if chunks.is_empty() { + input_buffer.clear(); + self.needs_input = true; + } else { + self.needs_input = false; + if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { + input_buffer.clear(); + input_buffer.extend_from_slice(&data[n_consumed_from_data..]); + } else { + input_buffer.drain(..consumed); + input_buffer.extend_from_slice(data); + } + } + + ret.map_err(DecompressError::Decompress) + } +} + +pub(super) enum DecompressError { + Decompress(E), + Eof(EofError), +} + +impl From for DecompressError { + fn from(err: E) -> Self { + Self::Decompress(err) + } +} + +pub(super) struct EofError; + +impl ToPyException for EofError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_eof_error("End of stream already reached".to_owned()) + } +} diff --git a/stdlib/src/compression/mod.rs b/stdlib/src/compression/mod.rs new file mode 100644 index 0000000000..e4ced7c00c --- /dev/null +++ b/stdlib/src/compression/mod.rs @@ -0,0 +1,5 @@ +mod generic; +use generic::*; + +pub mod bz2; +pub mod zlib; diff --git a/stdlib/src/zlib.rs b/stdlib/src/compression/zlib.rs similarity index 71% rename from stdlib/src/zlib.rs rename to stdlib/src/compression/zlib.rs index 0578f20c86..f032057026 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/compression/zlib.rs @@ -1,11 +1,12 @@ -// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker +// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz -pub(crate) use zlib::{DecompressArgs, make_module}; +pub(crate) use zlib::make_module; #[pymodule] mod zlib { - use super::generic::{ - DecompressError, DecompressState, DecompressStatus, Decompressor, FlushKind, flush_sync, + use super::super::{ + _decompress, CHUNKSIZE, DecompressArgs, DecompressError, DecompressState, DecompressStatus, + Decompressor, FlushKind, flush_sync, }; use crate::vm::{ PyObject, PyPayload, PyResult, VirtualMachine, @@ -144,113 +145,6 @@ mod zlib { } } - #[derive(Clone)] - pub(crate) struct Chunker<'a> { - data1: &'a [u8], - data2: &'a [u8], - } - impl<'a> Chunker<'a> { - pub(crate) fn new(data: &'a [u8]) -> Self { - Self { - data1: data, - data2: &[], - } - } - pub(crate) fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { - if data1.is_empty() { - Self { - data1: data2, - data2: &[], - } - } else { - Self { data1, data2 } - } - } - pub(crate) fn len(&self) -> usize { - self.data1.len() + self.data2.len() - } - pub(crate) fn is_empty(&self) -> bool { - self.data1.is_empty() - } - pub(crate) fn to_vec(&self) -> Vec { - [self.data1, self.data2].concat() - } - pub(crate) fn chunk(&self) -> &'a [u8] { - self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) - } - pub(crate) fn advance(&mut self, consumed: usize) { - self.data1 = &self.data1[consumed..]; - if self.data1.is_empty() { - self.data1 = std::mem::take(&mut self.data2); - } - } - } - - fn _decompress( - data: &[u8], - d: &mut D, - bufsize: usize, - max_length: Option, - calc_flush: impl Fn(bool) -> D::Flush, - ) -> Result<(Vec, bool), D::Error> { - let mut data = Chunker::new(data); - _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) - } - - pub(super) fn _decompress_chunks( - data: &mut Chunker<'_>, - d: &mut D, - bufsize: usize, - max_length: Option, - calc_flush: impl Fn(bool) -> D::Flush, - ) -> Result<(Vec, bool), D::Error> { - if data.is_empty() { - return Ok((Vec::new(), true)); - } - let max_length = max_length.unwrap_or(usize::MAX); - let mut buf = Vec::new(); - - 'outer: loop { - let chunk = data.chunk(); - let flush = calc_flush(chunk.len() == data.len()); - loop { - let additional = std::cmp::min(bufsize, max_length - buf.capacity()); - if additional == 0 { - return Ok((buf, false)); - } - buf.reserve_exact(additional); - - let prev_in = d.total_in(); - let res = d.decompress_vec(chunk, &mut buf, flush); - let consumed = d.total_in() - prev_in; - - data.advance(consumed as usize); - - match res { - Ok(status) => { - let stream_end = status.is_stream_end(); - if stream_end || data.is_empty() { - // we've reached the end of the stream, we're done - buf.shrink_to_fit(); - return Ok((buf, stream_end)); - } else if !chunk.is_empty() && consumed == 0 { - // we're gonna need a bigger buffer - continue; - } else { - // next chunk - continue 'outer; - } - } - Err(e) => { - d.maybe_set_dict(e)?; - // now try the next chunk - continue 'outer; - } - }; - } - } - } - #[derive(FromArgs)] struct PyFuncDecompressArgs { #[pyarg(positional)] @@ -434,25 +328,6 @@ mod zlib { } } - #[derive(FromArgs)] - pub(crate) struct DecompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(any, optional)] - max_length: OptionalArg, - } - - impl DecompressArgs { - pub(crate) fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { - self.data.borrow_buf() - } - pub(crate) fn max_length(&self) -> Option { - self.max_length - .into_option() - .and_then(|ArgSize { value }| usize::try_from(value).ok()) - } - } - #[derive(FromArgs)] #[allow(dead_code)] // FIXME: use args struct CompressobjArgs { @@ -533,8 +408,6 @@ mod zlib { // } } - const CHUNKSIZE: usize = u32::MAX as usize; - impl CompressInner { fn new(compress: Compress) -> Self { Self { @@ -734,7 +607,7 @@ mod zlib { #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length = args.max_length(); + let max_length = args.max_length_negative_is_none(); let data = &*args.data(); let inner = &mut *self.inner.lock(); @@ -754,147 +627,3 @@ mod zlib { // } } } - -mod generic { - use super::zlib::{_decompress_chunks, Chunker}; - use crate::vm::{ - VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytesRef}, - convert::ToPyException, - }; - - pub(crate) trait Decompressor { - type Flush: FlushKind; - type Status: DecompressStatus; - type Error; - - fn total_in(&self) -> u64; - fn decompress_vec( - &mut self, - input: &[u8], - output: &mut Vec, - flush: Self::Flush, - ) -> Result; - fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { - Err(err) - } - } - - pub(crate) trait DecompressStatus { - fn is_stream_end(&self) -> bool; - } - - pub(crate) trait FlushKind: Copy { - const SYNC: Self; - } - - impl FlushKind for () { - const SYNC: Self = (); - } - - pub(super) fn flush_sync(_final_chunk: bool) -> T { - T::SYNC - } - - #[derive(Debug)] - pub(crate) struct DecompressState { - decompress: D, - unused_data: PyBytesRef, - input_buffer: Vec, - eof: bool, - needs_input: bool, - } - - impl DecompressState { - pub(crate) fn new(decompress: D, vm: &VirtualMachine) -> Self { - Self { - decompress, - unused_data: vm.ctx.empty_bytes.clone(), - input_buffer: Vec::new(), - eof: false, - needs_input: true, - } - } - - pub(crate) fn eof(&self) -> bool { - self.eof - } - - pub(crate) fn unused_data(&self) -> PyBytesRef { - self.unused_data.clone() - } - - pub(crate) fn needs_input(&self) -> bool { - self.needs_input - } - - pub(crate) fn decompress( - &mut self, - data: &[u8], - max_length: Option, - bufsize: usize, - vm: &VirtualMachine, - ) -> Result, DecompressError> { - if self.eof { - return Err(DecompressError::Eof(EofError)); - } - - let input_buffer = &mut self.input_buffer; - let d = &mut self.decompress; - - let mut chunks = Chunker::chain(input_buffer, data); - - let prev_len = chunks.len(); - let (ret, stream_end) = - match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), - }; - let consumed = prev_len - chunks.len(); - - self.eof |= stream_end; - - if self.eof { - self.needs_input = false; - if !chunks.is_empty() { - self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); - } - } else if chunks.is_empty() { - input_buffer.clear(); - self.needs_input = true; - } else { - self.needs_input = false; - if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { - input_buffer.clear(); - input_buffer.extend_from_slice(&data[n_consumed_from_data..]); - } else { - input_buffer.drain(..consumed); - input_buffer.extend_from_slice(data); - } - } - - ret.map_err(DecompressError::Decompress) - } - } - - pub(crate) enum DecompressError { - Decompress(E), - Eof(EofError), - } - - impl From for DecompressError { - fn from(err: E) -> Self { - Self::Decompress(err) - } - } - - pub(crate) struct EofError; - - impl ToPyException for EofError { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_eof_error("End of stream already reached".to_owned()) - } - } -} - -pub(crate) use generic::{DecompressError, DecompressState, DecompressStatus, Decompressor}; diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 7b3124c50d..9b99a7ea4f 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -24,6 +24,7 @@ mod sha256; mod sha3; mod sha512; +mod compression; mod json; #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))] mod locale; @@ -37,13 +38,11 @@ mod statistics; mod suggestions; // TODO: maybe make this an extension module, if we ever get those // mod re; -mod bz2; #[cfg(not(target_arch = "wasm32"))] pub mod socket; #[cfg(all(unix, not(target_os = "redox")))] mod syslog; mod unicodedata; -mod zlib; mod faulthandler; #[cfg(any(unix, target_os = "wasi"))] @@ -112,7 +111,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "array" => array::make_module, "binascii" => binascii::make_module, "_bisect" => bisect::make_module, - "_bz2" => bz2::make_module, + "_bz2" => compression::bz2::make_module, "cmath" => cmath::make_module, "_contextvars" => contextvars::make_module, "_csv" => csv::make_module, @@ -133,7 +132,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, "unicodedata" => unicodedata::make_module, - "zlib" => zlib::make_module, + "zlib" => compression::zlib::make_module, "_statistics" => statistics::make_module, "_suggestions" => suggestions::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, From 48b08a2b7f1eaecf11a6f7bd126f578e10b9a7d1 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 7 May 2025 15:01:09 +0900 Subject: [PATCH 294/317] Revert "Split out common compression routines into separate file (#5728)" This reverts commit 9c88475b315541a5a86e5feb59432ab638a93d6d. --- stdlib/src/{compression => }/bz2.rs | 8 +- stdlib/src/compression/generic.rs | 268 ------------------------- stdlib/src/compression/mod.rs | 5 - stdlib/src/lib.rs | 7 +- stdlib/src/{compression => }/zlib.rs | 283 ++++++++++++++++++++++++++- 5 files changed, 285 insertions(+), 286 deletions(-) rename stdlib/src/{compression => }/bz2.rs (98%) delete mode 100644 stdlib/src/compression/generic.rs delete mode 100644 stdlib/src/compression/mod.rs rename stdlib/src/{compression => }/zlib.rs (71%) diff --git a/stdlib/src/compression/bz2.rs b/stdlib/src/bz2.rs similarity index 98% rename from stdlib/src/compression/bz2.rs rename to stdlib/src/bz2.rs index 27cfcb14ae..6339a44a24 100644 --- a/stdlib/src/compression/bz2.rs +++ b/stdlib/src/bz2.rs @@ -4,9 +4,6 @@ pub(crate) use _bz2::make_module; #[pymodule] mod _bz2 { - use super::super::{ - DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, - }; use crate::common::lock::PyMutex; use crate::vm::{ VirtualMachine, @@ -15,6 +12,9 @@ mod _bz2 { object::{PyPayload, PyResult}, types::Constructor, }; + use crate::zlib::{ + DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, + }; use bzip2::{Decompress, Status, write::BzEncoder}; use rustpython_vm::convert::ToPyException; use std::{fmt, io::Write}; @@ -74,7 +74,7 @@ mod _bz2 { impl BZ2Decompressor { #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length = args.max_length_negative_is_none(); + let max_length = args.max_length(); let data = &*args.data(); let mut state = self.state.lock(); diff --git a/stdlib/src/compression/generic.rs b/stdlib/src/compression/generic.rs deleted file mode 100644 index 64fc687723..0000000000 --- a/stdlib/src/compression/generic.rs +++ /dev/null @@ -1,268 +0,0 @@ -// cspell:ignore chunker -use crate::vm::{ - VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytesRef}, - convert::ToPyException, - function::{ArgBytesLike, ArgSize, OptionalArg}, -}; - -#[derive(FromArgs)] -pub(super) struct DecompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(any, optional)] - pub max_length: OptionalArg, -} - -impl DecompressArgs { - pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { - self.data.borrow_buf() - } - pub fn max_length_negative_is_none(&self) -> Option { - self.max_length - .into_option() - .and_then(|ArgSize { value }| usize::try_from(value).ok()) - } -} - -pub(super) trait Decompressor { - type Flush: FlushKind; - type Status: DecompressStatus; - type Error; - - fn total_in(&self) -> u64; - fn decompress_vec( - &mut self, - input: &[u8], - output: &mut Vec, - flush: Self::Flush, - ) -> Result; - fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { - Err(err) - } -} - -pub(super) trait DecompressStatus { - fn is_stream_end(&self) -> bool; -} - -pub(super) trait FlushKind: Copy { - const SYNC: Self; -} - -impl FlushKind for () { - const SYNC: Self = (); -} - -pub(super) fn flush_sync(_final_chunk: bool) -> T { - T::SYNC -} - -pub(super) const CHUNKSIZE: usize = u32::MAX as usize; - -#[derive(Clone)] -pub(super) struct Chunker<'a> { - data1: &'a [u8], - data2: &'a [u8], -} -impl<'a> Chunker<'a> { - pub fn new(data: &'a [u8]) -> Self { - Self { - data1: data, - data2: &[], - } - } - pub fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { - if data1.is_empty() { - Self { - data1: data2, - data2: &[], - } - } else { - Self { data1, data2 } - } - } - pub fn len(&self) -> usize { - self.data1.len() + self.data2.len() - } - pub fn is_empty(&self) -> bool { - self.data1.is_empty() - } - pub fn to_vec(&self) -> Vec { - [self.data1, self.data2].concat() - } - pub fn chunk(&self) -> &'a [u8] { - self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) - } - pub fn advance(&mut self, consumed: usize) { - self.data1 = &self.data1[consumed..]; - if self.data1.is_empty() { - self.data1 = std::mem::take(&mut self.data2); - } - } -} - -pub(super) fn _decompress( - data: &[u8], - d: &mut D, - bufsize: usize, - max_length: Option, - calc_flush: impl Fn(bool) -> D::Flush, -) -> Result<(Vec, bool), D::Error> { - let mut data = Chunker::new(data); - _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) -} - -pub(super) fn _decompress_chunks( - data: &mut Chunker<'_>, - d: &mut D, - bufsize: usize, - max_length: Option, - calc_flush: impl Fn(bool) -> D::Flush, -) -> Result<(Vec, bool), D::Error> { - if data.is_empty() { - return Ok((Vec::new(), true)); - } - let max_length = max_length.unwrap_or(usize::MAX); - let mut buf = Vec::new(); - - 'outer: loop { - let chunk = data.chunk(); - let flush = calc_flush(chunk.len() == data.len()); - loop { - let additional = std::cmp::min(bufsize, max_length - buf.capacity()); - if additional == 0 { - return Ok((buf, false)); - } - buf.reserve_exact(additional); - - let prev_in = d.total_in(); - let res = d.decompress_vec(chunk, &mut buf, flush); - let consumed = d.total_in() - prev_in; - - data.advance(consumed as usize); - - match res { - Ok(status) => { - let stream_end = status.is_stream_end(); - if stream_end || data.is_empty() { - // we've reached the end of the stream, we're done - buf.shrink_to_fit(); - return Ok((buf, stream_end)); - } else if !chunk.is_empty() && consumed == 0 { - // we're gonna need a bigger buffer - continue; - } else { - // next chunk - continue 'outer; - } - } - Err(e) => { - d.maybe_set_dict(e)?; - // now try the next chunk - continue 'outer; - } - }; - } - } -} - -#[derive(Debug)] -pub(super) struct DecompressState { - decompress: D, - unused_data: PyBytesRef, - input_buffer: Vec, - eof: bool, - needs_input: bool, -} - -impl DecompressState { - pub fn new(decompress: D, vm: &VirtualMachine) -> Self { - Self { - decompress, - unused_data: vm.ctx.empty_bytes.clone(), - input_buffer: Vec::new(), - eof: false, - needs_input: true, - } - } - - pub fn eof(&self) -> bool { - self.eof - } - - pub fn unused_data(&self) -> PyBytesRef { - self.unused_data.clone() - } - - pub fn needs_input(&self) -> bool { - self.needs_input - } - - pub fn decompress( - &mut self, - data: &[u8], - max_length: Option, - bufsize: usize, - vm: &VirtualMachine, - ) -> Result, DecompressError> { - if self.eof { - return Err(DecompressError::Eof(EofError)); - } - - let input_buffer = &mut self.input_buffer; - let d = &mut self.decompress; - - let mut chunks = Chunker::chain(input_buffer, data); - - let prev_len = chunks.len(); - let (ret, stream_end) = - match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), - }; - let consumed = prev_len - chunks.len(); - - self.eof |= stream_end; - - if self.eof { - self.needs_input = false; - if !chunks.is_empty() { - self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); - } - } else if chunks.is_empty() { - input_buffer.clear(); - self.needs_input = true; - } else { - self.needs_input = false; - if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { - input_buffer.clear(); - input_buffer.extend_from_slice(&data[n_consumed_from_data..]); - } else { - input_buffer.drain(..consumed); - input_buffer.extend_from_slice(data); - } - } - - ret.map_err(DecompressError::Decompress) - } -} - -pub(super) enum DecompressError { - Decompress(E), - Eof(EofError), -} - -impl From for DecompressError { - fn from(err: E) -> Self { - Self::Decompress(err) - } -} - -pub(super) struct EofError; - -impl ToPyException for EofError { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_eof_error("End of stream already reached".to_owned()) - } -} diff --git a/stdlib/src/compression/mod.rs b/stdlib/src/compression/mod.rs deleted file mode 100644 index e4ced7c00c..0000000000 --- a/stdlib/src/compression/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod generic; -use generic::*; - -pub mod bz2; -pub mod zlib; diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 9b99a7ea4f..7b3124c50d 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -24,7 +24,6 @@ mod sha256; mod sha3; mod sha512; -mod compression; mod json; #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))] mod locale; @@ -38,11 +37,13 @@ mod statistics; mod suggestions; // TODO: maybe make this an extension module, if we ever get those // mod re; +mod bz2; #[cfg(not(target_arch = "wasm32"))] pub mod socket; #[cfg(all(unix, not(target_os = "redox")))] mod syslog; mod unicodedata; +mod zlib; mod faulthandler; #[cfg(any(unix, target_os = "wasi"))] @@ -111,7 +112,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "array" => array::make_module, "binascii" => binascii::make_module, "_bisect" => bisect::make_module, - "_bz2" => compression::bz2::make_module, + "_bz2" => bz2::make_module, "cmath" => cmath::make_module, "_contextvars" => contextvars::make_module, "_csv" => csv::make_module, @@ -132,7 +133,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, "unicodedata" => unicodedata::make_module, - "zlib" => compression::zlib::make_module, + "zlib" => zlib::make_module, "_statistics" => statistics::make_module, "_suggestions" => suggestions::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, diff --git a/stdlib/src/compression/zlib.rs b/stdlib/src/zlib.rs similarity index 71% rename from stdlib/src/compression/zlib.rs rename to stdlib/src/zlib.rs index f032057026..0578f20c86 100644 --- a/stdlib/src/compression/zlib.rs +++ b/stdlib/src/zlib.rs @@ -1,12 +1,11 @@ -// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz +// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker -pub(crate) use zlib::make_module; +pub(crate) use zlib::{DecompressArgs, make_module}; #[pymodule] mod zlib { - use super::super::{ - _decompress, CHUNKSIZE, DecompressArgs, DecompressError, DecompressState, DecompressStatus, - Decompressor, FlushKind, flush_sync, + use super::generic::{ + DecompressError, DecompressState, DecompressStatus, Decompressor, FlushKind, flush_sync, }; use crate::vm::{ PyObject, PyPayload, PyResult, VirtualMachine, @@ -145,6 +144,113 @@ mod zlib { } } + #[derive(Clone)] + pub(crate) struct Chunker<'a> { + data1: &'a [u8], + data2: &'a [u8], + } + impl<'a> Chunker<'a> { + pub(crate) fn new(data: &'a [u8]) -> Self { + Self { + data1: data, + data2: &[], + } + } + pub(crate) fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { + if data1.is_empty() { + Self { + data1: data2, + data2: &[], + } + } else { + Self { data1, data2 } + } + } + pub(crate) fn len(&self) -> usize { + self.data1.len() + self.data2.len() + } + pub(crate) fn is_empty(&self) -> bool { + self.data1.is_empty() + } + pub(crate) fn to_vec(&self) -> Vec { + [self.data1, self.data2].concat() + } + pub(crate) fn chunk(&self) -> &'a [u8] { + self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) + } + pub(crate) fn advance(&mut self, consumed: usize) { + self.data1 = &self.data1[consumed..]; + if self.data1.is_empty() { + self.data1 = std::mem::take(&mut self.data2); + } + } + } + + fn _decompress( + data: &[u8], + d: &mut D, + bufsize: usize, + max_length: Option, + calc_flush: impl Fn(bool) -> D::Flush, + ) -> Result<(Vec, bool), D::Error> { + let mut data = Chunker::new(data); + _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) + } + + pub(super) fn _decompress_chunks( + data: &mut Chunker<'_>, + d: &mut D, + bufsize: usize, + max_length: Option, + calc_flush: impl Fn(bool) -> D::Flush, + ) -> Result<(Vec, bool), D::Error> { + if data.is_empty() { + return Ok((Vec::new(), true)); + } + let max_length = max_length.unwrap_or(usize::MAX); + let mut buf = Vec::new(); + + 'outer: loop { + let chunk = data.chunk(); + let flush = calc_flush(chunk.len() == data.len()); + loop { + let additional = std::cmp::min(bufsize, max_length - buf.capacity()); + if additional == 0 { + return Ok((buf, false)); + } + buf.reserve_exact(additional); + + let prev_in = d.total_in(); + let res = d.decompress_vec(chunk, &mut buf, flush); + let consumed = d.total_in() - prev_in; + + data.advance(consumed as usize); + + match res { + Ok(status) => { + let stream_end = status.is_stream_end(); + if stream_end || data.is_empty() { + // we've reached the end of the stream, we're done + buf.shrink_to_fit(); + return Ok((buf, stream_end)); + } else if !chunk.is_empty() && consumed == 0 { + // we're gonna need a bigger buffer + continue; + } else { + // next chunk + continue 'outer; + } + } + Err(e) => { + d.maybe_set_dict(e)?; + // now try the next chunk + continue 'outer; + } + }; + } + } + } + #[derive(FromArgs)] struct PyFuncDecompressArgs { #[pyarg(positional)] @@ -328,6 +434,25 @@ mod zlib { } } + #[derive(FromArgs)] + pub(crate) struct DecompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, optional)] + max_length: OptionalArg, + } + + impl DecompressArgs { + pub(crate) fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { + self.data.borrow_buf() + } + pub(crate) fn max_length(&self) -> Option { + self.max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()) + } + } + #[derive(FromArgs)] #[allow(dead_code)] // FIXME: use args struct CompressobjArgs { @@ -408,6 +533,8 @@ mod zlib { // } } + const CHUNKSIZE: usize = u32::MAX as usize; + impl CompressInner { fn new(compress: Compress) -> Self { Self { @@ -607,7 +734,7 @@ mod zlib { #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length = args.max_length_negative_is_none(); + let max_length = args.max_length(); let data = &*args.data(); let inner = &mut *self.inner.lock(); @@ -627,3 +754,147 @@ mod zlib { // } } } + +mod generic { + use super::zlib::{_decompress_chunks, Chunker}; + use crate::vm::{ + VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef}, + convert::ToPyException, + }; + + pub(crate) trait Decompressor { + type Flush: FlushKind; + type Status: DecompressStatus; + type Error; + + fn total_in(&self) -> u64; + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result; + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + Err(err) + } + } + + pub(crate) trait DecompressStatus { + fn is_stream_end(&self) -> bool; + } + + pub(crate) trait FlushKind: Copy { + const SYNC: Self; + } + + impl FlushKind for () { + const SYNC: Self = (); + } + + pub(super) fn flush_sync(_final_chunk: bool) -> T { + T::SYNC + } + + #[derive(Debug)] + pub(crate) struct DecompressState { + decompress: D, + unused_data: PyBytesRef, + input_buffer: Vec, + eof: bool, + needs_input: bool, + } + + impl DecompressState { + pub(crate) fn new(decompress: D, vm: &VirtualMachine) -> Self { + Self { + decompress, + unused_data: vm.ctx.empty_bytes.clone(), + input_buffer: Vec::new(), + eof: false, + needs_input: true, + } + } + + pub(crate) fn eof(&self) -> bool { + self.eof + } + + pub(crate) fn unused_data(&self) -> PyBytesRef { + self.unused_data.clone() + } + + pub(crate) fn needs_input(&self) -> bool { + self.needs_input + } + + pub(crate) fn decompress( + &mut self, + data: &[u8], + max_length: Option, + bufsize: usize, + vm: &VirtualMachine, + ) -> Result, DecompressError> { + if self.eof { + return Err(DecompressError::Eof(EofError)); + } + + let input_buffer = &mut self.input_buffer; + let d = &mut self.decompress; + + let mut chunks = Chunker::chain(input_buffer, data); + + let prev_len = chunks.len(); + let (ret, stream_end) = + match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = prev_len - chunks.len(); + + self.eof |= stream_end; + + if self.eof { + self.needs_input = false; + if !chunks.is_empty() { + self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); + } + } else if chunks.is_empty() { + input_buffer.clear(); + self.needs_input = true; + } else { + self.needs_input = false; + if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { + input_buffer.clear(); + input_buffer.extend_from_slice(&data[n_consumed_from_data..]); + } else { + input_buffer.drain(..consumed); + input_buffer.extend_from_slice(data); + } + } + + ret.map_err(DecompressError::Decompress) + } + } + + pub(crate) enum DecompressError { + Decompress(E), + Eof(EofError), + } + + impl From for DecompressError { + fn from(err: E) -> Self { + Self::Decompress(err) + } + } + + pub(crate) struct EofError; + + impl ToPyException for EofError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_eof_error("End of stream already reached".to_owned()) + } + } +} + +pub(crate) use generic::{DecompressError, DecompressState, DecompressStatus, Decompressor}; From 2a1ea45659a2c92a20fcb13ec42f613dc8bab57e Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 19 Apr 2025 11:42:46 -0700 Subject: [PATCH 295/317] add _lzma module with new dependency Signed-off-by: Ashwin Naren --- Cargo.lock | 32 ++++++++++++++++++++++++++++++++ stdlib/Cargo.toml | 1 + stdlib/src/lib.rs | 2 ++ stdlib/src/lzma.rs | 6 ++++++ 4 files changed, 41 insertions(+) create mode 100644 stdlib/src/lzma.rs diff --git a/Cargo.lock b/Cargo.lock index 38aad89fed..46c7135d83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -237,6 +237,12 @@ version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bzip2" version = "0.5.2" @@ -630,6 +636,21 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -1396,6 +1417,16 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mac_address" version = "1.1.8" @@ -2467,6 +2498,7 @@ dependencies = [ "libc", "libsqlite3-sys", "libz-rs-sys", + "lzma-rs", "mac_address", "malachite-bigint", "md-5", diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 8fcaaa24d9..70f70b8552 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -80,6 +80,7 @@ crc32fast = "1.3.2" flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } libz-sys = { package = "libz-rs-sys", version = "0.5" } bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] } +lzma-rs = "0.3" # tkinter tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 7b3124c50d..e8ba5570d5 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -83,6 +83,7 @@ mod uuid; #[cfg(feature = "tkinter")] mod tkinter; +mod lzma; use rustpython_common as common; use rustpython_vm as vm; @@ -120,6 +121,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "faulthandler" => faulthandler::make_module, "gc" => gc::make_module, "_hashlib" => hashlib::make_module, + "_lzma" => lzma::make_module, "_sha1" => sha1::make_module, "_sha3" => sha3::make_module, "_sha256" => sha256::make_module, diff --git a/stdlib/src/lzma.rs b/stdlib/src/lzma.rs new file mode 100644 index 0000000000..d80449621c --- /dev/null +++ b/stdlib/src/lzma.rs @@ -0,0 +1,6 @@ +pub(crate) use _lzma::make_module; + +#[pymodule] +mod _lzma { + +} \ No newline at end of file From 2042d877f9aeaec994e7ca0ab4c24517430a3977 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 19 Apr 2025 16:26:03 -0700 Subject: [PATCH 296/317] add lzma.py and test_lzma.py at 3.13.2 Signed-off-by: Ashwin Naren --- Lib/lzma.py | 364 +++++++ Lib/test/test_lzma.py | 2095 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2459 insertions(+) create mode 100644 Lib/lzma.py create mode 100644 Lib/test/test_lzma.py diff --git a/Lib/lzma.py b/Lib/lzma.py new file mode 100644 index 0000000000..6668921f00 --- /dev/null +++ b/Lib/lzma.py @@ -0,0 +1,364 @@ +"""Interface to the liblzma compression library. + +This module provides a class for reading and writing compressed files, +classes for incremental (de)compression, and convenience functions for +one-shot (de)compression. + +These classes and functions support both the XZ and legacy LZMA +container formats, as well as raw compressed data streams. +""" + +__all__ = [ + "CHECK_NONE", "CHECK_CRC32", "CHECK_CRC64", "CHECK_SHA256", + "CHECK_ID_MAX", "CHECK_UNKNOWN", + "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64", + "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC", + "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW", + "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4", + "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME", + + "LZMACompressor", "LZMADecompressor", "LZMAFile", "LZMAError", + "open", "compress", "decompress", "is_check_supported", +] + +import builtins +import io +import os +from _lzma import * +from _lzma import _encode_filter_properties, _decode_filter_properties +import _compression + + +# Value 0 no longer used +_MODE_READ = 1 +# Value 2 no longer used +_MODE_WRITE = 3 + + +class LZMAFile(_compression.BaseStream): + + """A file object providing transparent LZMA (de)compression. + + An LZMAFile can act as a wrapper for an existing file object, or + refer directly to a named file on disk. + + Note that LZMAFile provides a *binary* file interface - data read + is returned as bytes, and data to be written must be given as bytes. + """ + + def __init__(self, filename=None, mode="r", *, + format=None, check=-1, preset=None, filters=None): + """Open an LZMA-compressed file in binary mode. + + filename can be either an actual file name (given as a str, + bytes, or PathLike object), in which case the named file is + opened, or it can be an existing file object to read from or + write to. + + mode can be "r" for reading (default), "w" for (over)writing, + "x" for creating exclusively, or "a" for appending. These can + equivalently be given as "rb", "wb", "xb" and "ab" respectively. + + format specifies the container format to use for the file. + If mode is "r", this defaults to FORMAT_AUTO. Otherwise, the + default is FORMAT_XZ. + + check specifies the integrity check to use. This argument can + only be used when opening a file for writing. For FORMAT_XZ, + the default is CHECK_CRC64. FORMAT_ALONE and FORMAT_RAW do not + support integrity checks - for these formats, check must be + omitted, or be CHECK_NONE. + + When opening a file for reading, the *preset* argument is not + meaningful, and should be omitted. The *filters* argument should + also be omitted, except when format is FORMAT_RAW (in which case + it is required). + + When opening a file for writing, the settings used by the + compressor can be specified either as a preset compression + level (with the *preset* argument), or in detail as a custom + filter chain (with the *filters* argument). For FORMAT_XZ and + FORMAT_ALONE, the default is to use the PRESET_DEFAULT preset + level. For FORMAT_RAW, the caller must always specify a filter + chain; the raw compressor does not support preset compression + levels. + + preset (if provided) should be an integer in the range 0-9, + optionally OR-ed with the constant PRESET_EXTREME. + + filters (if provided) should be a sequence of dicts. Each dict + should have an entry for "id" indicating ID of the filter, plus + additional entries for options to the filter. + """ + self._fp = None + self._closefp = False + self._mode = None + + if mode in ("r", "rb"): + if check != -1: + raise ValueError("Cannot specify an integrity check " + "when opening a file for reading") + if preset is not None: + raise ValueError("Cannot specify a preset compression " + "level when opening a file for reading") + if format is None: + format = FORMAT_AUTO + mode_code = _MODE_READ + elif mode in ("w", "wb", "a", "ab", "x", "xb"): + if format is None: + format = FORMAT_XZ + mode_code = _MODE_WRITE + self._compressor = LZMACompressor(format=format, check=check, + preset=preset, filters=filters) + self._pos = 0 + else: + raise ValueError("Invalid mode: {!r}".format(mode)) + + if isinstance(filename, (str, bytes, os.PathLike)): + if "b" not in mode: + mode += "b" + self._fp = builtins.open(filename, mode) + self._closefp = True + self._mode = mode_code + elif hasattr(filename, "read") or hasattr(filename, "write"): + self._fp = filename + self._mode = mode_code + else: + raise TypeError("filename must be a str, bytes, file or PathLike object") + + if self._mode == _MODE_READ: + raw = _compression.DecompressReader(self._fp, LZMADecompressor, + trailing_error=LZMAError, format=format, filters=filters) + self._buffer = io.BufferedReader(raw) + + def close(self): + """Flush and close the file. + + May be called more than once without error. Once the file is + closed, any other operation on it will raise a ValueError. + """ + if self.closed: + return + try: + if self._mode == _MODE_READ: + self._buffer.close() + self._buffer = None + elif self._mode == _MODE_WRITE: + self._fp.write(self._compressor.flush()) + self._compressor = None + finally: + try: + if self._closefp: + self._fp.close() + finally: + self._fp = None + self._closefp = False + + @property + def closed(self): + """True if this file is closed.""" + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' + + def fileno(self): + """Return the file descriptor for the underlying file.""" + self._check_not_closed() + return self._fp.fileno() + + def seekable(self): + """Return whether the file supports seeking.""" + return self.readable() and self._buffer.seekable() + + def readable(self): + """Return whether the file was opened for reading.""" + self._check_not_closed() + return self._mode == _MODE_READ + + def writable(self): + """Return whether the file was opened for writing.""" + self._check_not_closed() + return self._mode == _MODE_WRITE + + def peek(self, size=-1): + """Return buffered data without advancing the file position. + + Always returns at least one byte of data, unless at EOF. + The exact number of bytes returned is unspecified. + """ + self._check_can_read() + # Relies on the undocumented fact that BufferedReader.peek() always + # returns at least one byte (except at EOF) + return self._buffer.peek(size) + + def read(self, size=-1): + """Read up to size uncompressed bytes from the file. + + If size is negative or omitted, read until EOF is reached. + Returns b"" if the file is already at EOF. + """ + self._check_can_read() + return self._buffer.read(size) + + def read1(self, size=-1): + """Read up to size uncompressed bytes, while trying to avoid + making multiple reads from the underlying stream. Reads up to a + buffer's worth of data if size is negative. + + Returns b"" if the file is at EOF. + """ + self._check_can_read() + if size < 0: + size = io.DEFAULT_BUFFER_SIZE + return self._buffer.read1(size) + + def readline(self, size=-1): + """Read a line of uncompressed bytes from the file. + + The terminating newline (if present) is retained. If size is + non-negative, no more than size bytes will be read (in which + case the line may be incomplete). Returns b'' if already at EOF. + """ + self._check_can_read() + return self._buffer.readline(size) + + def write(self, data): + """Write a bytes object to the file. + + Returns the number of uncompressed bytes written, which is + always the length of data in bytes. Note that due to buffering, + the file on disk may not reflect the data written until close() + is called. + """ + self._check_can_write() + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + # accept any data that supports the buffer protocol + data = memoryview(data) + length = data.nbytes + + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += length + return length + + def seek(self, offset, whence=io.SEEK_SET): + """Change the file position. + + The new position is specified by offset, relative to the + position indicated by whence. Possible values for whence are: + + 0: start of stream (default): offset must not be negative + 1: current stream position + 2: end of stream; offset must not be positive + + Returns the new file position. + + Note that seeking is emulated, so depending on the parameters, + this operation may be extremely slow. + """ + self._check_can_seek() + return self._buffer.seek(offset, whence) + + def tell(self): + """Return the current file position.""" + self._check_not_closed() + if self._mode == _MODE_READ: + return self._buffer.tell() + return self._pos + + +def open(filename, mode="rb", *, + format=None, check=-1, preset=None, filters=None, + encoding=None, errors=None, newline=None): + """Open an LZMA-compressed file in binary or text mode. + + filename can be either an actual file name (given as a str, bytes, + or PathLike object), in which case the named file is opened, or it + can be an existing file object to read from or write to. + + The mode argument can be "r", "rb" (default), "w", "wb", "x", "xb", + "a", or "ab" for binary mode, or "rt", "wt", "xt", or "at" for text + mode. + + The format, check, preset and filters arguments specify the + compression settings, as for LZMACompressor, LZMADecompressor and + LZMAFile. + + For binary mode, this function is equivalent to the LZMAFile + constructor: LZMAFile(filename, mode, ...). In this case, the + encoding, errors and newline arguments must not be provided. + + For text mode, an LZMAFile object is created, and wrapped in an + io.TextIOWrapper instance with the specified encoding, error + handling behavior, and line ending(s). + + """ + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") + + lz_mode = mode.replace("t", "") + binary_file = LZMAFile(filename, lz_mode, format=format, check=check, + preset=preset, filters=filters) + + if "t" in mode: + encoding = io.text_encoding(encoding) + return io.TextIOWrapper(binary_file, encoding, errors, newline) + else: + return binary_file + + +def compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None): + """Compress a block of data. + + Refer to LZMACompressor's docstring for a description of the + optional arguments *format*, *check*, *preset* and *filters*. + + For incremental compression, use an LZMACompressor instead. + """ + comp = LZMACompressor(format, check, preset, filters) + return comp.compress(data) + comp.flush() + + +def decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None): + """Decompress a block of data. + + Refer to LZMADecompressor's docstring for a description of the + optional arguments *format*, *check* and *filters*. + + For incremental decompression, use an LZMADecompressor instead. + """ + results = [] + while True: + decomp = LZMADecompressor(format, memlimit, filters) + try: + res = decomp.decompress(data) + except LZMAError: + if results: + break # Leftover data is not a valid LZMA/XZ stream; ignore it. + else: + raise # Error on the first iteration; bail out. + results.append(res) + if not decomp.eof: + raise LZMAError("Compressed data ended before the " + "end-of-stream marker was reached") + data = decomp.unused_data + if not data: + break + return b"".join(results) diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py new file mode 100644 index 0000000000..df6636c505 --- /dev/null +++ b/Lib/test/test_lzma.py @@ -0,0 +1,2095 @@ +import _compression +import array +from io import BytesIO, UnsupportedOperation, DEFAULT_BUFFER_SIZE +import os +import pickle +import random +import sys +from test import support +import unittest + +from test.support import _4G, bigmemtest +from test.support.import_helper import import_module +from test.support.os_helper import ( + TESTFN, unlink, FakePath +) + +lzma = import_module("lzma") +from lzma import LZMACompressor, LZMADecompressor, LZMAError, LZMAFile + + +class CompressorDecompressorTestCase(unittest.TestCase): + + # Test error cases. + + def test_simple_bad_args(self): + self.assertRaises(TypeError, LZMACompressor, []) + self.assertRaises(TypeError, LZMACompressor, format=3.45) + self.assertRaises(TypeError, LZMACompressor, check="") + self.assertRaises(TypeError, LZMACompressor, preset="asdf") + self.assertRaises(TypeError, LZMACompressor, filters=3) + # Can't specify FORMAT_AUTO when compressing. + self.assertRaises(ValueError, LZMACompressor, format=lzma.FORMAT_AUTO) + # Can't specify a preset and a custom filter chain at the same time. + with self.assertRaises(ValueError): + LZMACompressor(preset=7, filters=[{"id": lzma.FILTER_LZMA2}]) + + self.assertRaises(TypeError, LZMADecompressor, ()) + self.assertRaises(TypeError, LZMADecompressor, memlimit=b"qw") + with self.assertRaises(TypeError): + LZMADecompressor(lzma.FORMAT_RAW, filters="zzz") + # Cannot specify a memory limit with FILTER_RAW. + with self.assertRaises(ValueError): + LZMADecompressor(lzma.FORMAT_RAW, memlimit=0x1000000) + # Can only specify a custom filter chain with FILTER_RAW. + self.assertRaises(ValueError, LZMADecompressor, filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + LZMADecompressor(format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + LZMADecompressor(format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) + + lzc = LZMACompressor() + self.assertRaises(TypeError, lzc.compress) + self.assertRaises(TypeError, lzc.compress, b"foo", b"bar") + self.assertRaises(TypeError, lzc.flush, b"blah") + empty = lzc.flush() + self.assertRaises(ValueError, lzc.compress, b"quux") + self.assertRaises(ValueError, lzc.flush) + + lzd = LZMADecompressor() + self.assertRaises(TypeError, lzd.decompress) + self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar") + lzd.decompress(empty) + self.assertRaises(EOFError, lzd.decompress, b"quux") + + def test_bad_filter_spec(self): + self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"]) + self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}]) + self.assertRaises(ValueError, LZMACompressor, filters=[{"id": 98765}]) + with self.assertRaises(ValueError): + LZMACompressor(filters=[{"id": lzma.FILTER_LZMA2, "foo": 0}]) + with self.assertRaises(ValueError): + LZMACompressor(filters=[{"id": lzma.FILTER_DELTA, "foo": 0}]) + with self.assertRaises(ValueError): + LZMACompressor(filters=[{"id": lzma.FILTER_X86, "foo": 0}]) + + def test_decompressor_after_eof(self): + lzd = LZMADecompressor() + lzd.decompress(COMPRESSED_XZ) + self.assertRaises(EOFError, lzd.decompress, b"nyan") + + def test_decompressor_memlimit(self): + lzd = LZMADecompressor(memlimit=1024) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + lzd = LZMADecompressor(lzma.FORMAT_XZ, memlimit=1024) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + lzd = LZMADecompressor(lzma.FORMAT_ALONE, memlimit=1024) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE) + + # Test LZMADecompressor on known-good input data. + + def _test_decompressor(self, lzd, data, check, unused_data=b""): + self.assertFalse(lzd.eof) + out = lzd.decompress(data) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, check) + self.assertTrue(lzd.eof) + self.assertEqual(lzd.unused_data, unused_data) + + def test_decompressor_auto(self): + lzd = LZMADecompressor() + self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) + + lzd = LZMADecompressor() + self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) + + def test_decompressor_xz(self): + lzd = LZMADecompressor(lzma.FORMAT_XZ) + self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) + + def test_decompressor_alone(self): + lzd = LZMADecompressor(lzma.FORMAT_ALONE) + self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) + + def test_decompressor_raw_1(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE) + + def test_decompressor_raw_2(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE) + + def test_decompressor_raw_3(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE) + + def test_decompressor_raw_4(self): + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE) + + def test_decompressor_chunks(self): + lzd = LZMADecompressor() + out = [] + for i in range(0, len(COMPRESSED_XZ), 10): + self.assertFalse(lzd.eof) + out.append(lzd.decompress(COMPRESSED_XZ[i:i+10])) + out = b"".join(out) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertTrue(lzd.eof) + self.assertEqual(lzd.unused_data, b"") + + def test_decompressor_chunks_empty(self): + lzd = LZMADecompressor() + out = [] + for i in range(0, len(COMPRESSED_XZ), 10): + self.assertFalse(lzd.eof) + out.append(lzd.decompress(b'')) + out.append(lzd.decompress(b'')) + out.append(lzd.decompress(b'')) + out.append(lzd.decompress(COMPRESSED_XZ[i:i+10])) + out = b"".join(out) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertTrue(lzd.eof) + self.assertEqual(lzd.unused_data, b"") + + def test_decompressor_chunks_maxsize(self): + lzd = LZMADecompressor() + max_length = 100 + out = [] + + # Feed first half the input + len_ = len(COMPRESSED_XZ) // 2 + out.append(lzd.decompress(COMPRESSED_XZ[:len_], + max_length=max_length)) + self.assertFalse(lzd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data without providing more input + out.append(lzd.decompress(b'', max_length=max_length)) + self.assertFalse(lzd.needs_input) + self.assertEqual(len(out[-1]), max_length) + + # Retrieve more data while providing more input + out.append(lzd.decompress(COMPRESSED_XZ[len_:], + max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + # Retrieve remaining uncompressed data + while not lzd.eof: + out.append(lzd.decompress(b'', max_length=max_length)) + self.assertLessEqual(len(out[-1]), max_length) + + out = b"".join(out) + self.assertEqual(out, INPUT) + self.assertEqual(lzd.check, lzma.CHECK_CRC64) + self.assertEqual(lzd.unused_data, b"") + + def test_decompressor_inputbuf_1(self): + # Test reusing input buffer after moving existing + # contents to beginning + lzd = LZMADecompressor() + out = [] + + # Create input buffer and fill it + self.assertEqual(lzd.decompress(COMPRESSED_XZ[:100], + max_length=0), b'') + + # Retrieve some results, freeing capacity at beginning + # of input buffer + out.append(lzd.decompress(b'', 2)) + + # Add more data that fits into input buffer after + # moving existing data to beginning + out.append(lzd.decompress(COMPRESSED_XZ[100:105], 15)) + + # Decompress rest of data + out.append(lzd.decompress(COMPRESSED_XZ[105:])) + self.assertEqual(b''.join(out), INPUT) + + def test_decompressor_inputbuf_2(self): + # Test reusing input buffer by appending data at the + # end right away + lzd = LZMADecompressor() + out = [] + + # Create input buffer and empty it + self.assertEqual(lzd.decompress(COMPRESSED_XZ[:200], + max_length=0), b'') + out.append(lzd.decompress(b'')) + + # Fill buffer with new data + out.append(lzd.decompress(COMPRESSED_XZ[200:280], 2)) + + # Append some more data, not enough to require resize + out.append(lzd.decompress(COMPRESSED_XZ[280:300], 2)) + + # Decompress rest of data + out.append(lzd.decompress(COMPRESSED_XZ[300:])) + self.assertEqual(b''.join(out), INPUT) + + def test_decompressor_inputbuf_3(self): + # Test reusing input buffer after extending it + + lzd = LZMADecompressor() + out = [] + + # Create almost full input buffer + out.append(lzd.decompress(COMPRESSED_XZ[:200], 5)) + + # Add even more data to it, requiring resize + out.append(lzd.decompress(COMPRESSED_XZ[200:300], 5)) + + # Decompress rest of data + out.append(lzd.decompress(COMPRESSED_XZ[300:])) + self.assertEqual(b''.join(out), INPUT) + + def test_decompressor_unused_data(self): + lzd = LZMADecompressor() + extra = b"fooblibar" + self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64, + unused_data=extra) + + def test_decompressor_bad_input(self): + lzd = LZMADecompressor() + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) + + lzd = LZMADecompressor(lzma.FORMAT_XZ) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_ALONE) + + lzd = LZMADecompressor(lzma.FORMAT_ALONE) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + + def test_decompressor_bug_28275(self): + # Test coverage for Issue 28275 + lzd = LZMADecompressor() + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) + # Previously, a second call could crash due to internal inconsistency + self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) + + # Test that LZMACompressor->LZMADecompressor preserves the input data. + + def test_roundtrip_xz(self): + lzc = LZMACompressor() + cdata = lzc.compress(INPUT) + lzc.flush() + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + + def test_roundtrip_alone(self): + lzc = LZMACompressor(lzma.FORMAT_ALONE) + cdata = lzc.compress(INPUT) + lzc.flush() + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + + def test_roundtrip_raw(self): + lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + cdata = lzc.compress(INPUT) + lzc.flush() + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + + def test_roundtrip_raw_empty(self): + lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + cdata = lzc.compress(INPUT) + cdata += lzc.compress(b'') + cdata += lzc.compress(b'') + cdata += lzc.compress(b'') + cdata += lzc.flush() + lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + + def test_roundtrip_chunks(self): + lzc = LZMACompressor() + cdata = [] + for i in range(0, len(INPUT), 10): + cdata.append(lzc.compress(INPUT[i:i+10])) + cdata.append(lzc.flush()) + cdata = b"".join(cdata) + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + + def test_roundtrip_empty_chunks(self): + lzc = LZMACompressor() + cdata = [] + for i in range(0, len(INPUT), 10): + cdata.append(lzc.compress(INPUT[i:i+10])) + cdata.append(lzc.compress(b'')) + cdata.append(lzc.compress(b'')) + cdata.append(lzc.compress(b'')) + cdata.append(lzc.flush()) + cdata = b"".join(cdata) + lzd = LZMADecompressor() + self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + + # LZMADecompressor intentionally does not handle concatenated streams. + + def test_decompressor_multistream(self): + lzd = LZMADecompressor() + self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE, + lzma.CHECK_CRC64, unused_data=COMPRESSED_ALONE) + + # Test with inputs larger than 4GiB. + + @support.skip_if_pgo_task + @bigmemtest(size=_4G + 100, memuse=2) + def test_compressor_bigmem(self, size): + lzc = LZMACompressor() + cdata = lzc.compress(b"x" * size) + lzc.flush() + ddata = lzma.decompress(cdata) + try: + self.assertEqual(len(ddata), size) + self.assertEqual(len(ddata.strip(b"x")), 0) + finally: + ddata = None + + @support.skip_if_pgo_task + @bigmemtest(size=_4G + 100, memuse=3) + def test_decompressor_bigmem(self, size): + lzd = LZMADecompressor() + blocksize = min(10 * 1024 * 1024, size) + block = random.randbytes(blocksize) + try: + input = block * ((size-1) // blocksize + 1) + cdata = lzma.compress(input) + ddata = lzd.decompress(cdata) + self.assertEqual(ddata, input) + finally: + input = cdata = ddata = None + + # Pickling raises an exception; there's no way to serialize an lzma_stream. + + def test_pickle(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(TypeError): + pickle.dumps(LZMACompressor(), proto) + with self.assertRaises(TypeError): + pickle.dumps(LZMADecompressor(), proto) + + @support.refcount_test + def test_refleaks_in_decompressor___init__(self): + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + lzd = LZMADecompressor() + refs_before = gettotalrefcount() + for i in range(100): + lzd.__init__() + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_uninitialized_LZMADecompressor_crash(self): + self.assertEqual(LZMADecompressor.__new__(LZMADecompressor). + decompress(bytes()), b'') + + +class CompressDecompressFunctionTestCase(unittest.TestCase): + + # Test error cases: + + def test_bad_args(self): + self.assertRaises(TypeError, lzma.compress) + self.assertRaises(TypeError, lzma.compress, []) + self.assertRaises(TypeError, lzma.compress, b"", format="xz") + self.assertRaises(TypeError, lzma.compress, b"", check="none") + self.assertRaises(TypeError, lzma.compress, b"", preset="blah") + self.assertRaises(TypeError, lzma.compress, b"", filters=1024) + # Can't specify a preset and a custom filter chain at the same time. + with self.assertRaises(ValueError): + lzma.compress(b"", preset=3, filters=[{"id": lzma.FILTER_LZMA2}]) + + self.assertRaises(TypeError, lzma.decompress) + self.assertRaises(TypeError, lzma.decompress, []) + self.assertRaises(TypeError, lzma.decompress, b"", format="lzma") + self.assertRaises(TypeError, lzma.decompress, b"", memlimit=7.3e9) + with self.assertRaises(TypeError): + lzma.decompress(b"", format=lzma.FORMAT_RAW, filters={}) + # Cannot specify a memory limit with FILTER_RAW. + with self.assertRaises(ValueError): + lzma.decompress(b"", format=lzma.FORMAT_RAW, memlimit=0x1000000) + # Can only specify a custom filter chain with FILTER_RAW. + with self.assertRaises(ValueError): + lzma.decompress(b"", filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + lzma.decompress(b"", format=lzma.FORMAT_XZ, filters=FILTERS_RAW_1) + with self.assertRaises(ValueError): + lzma.decompress( + b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) + + def test_decompress_memlimit(self): + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_XZ, memlimit=1024) + with self.assertRaises(LZMAError): + lzma.decompress( + COMPRESSED_XZ, format=lzma.FORMAT_XZ, memlimit=1024) + with self.assertRaises(LZMAError): + lzma.decompress( + COMPRESSED_ALONE, format=lzma.FORMAT_ALONE, memlimit=1024) + + # Test LZMADecompressor on known-good input data. + + def test_decompress_good_input(self): + ddata = lzma.decompress(COMPRESSED_XZ) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress(COMPRESSED_ALONE) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress(COMPRESSED_XZ, lzma.FORMAT_XZ) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress(COMPRESSED_ALONE, lzma.FORMAT_ALONE) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_1, lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_2, lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_3, lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + self.assertEqual(ddata, INPUT) + + ddata = lzma.decompress( + COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self.assertEqual(ddata, INPUT) + + def test_decompress_incomplete_input(self): + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128]) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128]) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_1[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_1) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_2[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_3[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) + self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128], + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + + def test_decompress_bad_input(self): + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_BOGUS) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_RAW_1) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_ALONE, format=lzma.FORMAT_XZ) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_XZ, format=lzma.FORMAT_ALONE) + with self.assertRaises(LZMAError): + lzma.decompress(COMPRESSED_XZ, format=lzma.FORMAT_RAW, + filters=FILTERS_RAW_1) + + # Test that compress()->decompress() preserves the input data. + + def test_roundtrip(self): + cdata = lzma.compress(INPUT) + ddata = lzma.decompress(cdata) + self.assertEqual(ddata, INPUT) + + cdata = lzma.compress(INPUT, lzma.FORMAT_XZ) + ddata = lzma.decompress(cdata) + self.assertEqual(ddata, INPUT) + + cdata = lzma.compress(INPUT, lzma.FORMAT_ALONE) + ddata = lzma.decompress(cdata) + self.assertEqual(ddata, INPUT) + + cdata = lzma.compress(INPUT, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + ddata = lzma.decompress(cdata, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + self.assertEqual(ddata, INPUT) + + # Unlike LZMADecompressor, decompress() *does* handle concatenated streams. + + def test_decompress_multistream(self): + ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_ALONE) + self.assertEqual(ddata, INPUT * 2) + + # Test robust handling of non-LZMA data following the compressed stream(s). + + def test_decompress_trailing_junk(self): + ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_BOGUS) + self.assertEqual(ddata, INPUT) + + def test_decompress_multistream_trailing_junk(self): + ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS) + self.assertEqual(ddata, INPUT * 3) + + +class TempFile: + """Context manager - creates a file, and deletes it on __exit__.""" + + def __init__(self, filename, data=b""): + self.filename = filename + self.data = data + + def __enter__(self): + with open(self.filename, "wb") as f: + f.write(self.data) + + def __exit__(self, *args): + unlink(self.filename) + + +class FileTestCase(unittest.TestCase): + + def test_init(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "rb") + with LZMAFile(BytesIO(), "w") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(BytesIO(), "x") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(BytesIO(), "a") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + + def test_init_with_PathLike_filename(self): + filename = FakePath(TESTFN) + with TempFile(filename, COMPRESSED_XZ): + with LZMAFile(filename) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) + with LZMAFile(filename, "a") as f: + f.write(INPUT) + self.assertEqual(f.name, TESTFN) + with LZMAFile(filename) as f: + self.assertEqual(f.read(), INPUT * 2) + self.assertEqual(f.name, TESTFN) + + def test_init_with_filename(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with LZMAFile(TESTFN) as f: + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'rb') + with LZMAFile(TESTFN, "w") as f: + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') + with LZMAFile(TESTFN, "a") as f: + self.assertEqual(f.name, TESTFN) + self.assertEqual(f.mode, 'wb') + + def test_init_mode(self): + with TempFile(TESTFN): + with LZMAFile(TESTFN, "r") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "rb") + with LZMAFile(TESTFN, "rb") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "rb") + with LZMAFile(TESTFN, "w") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(TESTFN, "wb") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(TESTFN, "a") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + with LZMAFile(TESTFN, "ab") as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, "wb") + + def test_init_with_x_mode(self): + self.addCleanup(unlink, TESTFN) + for mode in ("x", "xb"): + unlink(TESTFN) + with LZMAFile(TESTFN, mode) as f: + self.assertIsInstance(f, LZMAFile) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(FileExistsError): + LZMAFile(TESTFN, mode) + + def test_init_bad_mode(self): + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), (3, "x")) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "xt") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "x+") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "rx") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "wx") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "rt") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "r+") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "wt") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "w+") + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), "rw") + + def test_init_bad_check(self): + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", check=b"asd") + # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid. + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", check=lzma.CHECK_UNKNOWN) + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", check=lzma.CHECK_ID_MAX + 3) + # Cannot specify a check with mode="r". + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_NONE) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_CRC32) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_CRC64) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_SHA256) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), check=lzma.CHECK_UNKNOWN) + + def test_init_bad_preset(self): + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", preset=4.39) + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", preset=10) + with self.assertRaises(LZMAError): + LZMAFile(BytesIO(), "w", preset=23) + with self.assertRaises(OverflowError): + LZMAFile(BytesIO(), "w", preset=-1) + with self.assertRaises(OverflowError): + LZMAFile(BytesIO(), "w", preset=-7) + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", preset="foo") + # Cannot specify a preset with mode="r". + with self.assertRaises(ValueError): + LZMAFile(BytesIO(COMPRESSED_XZ), preset=3) + + def test_init_bad_filter_spec(self): + with self.assertRaises(TypeError): + LZMAFile(BytesIO(), "w", filters=[b"wobsite"]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", filters=[{"xyzzy": 3}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", filters=[{"id": 98765}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", + filters=[{"id": lzma.FILTER_LZMA2, "foo": 0}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", + filters=[{"id": lzma.FILTER_DELTA, "foo": 0}]) + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", + filters=[{"id": lzma.FILTER_X86, "foo": 0}]) + + def test_init_with_preset_and_filters(self): + with self.assertRaises(ValueError): + LZMAFile(BytesIO(), "w", format=lzma.FORMAT_RAW, + preset=6, filters=FILTERS_RAW_1) + + def test_close(self): + with BytesIO(COMPRESSED_XZ) as src: + f = LZMAFile(src) + f.close() + # LZMAFile.close() should not close the underlying file object. + self.assertFalse(src.closed) + # Try closing an already-closed LZMAFile. + f.close() + self.assertFalse(src.closed) + + # Test with a real file on disk, opened directly by LZMAFile. + with TempFile(TESTFN, COMPRESSED_XZ): + f = LZMAFile(TESTFN) + fp = f._fp + f.close() + # Here, LZMAFile.close() *should* close the underlying file object. + self.assertTrue(fp.closed) + # Try closing an already-closed LZMAFile. + f.close() + + def test_closed(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertFalse(f.closed) + f.read() + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertFalse(f.closed) + finally: + f.close() + self.assertTrue(f.closed) + + def test_fileno(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertRaises(UnsupportedOperation, f.fileno) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + with TempFile(TESTFN, COMPRESSED_XZ): + f = LZMAFile(TESTFN) + try: + self.assertEqual(f.fileno(), f._fp.fileno()) + self.assertIsInstance(f.fileno(), int) + finally: + f.close() + self.assertRaises(ValueError, f.fileno) + + def test_seekable(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertTrue(f.seekable()) + f.read() + self.assertTrue(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + src = BytesIO(COMPRESSED_XZ) + src.seekable = lambda: False + f = LZMAFile(src) + try: + self.assertFalse(f.seekable()) + finally: + f.close() + self.assertRaises(ValueError, f.seekable) + + def test_readable(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertTrue(f.readable()) + f.read() + self.assertTrue(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertFalse(f.readable()) + finally: + f.close() + self.assertRaises(ValueError, f.readable) + + def test_writable(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + try: + self.assertFalse(f.writable()) + f.read() + self.assertFalse(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + f = LZMAFile(BytesIO(), "w") + try: + self.assertTrue(f.writable()) + finally: + f.close() + self.assertRaises(ValueError, f.writable) + + def test_read(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f: + self.assertEqual(f.read(), INPUT) + with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_1), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_1) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_2), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_3), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + with LZMAFile(BytesIO(COMPRESSED_RAW_4), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + + def test_read_0(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertEqual(f.read(0), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f: + self.assertEqual(f.read(0), b"") + with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f: + self.assertEqual(f.read(0), b"") + with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f: + self.assertEqual(f.read(0), b"") + + def test_read_10(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + chunks = [] + while result := f.read(10): + self.assertLessEqual(len(result), 10) + chunks.append(result) + self.assertEqual(b"".join(chunks), INPUT) + + def test_read_multistream(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: + self.assertEqual(f.read(), INPUT * 5) + with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_ALONE)) as f: + self.assertEqual(f.read(), INPUT * 2) + with LZMAFile(BytesIO(COMPRESSED_RAW_3 * 4), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_3) as f: + self.assertEqual(f.read(), INPUT * 4) + + def test_read_multistream_buffer_size_aligned(self): + # Test the case where a stream boundary coincides with the end + # of the raw read buffer. + saved_buffer_size = _compression.BUFFER_SIZE + _compression.BUFFER_SIZE = len(COMPRESSED_XZ) + try: + with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: + self.assertEqual(f.read(), INPUT * 5) + finally: + _compression.BUFFER_SIZE = saved_buffer_size + + def test_read_trailing_junk(self): + with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f: + self.assertEqual(f.read(), INPUT) + + def test_read_multistream_trailing_junk(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f: + self.assertEqual(f.read(), INPUT * 5) + + def test_read_from_file(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with LZMAFile(TESTFN) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_file_with_bytes_filename(self): + bytes_filename = os.fsencode(TESTFN) + with TempFile(TESTFN, COMPRESSED_XZ): + with LZMAFile(bytes_filename) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, bytes_filename) + + def test_read_from_fileobj(self): + with TempFile(TESTFN, COMPRESSED_XZ): + with open(TESTFN, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_from_fileobj_with_int_name(self): + with TempFile(TESTFN, COMPRESSED_XZ): + fd = os.open(TESTFN, os.O_RDONLY) + with open(fd, 'rb') as raw: + with LZMAFile(raw) as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.read(), b"") + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def test_read_incomplete(self): + with LZMAFile(BytesIO(COMPRESSED_XZ[:128])) as f: + self.assertRaises(EOFError, f.read) + + def test_read_truncated(self): + # Drop stream footer: CRC (4 bytes), index size (4 bytes), + # flags (2 bytes) and magic number (2 bytes). + truncated = COMPRESSED_XZ[:-12] + with LZMAFile(BytesIO(truncated)) as f: + self.assertRaises(EOFError, f.read) + with LZMAFile(BytesIO(truncated)) as f: + self.assertEqual(f.read(len(INPUT)), INPUT) + self.assertRaises(EOFError, f.read, 1) + # Incomplete 12-byte header. + for i in range(12): + with LZMAFile(BytesIO(truncated[:i])) as f: + self.assertRaises(EOFError, f.read, 1) + + def test_read_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.read) + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertRaises(TypeError, f.read, float()) + + def test_read_bad_data(self): + with LZMAFile(BytesIO(COMPRESSED_BOGUS)) as f: + self.assertRaises(LZMAError, f.read) + + def test_read1(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + blocks = [] + while result := f.read1(): + blocks.append(result) + self.assertEqual(b"".join(blocks), INPUT) + self.assertEqual(f.read1(), b"") + + def test_read1_0(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertEqual(f.read1(0), b"") + + def test_read1_10(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + blocks = [] + while result := f.read1(10): + blocks.append(result) + self.assertEqual(b"".join(blocks), INPUT) + self.assertEqual(f.read1(), b"") + + def test_read1_multistream(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: + blocks = [] + while result := f.read1(): + blocks.append(result) + self.assertEqual(b"".join(blocks), INPUT * 5) + self.assertEqual(f.read1(), b"") + + def test_read1_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.read1) + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.read1) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertRaises(TypeError, f.read1, None) + + def test_peek(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + result = f.peek() + self.assertGreater(len(result), 0) + self.assertTrue(INPUT.startswith(result)) + self.assertEqual(f.read(), INPUT) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + result = f.peek(10) + self.assertGreater(len(result), 0) + self.assertTrue(INPUT.startswith(result)) + self.assertEqual(f.read(), INPUT) + + def test_peek_bad_args(self): + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.peek) + + def test_iterator(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_ALONE)) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_XZ), format=lzma.FORMAT_XZ) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_ALONE), format=lzma.FORMAT_ALONE) as f: + self.assertListEqual(list(iter(f)), lines) + with LZMAFile(BytesIO(COMPRESSED_RAW_2), + format=lzma.FORMAT_RAW, filters=FILTERS_RAW_2) as f: + self.assertListEqual(list(iter(f)), lines) + + def test_readline(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + for line in lines: + self.assertEqual(f.readline(), line) + + def test_readlines(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertListEqual(f.readlines(), lines) + + def test_decompress_limited(self): + """Decompressed data buffering should be limited""" + bomb = lzma.compress(b'\0' * int(2e6), preset=6) + self.assertLess(len(bomb), _compression.BUFFER_SIZE) + + decomp = LZMAFile(BytesIO(bomb)) + self.assertEqual(decomp.read(1), b'\0') + max_decomp = 1 + DEFAULT_BUFFER_SIZE + self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, + "Excessive amount of data was decompressed") + + def test_write(self): + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + f.write(INPUT) + with self.assertRaises(AttributeError): + f.name + expected = lzma.compress(INPUT) + self.assertEqual(dst.getvalue(), expected) + with BytesIO() as dst: + with LZMAFile(dst, "w", format=lzma.FORMAT_XZ) as f: + f.write(INPUT) + expected = lzma.compress(INPUT, format=lzma.FORMAT_XZ) + self.assertEqual(dst.getvalue(), expected) + with BytesIO() as dst: + with LZMAFile(dst, "w", format=lzma.FORMAT_ALONE) as f: + f.write(INPUT) + expected = lzma.compress(INPUT, format=lzma.FORMAT_ALONE) + self.assertEqual(dst.getvalue(), expected) + with BytesIO() as dst: + with LZMAFile(dst, "w", format=lzma.FORMAT_RAW, + filters=FILTERS_RAW_2) as f: + f.write(INPUT) + expected = lzma.compress(INPUT, format=lzma.FORMAT_RAW, + filters=FILTERS_RAW_2) + self.assertEqual(dst.getvalue(), expected) + + def test_write_10(self): + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + for start in range(0, len(INPUT), 10): + f.write(INPUT[start:start+10]) + expected = lzma.compress(INPUT) + self.assertEqual(dst.getvalue(), expected) + + def test_write_append(self): + part1 = INPUT[:1024] + part2 = INPUT[1024:1536] + part3 = INPUT[1536:] + expected = b"".join(lzma.compress(x) for x in (part1, part2, part3)) + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + f.write(part1) + self.assertEqual(f.mode, 'wb') + with LZMAFile(dst, "a") as f: + f.write(part2) + self.assertEqual(f.mode, 'wb') + with LZMAFile(dst, "a") as f: + f.write(part3) + self.assertEqual(f.mode, 'wb') + self.assertEqual(dst.getvalue(), expected) + + def test_write_to_file(self): + try: + with LZMAFile(TESTFN, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, TESTFN) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_file_with_bytes_filename(self): + bytes_filename = os.fsencode(TESTFN) + try: + with LZMAFile(bytes_filename, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, bytes_filename) + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj(self): + try: + with open(TESTFN, "wb") as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_to_fileobj_with_int_name(self): + try: + fd = os.open(TESTFN, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with LZMAFile(raw, "w") as f: + f.write(INPUT) + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + expected = lzma.compress(INPUT) + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_append_to_file(self): + part1 = INPUT[:1024] + part2 = INPUT[1024:1536] + part3 = INPUT[1536:] + expected = b"".join(lzma.compress(x) for x in (part1, part2, part3)) + try: + with LZMAFile(TESTFN, "w") as f: + f.write(part1) + self.assertEqual(f.mode, 'wb') + with LZMAFile(TESTFN, "a") as f: + f.write(part2) + self.assertEqual(f.mode, 'wb') + with LZMAFile(TESTFN, "a") as f: + f.write(part3) + self.assertEqual(f.mode, 'wb') + with open(TESTFN, "rb") as f: + self.assertEqual(f.read(), expected) + finally: + unlink(TESTFN) + + def test_write_bad_args(self): + f = LZMAFile(BytesIO(), "w") + f.close() + self.assertRaises(ValueError, f.write, b"foo") + with LZMAFile(BytesIO(COMPRESSED_XZ), "r") as f: + self.assertRaises(ValueError, f.write, b"bar") + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(TypeError, f.write, None) + self.assertRaises(TypeError, f.write, "text") + self.assertRaises(TypeError, f.write, 789) + + def test_writelines(self): + with BytesIO(INPUT) as f: + lines = f.readlines() + with BytesIO() as dst: + with LZMAFile(dst, "w") as f: + f.writelines(lines) + expected = lzma.compress(INPUT) + self.assertEqual(dst.getvalue(), expected) + + def test_seek_forward(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(555) + self.assertEqual(f.read(), INPUT[555:]) + + def test_seek_forward_across_streams(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 2)) as f: + f.seek(len(INPUT) + 123) + self.assertEqual(f.read(), INPUT[123:]) + + def test_seek_forward_relative_to_current(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.read(100) + f.seek(1236, 1) + self.assertEqual(f.read(), INPUT[1336:]) + + def test_seek_forward_relative_to_end(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(-555, 2) + self.assertEqual(f.read(), INPUT[-555:]) + + def test_seek_backward(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.read(1001) + f.seek(211) + self.assertEqual(f.read(), INPUT[211:]) + + def test_seek_backward_across_streams(self): + with LZMAFile(BytesIO(COMPRESSED_XZ * 2)) as f: + f.read(len(INPUT) + 333) + f.seek(737) + self.assertEqual(f.read(), INPUT[737:] + INPUT) + + def test_seek_backward_relative_to_end(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(-150, 2) + self.assertEqual(f.read(), INPUT[-150:]) + + def test_seek_past_end(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(len(INPUT) + 9001) + self.assertEqual(f.tell(), len(INPUT)) + self.assertEqual(f.read(), b"") + + def test_seek_past_start(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + f.seek(-88) + self.assertEqual(f.tell(), 0) + self.assertEqual(f.read(), INPUT) + + def test_seek_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.seek, 0) + with LZMAFile(BytesIO(), "w") as f: + self.assertRaises(ValueError, f.seek, 0) + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + self.assertRaises(ValueError, f.seek, 0, 3) + # io.BufferedReader raises TypeError instead of ValueError + self.assertRaises((TypeError, ValueError), f.seek, 9, ()) + self.assertRaises(TypeError, f.seek, None) + self.assertRaises(TypeError, f.seek, b"derp") + + def test_tell(self): + with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: + pos = 0 + while True: + self.assertEqual(f.tell(), pos) + result = f.read(183) + if not result: + break + pos += len(result) + self.assertEqual(f.tell(), len(INPUT)) + with LZMAFile(BytesIO(), "w") as f: + for pos in range(0, len(INPUT), 144): + self.assertEqual(f.tell(), pos) + f.write(INPUT[pos:pos+144]) + self.assertEqual(f.tell(), len(INPUT)) + + def test_tell_bad_args(self): + f = LZMAFile(BytesIO(COMPRESSED_XZ)) + f.close() + self.assertRaises(ValueError, f.tell) + + def test_issue21872(self): + # sometimes decompress data incompletely + + # --------------------- + # when max_length == -1 + # --------------------- + d1 = LZMADecompressor() + entire = d1.decompress(ISSUE_21872_DAT, max_length=-1) + self.assertEqual(len(entire), 13160) + self.assertTrue(d1.eof) + + # --------------------- + # when max_length > 0 + # --------------------- + d2 = LZMADecompressor() + + # When this value of max_length is used, the input and output + # buffers are exhausted at the same time, and lzs's internal + # state still have 11 bytes can be output. + out1 = d2.decompress(ISSUE_21872_DAT, max_length=13149) + self.assertFalse(d2.needs_input) # ensure needs_input mechanism works + self.assertFalse(d2.eof) + + # simulate needs_input mechanism + # output internal state's 11 bytes + out2 = d2.decompress(b'') + self.assertEqual(len(out2), 11) + self.assertTrue(d2.eof) + self.assertEqual(out1 + out2, entire) + + def test_issue44439(self): + q = array.array('Q', [1, 2, 3, 4, 5]) + LENGTH = len(q) * q.itemsize + + with LZMAFile(BytesIO(), 'w') as f: + self.assertEqual(f.write(q), LENGTH) + self.assertEqual(f.tell(), LENGTH) + + +class OpenTestCase(unittest.TestCase): + + def test_binary_modes(self): + with lzma.open(BytesIO(COMPRESSED_XZ), "rb") as f: + self.assertEqual(f.read(), INPUT) + with BytesIO() as bio: + with lzma.open(bio, "wb") as f: + f.write(INPUT) + file_data = lzma.decompress(bio.getvalue()) + self.assertEqual(file_data, INPUT) + with lzma.open(bio, "ab") as f: + f.write(INPUT) + file_data = lzma.decompress(bio.getvalue()) + self.assertEqual(file_data, INPUT * 2) + + def test_text_modes(self): + uncompressed = INPUT.decode("ascii") + uncompressed_raw = uncompressed.replace("\n", os.linesep) + with lzma.open(BytesIO(COMPRESSED_XZ), "rt", encoding="ascii") as f: + self.assertEqual(f.read(), uncompressed) + with BytesIO() as bio: + with lzma.open(bio, "wt", encoding="ascii") as f: + f.write(uncompressed) + file_data = lzma.decompress(bio.getvalue()).decode("ascii") + self.assertEqual(file_data, uncompressed_raw) + with lzma.open(bio, "at", encoding="ascii") as f: + f.write(uncompressed) + file_data = lzma.decompress(bio.getvalue()).decode("ascii") + self.assertEqual(file_data, uncompressed_raw * 2) + + def test_filename(self): + with TempFile(TESTFN): + with lzma.open(TESTFN, "wb") as f: + f.write(INPUT) + with open(TESTFN, "rb") as f: + file_data = lzma.decompress(f.read()) + self.assertEqual(file_data, INPUT) + with lzma.open(TESTFN, "rb") as f: + self.assertEqual(f.read(), INPUT) + with lzma.open(TESTFN, "ab") as f: + f.write(INPUT) + with lzma.open(TESTFN, "rb") as f: + self.assertEqual(f.read(), INPUT * 2) + + def test_with_pathlike_filename(self): + filename = FakePath(TESTFN) + with TempFile(filename): + with lzma.open(filename, "wb") as f: + f.write(INPUT) + self.assertEqual(f.name, TESTFN) + with open(filename, "rb") as f: + file_data = lzma.decompress(f.read()) + self.assertEqual(file_data, INPUT) + with lzma.open(filename, "rb") as f: + self.assertEqual(f.read(), INPUT) + self.assertEqual(f.name, TESTFN) + + def test_bad_params(self): + # Test invalid parameter combinations. + with self.assertRaises(ValueError): + lzma.open(TESTFN, "") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rbt") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rb", encoding="utf-8") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rb", errors="ignore") + with self.assertRaises(ValueError): + lzma.open(TESTFN, "rb", newline="\n") + + def test_format_and_filters(self): + # Test non-default format and filter chain. + options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1} + with lzma.open(BytesIO(COMPRESSED_RAW_1), "rb", **options) as f: + self.assertEqual(f.read(), INPUT) + with BytesIO() as bio: + with lzma.open(bio, "wb", **options) as f: + f.write(INPUT) + file_data = lzma.decompress(bio.getvalue(), **options) + self.assertEqual(file_data, INPUT) + + def test_encoding(self): + # Test non-default encoding. + uncompressed = INPUT.decode("ascii") + uncompressed_raw = uncompressed.replace("\n", os.linesep) + with BytesIO() as bio: + with lzma.open(bio, "wt", encoding="utf-16-le") as f: + f.write(uncompressed) + file_data = lzma.decompress(bio.getvalue()).decode("utf-16-le") + self.assertEqual(file_data, uncompressed_raw) + bio.seek(0) + with lzma.open(bio, "rt", encoding="utf-16-le") as f: + self.assertEqual(f.read(), uncompressed) + + def test_encoding_error_handler(self): + # Test with non-default encoding error handler. + with BytesIO(lzma.compress(b"foo\xffbar")) as bio: + with lzma.open(bio, "rt", encoding="ascii", errors="ignore") as f: + self.assertEqual(f.read(), "foobar") + + def test_newline(self): + # Test with explicit newline (universal newline mode disabled). + text = INPUT.decode("ascii") + with BytesIO() as bio: + with lzma.open(bio, "wt", encoding="ascii", newline="\n") as f: + f.write(text) + bio.seek(0) + with lzma.open(bio, "rt", encoding="ascii", newline="\r") as f: + self.assertEqual(f.readlines(), [text]) + + def test_x_mode(self): + self.addCleanup(unlink, TESTFN) + for mode in ("x", "xb", "xt"): + unlink(TESTFN) + encoding = "ascii" if "t" in mode else None + with lzma.open(TESTFN, mode, encoding=encoding): + pass + with self.assertRaises(FileExistsError): + with lzma.open(TESTFN, mode): + pass + + +class MiscellaneousTestCase(unittest.TestCase): + + def test_is_check_supported(self): + # CHECK_NONE and CHECK_CRC32 should always be supported, + # regardless of the options liblzma was compiled with. + self.assertTrue(lzma.is_check_supported(lzma.CHECK_NONE)) + self.assertTrue(lzma.is_check_supported(lzma.CHECK_CRC32)) + + # The .xz format spec cannot store check IDs above this value. + self.assertFalse(lzma.is_check_supported(lzma.CHECK_ID_MAX + 1)) + + # This value should not be a valid check ID. + self.assertFalse(lzma.is_check_supported(lzma.CHECK_UNKNOWN)) + + def test__encode_filter_properties(self): + with self.assertRaises(TypeError): + lzma._encode_filter_properties(b"not a dict") + with self.assertRaises(ValueError): + lzma._encode_filter_properties({"id": 0x100}) + with self.assertRaises(ValueError): + lzma._encode_filter_properties({"id": lzma.FILTER_LZMA2, "junk": 12}) + with self.assertRaises(lzma.LZMAError): + lzma._encode_filter_properties({"id": lzma.FILTER_DELTA, + "dist": 9001}) + + # Test with parameters used by zipfile module. + props = lzma._encode_filter_properties({ + "id": lzma.FILTER_LZMA1, + "pb": 2, + "lp": 0, + "lc": 3, + "dict_size": 8 << 20, + }) + self.assertEqual(props, b"]\x00\x00\x80\x00") + + def test__decode_filter_properties(self): + with self.assertRaises(TypeError): + lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes}) + with self.assertRaises(lzma.LZMAError): + lzma._decode_filter_properties(lzma.FILTER_DELTA, b"too long") + + # Test with parameters used by zipfile module. + filterspec = lzma._decode_filter_properties( + lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") + self.assertEqual(filterspec["id"], lzma.FILTER_LZMA1) + self.assertEqual(filterspec["pb"], 2) + self.assertEqual(filterspec["lp"], 0) + self.assertEqual(filterspec["lc"], 3) + self.assertEqual(filterspec["dict_size"], 8 << 20) + + # see gh-104282 + filters = [lzma.FILTER_X86, lzma.FILTER_POWERPC, + lzma.FILTER_IA64, lzma.FILTER_ARM, + lzma.FILTER_ARMTHUMB, lzma.FILTER_SPARC] + for f in filters: + filterspec = lzma._decode_filter_properties(f, b"") + self.assertEqual(filterspec, {"id": f}) + + def test_filter_properties_roundtrip(self): + spec1 = lzma._decode_filter_properties( + lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") + reencoded = lzma._encode_filter_properties(spec1) + spec2 = lzma._decode_filter_properties(lzma.FILTER_LZMA1, reencoded) + self.assertEqual(spec1, spec2) + + +# Test data: + +INPUT = b""" +LAERTES + + O, fear me not. + I stay too long: but here my father comes. + + Enter POLONIUS + + A double blessing is a double grace, + Occasion smiles upon a second leave. + +LORD POLONIUS + + Yet here, Laertes! aboard, aboard, for shame! + The wind sits in the shoulder of your sail, + And you are stay'd for. There; my blessing with thee! + And these few precepts in thy memory + See thou character. Give thy thoughts no tongue, + Nor any unproportioned thought his act. + Be thou familiar, but by no means vulgar. + Those friends thou hast, and their adoption tried, + Grapple them to thy soul with hoops of steel; + But do not dull thy palm with entertainment + Of each new-hatch'd, unfledged comrade. Beware + Of entrance to a quarrel, but being in, + Bear't that the opposed may beware of thee. + Give every man thy ear, but few thy voice; + Take each man's censure, but reserve thy judgment. + Costly thy habit as thy purse can buy, + But not express'd in fancy; rich, not gaudy; + For the apparel oft proclaims the man, + And they in France of the best rank and station + Are of a most select and generous chief in that. + Neither a borrower nor a lender be; + For loan oft loses both itself and friend, + And borrowing dulls the edge of husbandry. + This above all: to thine ownself be true, + And it must follow, as the night the day, + Thou canst not then be false to any man. + Farewell: my blessing season this in thee! + +LAERTES + + Most humbly do I take my leave, my lord. + +LORD POLONIUS + + The time invites you; go; your servants tend. + +LAERTES + + Farewell, Ophelia; and remember well + What I have said to you. + +OPHELIA + + 'Tis in my memory lock'd, + And you yourself shall keep the key of it. + +LAERTES + + Farewell. +""" + +COMPRESSED_BOGUS = b"this is not a valid lzma stream" + +COMPRESSED_XZ = ( + b"\xfd7zXZ\x00\x00\x04\xe6\xd6\xb4F\x02\x00!\x01\x16\x00\x00\x00t/\xe5\xa3" + b"\xe0\x07\x80\x03\xdf]\x00\x05\x14\x07bX\x19\xcd\xddn\x98\x15\xe4\xb4\x9d" + b"o\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8\xe2\xfc\xe7\xd9\xfe6\xb8(" + b"\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02\x17/\xa6=\xf0\xa2\xdf/M\x89" + b"\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ\x15\x80\x8c\xf8\x8do\xfa\x12" + b"\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t\xca6 BF$\xe5Q\xa4\x98\xee\xde" + b"l\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81\xe4N\xc8\x86\x153\xf5x2\xa2O" + b"\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z\xc4\xcdS\xb6t<\x16\xf2\x9cI#" + b"\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0\xaa\x96-Pe\xade:\x04\t\x1b\xf7" + b"\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b" + b"\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa" + b"\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8\x84b\xf6\xc3\xd4c-H\x93oJl\xd0iQ\xe4k" + b"\x84\x0b\xc1\xb7\xbc\xb1\x17\x88\xb1\xca?@\xf6\x07\xea\xe6x\xf1H12P\x0f" + b"\x8a\xc9\xeauw\xe3\xbe\xaai\xa9W\xd0\x80\xcd#cb5\x99\xd8]\xa9d\x0c\xbd" + b"\xa2\xdcWl\xedUG\xbf\x89yF\xf77\x81v\xbd5\x98\xbeh8\x18W\x08\xf0\x1b\x99" + b"5:\x1a?rD\x96\xa1\x04\x0f\xae\xba\x85\xeb\x9d5@\xf5\x83\xd37\x83\x8ac" + b"\x06\xd4\x97i\xcdt\x16S\x82k\xf6K\x01vy\x88\x91\x9b6T\xdae\r\xfd]:k\xbal" + b"\xa9\xbba\xc34\xf9r\xeb}r\xdb\xc7\xdb*\x8f\x03z\xdc8h\xcc\xc9\xd3\xbcl" + b"\xa5-\xcb\xeaK\xa2\xc5\x15\xc0\xe3\xc1\x86Z\xfb\xebL\xe13\xcf\x9c\xe3" + b"\x1d\xc9\xed\xc2\x06\xcc\xce!\x92\xe5\xfe\x9c^\xa59w \x9bP\xa3PK\x08d" + b"\xf9\xe2Z}\xa7\xbf\xed\xeb%$\x0c\x82\xb8/\xb0\x01\xa9&,\xf7qh{Q\x96)\xf2" + b"q\x96\xc3\x80\xb4\x12\xb0\xba\xe6o\xf4!\xb4[\xd4\x8aw\x10\xf7t\x0c\xb3" + b"\xd9\xd5\xc3`^\x81\x11??\\\xa4\x99\x85R\xd4\x8e\x83\xc9\x1eX\xbfa\xf1" + b"\xac\xb0\xea\xea\xd7\xd0\xab\x18\xe2\xf2\xed\xe1\xb7\xc9\x18\xcbS\xe4>" + b"\xc9\x95H\xe8\xcb\t\r%\xeb\xc7$.o\xf1\xf3R\x17\x1db\xbb\xd8U\xa5^\xccS" + b"\x16\x01\x87\xf3/\x93\xd1\xf0v\xc0r\xd7\xcc\xa2Gkz\xca\x80\x0e\xfd\xd0" + b"\x8b\xbb\xd2Ix\xb3\x1ey\xca-0\xe3z^\xd6\xd6\x8f_\xf1\x9dP\x9fi\xa7\xd1" + b"\xe8\x90\x84\xdc\xbf\xcdky\x8e\xdc\x81\x7f\xa3\xb2+\xbf\x04\xef\xd8\\" + b"\xc4\xdf\xe1\xb0\x01\xe9\x93\xe3Y\xf1\x1dY\xe8h\x81\xcf\xf1w\xcc\xb4\xef" + b" \x8b|\x04\xea\x83ej\xbe\x1f\xd4z\x9c`\xd3\x1a\x92A\x06\xe5\x8f\xa9\x13" + b"\t\x9e=\xfa\x1c\xe5_\x9f%v\x1bo\x11ZO\xd8\xf4\t\xddM\x16-\x04\xfc\x18<\"" + b"CM\xddg~b\xf6\xef\x8e\x0c\xd0\xde|\xa0'\x8a\x0c\xd6x\xae!J\xa6F\x88\x15u" + b"\x008\x17\xbc7y\xb3\xd8u\xac_\x85\x8d\xe7\xc1@\x9c\xecqc\xa3#\xad\xf1" + b"\x935\xb5)_\r\xec3]\x0fo]5\xd0my\x07\x9b\xee\x81\xb5\x0f\xcfK+\x00\xc0" + b"\xe4b\x10\xe4\x0c\x1a \x9b\xe0\x97t\xf6\xa1\x9e\x850\xba\x0c\x9a\x8d\xc8" + b"\x8f\x07\xd7\xae\xc8\xf9+i\xdc\xb9k\xb0>f\x19\xb8\r\xa8\xf8\x1f$\xa5{p" + b"\xc6\x880\xce\xdb\xcf\xca_\x86\xac\x88h6\x8bZ%'\xd0\n\xbf\x0f\x9c\"\xba" + b"\xe5\x86\x9f\x0f7X=mNX[\xcc\x19FU\xc9\x860\xbc\x90a+* \xae_$\x03\x1e\xd3" + b"\xcd_\xa0\x9c\xde\xaf46q\xa5\xc9\x92\xd7\xca\xe3`\x9d\x85}\xb4\xff\xb3" + b"\x83\xfb\xb6\xca\xae`\x0bw\x7f\xfc\xd8\xacVe\x19\xc8\x17\x0bZ\xad\x88" + b"\xeb#\x97\x03\x13\xb1d\x0f{\x0c\x04w\x07\r\x97\xbd\xd6\xc1\xc3B:\x95\x08" + b"^\x10V\xaeaH\x02\xd9\xe3\n\\\x01X\xf6\x9c\x8a\x06u#%\xbe*\xa1\x18v\x85" + b"\xec!\t4\x00\x00\x00\x00Vj?uLU\xf3\xa6\x00\x01\xfb\x07\x81\x0f\x00\x00tw" + b"\x99P\xb1\xc4g\xfb\x02\x00\x00\x00\x00\x04YZ" +) + +COMPRESSED_ALONE = ( + b"]\x00\x00\x80\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x05\x14\x07bX\x19" + b"\xcd\xddn\x98\x15\xe4\xb4\x9do\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8" + b"\xe2\xfc\xe7\xd9\xfe6\xb8(\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02" + b"\x17/\xa6=\xf0\xa2\xdf/M\x89\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ" + b"\x15\x80\x8c\xf8\x8do\xfa\x12\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t" + b"\xca6 BF$\xe5Q\xa4\x98\xee\xdel\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81" + b"\xe4N\xc8\x86\x153\xf5x2\xa2O\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z" + b"\xc4\xcdS\xb6t<\x16\xf2\x9cI#\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0" + b"\xaa\x96-Pe\xade:\x04\t\x1b\xf7\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9" + b"\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7" + b"\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8" + b"\x84b\xf8\x1epl\xeajr\xd1=\t\x03\xdd\x13\x1b3!E\xf9vV\xdaF\xf3\xd7\xb4" + b"\x0c\xa9P~\xec\xdeE\xe37\xf6\x1d\xc6\xbb\xddc%\xb6\x0fI\x07\xf0;\xaf\xe7" + b"\xa0\x8b\xa7Z\x99(\xe9\xe2\xf0o\x18>`\xe1\xaa\xa8\xd9\xa1\xb2}\xe7\x8d" + b"\x834T\xb6\xef\xc1\xde\xe3\x98\xbcD\x03MA@\xd8\xed\xdc\xc8\x93\x03\x1a" + b"\x93\x0b\x7f\x94\x12\x0b\x02Sa\x18\xc9\xc5\x9bTJE}\xf6\xc8g\x17#ZV\x01" + b"\xc9\x9dc\x83\x0e>0\x16\x90S\xb8/\x03y_\x18\xfa(\xd7\x0br\xa2\xb0\xba?" + b"\x8c\xe6\x83@\x84\xdf\x02:\xc5z\x9e\xa6\x84\xc9\xf5BeyX\x83\x1a\xf1 :\t" + b"\xf7\x19\xfexD\\&G\xf3\x85Y\xa2J\xf9\x0bv{\x89\xf6\xe7)A\xaf\x04o\x00" + b"\x075\xd3\xe0\x7f\x97\x98F\x0f?v\x93\xedVtTf\xb5\x97\x83\xed\x19\xd7\x1a" + b"'k\xd7\xd9\xc5\\Y\xd1\xdc\x07\x15|w\xbc\xacd\x87\x08d\xec\xa7\xf6\x82" + b"\xfc\xb3\x93\xeb\xb9 \x8d\xbc ,\xb3X\xb0\xd2s\xd7\xd1\xffv\x05\xdf}\xa2" + b"\x96\xfb%\n\xdf\xa2\x7f\x08.\xa16\n\xe0\x19\x93\x7fh\n\x1c\x8c\x0f \x11" + b"\xc6Bl\x95\x19U}\xe4s\xb5\x10H\xea\x86pB\xe88\x95\xbe\x8cZ\xdb\xe4\x94A" + b"\x92\xb9;z\xaa\xa7{\x1c5!\xc0\xaf\xc1A\xf9\xda\xf0$\xb0\x02qg\xc8\xc7/|" + b"\xafr\x99^\x91\x88\xbf\x03\xd9=\xd7n\xda6{>8\n\xc7:\xa9'\xba.\x0b\xe2" + b"\xb5\x1d\x0e\n\x9a\x8e\x06\x8f:\xdd\x82'[\xc3\"wD$\xa7w\xecq\x8c,1\x93" + b"\xd0,\xae2w\x93\x12$Jd\x19mg\x02\x93\x9cA\x95\x9d&\xca8i\x9c\xb0;\xe7NQ" + b"\x1frh\x8beL;\xb0m\xee\x07Q\x9b\xc6\xd8\x03\xb5\xdeN\xd4\xfe\x98\xd0\xdc" + b"\x1a[\x04\xde\x1a\xf6\x91j\xf8EOli\x8eB^\x1d\x82\x07\xb2\xb5R]\xb7\xd7" + b"\xe9\xa6\xc3.\xfb\xf0-\xb4e\x9b\xde\x03\x88\xc6\xc1iN\x0e\x84wbQ\xdf~" + b"\xe9\xa4\x884\x96kM\xbc)T\xf3\x89\x97\x0f\x143\xe7)\xa0\xb3B\x00\xa8\xaf" + b"\x82^\xcb\xc7..\xdb\xc7\t\x9dH\xee5\xe9#\xe6NV\x94\xcb$Kk\xe3\x7f\r\xe3t" + b"\x12\xcf'\xefR\x8b\xf42\xcf-LH\xac\xe5\x1f0~?SO\xeb\xc1E\x1a\x1c]\xf2" + b"\xc4<\x11\x02\x10Z0a*?\xe4r\xff\xfb\xff\xf6\x14nG\xead^\xd6\xef8\xb6uEI" + b"\x99\nV\xe2\xb3\x95\x8e\x83\xf6i!\xb5&1F\xb1DP\xf4 SO3D!w\x99_G\x7f+\x90" + b".\xab\xbb]\x91>\xc9#h;\x0f5J\x91K\xf4^-[\x9e\x8a\\\x94\xca\xaf\xf6\x19" + b"\xd4\xa1\x9b\xc4\xb8p\xa1\xae\x15\xe9r\x84\xe0\xcar.l []\x8b\xaf+0\xf2g" + b"\x01aKY\xdfI\xcf,\n\xe8\xf0\xe7V\x80_#\xb2\xf2\xa9\x06\x8c>w\xe2W,\xf4" + b"\x8c\r\xf963\xf5J\xcc2\x05=kT\xeaUti\xe5_\xce\x1b\xfa\x8dl\x02h\xef\xa8" + b"\xfbf\x7f\xff\xf0\x19\xeax" +) + +FILTERS_RAW_1 = [{"id": lzma.FILTER_LZMA2, "preset": 3}] +COMPRESSED_RAW_1 = ( + b"\xe0\x07\x80\x03\xfd]\x00\x05\x14\x07bX\x19\xcd\xddn\x96cyq\xa1\xdd\xee" + b"\xf8\xfam\xe3'\x88\xd3\xff\xe4\x9e \xceQ\x91\xa4\x14I\xf6\xb9\x9dVL8\x15" + b"_\x0e\x12\xc3\xeb\xbc\xa5\xcd\nW\x1d$=R;\x1d\xf8k8\t\xb1{\xd4\xc5+\x9d" + b"\x87c\xe5\xef\x98\xb4\xd7S3\xcd\xcc\xd2\xed\xa4\x0em\xe5\xf4\xdd\xd0b" + b"\xbe4*\xaa\x0b\xc5\x08\x10\x85+\x81.\x17\xaf9\xc9b\xeaZrA\xe20\x7fs\"r" + b"\xdaG\x81\xde\x90cu\xa5\xdb\xa9.A\x08l\xb0<\xf6\x03\xddOi\xd0\xc5\xb4" + b"\xec\xecg4t6\"\xa6\xb8o\xb5?\x18^}\xb6}\x03[:\xeb\x03\xa9\n[\x89l\x19g" + b"\x16\xc82\xed\x0b\xfb\x86n\xa2\x857@\x93\xcd6T\xc3u\xb0\t\xf9\x1b\x918" + b"\xfc[\x1b\x1e4\xb3\x14\x06PCV\xa8\"\xf5\x81x~\xe9\xb5N\x9cK\x9f\xc6\xc3%" + b"\xc8k:{6\xe7\xf7\xbd\x05\x02\xb4\xc4\xc3\xd3\xfd\xc3\xa8\\\xfc@\xb1F_" + b"\xc8\x90\xd9sU\x98\xad8\x05\x07\xde7J\x8bM\xd0\xb3;X\xec\x87\xef\xae\xb3" + b"eO,\xb1z,d\x11y\xeejlB\x02\x1d\xf28\x1f#\x896\xce\x0b\xf0\xf5\xa9PK\x0f" + b"\xb3\x13P\xd8\x88\xd2\xa1\x08\x04C?\xdb\x94_\x9a\"\xe9\xe3e\x1d\xde\x9b" + b"\xa1\xe8>H\x98\x10;\xc5\x03#\xb5\x9d4\x01\xe7\xc5\xba%v\xa49\x97A\xe0\"" + b"\x8c\xc22\xe3i\xc1\x9d\xab3\xdf\xbe\xfdDm7\x1b\x9d\xab\xb5\x15o:J\x92" + b"\xdb\x816\x17\xc2O\x99\x1b\x0e\x8d\xf3\tQ\xed\x8e\x95S/\x16M\xb2S\x04" + b"\x0f\xc3J\xc6\xc7\xe4\xcb\xc5\xf4\xe7d\x14\xe4=^B\xfb\xd3E\xd3\x1e\xcd" + b"\x91\xa5\xd0G\x8f.\xf6\xf9\x0bb&\xd9\x9f\xc2\xfdj\xa2\x9e\xc4\\\x0e\x1dC" + b"v\xe8\xd2\x8a?^H\xec\xae\xeb>\xfe\xb8\xab\xd4IqY\x8c\xd4K7\x11\xf4D\xd0W" + b"\xa5\xbe\xeaO\xbf\xd0\x04\xfdl\x10\xae5\xd4U\x19\x06\xf9{\xaa\xe0\x81" + b"\x0f\xcf\xa3k{\x95\xbd\x19\xa2\xf8\xe4\xa3\x08O*\xf1\xf1B-\xc7(\x0eR\xfd" + b"@E\x9f\xd3\x1e:\xfdV\xb7\x04Y\x94\xeb]\x83\xc4\xa5\xd7\xc0gX\x98\xcf\x0f" + b"\xcd3\x00]n\x17\xec\xbd\xa3Y\x86\xc5\xf3u\xf6*\xbdT\xedA$A\xd9A\xe7\x98" + b"\xef\x14\x02\x9a\xfdiw\xec\xa0\x87\x11\xd9%\xc5\xeb\x8a=\xae\xc0\xc4\xc6" + b"D\x80\x8f\xa8\xd1\xbbq\xb2\xc0\xa0\xf5Cqp\xeeL\xe3\xe5\xdc \x84\"\xe9" + b"\x80t\x83\x05\xba\xf1\xc5~\x93\xc9\xf0\x01c\xceix\x9d\xed\xc5)l\x16)\xd1" + b"\x03@l\x04\x7f\x87\xa5yn\x1b\x01D\xaa:\xd2\x96\xb4\xb3?\xb0\xf9\xce\x07" + b"\xeb\x81\x00\xe4\xc3\xf5%_\xae\xd4\xf9\xeb\xe2\rh\xb2#\xd67Q\x16D\x82hn" + b"\xd1\xa3_?q\xf0\xe2\xac\xf317\x9e\xd0_\x83|\xf1\xca\xb7\x95S\xabW\x12" + b"\xff\xddt\xf69L\x01\xf2|\xdaW\xda\xees\x98L\x18\xb8_\xe8$\x82\xea\xd6" + b"\xd1F\xd4\x0b\xcdk\x01vf\x88h\xc3\xae\xb91\xc7Q\x9f\xa5G\xd9\xcc\x1f\xe3" + b"5\xb1\xdcy\x7fI\x8bcw\x8e\x10rIp\x02:\x19p_\xc8v\xcea\"\xc1\xd9\x91\x03" + b"\xbfe\xbe\xa6\xb3\xa8\x14\x18\xc3\xabH*m}\xc2\xc1\x9a}>l%\xce\x84\x99" + b"\xb3d\xaf\xd3\x82\x15\xdf\xc1\xfc5fOg\x9b\xfc\x8e^&\t@\xce\x9f\x06J\xb8" + b"\xb5\x86\x1d\xda{\x9f\xae\xb0\xff\x02\x81r\x92z\x8cM\xb7ho\xc9^\x9c\xb6" + b"\x9c\xae\xd1\xc9\xf4\xdfU7\xd6\\!\xea\x0b\x94k\xb9Ud~\x98\xe7\x86\x8az" + b"\x10;\xe3\x1d\xe5PG\xf8\xa4\x12\x05w\x98^\xc4\xb1\xbb\xfb\xcf\xe0\x7f" + b"\x033Sf\x0c \xb1\xf6@\x94\xe5\xa3\xb2\xa7\x10\x9a\xc0\x14\xc3s\xb5xRD" + b"\xf4`W\xd9\xe5\xd3\xcf\x91\rTZ-X\xbe\xbf\xb5\xe2\xee|\x1a\xbf\xfb\x08" + b"\x91\xe1\xfc\x9a\x18\xa3\x8b\xd6^\x89\xf5[\xef\x87\xd1\x06\x1c7\xd6\xa2" + b"\t\tQ5/@S\xc05\xd2VhAK\x03VC\r\x9b\x93\xd6M\xf1xO\xaaO\xed\xb9<\x0c\xdae" + b"*\xd0\x07Hk6\x9fG+\xa1)\xcd\x9cl\x87\xdb\xe1\xe7\xefK}\x875\xab\xa0\x19u" + b"\xf6*F\xb32\x00\x00\x00" +) + +FILTERS_RAW_2 = [{"id": lzma.FILTER_DELTA, "dist": 2}, + {"id": lzma.FILTER_LZMA2, + "preset": lzma.PRESET_DEFAULT | lzma.PRESET_EXTREME}] +COMPRESSED_RAW_2 = ( + b"\xe0\x07\x80\x05\x91]\x00\x05\x14\x06-\xd4\xa8d?\xef\xbe\xafH\xee\x042" + b"\xcb.\xb5g\x8f\xfb\x14\xab\xa5\x9f\x025z\xa4\xdd\xd8\t[}W\xf8\x0c\x1dmH" + b"\xfa\x05\xfcg\xba\xe5\x01Q\x0b\x83R\xb6A\x885\xc0\xba\xee\n\x1cv~\xde:o" + b"\x06:J\xa7\x11Cc\xea\xf7\xe5*o\xf7\x83\\l\xbdE\x19\x1f\r\xa8\x10\xb42" + b"\x0caU{\xd7\xb8w\xdc\xbe\x1b\xfc8\xb4\xcc\xd38\\\xf6\x13\xf6\xe7\x98\xfa" + b"\xc7[\x17_9\x86%\xa8\xf8\xaa\xb8\x8dfs#\x1e=\xed<\x92\x10\\t\xff\x86\xfb" + b"=\x9e7\x18\x1dft\\\xb5\x01\x95Q\xc5\x19\xb38\xe0\xd4\xaa\x07\xc3\x7f\xd8" + b"\xa2\x00>-\xd3\x8e\xa1#\xfa\x83ArAm\xdbJ~\x93\xa3B\x82\xe0\xc7\xcc(\x08`" + b"WK\xad\x1b\x94kaj\x04 \xde\xfc\xe1\xed\xb0\x82\x91\xefS\x84%\x86\xfbi" + b"\x99X\xf1B\xe7\x90;E\xfde\x98\xda\xca\xd6T\xb4bg\xa4\n\x9aj\xd1\x83\x9e]" + b"\"\x7fM\xb5\x0fr\xd2\\\xa5j~P\x10GH\xbfN*Z\x10.\x81\tpE\x8a\x08\xbe1\xbd" + b"\xcd\xa9\xe1\x8d\x1f\x04\xf9\x0eH\xb9\xae\xd6\xc3\xc1\xa5\xa9\x95P\xdc~" + b"\xff\x01\x930\xa9\x04\xf6\x03\xfe\xb5JK\xc3]\xdd9\xb1\xd3\xd7F\xf5\xd1" + b"\x1e\xa0\x1c_\xed[\x0c\xae\xd4\x8b\x946\xeb\xbf\xbb\xe3$kS{\xb5\x80,f:Sj" + b"\x0f\x08z\x1c\xf5\xe8\xe6\xae\x98\xb0Q~r\x0f\xb0\x05?\xb6\x90\x19\x02&" + b"\xcb\x80\t\xc4\xea\x9c|x\xce\x10\x9c\xc5|\xcbdhh+\x0c'\xc5\x81\xc33\xb5" + b"\x14q\xd6\xc5\xe3`Z#\xdc\x8a\xab\xdd\xea\x08\xc2I\xe7\x02l{\xec\x196\x06" + b"\x91\x8d\xdc\xd5\xb3x\xe1hz%\xd1\xf8\xa5\xdd\x98!\x8c\x1c\xc1\x17RUa\xbb" + b"\x95\x0f\xe4X\xea1\x0c\xf1=R\xbe\xc60\xe3\xa4\x9a\x90bd\x97$]B\x01\xdd" + b"\x1f\xe3h2c\x1e\xa0L`4\xc6x\xa3Z\x8a\r\x14]T^\xd8\x89\x1b\x92\r;\xedY" + b"\x0c\xef\x8d9z\xf3o\xb6)f\xa9]$n\rp\x93\xd0\x10\xa4\x08\xb8\xb2\x8b\xb6" + b"\x8f\x80\xae;\xdcQ\xf1\xfa\x9a\x06\x8e\xa5\x0e\x8cK\x9c @\xaa:UcX\n!\xc6" + b"\x02\x12\xcb\x1b\"=\x16.\x1f\x176\xf2g=\xe1Wn\xe9\xe1\xd4\xf1O\xad\x15" + b"\x86\xe9\xa3T\xaf\xa9\xd7D\xb5\xd1W3pnt\x11\xc7VOj\xb7M\xc4i\xa1\xf1$3" + b"\xbb\xdc\x8af\xb0\xc5Y\r\xd1\xfb\xf2\xe7K\xe6\xc5hwO\xfe\x8c2^&\x07\xd5" + b"\x1fV\x19\xfd\r\x14\xd2i=yZ\xe6o\xaf\xc6\xb6\x92\x9d\xc4\r\xb3\xafw\xac%" + b"\xcfc\x1a\xf1`]\xf2\x1a\x9e\x808\xedm\xedQ\xb2\xfe\xe4h`[q\xae\xe0\x0f" + b"\xba0g\xb6\"N\xc3\xfb\xcfR\x11\xc5\x18)(\xc40\\\xa3\x02\xd9G!\xce\x1b" + b"\xc1\x96x\xb5\xc8z\x1f\x01\xb4\xaf\xde\xc2\xcd\x07\xe7H\xb3y\xa8M\n\\A\t" + b"ar\xddM\x8b\x9a\xea\x84\x9b!\xf1\x8d\xb1\xf1~\x1e\r\xa5H\xba\xf1\x84o" + b"\xda\x87\x01h\xe9\xa2\xbe\xbeqN\x9d\x84\x0b!WG\xda\xa1\xa5A\xb7\xc7`j" + b"\x15\xf2\xe9\xdd?\x015B\xd2~E\x06\x11\xe0\x91!\x05^\x80\xdd\xa8y\x15}" + b"\xa1)\xb1)\x81\x18\xf4\xf4\xf8\xc0\xefD\xe3\xdb2f\x1e\x12\xabu\xc9\x97" + b"\xcd\x1e\xa7\x0c\x02x4_6\x03\xc4$t\xf39\x94\x1d=\xcb\xbfv\\\xf5\xa3\x1d" + b"\x9d8jk\x95\x13)ff\xf9n\xc4\xa9\xe3\x01\xb8\xda\xfb\xab\xdfM\x99\xfb\x05" + b"\xe0\xe9\xb0I\xf4E\xab\xe2\x15\xa3\x035\xe7\xdeT\xee\x82p\xb4\x88\xd3" + b"\x893\x9c/\xc0\xd6\x8fou;\xf6\x95PR\xa9\xb2\xc1\xefFj\xe2\xa7$\xf7h\xf1" + b"\xdfK(\xc9c\xba7\xe8\xe3)\xdd\xb2,\x83\xfb\x84\x18.y\x18Qi\x88\xf8`h-" + b"\xef\xd5\xed\x8c\t\xd8\xc3^\x0f\x00\xb7\xd0[!\xafM\x9b\xd7.\x07\xd8\xfb" + b"\xd9\xe2-S+\xaa8,\xa0\x03\x1b \xea\xa8\x00\xc3\xab~\xd0$e\xa5\x7f\xf7" + b"\x95P]\x12\x19i\xd9\x7fo\x0c\xd8g^\rE\xa5\x80\x18\xc5\x01\x80\xaek`\xff~" + b"\xb6y\xe7+\xe5\x11^D\xa7\x85\x18\"!\xd6\xd2\xa7\xf4\x1eT\xdb\x02\xe15" + b"\x02Y\xbc\x174Z\xe7\x9cH\x1c\xbf\x0f\xc6\xe9f]\xcf\x8cx\xbc\xe5\x15\x94" + b"\xfc3\xbc\xa7TUH\xf1\x84\x1b\xf7\xa9y\xc07\x84\xf8X\xd8\xef\xfc \x1c\xd8" + b"( /\xf2\xb7\xec\xc1\\\x8c\xf6\x95\xa1\x03J\x83vP8\xe1\xe3\xbb~\xc24kA" + b"\x98y\xa1\xf2P\xe9\x9d\xc9J\xf8N\x99\xb4\xceaO\xde\x16\x1e\xc2\x19\xa7" + b"\x03\xd2\xe0\x8f:\x15\xf3\x84\x9e\xee\xe6e\xb8\x02q\xc7AC\x1emw\xfd\t" + b"\x9a\x1eu\xc1\xa9\xcaCwUP\x00\xa5\xf78L4w!\x91L2 \x87\xd0\xf2\x06\x81j" + b"\x80;\x03V\x06\x87\x92\xcb\x90lv@E\x8d\x8d\xa5\xa6\xe7Z[\xdf\xd6E\x03`>" + b"\x8f\xde\xa1bZ\x84\xd0\xa9`\x05\x0e{\x80;\xe3\xbef\x8d\x1d\xebk1.\xe3" + b"\xe9N\x15\xf7\xd4(\xfa\xbb\x15\xbdu\xf7\x7f\x86\xae!\x03L\x1d\xb5\xc1" + b"\xb9\x11\xdb\xd0\x93\xe4\x02\xe1\xd2\xcbBjc_\xe8}d\xdb\xc3\xa0Y\xbe\xc9/" + b"\x95\x01\xa3,\xe6bl@\x01\xdbp\xc2\xce\x14\x168\xc2q\xe3uH\x89X\xa4\xa9" + b"\x19\x1d\xc1}\x7fOX\x19\x9f\xdd\xbe\x85\x83\xff\x96\x1ee\x82O`CF=K\xeb$I" + b"\x17_\xefX\x8bJ'v\xde\x1f+\xd9.v\xf8Tv\x17\xf2\x9f5\x19\xe1\xb9\x91\xa8S" + b"\x86\xbd\x1a\"(\xa5x\x8dC\x03X\x81\x91\xa8\x11\xc4pS\x13\xbc\xf2'J\xae!" + b"\xef\xef\x84G\t\x8d\xc4\x10\x132\x00oS\x9e\xe0\xe4d\x8f\xb8y\xac\xa6\x9f" + b",\xb8f\x87\r\xdf\x9eE\x0f\xe1\xd0\\L\x00\xb2\xe1h\x84\xef}\x98\xa8\x11" + b"\xccW#\\\x83\x7fo\xbbz\x8f\x00" +) + +FILTERS_RAW_3 = [{"id": lzma.FILTER_IA64, "start_offset": 0x100}, + {"id": lzma.FILTER_LZMA2}] +COMPRESSED_RAW_3 = ( + b"\xe0\x07\x80\x03\xdf]\x00\x05\x14\x07bX\x19\xcd\xddn\x98\x15\xe4\xb4\x9d" + b"o\x1d\xc4\xe5\n\x03\xcc2h\xc7\\\x86\xff\xf8\xe2\xfc\xe7\xd9\xfe6\xb8(" + b"\xa8wd\xc2\"u.n\x1e\xc3\xf2\x8e\x8d\x8f\x02\x17/\xa6=\xf0\xa2\xdf/M\x89" + b"\xbe\xde\xa7\x1cz\x18-]\xd5\xef\x13\x8frZ\x15\x80\x8c\xf8\x8do\xfa\x12" + b"\x9b#z/\xef\xf0\xfaF\x01\x82\xa3M\x8e\xa1t\xca6 BF$\xe5Q\xa4\x98\xee\xde" + b"l\xe8\x7f\xf0\x9d,bn\x0b\x13\xd4\xa8\x81\xe4N\xc8\x86\x153\xf5x2\xa2O" + b"\x13@Q\xa1\x00/\xa5\xd0O\x97\xdco\xae\xf7z\xc4\xcdS\xb6t<\x16\xf2\x9cI#" + b"\x89ud\xc66Y\xd9\xee\xe6\xce\x12]\xe5\xf0\xaa\x96-Pe\xade:\x04\t\x1b\xf7" + b"\xdb7\n\x86\x1fp\xc8J\xba\xf4\xf0V\xa9\xdc\xf0\x02%G\xf9\xdf=?\x15\x1b" + b"\xe1(\xce\x82=\xd6I\xac3\x12\x0cR\xb7\xae\r\xb1i\x03\x95\x01\xbd\xbe\xfa" + b"\x02s\x01P\x9d\x96X\xb12j\xc8L\xa8\x84b\xf6\xc3\xd4c-H\x93oJl\xd0iQ\xe4k" + b"\x84\x0b\xc1\xb7\xbc\xb1\x17\x88\xb1\xca?@\xf6\x07\xea\xe6x\xf1H12P\x0f" + b"\x8a\xc9\xeauw\xe3\xbe\xaai\xa9W\xd0\x80\xcd#cb5\x99\xd8]\xa9d\x0c\xbd" + b"\xa2\xdcWl\xedUG\xbf\x89yF\xf77\x81v\xbd5\x98\xbeh8\x18W\x08\xf0\x1b\x99" + b"5:\x1a?rD\x96\xa1\x04\x0f\xae\xba\x85\xeb\x9d5@\xf5\x83\xd37\x83\x8ac" + b"\x06\xd4\x97i\xcdt\x16S\x82k\xf6K\x01vy\x88\x91\x9b6T\xdae\r\xfd]:k\xbal" + b"\xa9\xbba\xc34\xf9r\xeb}r\xdb\xc7\xdb*\x8f\x03z\xdc8h\xcc\xc9\xd3\xbcl" + b"\xa5-\xcb\xeaK\xa2\xc5\x15\xc0\xe3\xc1\x86Z\xfb\xebL\xe13\xcf\x9c\xe3" + b"\x1d\xc9\xed\xc2\x06\xcc\xce!\x92\xe5\xfe\x9c^\xa59w \x9bP\xa3PK\x08d" + b"\xf9\xe2Z}\xa7\xbf\xed\xeb%$\x0c\x82\xb8/\xb0\x01\xa9&,\xf7qh{Q\x96)\xf2" + b"q\x96\xc3\x80\xb4\x12\xb0\xba\xe6o\xf4!\xb4[\xd4\x8aw\x10\xf7t\x0c\xb3" + b"\xd9\xd5\xc3`^\x81\x11??\\\xa4\x99\x85R\xd4\x8e\x83\xc9\x1eX\xbfa\xf1" + b"\xac\xb0\xea\xea\xd7\xd0\xab\x18\xe2\xf2\xed\xe1\xb7\xc9\x18\xcbS\xe4>" + b"\xc9\x95H\xe8\xcb\t\r%\xeb\xc7$.o\xf1\xf3R\x17\x1db\xbb\xd8U\xa5^\xccS" + b"\x16\x01\x87\xf3/\x93\xd1\xf0v\xc0r\xd7\xcc\xa2Gkz\xca\x80\x0e\xfd\xd0" + b"\x8b\xbb\xd2Ix\xb3\x1ey\xca-0\xe3z^\xd6\xd6\x8f_\xf1\x9dP\x9fi\xa7\xd1" + b"\xe8\x90\x84\xdc\xbf\xcdky\x8e\xdc\x81\x7f\xa3\xb2+\xbf\x04\xef\xd8\\" + b"\xc4\xdf\xe1\xb0\x01\xe9\x93\xe3Y\xf1\x1dY\xe8h\x81\xcf\xf1w\xcc\xb4\xef" + b" \x8b|\x04\xea\x83ej\xbe\x1f\xd4z\x9c`\xd3\x1a\x92A\x06\xe5\x8f\xa9\x13" + b"\t\x9e=\xfa\x1c\xe5_\x9f%v\x1bo\x11ZO\xd8\xf4\t\xddM\x16-\x04\xfc\x18<\"" + b"CM\xddg~b\xf6\xef\x8e\x0c\xd0\xde|\xa0'\x8a\x0c\xd6x\xae!J\xa6F\x88\x15u" + b"\x008\x17\xbc7y\xb3\xd8u\xac_\x85\x8d\xe7\xc1@\x9c\xecqc\xa3#\xad\xf1" + b"\x935\xb5)_\r\xec3]\x0fo]5\xd0my\x07\x9b\xee\x81\xb5\x0f\xcfK+\x00\xc0" + b"\xe4b\x10\xe4\x0c\x1a \x9b\xe0\x97t\xf6\xa1\x9e\x850\xba\x0c\x9a\x8d\xc8" + b"\x8f\x07\xd7\xae\xc8\xf9+i\xdc\xb9k\xb0>f\x19\xb8\r\xa8\xf8\x1f$\xa5{p" + b"\xc6\x880\xce\xdb\xcf\xca_\x86\xac\x88h6\x8bZ%'\xd0\n\xbf\x0f\x9c\"\xba" + b"\xe5\x86\x9f\x0f7X=mNX[\xcc\x19FU\xc9\x860\xbc\x90a+* \xae_$\x03\x1e\xd3" + b"\xcd_\xa0\x9c\xde\xaf46q\xa5\xc9\x92\xd7\xca\xe3`\x9d\x85}\xb4\xff\xb3" + b"\x83\xfb\xb6\xca\xae`\x0bw\x7f\xfc\xd8\xacVe\x19\xc8\x17\x0bZ\xad\x88" + b"\xeb#\x97\x03\x13\xb1d\x0f{\x0c\x04w\x07\r\x97\xbd\xd6\xc1\xc3B:\x95\x08" + b"^\x10V\xaeaH\x02\xd9\xe3\n\\\x01X\xf6\x9c\x8a\x06u#%\xbe*\xa1\x18v\x85" + b"\xec!\t4\x00\x00\x00" +) + +FILTERS_RAW_4 = [{"id": lzma.FILTER_DELTA, "dist": 4}, + {"id": lzma.FILTER_X86, "start_offset": 0x40}, + {"id": lzma.FILTER_LZMA2, "preset": 4, "lc": 2}] +COMPRESSED_RAW_4 = ( + b"\xe0\x07\x80\x06\x0e\\\x00\x05\x14\x07bW\xaah\xdd\x10\xdc'\xd6\x90,\xc6v" + b"Jq \x14l\xb7\x83xB\x0b\x97f=&fx\xba\n>Tn\xbf\x8f\xfb\x1dF\xca\xc3v_\xca?" + b"\xfbV<\x92#\xd4w\xa6\x8a\xeb\xf6\x03\xc0\x01\x94\xd8\x9e\x13\x12\x98\xd1" + b"*\xfa]c\xe8\x1e~\xaf\xb5]Eg\xfb\x9e\x01\"8\xb2\x90\x06=~\xe9\x91W\xcd" + b"\xecD\x12\xc7\xfa\xe1\x91\x06\xc7\x99\xb9\xe3\x901\x87\x19u\x0f\x869\xff" + b"\xc1\xb0hw|\xb0\xdcl\xcck\xb16o7\x85\xee{Y_b\xbf\xbc$\xf3=\x8d\x8bw\xe5Z" + b"\x08@\xc4kmE\xad\xfb\xf6*\xd8\xad\xa1\xfb\xc5{\xdej,)\x1emB\x1f<\xaeca" + b"\x80(\xee\x07 \xdf\xe9\xf8\xeb\x0e-\x97\x86\x90c\xf9\xea'B\xf7`\xd7\xb0" + b"\x92\xbd\xa0\x82]\xbd\x0e\x0eB\x19\xdc\x96\xc6\x19\xd86D\xf0\xd5\x831" + b"\x03\xb7\x1c\xf7&5\x1a\x8f PZ&j\xf8\x98\x1bo\xcc\x86\x9bS\xd3\xa5\xcdu" + b"\xf9$\xcc\x97o\xe5V~\xfb\x97\xb5\x0b\x17\x9c\xfdxW\x10\xfep4\x80\xdaHDY" + b"\xfa)\xfet\xb5\"\xd4\xd3F\x81\xf4\x13\x1f\xec\xdf\xa5\x13\xfc\"\x91x\xb7" + b"\x99\xce\xc8\x92\n\xeb[\x10l*Y\xd8\xb1@\x06\xc8o\x8d7r\xebu\xfd5\x0e\x7f" + b"\xf1$U{\t}\x1fQ\xcfxN\x9d\x9fXX\xe9`\x83\xc1\x06\xf4\x87v-f\x11\xdb/\\" + b"\x06\xff\xd7)B\xf3g\x06\x88#2\x1eB244\x7f4q\t\xc893?mPX\x95\xa6a\xfb)d" + b"\x9b\xfc\x98\x9aj\x04\xae\x9b\x9d\x19w\xba\xf92\xfaA\x11\\\x17\x97C3\xa4" + b"\xbc!\x88\xcdo[\xec:\x030\x91.\x85\xe0@\\4\x16\x12\x9d\xcaJv\x97\xb04" + b"\xack\xcbkf\xa3ss\xfc\x16^\x8ce\x85a\xa5=&\xecr\xb3p\xd1E\xd5\x80y\xc7" + b"\xda\xf6\xfek\xbcT\xbfH\xee\x15o\xc5\x8c\x830\xec\x1d\x01\xae\x0c-e\\" + b"\x91\x90\x94\xb2\xf8\x88\x91\xe8\x0b\xae\xa7>\x98\xf6\x9ck\xd2\xc6\x08" + b"\xe6\xab\t\x98\xf2!\xa0\x8c^\xacqA\x99<\x1cEG\x97\xc8\xf1\xb6\xb9\x82" + b"\x8d\xf7\x08s\x98a\xff\xe3\xcc\x92\x0e\xd2\xb6U\xd7\xd9\x86\x7fa\xe5\x1c" + b"\x8dTG@\t\x1e\x0e7*\xfc\xde\xbc]6N\xf7\xf1\x84\x9e\x9f\xcf\xe9\x1e\xb5'" + b"\xf4<\xdf\x99sq\xd0\x9d\xbd\x99\x0b\xb4%p4\xbf{\xbb\x8a\xd2\x0b\xbc=M" + b"\x94H:\xf5\xa8\xd6\xa4\xc90\xc2D\xb9\xd3\xa8\xb0S\x87 `\xa2\xeb\xf3W\xce" + b" 7\xf9N#\r\xe6\xbe\t\x9d\xe7\x811\xf9\x10\xc1\xc2\x14\xf6\xfc\xcba\xb7" + b"\xb1\x7f\x95l\xe4\tjA\xec:\x10\xe5\xfe\xc2\\=D\xe2\x0c\x0b3]\xf7\xc1\xf7" + b"\xbceZ\xb1A\xea\x16\xe5\xfddgFQ\xed\xaf\x04\xa3\xd3\xf8\xa2q\x19B\xd4r" + b"\xc5\x0c\x9a\x14\x94\xea\x91\xc4o\xe4\xbb\xb4\x99\xf4@\xd1\xe6\x0c\xe3" + b"\xc6d\xa0Q\n\xf2/\xd8\xb8S5\x8a\x18:\xb5g\xac\x95D\xce\x17\x07\xd4z\xda" + b"\x90\xe65\x07\x19H!\t\xfdu\x16\x8e\x0eR\x19\xf4\x8cl\x0c\xf9Q\xf1\x80" + b"\xe3\xbf\xd7O\xf8\x8c\x18\x0b\x9c\xf1\x1fb\xe1\tR\xb2\xf1\xe1A\xea \xcf-" + b"IGE\xf1\x14\x98$\x83\x15\xc9\xd8j\xbf\x19\x0f\xd5\xd1\xaa\xb3\xf3\xa5I2s" + b"\x8d\x145\xca\xd5\xd93\x9c\xb8D0\xe6\xaa%\xd0\xc0P}JO^h\x8e\x08\xadlV." + b"\x18\x88\x13\x05o\xb0\x07\xeaw\xe0\xb6\xa4\xd5*\xe4r\xef\x07G+\xc1\xbei[" + b"w\xe8\xab@_\xef\x15y\xe5\x12\xc9W\x1b.\xad\x85-\xc2\xf7\xe3mU6g\x8eSA" + b"\x01(\xd3\xdb\x16\x13=\xde\x92\xf9,D\xb8\x8a\xb2\xb4\xc9\xc3\xefnE\xe8\\" + b"\xa6\xe2Y\xd2\xcf\xcb\x8c\xb6\xd5\xe9\x1d\x1e\x9a\x8b~\xe2\xa6\rE\x84uV" + b"\xed\xc6\x99\xddm<\x10[\x0fu\x1f\xc1\x1d1\n\xcfw\xb2%!\xf0[\xce\x87\x83B" + b"\x08\xaa,\x08%d\xcef\x94\"\xd9g.\xc83\xcbXY+4\xec\x85qA\n\x1d=9\xf0*\xb1" + b"\x1f/\xf3s\xd61b\x7f@\xfb\x9d\xe3FQ\\\xbd\x82\x1e\x00\xf3\xce\xd3\xe1" + b"\xca,E\xfd7[\xab\xb6\xb7\xac!mA}\xbd\x9d3R5\x9cF\xabH\xeb\x92)cc\x13\xd0" + b"\xbd\xee\xe9n{\x1dIJB\xa5\xeb\x11\xe8`w&`\x8b}@Oxe\t\x8a\x07\x02\x95\xf2" + b"\xed\xda|\xb1e\xbe\xaa\xbbg\x19@\xe1Y\x878\x84\x0f\x8c\xe3\xc98\xf2\x9e" + b"\xd5N\xb5J\xef\xab!\xe2\x8dq\xe1\xe5q\xc5\xee\x11W\xb7\xe4k*\x027\xa0" + b"\xa3J\xf4\xd8m\xd0q\x94\xcb\x07\n:\xb6`.\xe4\x9c\x15+\xc0)\xde\x80X\xd4" + b"\xcfQm\x01\xc2cP\x1cA\x85'\xc9\xac\x8b\xe6\xb2)\xe6\x84t\x1c\x92\xe4Z" + b"\x1cR\xb0\x9e\x96\xd1\xfb\x1c\xa6\x8b\xcb`\x10\x12]\xf2gR\x9bFT\xe0\xc8H" + b"S\xfb\xac<\x04\xc7\xc1\xe8\xedP\xf4\x16\xdb\xc0\xd7e\xc2\x17J^\x1f\xab" + b"\xff[\x08\x19\xb4\xf5\xfb\x19\xb4\x04\xe5c~']\xcb\xc2A\xec\x90\xd0\xed" + b"\x06,\xc5K{\x86\x03\xb1\xcdMx\xdeQ\x8c3\xf9\x8a\xea=\x89\xaba\xd2\xc89a" + b"\xd72\xf0\xc3\x19\x8a\xdfs\xd4\xfd\xbb\x81b\xeaE\"\xd8\xf4d\x0cD\xf7IJ!" + b"\xe5d\xbbG\xe9\xcam\xaa\x0f_r\x95\x91NBq\xcaP\xce\xa7\xa9\xb5\x10\x94eP!" + b"|\x856\xcd\xbfIir\xb8e\x9bjP\x97q\xabwS7\x1a\x0ehM\xe7\xca\x86?\xdeP}y~" + b"\x0f\x95I\xfc\x13\xe1r\xa9k\x88\xcb" + b"\xfd\xc3v\xe2\xb9\x8a\x02\x8eq\x92I\xf8\xf6\xf1\x03s\x9b\xb8\xe3\"\xe3" + b"\xa9\xa5>D\xb8\x96;\xe7\x92\xd133\xe8\xdd'e\xc9.\xdc;\x17\x1f\xf5H\x13q" + b"\xa4W\x0c\xdb~\x98\x01\xeb\xdf\xe32\x13\x0f\xddx\n6\xa0\t\x10\xb6\xbb" + b"\xb0\xc3\x18\xb6;\x9fj[\xd9\xd5\xc9\x06\x8a\x87\xcd\xe5\xee\xfc\x9c-%@" + b"\xee\xe0\xeb\xd2\xe3\xe8\xfb\xc0\x122\\\xc7\xaf\xc2\xa1Oth\xb3\x8f\x82" + b"\xb3\x18\xa8\x07\xd5\xee_\xbe\xe0\x1cA\x1e_\r\x9a\xb0\x17W&\xa2D\x91\x94" + b"\x1a\xb2\xef\xf2\xdc\x85;X\xb0,\xeb>-7S\xe5\xca\x07)\x1fp\x7f\xcaQBL\xca" + b"\xf3\xb9d\xfc\xb5su\xb0\xc8\x95\x90\xeb*)\xa0v\xe4\x9a{FW\xf4l\xde\xcdj" + b"\x00" +) + +ISSUE_21872_DAT = ( + b']\x00\x00@\x00h3\x00\x00\x00\x00\x00\x00\x00\x00`D\x0c\x99\xc8' + b'\xd1\xbbZ^\xc43+\x83\xcd\xf1\xc6g\xec-\x061F\xb1\xbb\xc7\x17%-\xea' + b'\xfap\xfb\x8fs\x128\xb2,\x88\xe4\xc0\x12|*x\xd0\xa2\xc4b\x1b!\x02c' + b'\xab\xd9\x87U\xb8n \xfaVJ\x9a"\xb78\xff%_\x17`?@*\xc2\x82' + b"\xf2^\x1b\xb8\x04&\xc0\xbb\x03g\x9d\xca\xe9\xa4\xc9\xaf'\xe5\x8e}" + b'F\xdd\x11\xf3\x86\xbe\x1fN\x95\\\xef\xa2Mz-\xcb\x9a\xe3O@' + b"\x19\x07\xf6\xee\x9e\x9ag\xc6\xa5w\rnG'\x99\xfd\xfeGI\xb0" + b'\xbb\xf9\xc2\xe1\xff\xc5r\xcf\x85y[\x01\xa1\xbd\xcc/\xa3\x1b\x83\xaa' + b'\xc6\xf9\x99\x0c\xb6_\xc9MQ+x\xa2F\xda]\xdd\xe8\xfb\x1a&' + b',\xc4\x19\x1df\x81\x1e\x90\xf3\xb8Hgr\x85v\xbe\xa3qx\x01Y\xb5\x9fF' + b"\x13\x18\x01\xe69\x9b\xc8'\x1e\x9d\xd6\xe4F\x84\xac\xf8d<\x11\xd5" + b'\\\x0b\xeb\x0e\x82\xab\xb1\xe6\x1fka\xe1i\xc4 C\xb1"4)\xd6\xa7`\x02' + b'\xec\x11\x8c\xf0\x14\xb0\x1d\x1c\xecy\xf0\xb7|\x11j\x85X\xb2!\x1c' + b'\xac\xb5N\xc7\x85j\x9ev\xf5\xe6\x0b\xc1]c\xc15\x16\x9f\xd5\x99' + b"\xfei^\xd2G\x9b\xbdl\xab:\xbe,\xa9'4\x82\xe5\xee\xb3\xc1" + b'$\x93\x95\xa8Y\x16\xf5\xbf\xacw\x91\x04\x1d\x18\x06\xe9\xc5\xfdk\x06' + b'\xe8\xfck\xc5\x86>\x8b~\xa4\xcb\xf1\xb3\x04\xf1\x04G5\xe2\xcc]' + b'\x16\xbf\x140d\x18\xe2\xedw#(3\xca\xa1\x80bX\x7f\xb3\x84' + b'\x9d\xdb\xe7\x08\x97\xcd\x16\xb9\xf1\xd5r+m\x1e\xcb3q\xc5\x9e\x92' + b"\x7f\x8e*\xc7\xde\xe9\xe26\xcds\xb1\x10-\xf6r\x02?\x9d\xddCgJN'" + b'\x11M\xfa\nQ\n\xe6`m\xb8N\xbbq\x8el\x0b\x02\xc7:q\x04G\xa1T' + b'\xf1\xfe!0\x85~\xe5\x884\xe9\x89\xfb\x13J8\x15\xe42\xb6\xad' + b'\x877A\x9a\xa6\xbft]\xd0\xe35M\xb0\x0cK\xc8\xf6\x88\xae\xed\xa9,j7' + b'\x81\x13\xa0(\xcb\xe1\xe9l2\x7f\xcd\xda\x95(\xa70B\xbd\xf4\xe3' + b'hp\x94\xbdJ\xd7\t\xc7g\xffo?\x89?\xf8}\x7f\xbc\x1c\x87' + b'\x14\xc0\xcf\x8cV:\x9a\x0e\xd0\xb2\x1ck\xffk\xb9\xe0=\xc7\x8d/' + b'\xb8\xff\x7f\x1d\x87`\x19.\x98X*~\xa7j\xb9\x0b"\xf4\xe4;V`\xb9\xd7' + b'\x03\x1e\xd0t0\xd3\xde\x1fd\xb9\xe2)\x16\x81}\xb1\\b\x7fJ' + b'\x92\xf4\xff\n+V!\xe9\xde\x98\xa0\x8fK\xdf7\xb9\xc0\x12\x1f\xe2' + b'\xe9\xb0`\xae\x14\r\xa7\xc4\x81~\xd8\x8d\xc5\x06\xd8m\xb0Y\x8a)' + b'\x06/\xbb\xf9\xac\xeaP\xe0\x91\x05m[\xe5z\xe6Z\xf3\x9f\xc7\xd0' + b'\xd3\x8b\xf3\x8a\x1b\xfa\xe4Pf\xbc0\x17\x10\xa9\xd0\x95J{\xb3\xc3' + b'\xfdW\x9bop\x0f\xbe\xaee\xa3]\x93\x9c\xda\xb75<\xf6g!\xcc\xb1\xfc\\' + b'7\x152Mc\x17\x84\x9d\xcd35\r0\xacL-\xf3\xfb\xcb\x96\x1e\xe9U\x7f' + b'\xd7\xca\xb0\xcc\x89\x0c*\xce\x14\xd1P\xf1\x03\xb6.~9o?\xe8' + b'\r\x86\xe0\x92\x87}\xa3\x84\x03P\xe0\xc2\x7f\n;m\x9d\x9e\xb4|' + b'\x8c\x18\xc0#0\xfe3\x07<\xda\xd8\xcf^\xd4Hi\xd6\xb3\x0bT' + b'\x1dF\x88\x85q}\x02\xc6&\xc4\xae\xce\x9cU\xfa\x0f\xcc\xb6\x1f\x11' + b'drw\x9eN\x19\xbd\xffz\x0f\xf0\x04s\xadR\xc1\xc0\xbfl\xf1\xba\xf95^' + b'e\xb1\xfbVY\xd9\x9f\x1c\xbf*\xc4\xa86\x08+\xd6\x88[\xc4_rc\xf0f' + b'\xb8\xd4\xec\x1dx\x19|\xbf\xa7\xe0\x82\x0b\x8c~\x10L/\x90\xd6\xfb' + b'\x81\xdb\x98\xcc\x02\x14\xa5C\xb2\xa7i\xfd\xcd\x1fO\xf7\xe9\x89t\xf0' + b'\x17\xa5\x1c\xad\xfe' + b'"\xd1v`\x82\xd5$\x89\xcf^\xd52.\xafd\xe8d@\xaa\xd5Y|\x90\x84' + b'j\xdb}\x84riV\x8e\xf0X4rB\xf2NPS[\x8e\x88\xd4\x0fI\xb8' + b'\xdd\xcb\x1d\xf2(\xdf;9\x9e|\xef^0;.*[\x9fl\x7f\xa2_X\xaff!\xbb\x03' + b'\xff\x19\x8f\x88\xb5\xb6\x884\xa3\x05\xde3D{\xe3\xcb\xce\xe4t]' + b'\x875\xe3Uf\xae\xea\x88\x1c\x03b\n\xb1,Q\xec\xcf\x08\t\xde@\x83\xaa<' + b',-\xe4\xee\x9b\x843\xe5\x007\tK\xac\x057\xd6*X\xa3\xc6~\xba\xe6O' + b'\x81kz"\xbe\xe43sL\xf1\xfa;\xf4^\x1e\xb4\x80\xe2\xbd\xaa\x17Z\xe1f' + b'\xda\xa6\xb9\x07:]}\x9fa\x0b?\xba\xe7\xf15\x04M\xe3\n}M\xa4\xcb\r' + b'2\x8a\x88\xa9\xa7\x92\x93\x84\x81Yo\x00\xcc\xc4\xab\x9aT\x96\x0b\xbe' + b'U\xac\x1d\x8d\x1b\x98"\xf8\x8f\xf1u\xc1n\xcc\xfcA\xcc\x90\xb7i' + b'\x83\x9c\x9c~\x1d4\xa2\xf0*J\xe7t\x12\xb4\xe3\xa0u\xd7\x95Z' + b'\xf7\xafG\x96~ST,\xa7\rC\x06\xf4\xf0\xeb`2\x9e>Q\x0e\xf6\xf5\xc5' + b'\x9b\xb5\xaf\xbe\xa3\x8f\xc0\xa3hu\x14\x12 \x97\x99\x04b\x8e\xc7\x1b' + b'VKc\xc1\xf3 \xde\x85-:\xdc\x1f\xac\xce*\x06\xb3\x80;`' + b'\xdb\xdd\x97\xfdg\xbf\xe7\xa8S\x08}\xf55e7\xb8/\xf0!\xc8' + b"Y\xa8\x9a\x07'\xe2\xde\r\x02\xe1\xb2\x0c\xf4C\xcd\xf9\xcb(\xe8\x90" + b'\xd3bTD\x15_\xf6\xc3\xfb\xb3E\xfc\xd6\x98{\xc6\\fz\x81\xa99\x85\xcb' + b'\xa5\xb1\x1d\x94bqW\x1a!;z~\x18\x88\xe8i\xdb\x1b\x8d\x8d' + b'\x06\xaa\x0e\x99s+5k\x00\xe4\xffh\xfe\xdbt\xa6\x1bU\xde\xa3' + b'\xef\xcb\x86\x9e\x81\x16j\n\x9d\xbc\xbbC\x80?\x010\xc7Jj;' + b'\xc4\xe5\x86\xd5\x0e0d#\xc6;\xb8\xd1\xc7c\xb5&8?\xd9J\xe5\xden\xb9' + b'\xe9cb4\xbb\xe6\x14\xe0\xe7l\x1b\x85\x94\x1fh\xf1n\xdeZ\xbe' + b'\x88\xff\xc2e\xca\xdc,B-\x8ac\xc9\xdf\xf5|&\xe4LL\xf0\x1f\xaa8\xbd' + b'\xc26\x94bVi\xd3\x0c\x1c\xb6\xbb\x99F\x8f\x0e\xcc\x8e4\xc6/^W\xf5?' + b'\xdc\x84(\x14dO\x9aD6\x0f4\xa3,\x0c\x0bS\x9fJ\xe1\xacc^\x8a0\t\x80D[' + b'\xb8\xe6\x86\xb0\xe8\xd4\xf9\x1en\xf1\xf5^\xeb\xb8\xb8\xf8' + b')\xa8\xbf\xaa\x84\x86\xb1a \x95\x16\x08\x1c\xbb@\xbd+\r/\xfb' + b'\x92\xfbh\xf1\x8d3\xf9\x92\xde`\xf1\x86\x03\xaa+\xd9\xd9\xc6P\xaf' + b'\xe3-\xea\xa5\x0fB\xca\xde\xd5n^\xe3/\xbf\xa6w\xc8\x0e' + b'\xc6\xfd\x97_+D{\x03\x15\xe8s\xb1\xc8HAG\xcf\xf4\x1a\xdd' + b'\xad\x11\xbf\x157q+\xdeW\x89g"X\x82\xfd~\xf7\xab4\xf6`\xab\xf1q' + b')\x82\x10K\xe9sV\xfe\xe45\xafs*\x14\xa7;\xac{\x06\x9d<@\x93G' + b'j\x1d\xefL\xe9\xd8\x92\x19&\xa1\x16\x19\x04\tu5\x01]\xf6\xf4' + b'\xcd\\\xd8A|I\xd4\xeb\x05\x88C\xc6e\xacQ\xe9*\x97~\x9au\xf8Xy' + b"\x17P\x10\x9f\n\x8c\xe2fZEu>\x9b\x1e\x91\x0b'`\xbd\xc0\xa8\x86c\x1d" + b'Z\xe2\xdc8j\x95\xffU\x90\x1e\xf4o\xbc\xe5\xe3e>\xd2R\xc0b#\xbc\x15' + b'H-\xb9!\xde\x9d\x90k\xdew\x9b{\x99\xde\xf7/K)A\xfd\xf5\xe6:\xda' + b'UM\xcc\xbb\xa2\x0b\x9a\x93\xf5{9\xc0 \xd2((6i\xc0\xbbu\xd8\x9e\x8d' + b'\xf8\x04q\x10\xd4\x14\x9e7-\xb9B\xea\x01Q8\xc8v\x9a\x12A\x88Cd\x92' + b"\x1c\x8c!\xf4\x94\x87'\xe3\xcd\xae\xf7\xd8\x93\xfa\xde\xa8b\x9e\xee2" + b'K\xdb\x00l\x9d\t\xb1|D\x05U\xbb\xf4>\xf1w\x887\xd1}W\x9d|g|1\xb0\x13' + b"\xa3 \xe5\xbfm@\xc06+\xb7\t\xcf\x15D\x9a \x1fM\x1f\xd2\xb5'\xa9\xbb" + b'~Co\x82\xfa\xc2\t\xe6f\xfc\xbeI\xae1\x8e\xbe\xb8\xcf\x86\x17' + b'\x9f\xe2`\xbd\xaf\xba\xb9\xbc\x1b\xa3\xcd\x82\x8fwc\xefd\xa9\xd5\x14' + b'\xe2C\xafUE\xb6\x11MJH\xd0=\x05\xd4*I\xff"\r\x1b^\xcaS6=\xec@\xd5' + b'\x11,\xe0\x87Gr\xaa[\xb8\xbc>n\xbd\x81\x0c\x07<\xe9\x92(' + b'\xb2\xff\xac}\xe7\xb6\x15\x90\x9f~4\x9a\xe6\xd6\xd8s\xed\x99tf' + b'\xa0f\xf8\xf1\x87\t\x96/)\x85\xb6\n\xd7\xb2w\x0b\xbc\xba\x99\xee' + b'Q\xeen\x1d\xad\x03\xc3s\xd1\xfd\xa2\xc6\xb7\x9a\x9c(G<6\xad[~H ' + b'\x16\x89\x89\xd0\xc3\xd2\xca~\xac\xea\xa5\xed\xe5\xfb\r:' + b'\x8e\xa6\xf1e\xbb\xba\xbd\xe0(\xa3\x89_\x01(\xb5c\xcc\x9f\x1fg' + b'v\xfd\x17\xb3\x08S=S\xee\xfc\x85>\x91\x8d\x8d\nYR\xb3G\xd1A\xa2\xb1' + b'\xec\xb0\x01\xd2\xcd\xf9\xfe\x82\x06O\xb3\xecd\xa9c\xe0\x8eP\x90\xce' + b'\xe0\xcd\xd8\xd8\xdc\x9f\xaa\x01"[Q~\xe4\x88\xa1#\xc1\x12C\xcf' + b'\xbe\x80\x11H\xbf\x86\xd8\xbem\xcfWFQ(X\x01DK\xdfB\xaa\x10.-' + b'\xd5\x9e|\x86\x15\x86N]\xc7Z\x17\xcd=\xd7)M\xde\x15\xa4LTi\xa0\x15' + b'\xd1\xe7\xbdN\xa4?\xd1\xe7\x02\xfe4\xe4O\x89\x98&\x96\x0f\x02\x9c' + b'\x9e\x19\xaa\x13u7\xbd0\xdc\xd8\x93\xf4BNE\x1d\x93\x82\x81\x16' + b'\xe5y\xcf\x98D\xca\x9a\xe2\xfd\xcdL\xcc\xd1\xfc_\x0b\x1c\xa0]\xdc' + b'\xa91 \xc9c\xd8\xbf\x97\xcfp\xe6\x19-\xad\xff\xcc\xd1N(\xe8' + b'\xeb#\x182\x96I\xf7l\xf3r\x00' +) + + +if __name__ == "__main__": + unittest.main() From a5016446f40a1aa7eb2e03b268c754524599a6ec Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 19 Apr 2025 16:26:21 -0700 Subject: [PATCH 297/317] _lzma implementation and test marking Signed-off-by: Ashwin Naren --- Cargo.lock | 44 +-- Lib/tarfile.py | 6 +- Lib/test/support/__init__.py | 10 +- Lib/test/test_lzma.py | 104 ++++++- Lib/test/test_tarfile.py | 4 +- Lib/test/test_zoneinfo/test_zoneinfo.py | 8 + stdlib/Cargo.toml | 3 +- stdlib/src/{ => compression}/bz2.rs | 6 +- stdlib/src/compression/lzma.rs | 395 ++++++++++++++++++++++++ stdlib/src/compression/mod.rs | 363 ++++++++++++++++++++++ stdlib/src/{ => compression}/zlib.rs | 370 +++------------------- stdlib/src/lib.rs | 13 +- stdlib/src/lzma.rs | 6 - 13 files changed, 962 insertions(+), 370 deletions(-) rename stdlib/src/{ => compression}/bz2.rs (99%) create mode 100644 stdlib/src/compression/lzma.rs create mode 100644 stdlib/src/compression/mod.rs rename stdlib/src/{ => compression}/zlib.rs (65%) delete mode 100644 stdlib/src/lzma.rs diff --git a/Cargo.lock b/Cargo.lock index 46c7135d83..a9e57ed825 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -237,12 +237,6 @@ version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bzip2" version = "0.5.2" @@ -636,21 +630,6 @@ dependencies = [ "target-lexicon", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1418,13 +1397,14 @@ dependencies = [ ] [[package]] -name = "lzma-rs" -version = "0.3.0" +name = "lzma-sys" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" dependencies = [ - "byteorder", - "crc", + "cc", + "libc", + "pkg-config", ] [[package]] @@ -2498,7 +2478,7 @@ dependencies = [ "libc", "libsqlite3-sys", "libz-rs-sys", - "lzma-rs", + "lzma-sys", "mac_address", "malachite-bigint", "md-5", @@ -2544,6 +2524,7 @@ dependencies = [ "widestring", "windows-sys 0.59.0", "xml-rs", + "xz2", ] [[package]] @@ -3707,6 +3688,15 @@ version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 3bbbcaa621..ab2266b285 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -392,7 +392,8 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, elif comptype == "xz": try: - import lzma + # TODO: RUSTPYTHON remove underscore + import lzma_ except ImportError: raise CompressionError("lzma module is not available") from None if mode == "r": @@ -1923,7 +1924,8 @@ def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): raise ValueError("mode must be 'r', 'w' or 'x'") try: - from lzma import LZMAFile, LZMAError + # TODO: RUSTPYTHON remove underscore + from lzma_ import LZMAFile, LZMAError except ImportError: raise CompressionError("lzma module is not available") from None diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 106265ab2f..ee94201b0d 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -494,10 +494,12 @@ def requires_bz2(reason='requires bz2'): return unittest.skipUnless(bz2, reason) def requires_lzma(reason='requires lzma'): - try: - import lzma - except ImportError: - lzma = None + # try: + # import lzma + # except ImportError: + # lzma = None + # TODO: RUSTPYTHON + lzma = None return unittest.skipUnless(lzma, reason) def has_no_debug_ranges(): diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index df6636c505..1bac61f59e 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -22,6 +22,8 @@ class CompressorDecompressorTestCase(unittest.TestCase): # Test error cases. + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_simple_bad_args(self): self.assertRaises(TypeError, LZMACompressor, []) self.assertRaises(TypeError, LZMACompressor, format=3.45) @@ -62,6 +64,8 @@ def test_simple_bad_args(self): lzd.decompress(empty) self.assertRaises(EOFError, lzd.decompress, b"quux") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_bad_filter_spec(self): self.assertRaises(TypeError, LZMACompressor, filters=[b"wobsite"]) self.assertRaises(ValueError, LZMACompressor, filters=[{"xyzzy": 3}]) @@ -78,6 +82,8 @@ def test_decompressor_after_eof(self): lzd.decompress(COMPRESSED_XZ) self.assertRaises(EOFError, lzd.decompress, b"nyan") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_memlimit(self): lzd = LZMADecompressor(memlimit=1024) self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) @@ -98,6 +104,8 @@ def _test_decompressor(self, lzd, data, check, unused_data=b""): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, unused_data) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_auto(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) @@ -105,30 +113,44 @@ def test_decompressor_auto(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_xz(self): lzd = LZMADecompressor(lzma.FORMAT_XZ) self._test_decompressor(lzd, COMPRESSED_XZ, lzma.CHECK_CRC64) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_alone(self): lzd = LZMADecompressor(lzma.FORMAT_ALONE) self._test_decompressor(lzd, COMPRESSED_ALONE, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_raw_1(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) self._test_decompressor(lzd, COMPRESSED_RAW_1, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_raw_2(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_2) self._test_decompressor(lzd, COMPRESSED_RAW_2, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_raw_3(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_3) self._test_decompressor(lzd, COMPRESSED_RAW_3, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_raw_4(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, COMPRESSED_RAW_4, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_chunks(self): lzd = LZMADecompressor() out = [] @@ -141,6 +163,8 @@ def test_decompressor_chunks(self): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, b"") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_chunks_empty(self): lzd = LZMADecompressor() out = [] @@ -156,6 +180,8 @@ def test_decompressor_chunks_empty(self): self.assertTrue(lzd.eof) self.assertEqual(lzd.unused_data, b"") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_chunks_maxsize(self): lzd = LZMADecompressor() max_length = 100 @@ -247,12 +273,16 @@ def test_decompressor_inputbuf_3(self): out.append(lzd.decompress(COMPRESSED_XZ[300:])) self.assertEqual(b''.join(out), INPUT) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_unused_data(self): lzd = LZMADecompressor() extra = b"fooblibar" self._test_decompressor(lzd, COMPRESSED_XZ + extra, lzma.CHECK_CRC64, unused_data=extra) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_bad_input(self): lzd = LZMADecompressor() self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_RAW_1) @@ -266,6 +296,8 @@ def test_decompressor_bad_input(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_1) self.assertRaises(LZMAError, lzd.decompress, COMPRESSED_XZ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_bug_28275(self): # Test coverage for Issue 28275 lzd = LZMADecompressor() @@ -275,24 +307,32 @@ def test_decompressor_bug_28275(self): # Test that LZMACompressor->LZMADecompressor preserves the input data. + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip_xz(self): lzc = LZMACompressor() cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip_alone(self): lzc = LZMACompressor(lzma.FORMAT_ALONE) cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip_raw(self): lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) cdata = lzc.compress(INPUT) + lzc.flush() lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip_raw_empty(self): lzc = LZMACompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) cdata = lzc.compress(INPUT) @@ -303,6 +343,8 @@ def test_roundtrip_raw_empty(self): lzd = LZMADecompressor(lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self._test_decompressor(lzd, cdata, lzma.CHECK_NONE) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip_chunks(self): lzc = LZMACompressor() cdata = [] @@ -313,6 +355,8 @@ def test_roundtrip_chunks(self): lzd = LZMADecompressor() self._test_decompressor(lzd, cdata, lzma.CHECK_CRC64) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip_empty_chunks(self): lzc = LZMACompressor() cdata = [] @@ -328,6 +372,8 @@ def test_roundtrip_empty_chunks(self): # LZMADecompressor intentionally does not handle concatenated streams. + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompressor_multistream(self): lzd = LZMADecompressor() self._test_decompressor(lzd, COMPRESSED_XZ + COMPRESSED_ALONE, @@ -363,6 +409,8 @@ def test_decompressor_bigmem(self, size): # Pickling raises an exception; there's no way to serialize an lzma_stream. + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.assertRaises(TypeError): @@ -388,6 +436,8 @@ class CompressDecompressFunctionTestCase(unittest.TestCase): # Test error cases: + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_bad_args(self): self.assertRaises(TypeError, lzma.compress) self.assertRaises(TypeError, lzma.compress, []) @@ -417,6 +467,8 @@ def test_bad_args(self): lzma.decompress( b"", format=lzma.FORMAT_ALONE, filters=FILTERS_RAW_1) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_memlimit(self): with self.assertRaises(LZMAError): lzma.decompress(COMPRESSED_XZ, memlimit=1024) @@ -429,6 +481,8 @@ def test_decompress_memlimit(self): # Test LZMADecompressor on known-good input data. + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_good_input(self): ddata = lzma.decompress(COMPRESSED_XZ) self.assertEqual(ddata, INPUT) @@ -458,6 +512,8 @@ def test_decompress_good_input(self): COMPRESSED_RAW_4, lzma.FORMAT_RAW, filters=FILTERS_RAW_4) self.assertEqual(ddata, INPUT) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_incomplete_input(self): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_XZ[:128]) self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_ALONE[:128]) @@ -470,6 +526,8 @@ def test_decompress_incomplete_input(self): self.assertRaises(LZMAError, lzma.decompress, COMPRESSED_RAW_4[:128], format=lzma.FORMAT_RAW, filters=FILTERS_RAW_4) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_bad_input(self): with self.assertRaises(LZMAError): lzma.decompress(COMPRESSED_BOGUS) @@ -485,6 +543,8 @@ def test_decompress_bad_input(self): # Test that compress()->decompress() preserves the input data. + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_roundtrip(self): cdata = lzma.compress(INPUT) ddata = lzma.decompress(cdata) @@ -510,10 +570,14 @@ def test_decompress_multistream(self): # Test robust handling of non-LZMA data following the compressed stream(s). + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ + COMPRESSED_BOGUS) self.assertEqual(ddata, INPUT) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_decompress_multistream_trailing_junk(self): ddata = lzma.decompress(COMPRESSED_XZ * 3 + COMPRESSED_BOGUS) self.assertEqual(ddata, INPUT * 3) @@ -550,6 +614,8 @@ def test_init(self): self.assertIsInstance(f, LZMAFile) self.assertEqual(f.mode, "wb") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_init_with_PathLike_filename(self): filename = FakePath(TESTFN) with TempFile(filename, COMPRESSED_XZ): @@ -630,6 +696,8 @@ def test_init_bad_mode(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(COMPRESSED_XZ), "rw") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_init_bad_check(self): with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", check=b"asd") @@ -667,6 +735,8 @@ def test_init_bad_preset(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(COMPRESSED_XZ), preset=3) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_init_bad_filter_spec(self): with self.assertRaises(TypeError): LZMAFile(BytesIO(), "w", filters=[b"wobsite"]) @@ -684,6 +754,8 @@ def test_init_bad_filter_spec(self): LZMAFile(BytesIO(), "w", filters=[{"id": lzma.FILTER_X86, "foo": 0}]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_init_with_preset_and_filters(self): with self.assertRaises(ValueError): LZMAFile(BytesIO(), "w", format=lzma.FORMAT_RAW, @@ -802,6 +874,8 @@ def test_writable(self): f.close() self.assertRaises(ValueError, f.writable) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_read(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: self.assertEqual(f.read(), INPUT) @@ -849,6 +923,8 @@ def test_read_10(self): chunks.append(result) self.assertEqual(b"".join(chunks), INPUT) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_read_multistream(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: self.assertEqual(f.read(), INPUT * 5) @@ -869,10 +945,14 @@ def test_read_multistream_buffer_size_aligned(self): finally: _compression.BUFFER_SIZE = saved_buffer_size + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_read_trailing_junk(self): with LZMAFile(BytesIO(COMPRESSED_XZ + COMPRESSED_BOGUS)) as f: self.assertEqual(f.read(), INPUT) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_read_multistream_trailing_junk(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5 + COMPRESSED_BOGUS)) as f: self.assertEqual(f.read(), INPUT * 5) @@ -978,6 +1058,8 @@ def test_read_bad_args(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: self.assertRaises(TypeError, f.read, float()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_read_bad_data(self): with LZMAFile(BytesIO(COMPRESSED_BOGUS)) as f: self.assertRaises(LZMAError, f.read) @@ -1035,6 +1117,8 @@ def test_peek_bad_args(self): with LZMAFile(BytesIO(), "w") as f: self.assertRaises(ValueError, f.peek) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_iterator(self): with BytesIO(INPUT) as f: lines = f.readlines() @@ -1074,6 +1158,8 @@ def test_decompress_limited(self): self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp, "Excessive amount of data was decompressed") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_write(self): with BytesIO() as dst: with LZMAFile(dst, "w") as f: @@ -1342,6 +1428,8 @@ def test_tell_bad_args(self): f.close() self.assertRaises(ValueError, f.tell) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_issue21872(self): # sometimes decompress data incompletely @@ -1425,6 +1513,8 @@ def test_filename(self): with lzma.open(TESTFN, "rb") as f: self.assertEqual(f.read(), INPUT * 2) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_with_pathlike_filename(self): filename = FakePath(TESTFN) with TempFile(filename): @@ -1451,6 +1541,8 @@ def test_bad_params(self): with self.assertRaises(ValueError): lzma.open(TESTFN, "rb", newline="\n") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_format_and_filters(self): # Test non-default format and filter chain. options = {"format": lzma.FORMAT_RAW, "filters": FILTERS_RAW_1} @@ -1481,6 +1573,8 @@ def test_encoding_error_handler(self): with lzma.open(bio, "rt", encoding="ascii", errors="ignore") as f: self.assertEqual(f.read(), "foobar") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_newline(self): # Test with explicit newline (universal newline mode disabled). text = INPUT.decode("ascii") @@ -1505,6 +1599,8 @@ def test_x_mode(self): class MiscellaneousTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_is_check_supported(self): # CHECK_NONE and CHECK_CRC32 should always be supported, # regardless of the options liblzma was compiled with. @@ -1517,6 +1613,8 @@ def test_is_check_supported(self): # This value should not be a valid check ID. self.assertFalse(lzma.is_check_supported(lzma.CHECK_UNKNOWN)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test__encode_filter_properties(self): with self.assertRaises(TypeError): lzma._encode_filter_properties(b"not a dict") @@ -1538,6 +1636,8 @@ def test__encode_filter_properties(self): }) self.assertEqual(props, b"]\x00\x00\x80\x00") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test__decode_filter_properties(self): with self.assertRaises(TypeError): lzma._decode_filter_properties(lzma.FILTER_X86, {"should be": bytes}) @@ -1561,6 +1661,8 @@ def test__decode_filter_properties(self): filterspec = lzma._decode_filter_properties(f, b"") self.assertEqual(filterspec, {"id": f}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_filter_properties_roundtrip(self): spec1 = lzma._decode_filter_properties( lzma.FILTER_LZMA1, b"]\x00\x00\x80\x00") @@ -2092,4 +2194,4 @@ def test_filter_properties_roundtrip(self): if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 63f7b347ad..bff67ba221 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -28,7 +28,9 @@ except ImportError: bz2 = None try: - import lzma + # TODO: RUSTPYTHON + lzma = None + # import lzma except ImportError: lzma = None diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 8414721555..e05bd046e8 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1728,6 +1728,8 @@ def test_env_variable_relative_paths(self): with self.subTest("filtered", path_var=path_var): self.assertSequenceEqual(tzpath, expected_paths) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_env_variable_relative_paths_warning_location(self): path_var = "path/to/somewhere" @@ -1822,6 +1824,8 @@ def test_getattr_error(self): with self.assertRaises(AttributeError): self.module.NOATTRIBUTE + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dir_contains_all(self): """dir(self.module) should at least contain everything in __all__.""" module_all_set = set(self.module.__all__) @@ -1925,12 +1929,16 @@ class ExtensionBuiltTest(unittest.TestCase): rely on these tests as an indication of stable properties of these classes. """ + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cache_location(self): # The pure Python version stores caches on attributes, but the C # extension stores them in C globals (at least for now) self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_gc_tracked(self): import gc diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 70f70b8552..f051ea7b2b 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -80,7 +80,6 @@ crc32fast = "1.3.2" flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } libz-sys = { package = "libz-rs-sys", version = "0.5" } bzip2 = { version = "0.5", features = ["libbz2-rs-sys"] } -lzma-rs = "0.3" # tkinter tk-sys = { git = "https://github.com/arihant2math/tkinter.git", tag = "v0.2.0", optional = true } @@ -113,6 +112,8 @@ foreign-types-shared = { version = "0.1.1", optional = true } [target.'cfg(not(any(target_os = "android", target_arch = "wasm32")))'.dependencies] libsqlite3-sys = { version = "0.28", features = ["bundled"], optional = true } +lzma-sys = "0.1" +xz2 = "0.1" [target.'cfg(windows)'.dependencies] junction = { workspace = true } diff --git a/stdlib/src/bz2.rs b/stdlib/src/compression/bz2.rs similarity index 99% rename from stdlib/src/bz2.rs rename to stdlib/src/compression/bz2.rs index 6339a44a24..ca7d1250ce 100644 --- a/stdlib/src/bz2.rs +++ b/stdlib/src/compression/bz2.rs @@ -5,6 +5,9 @@ pub(crate) use _bz2::make_module; #[pymodule] mod _bz2 { use crate::common::lock::PyMutex; + use crate::compression::{ + DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, + }; use crate::vm::{ VirtualMachine, builtins::{PyBytesRef, PyTypeRef}, @@ -12,9 +15,6 @@ mod _bz2 { object::{PyPayload, PyResult}, types::Constructor, }; - use crate::zlib::{ - DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, - }; use bzip2::{Decompress, Status, write::BzEncoder}; use rustpython_vm::convert::ToPyException; use std::{fmt, io::Write}; diff --git a/stdlib/src/compression/lzma.rs b/stdlib/src/compression/lzma.rs new file mode 100644 index 0000000000..ed8be9a695 --- /dev/null +++ b/stdlib/src/compression/lzma.rs @@ -0,0 +1,395 @@ +// spell-checker:ignore ARMTHUMB + +pub(crate) use _lzma::make_module; + +#[pymodule] +mod _lzma { + use crate::compression::{ + CompressFlushKind, CompressState, CompressStatusKind, Compressor, DecompressArgs, + DecompressError, DecompressState, DecompressStatus, Decompressor, + }; + #[pyattr] + use lzma_sys::{ + LZMA_CHECK_CRC32 as CHECK_CRC32, LZMA_CHECK_CRC64 as CHECK_CRC64, + LZMA_CHECK_NONE as CHECK_NONE, LZMA_CHECK_SHA256 as CHECK_SHA256, + }; + #[pyattr] + use lzma_sys::{ + LZMA_FILTER_ARM as FILTER_ARM, LZMA_FILTER_ARMTHUMB as FILTER_ARMTHUMB, + LZMA_FILTER_IA64 as FILTER_IA64, LZMA_FILTER_LZMA1 as FILTER_LZMA1, + LZMA_FILTER_LZMA2 as FILTER_LZMA2, LZMA_FILTER_POWERPC as FILTER_POWERPC, + LZMA_FILTER_SPARC as FILTER_SPARC, LZMA_FILTER_X86 as FILTER_X86, + }; + #[pyattr] + use lzma_sys::{ + LZMA_MF_BT2 as MF_BT2, LZMA_MF_BT3 as MF_BT3, LZMA_MF_BT4 as MF_BT4, LZMA_MF_HC3 as MF_HC3, + LZMA_MF_HC4 as MF_HC4, + }; + #[pyattr] + use lzma_sys::{LZMA_MODE_FAST as MODE_FAST, LZMA_MODE_NORMAL as MODE_NORMAL}; + #[pyattr] + use lzma_sys::{ + LZMA_PRESET_DEFAULT as PRESET_DEFAULT, LZMA_PRESET_EXTREME as PRESET_EXTREME, + LZMA_PRESET_LEVEL_MASK as PRESET_LEVEL_MASK, + }; + use rustpython_common::lock::PyMutex; + use rustpython_vm::builtins::{PyBaseExceptionRef, PyBytesRef, PyTypeRef}; + use rustpython_vm::convert::ToPyException; + use rustpython_vm::function::ArgBytesLike; + use rustpython_vm::types::Constructor; + use rustpython_vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use std::fmt; + use xz2::stream::{Action, Check, Error, Filters, Status, Stream}; + + #[cfg(windows)] + type EnumVal = i32; + #[cfg(not(windows))] + type EnumVal = u32; + + const BUFSIZ: usize = 8192; + // TODO: can't find this in lzma-sys, but find way not to hardcode this + #[pyattr] + const FILTER_DELTA: i32 = 3; + #[pyattr] + const CHECK_UNKNOWN: i32 = 16; + + // the variant ids are hardcoded to be equivalent to the C enum values + enum Format { + Auto = 0, + Xz = 1, + Alone = 2, + Raw = 3, + } + + #[pyattr] + const FORMAT_AUTO: i32 = Format::Auto as i32; + #[pyattr] + const FORMAT_XZ: i32 = Format::Xz as i32; + #[pyattr] + const FORMAT_ALONE: i32 = Format::Alone as i32; + #[pyattr] + const FORMAT_RAW: i32 = Format::Raw as i32; + + #[pyattr(once, name = "LZMAError")] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "lzma", + "LZMAError", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } + + fn new_lzma_error(message: impl Into, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_exception_msg(vm.class("lzma", "LZMAError"), message.into()) + } + + #[pyfunction] + fn is_check_supported(check: i32) -> bool { + unsafe { lzma_sys::lzma_check_is_supported(check as _) != 0 } + } + + // TODO: To implement these we need a function to convert a pyobject to a lzma filter and related structs + #[pyfunction] + fn _encode_filter_properties() -> PyResult<()> { + Ok(()) + } + + #[pyfunction] + fn _decode_filter_properties(_filter_id: u64, _buffer: ArgBytesLike) -> PyResult<()> { + Ok(()) + } + + #[pyattr] + #[pyclass(name = "LZMADecompressor")] + #[derive(PyPayload)] + struct LZMADecompressor { + state: PyMutex>, + } + + impl Decompressor for Stream { + type Flush = (); + type Status = Status; + type Error = Error; + + fn total_in(&self) -> u64 { + self.total_in() + } + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + (): Self::Flush, + ) -> Result { + self.process_vec(input, output, Action::Run) + } + } + + impl DecompressStatus for Status { + fn is_stream_end(&self) -> bool { + *self == Status::StreamEnd + } + } + + impl fmt::Debug for LZMADecompressor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_lzma.LZMADecompressor") + } + } + #[derive(FromArgs)] + pub struct LZMADecompressorConstructorArgs { + #[pyarg(any, default = FORMAT_AUTO)] + format: i32, + #[pyarg(any, optional)] + memlimit: Option, + #[pyarg(any, optional)] + filters: Option, + } + + impl Constructor for LZMADecompressor { + type Args = LZMADecompressorConstructorArgs; + + fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let memlimit = args.memlimit.unwrap_or(u64::MAX); + let filters = args.filters.unwrap_or(0); + let stream_result = match args.format { + FORMAT_AUTO => Stream::new_auto_decoder(memlimit, filters), + FORMAT_XZ => Stream::new_stream_decoder(memlimit, filters), + FORMAT_ALONE => Stream::new_lzma_decoder(memlimit), + // TODO: FORMAT_RAW + _ => return Err(new_lzma_error("Invalid format", vm)), + }; + Self { + state: PyMutex::new(DecompressState::new( + stream_result + .map_err(|_| new_lzma_error("Failed to initialize decoder", vm))?, + vm, + )), + } + .into_ref_with_type(vm, cls) + .map(Into::into) + } + } + + #[pyclass(with(Constructor))] + impl LZMADecompressor { + #[pymethod] + fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { + let max_length = args.max_length(); + let data = &*args.data(); + + let mut state = self.state.lock(); + state + .decompress(data, max_length, BUFSIZ, vm) + .map_err(|e| match e { + DecompressError::Decompress(err) => vm.new_os_error(err.to_string()), + DecompressError::Eof(err) => err.to_pyexception(vm), + }) + } + + #[pygetset] + fn eof(&self) -> bool { + self.state.lock().eof() + } + + #[pygetset] + fn unused_data(&self) -> PyBytesRef { + self.state.lock().unused_data() + } + + #[pygetset] + fn needs_input(&self) -> bool { + // False if the decompress() method can provide more + // decompressed data before requiring new uncompressed input. + self.state.lock().needs_input() + } + + // TODO: mro()? + } + + struct CompressorInner { + stream: Stream, + } + + impl CompressStatusKind for Status { + const OK: Self = Status::Ok; + const EOF: Self = Status::StreamEnd; + + fn to_usize(self) -> usize { + self as usize + } + } + + impl CompressFlushKind for Action { + const NONE: Self = Action::Run; + const FINISH: Self = Action::Finish; + + fn to_usize(self) -> usize { + self as usize + } + } + + impl Compressor for CompressorInner { + type Status = Status; + type Flush = Action; + const CHUNKSIZE: usize = u32::MAX as usize; + const DEF_BUF_SIZE: usize = 16 * 1024; + + fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + vm: &VirtualMachine, + ) -> PyResult { + self.stream + .process_vec(input, output, flush) + .map_err(|_| new_lzma_error("Failed to compress data", vm)) + } + + fn total_in(&mut self) -> usize { + self.stream.total_in() as usize + } + + fn new_error(message: impl Into, vm: &VirtualMachine) -> PyBaseExceptionRef { + new_lzma_error(message, vm) + } + } + + impl CompressorInner { + fn new(stream: Stream) -> Self { + Self { stream } + } + } + + #[pyattr] + #[pyclass(name = "LZMACompressor")] + #[derive(PyPayload)] + struct LZMACompressor { + state: PyMutex>, + } + + impl fmt::Debug for LZMACompressor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "_lzma.LZMACompressor") + } + } + + fn int_to_check(check: i32) -> Option { + if check == -1 { + return Some(Check::None); + } + match check as EnumVal { + CHECK_NONE => Some(Check::None), + CHECK_CRC32 => Some(Check::Crc32), + CHECK_CRC64 => Some(Check::Crc64), + CHECK_SHA256 => Some(Check::Sha256), + _ => None, + } + } + + fn parse_filter_chain_spec( + filter_specs: Vec, + vm: &VirtualMachine, + ) -> PyResult { + // TODO: don't hardcode + const LZMA_FILTERS_MAX: usize = 4; + if filter_specs.len() > LZMA_FILTERS_MAX { + return Err(new_lzma_error( + format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), + vm, + )); + } + let filters = Filters::new(); + for _item in filter_specs {} + Ok(filters) + } + + impl LZMACompressor { + fn init_xz( + check: i32, + preset: u32, + filters: Option>, + vm: &VirtualMachine, + ) -> PyResult { + let real_check = int_to_check(check) + .ok_or_else(|| vm.new_type_error("Invalid check value".to_string()))?; + if let Some(filters) = filters { + let filters = parse_filter_chain_spec(filters, vm)?; + Ok(Stream::new_stream_encoder(&filters, real_check) + .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?) + } else { + Ok(Stream::new_easy_encoder(preset, real_check) + .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?) + } + } + } + + #[derive(FromArgs)] + pub struct LZMACompressorConstructorArgs { + // format=FORMAT_XZ, check=-1, preset=None, filters=None + // {'format': 3, 'filters': [{'id': 3, 'dist': 2}, {'id': 33, 'preset': 2147483654}]} + #[pyarg(any, default = FORMAT_XZ)] + format: i32, + #[pyarg(any, default = -1)] + check: i32, + #[pyarg(any, optional)] + preset: Option, + #[pyarg(any, optional)] + filters: Option>, + #[pyarg(any, optional)] + _filter_specs: Option>, + #[pyarg(positional, optional)] + preset_obj: Option, + } + + impl Constructor for LZMACompressor { + type Args = LZMACompressorConstructorArgs; + + fn py_new(_cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + let preset = args.preset.unwrap_or(PRESET_DEFAULT); + #[allow(clippy::unnecessary_cast)] + if args.format != FORMAT_XZ as i32 + && args.check != -1 + && args.check != CHECK_NONE as i32 + { + return Err(new_lzma_error( + "Integrity checks are only supported by FORMAT_XZ", + vm, + )); + } + if args.preset_obj.is_some() && args.filters.is_some() { + return Err(new_lzma_error( + "Cannot specify both preset and filter chain", + vm, + )); + } + let stream = match args.format { + FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?, + // TODO: ALONE AND RAW + _ => return Err(new_lzma_error("Invalid format", vm)), + }; + Ok(Self { + state: PyMutex::new(CompressState::new(CompressorInner::new(stream))), + } + .into_pyobject(vm)) + } + } + + #[pyclass(with(Constructor))] + impl LZMACompressor { + #[pymethod] + fn compress(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult> { + let mut state = self.state.lock(); + // TODO: Flush check + state.compress(&data.borrow_buf(), vm) + } + + #[pymethod] + fn flush(&self, vm: &VirtualMachine) -> PyResult> { + // TODO: flush check + let mut state = self.state.lock(); + // TODO: check if action is correct + state.flush(Action::Finish, vm) + } + } +} diff --git a/stdlib/src/compression/mod.rs b/stdlib/src/compression/mod.rs new file mode 100644 index 0000000000..fff7130022 --- /dev/null +++ b/stdlib/src/compression/mod.rs @@ -0,0 +1,363 @@ +// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker + +use crate::vm::function::{ArgBytesLike, ArgSize, OptionalArg}; +use crate::vm::{ + PyResult, VirtualMachine, + builtins::{PyBaseExceptionRef, PyBytesRef}, + convert::ToPyException, +}; +pub(crate) mod bz2; +#[cfg(not(any(target_os = "android", target_arch = "wasm32")))] +pub(crate) mod lzma; +pub(crate) mod zlib; + +pub const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; +// TODO: don't hardcode +const CHUNKSIZE: usize = u32::MAX as usize; + +#[derive(Clone)] +pub struct Chunker<'a> { + data1: &'a [u8], + data2: &'a [u8], +} +impl<'a> Chunker<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { + data1: data, + data2: &[], + } + } + pub fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { + if data1.is_empty() { + Self { + data1: data2, + data2: &[], + } + } else { + Self { data1, data2 } + } + } + pub fn len(&self) -> usize { + self.data1.len() + self.data2.len() + } + pub fn is_empty(&self) -> bool { + self.data1.is_empty() + } + pub fn to_vec(&self) -> Vec { + [self.data1, self.data2].concat() + } + pub fn chunk(&self) -> &'a [u8] { + self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) + } + pub fn advance(&mut self, consumed: usize) { + self.data1 = &self.data1[consumed..]; + if self.data1.is_empty() { + self.data1 = std::mem::take(&mut self.data2); + } + } +} + +pub fn _decompress_chunks( + data: &mut Chunker<'_>, + d: &mut D, + bufsize: usize, + max_length: Option, + calc_flush: impl Fn(bool) -> D::Flush, +) -> Result<(Vec, bool), D::Error> { + if data.is_empty() { + return Ok((Vec::new(), true)); + } + let max_length = max_length.unwrap_or(usize::MAX); + let mut buf = Vec::new(); + + 'outer: loop { + let chunk = data.chunk(); + let flush = calc_flush(chunk.len() == data.len()); + loop { + let additional = std::cmp::min(bufsize, max_length - buf.capacity()); + if additional == 0 { + return Ok((buf, false)); + } + buf.reserve_exact(additional); + + let prev_in = d.total_in(); + let res = d.decompress_vec(chunk, &mut buf, flush); + let consumed = d.total_in() - prev_in; + + data.advance(consumed as usize); + + match res { + Ok(status) => { + let stream_end = status.is_stream_end(); + if stream_end || data.is_empty() { + // we've reached the end of the stream, we're done + buf.shrink_to_fit(); + return Ok((buf, stream_end)); + } else if !chunk.is_empty() && consumed == 0 { + // we're gonna need a bigger buffer + continue; + } else { + // next chunk + continue 'outer; + } + } + Err(e) => { + d.maybe_set_dict(e)?; + // now try the next chunk + continue 'outer; + } + }; + } + } +} + +pub trait Compressor { + type Status: CompressStatusKind; + type Flush: CompressFlushKind; + const CHUNKSIZE: usize; + const DEF_BUF_SIZE: usize; + + fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + vm: &VirtualMachine, + ) -> PyResult; + + fn total_in(&mut self) -> usize; + + fn new_error(message: impl Into, vm: &VirtualMachine) -> PyBaseExceptionRef; +} + +pub trait CompressFlushKind: Copy { + const NONE: Self; + const FINISH: Self; + + fn to_usize(self) -> usize; +} + +pub trait CompressStatusKind: Copy { + const OK: Self; + const EOF: Self; + + fn to_usize(self) -> usize; +} + +#[derive(Debug)] +pub struct CompressState { + compressor: Option, +} + +impl CompressState { + pub fn new(compressor: C) -> Self { + Self { + compressor: Some(compressor), + } + } + + fn get_compressor(&mut self, vm: &VirtualMachine) -> PyResult<&mut C> { + self.compressor + .as_mut() + .ok_or_else(|| C::new_error(USE_AFTER_FINISH_ERR, vm)) + } + + pub fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult> { + let mut buf = Vec::new(); + let compressor = self.get_compressor(vm)?; + + for mut chunk in data.chunks(C::CHUNKSIZE) { + while !chunk.is_empty() { + buf.reserve(C::DEF_BUF_SIZE); + let prev_in = compressor.total_in(); + compressor.compress_vec(chunk, &mut buf, C::Flush::NONE, vm)?; + let consumed = compressor.total_in() - prev_in; + chunk = &chunk[consumed..]; + } + } + + buf.shrink_to_fit(); + Ok(buf) + } + + pub fn flush(&mut self, mode: C::Flush, vm: &VirtualMachine) -> PyResult> { + let mut buf = Vec::new(); + let compressor = self.get_compressor(vm)?; + + let status = loop { + if buf.len() == buf.capacity() { + buf.reserve(C::DEF_BUF_SIZE); + } + let status = compressor.compress_vec(&[], &mut buf, mode, vm)?; + if buf.len() != buf.capacity() { + break status; + } + }; + + if status.to_usize() == C::Status::EOF.to_usize() { + if mode.to_usize() == C::Flush::FINISH.to_usize() { + self.compressor = None; + } else { + return Err(C::new_error("unexpected eof", vm)); + } + } + + buf.shrink_to_fit(); + Ok(buf) + } +} + +pub trait Decompressor { + type Flush: DecompressFlushKind; + type Status: DecompressStatus; + type Error; + + fn total_in(&self) -> u64; + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result; + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + Err(err) + } +} + +pub trait DecompressStatus { + fn is_stream_end(&self) -> bool; +} + +pub trait DecompressFlushKind: Copy { + const SYNC: Self; +} + +impl DecompressFlushKind for () { + const SYNC: Self = (); +} + +pub fn flush_sync(_final_chunk: bool) -> T { + T::SYNC +} + +#[derive(Debug)] +pub struct DecompressState { + decompress: D, + unused_data: PyBytesRef, + input_buffer: Vec, + eof: bool, + needs_input: bool, +} + +impl DecompressState { + pub fn new(decompress: D, vm: &VirtualMachine) -> Self { + Self { + decompress, + unused_data: vm.ctx.empty_bytes.clone(), + input_buffer: Vec::new(), + eof: false, + needs_input: true, + } + } + + pub fn eof(&self) -> bool { + self.eof + } + + pub fn unused_data(&self) -> PyBytesRef { + self.unused_data.clone() + } + + pub fn needs_input(&self) -> bool { + self.needs_input + } + + pub fn decompress( + &mut self, + data: &[u8], + max_length: Option, + bufsize: usize, + vm: &VirtualMachine, + ) -> Result, DecompressError> { + if self.eof { + return Err(DecompressError::Eof(EofError)); + } + + let input_buffer = &mut self.input_buffer; + let d = &mut self.decompress; + + let mut chunks = Chunker::chain(input_buffer, data); + + let prev_len = chunks.len(); + let (ret, stream_end) = + match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { + Ok((buf, stream_end)) => (Ok(buf), stream_end), + Err(err) => (Err(err), false), + }; + let consumed = prev_len - chunks.len(); + + self.eof |= stream_end; + + if self.eof { + self.needs_input = false; + if !chunks.is_empty() { + self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); + } + } else if chunks.is_empty() { + input_buffer.clear(); + self.needs_input = true; + } else { + self.needs_input = false; + if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { + input_buffer.clear(); + input_buffer.extend_from_slice(&data[n_consumed_from_data..]); + } else { + input_buffer.drain(..consumed); + input_buffer.extend_from_slice(data); + } + } + + ret.map_err(DecompressError::Decompress) + } +} + +pub enum DecompressError { + Decompress(E), + Eof(EofError), +} + +impl From for DecompressError { + fn from(err: E) -> Self { + Self::Decompress(err) + } +} + +pub struct EofError; + +impl ToPyException for EofError { + fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { + vm.new_eof_error("End of stream already reached".to_owned()) + } +} + +#[derive(FromArgs)] +pub struct DecompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, optional)] + max_length: OptionalArg, +} + +impl DecompressArgs { + pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { + self.data.borrow_buf() + } + pub fn raw_max_length(&self) -> Option { + self.max_length.into_option().map(|ArgSize { value }| value) + } + pub fn max_length(&self) -> Option { + self.max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()) + } +} diff --git a/stdlib/src/zlib.rs b/stdlib/src/compression/zlib.rs similarity index 65% rename from stdlib/src/zlib.rs rename to stdlib/src/compression/zlib.rs index 0578f20c86..d4bc30389f 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/compression/zlib.rs @@ -1,11 +1,13 @@ // spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker -pub(crate) use zlib::{DecompressArgs, make_module}; +pub(crate) use zlib::make_module; #[pymodule] mod zlib { - use super::generic::{ - DecompressError, DecompressState, DecompressStatus, Decompressor, FlushKind, flush_sync, + use crate::compression::{ + _decompress_chunks, Chunker, CompressFlushKind, CompressState, CompressStatusKind, + Compressor, DecompressArgs, DecompressError, DecompressFlushKind, DecompressState, + DecompressStatus, Decompressor, USE_AFTER_FINISH_ERR, flush_sync, }; use crate::vm::{ PyObject, PyPayload, PyResult, VirtualMachine, @@ -144,48 +146,6 @@ mod zlib { } } - #[derive(Clone)] - pub(crate) struct Chunker<'a> { - data1: &'a [u8], - data2: &'a [u8], - } - impl<'a> Chunker<'a> { - pub(crate) fn new(data: &'a [u8]) -> Self { - Self { - data1: data, - data2: &[], - } - } - pub(crate) fn chain(data1: &'a [u8], data2: &'a [u8]) -> Self { - if data1.is_empty() { - Self { - data1: data2, - data2: &[], - } - } else { - Self { data1, data2 } - } - } - pub(crate) fn len(&self) -> usize { - self.data1.len() + self.data2.len() - } - pub(crate) fn is_empty(&self) -> bool { - self.data1.is_empty() - } - pub(crate) fn to_vec(&self) -> Vec { - [self.data1, self.data2].concat() - } - pub(crate) fn chunk(&self) -> &'a [u8] { - self.data1.get(..CHUNKSIZE).unwrap_or(self.data1) - } - pub(crate) fn advance(&mut self, consumed: usize) { - self.data1 = &self.data1[consumed..]; - if self.data1.is_empty() { - self.data1 = std::mem::take(&mut self.data2); - } - } - } - fn _decompress( data: &[u8], d: &mut D, @@ -197,60 +157,6 @@ mod zlib { _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) } - pub(super) fn _decompress_chunks( - data: &mut Chunker<'_>, - d: &mut D, - bufsize: usize, - max_length: Option, - calc_flush: impl Fn(bool) -> D::Flush, - ) -> Result<(Vec, bool), D::Error> { - if data.is_empty() { - return Ok((Vec::new(), true)); - } - let max_length = max_length.unwrap_or(usize::MAX); - let mut buf = Vec::new(); - - 'outer: loop { - let chunk = data.chunk(); - let flush = calc_flush(chunk.len() == data.len()); - loop { - let additional = std::cmp::min(bufsize, max_length - buf.capacity()); - if additional == 0 { - return Ok((buf, false)); - } - buf.reserve_exact(additional); - - let prev_in = d.total_in(); - let res = d.decompress_vec(chunk, &mut buf, flush); - let consumed = d.total_in() - prev_in; - - data.advance(consumed as usize); - - match res { - Ok(status) => { - let stream_end = status.is_stream_end(); - if stream_end || data.is_empty() { - // we've reached the end of the stream, we're done - buf.shrink_to_fit(); - return Ok((buf, stream_end)); - } else if !chunk.is_empty() && consumed == 0 { - // we're gonna need a bigger buffer - continue; - } else { - // next chunk - continue 'outer; - } - } - Err(e) => { - d.maybe_set_dict(e)?; - // now try the next chunk - continue 'outer; - } - }; - } - } - } - #[derive(FromArgs)] struct PyFuncDecompressArgs { #[pyarg(positional)] @@ -393,11 +299,10 @@ mod zlib { #[pymethod] fn decompress(&self, args: DecompressArgs, vm: &VirtualMachine) -> PyResult> { - let max_length: usize = args - .max_length - .map_or(0, |x| x.value) - .try_into() - .map_err(|_| vm.new_value_error("must be non-negative".to_owned()))?; + let max_length: usize = + args.raw_max_length().unwrap_or(0).try_into().map_err(|_| { + vm.new_value_error("max_length must be non-negative".to_owned()) + })?; let max_length = (max_length != 0).then_some(max_length); let data = &*args.data(); @@ -434,25 +339,6 @@ mod zlib { } } - #[derive(FromArgs)] - pub(crate) struct DecompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(any, optional)] - max_length: OptionalArg, - } - - impl DecompressArgs { - pub(crate) fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { - self.data.borrow_buf() - } - pub(crate) fn max_length(&self) -> Option { - self.max_length - .into_option() - .and_then(|ArgSize { value }| usize::try_from(value).ok()) - } - } - #[derive(FromArgs)] #[allow(dead_code)] // FIXME: use args struct CompressobjArgs { @@ -487,20 +373,20 @@ mod zlib { zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap()); } Ok(PyCompress { - inner: PyMutex::new(CompressInner::new(compress)), + inner: PyMutex::new(CompressState::new(CompressInner::new(compress))), }) } #[derive(Debug)] struct CompressInner { - compress: Option, + compress: Compress, } #[pyattr] #[pyclass(name = "Compress")] #[derive(Debug, PyPayload)] struct PyCompress { - inner: PyMutex, + inner: PyMutex>, } #[pyclass] @@ -537,60 +423,52 @@ mod zlib { impl CompressInner { fn new(compress: Compress) -> Self { - Self { - compress: Some(compress), - } + Self { compress } } + } - fn get_compress(&mut self, vm: &VirtualMachine) -> PyResult<&mut Compress> { - self.compress - .as_mut() - .ok_or_else(|| new_zlib_error(USE_AFTER_FINISH_ERR, vm)) - } - - fn compress(&mut self, data: &[u8], vm: &VirtualMachine) -> PyResult> { - let c = self.get_compress(vm)?; - let mut buf = Vec::new(); - - for mut chunk in data.chunks(CHUNKSIZE) { - while !chunk.is_empty() { - buf.reserve(DEF_BUF_SIZE); - let prev_in = c.total_in(); - c.compress_vec(chunk, &mut buf, FlushCompress::None) - .map_err(|_| new_zlib_error("error while compressing", vm))?; - let consumed = c.total_in() - prev_in; - chunk = &chunk[consumed as usize..]; - } - } + impl CompressStatusKind for Status { + const OK: Self = Status::Ok; + const EOF: Self = Status::StreamEnd; - buf.shrink_to_fit(); - Ok(buf) + fn to_usize(self) -> usize { + self as usize } + } - fn flush(&mut self, mode: FlushCompress, vm: &VirtualMachine) -> PyResult> { - let c = self.get_compress(vm)?; - let mut buf = Vec::new(); + impl CompressFlushKind for FlushCompress { + const NONE: Self = FlushCompress::None; + const FINISH: Self = FlushCompress::Finish; - let status = loop { - if buf.len() == buf.capacity() { - buf.reserve(DEF_BUF_SIZE); - } - let status = c - .compress_vec(&[], &mut buf, mode) - .map_err(|_| new_zlib_error("error while compressing", vm))?; - if buf.len() != buf.capacity() { - break status; - } - }; + fn to_usize(self) -> usize { + self as usize + } + } - match status { - Status::Ok | Status::BufError => {} - Status::StreamEnd if mode == FlushCompress::Finish => self.compress = None, - Status::StreamEnd => return Err(new_zlib_error("unexpected eof", vm)), - } + impl Compressor for CompressInner { + type Status = Status; + type Flush = FlushCompress; + const CHUNKSIZE: usize = CHUNKSIZE; + const DEF_BUF_SIZE: usize = DEF_BUF_SIZE; - buf.shrink_to_fit(); - Ok(buf) + fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + vm: &VirtualMachine, + ) -> PyResult { + self.compress + .compress_vec(input, output, flush) + .map_err(|_| new_zlib_error("error while compressing", vm)) + } + + fn total_in(&mut self) -> usize { + self.compress.total_in() as usize + } + + fn new_error(message: impl Into, vm: &VirtualMachine) -> PyBaseExceptionRef { + new_zlib_error(message, vm) } } @@ -598,8 +476,6 @@ mod zlib { vm.new_exception_msg(vm.class("zlib", "error"), message.into()) } - const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; - struct Level(Option); impl Level { @@ -647,7 +523,7 @@ mod zlib { } } - impl FlushKind for FlushDecompress { + impl DecompressFlushKind for FlushDecompress { const SYNC: Self = FlushDecompress::Sync; } @@ -754,147 +630,3 @@ mod zlib { // } } } - -mod generic { - use super::zlib::{_decompress_chunks, Chunker}; - use crate::vm::{ - VirtualMachine, - builtins::{PyBaseExceptionRef, PyBytesRef}, - convert::ToPyException, - }; - - pub(crate) trait Decompressor { - type Flush: FlushKind; - type Status: DecompressStatus; - type Error; - - fn total_in(&self) -> u64; - fn decompress_vec( - &mut self, - input: &[u8], - output: &mut Vec, - flush: Self::Flush, - ) -> Result; - fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { - Err(err) - } - } - - pub(crate) trait DecompressStatus { - fn is_stream_end(&self) -> bool; - } - - pub(crate) trait FlushKind: Copy { - const SYNC: Self; - } - - impl FlushKind for () { - const SYNC: Self = (); - } - - pub(super) fn flush_sync(_final_chunk: bool) -> T { - T::SYNC - } - - #[derive(Debug)] - pub(crate) struct DecompressState { - decompress: D, - unused_data: PyBytesRef, - input_buffer: Vec, - eof: bool, - needs_input: bool, - } - - impl DecompressState { - pub(crate) fn new(decompress: D, vm: &VirtualMachine) -> Self { - Self { - decompress, - unused_data: vm.ctx.empty_bytes.clone(), - input_buffer: Vec::new(), - eof: false, - needs_input: true, - } - } - - pub(crate) fn eof(&self) -> bool { - self.eof - } - - pub(crate) fn unused_data(&self) -> PyBytesRef { - self.unused_data.clone() - } - - pub(crate) fn needs_input(&self) -> bool { - self.needs_input - } - - pub(crate) fn decompress( - &mut self, - data: &[u8], - max_length: Option, - bufsize: usize, - vm: &VirtualMachine, - ) -> Result, DecompressError> { - if self.eof { - return Err(DecompressError::Eof(EofError)); - } - - let input_buffer = &mut self.input_buffer; - let d = &mut self.decompress; - - let mut chunks = Chunker::chain(input_buffer, data); - - let prev_len = chunks.len(); - let (ret, stream_end) = - match _decompress_chunks(&mut chunks, d, bufsize, max_length, flush_sync) { - Ok((buf, stream_end)) => (Ok(buf), stream_end), - Err(err) => (Err(err), false), - }; - let consumed = prev_len - chunks.len(); - - self.eof |= stream_end; - - if self.eof { - self.needs_input = false; - if !chunks.is_empty() { - self.unused_data = vm.ctx.new_bytes(chunks.to_vec()); - } - } else if chunks.is_empty() { - input_buffer.clear(); - self.needs_input = true; - } else { - self.needs_input = false; - if let Some(n_consumed_from_data) = consumed.checked_sub(input_buffer.len()) { - input_buffer.clear(); - input_buffer.extend_from_slice(&data[n_consumed_from_data..]); - } else { - input_buffer.drain(..consumed); - input_buffer.extend_from_slice(data); - } - } - - ret.map_err(DecompressError::Decompress) - } - } - - pub(crate) enum DecompressError { - Decompress(E), - Eof(EofError), - } - - impl From for DecompressError { - fn from(err: E) -> Self { - Self::Decompress(err) - } - } - - pub(crate) struct EofError; - - impl ToPyException for EofError { - fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_eof_error("End of stream already reached".to_owned()) - } - } -} - -pub(crate) use generic::{DecompressError, DecompressState, DecompressStatus, Decompressor}; diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index e8ba5570d5..ed4d01f509 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -11,6 +11,7 @@ pub mod array; mod binascii; mod bisect; mod cmath; +mod compression; mod contextvars; mod csv; mod dis; @@ -37,13 +38,11 @@ mod statistics; mod suggestions; // TODO: maybe make this an extension module, if we ever get those // mod re; -mod bz2; #[cfg(not(target_arch = "wasm32"))] pub mod socket; #[cfg(all(unix, not(target_os = "redox")))] mod syslog; mod unicodedata; -mod zlib; mod faulthandler; #[cfg(any(unix, target_os = "wasi"))] @@ -83,7 +82,6 @@ mod uuid; #[cfg(feature = "tkinter")] mod tkinter; -mod lzma; use rustpython_common as common; use rustpython_vm as vm; @@ -113,7 +111,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "array" => array::make_module, "binascii" => binascii::make_module, "_bisect" => bisect::make_module, - "_bz2" => bz2::make_module, + "_bz2" => compression::bz2::make_module, "cmath" => cmath::make_module, "_contextvars" => contextvars::make_module, "_csv" => csv::make_module, @@ -121,7 +119,6 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "faulthandler" => faulthandler::make_module, "gc" => gc::make_module, "_hashlib" => hashlib::make_module, - "_lzma" => lzma::make_module, "_sha1" => sha1::make_module, "_sha3" => sha3::make_module, "_sha256" => sha256::make_module, @@ -135,7 +132,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, "unicodedata" => unicodedata::make_module, - "zlib" => zlib::make_module, + "zlib" => compression::zlib::make_module, "_statistics" => statistics::make_module, "_suggestions" => suggestions::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, @@ -153,6 +150,10 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_multiprocessing" => multiprocessing::make_module, "_socket" => socket::make_module, } + #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] + { + "_lzma" => compression::lzma::make_module, + } #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))] { "_sqlite3" => sqlite::make_module, diff --git a/stdlib/src/lzma.rs b/stdlib/src/lzma.rs deleted file mode 100644 index d80449621c..0000000000 --- a/stdlib/src/lzma.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub(crate) use _lzma::make_module; - -#[pymodule] -mod _lzma { - -} \ No newline at end of file From acae154f1b1f4ad95a477ddf26d7644dc45aefa7 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 6 May 2025 14:30:40 +0900 Subject: [PATCH 298/317] more diff-friendly disabling xz --- Lib/tarfile.py | 13 +++++++++---- Lib/test/support/__init__.py | 10 +++++----- Lib/test/test_tarfile.py | 6 +++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index ab2266b285..04fda11597 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -392,10 +392,13 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, elif comptype == "xz": try: - # TODO: RUSTPYTHON remove underscore - import lzma_ + import lzma except ImportError: raise CompressionError("lzma module is not available") from None + + # XXX: RUSTPYTHON; xz is not supported yet + raise CompressionError("lzma module is not available") from None + if mode == "r": self.dbuf = b"" self.cmp = lzma.LZMADecompressor() @@ -1924,11 +1927,13 @@ def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs): raise ValueError("mode must be 'r', 'w' or 'x'") try: - # TODO: RUSTPYTHON remove underscore - from lzma_ import LZMAFile, LZMAError + from lzma import LZMAFile, LZMAError except ImportError: raise CompressionError("lzma module is not available") from None + # XXX: RUSTPYTHON; xz is not supported yet + raise CompressionError("lzma module is not available") from None + fileobj = LZMAFile(fileobj or name, mode, preset=preset) try: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ee94201b0d..3768a979b2 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -494,11 +494,11 @@ def requires_bz2(reason='requires bz2'): return unittest.skipUnless(bz2, reason) def requires_lzma(reason='requires lzma'): - # try: - # import lzma - # except ImportError: - # lzma = None - # TODO: RUSTPYTHON + try: + import lzma + except ImportError: + lzma = None + # XXX: RUSTPYTHON; xz is not supported yet lzma = None return unittest.skipUnless(lzma, reason) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index bff67ba221..0fed89c773 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -28,11 +28,11 @@ except ImportError: bz2 = None try: - # TODO: RUSTPYTHON - lzma = None - # import lzma + import lzma except ImportError: lzma = None +# XXX: RUSTPYTHON; xz is not supported yet +lzma = None def sha256sum(data): return sha256(data).hexdigest() From d9c18c559380f1458b448cba43fd6b537dc576aa Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 7 May 2025 15:36:27 +0900 Subject: [PATCH 299/317] flatten compression modules --- stdlib/src/{compression => }/bz2.rs | 2 +- .../{compression/mod.rs => compression.rs} | 131 ++++++++++-------- stdlib/src/lib.rs | 13 +- stdlib/src/{compression => }/lzma.rs | 0 stdlib/src/{compression => }/zlib.rs | 17 +-- 5 files changed, 84 insertions(+), 79 deletions(-) rename stdlib/src/{compression => }/bz2.rs (99%) rename stdlib/src/{compression/mod.rs => compression.rs} (95%) rename stdlib/src/{compression => }/lzma.rs (100%) rename stdlib/src/{compression => }/zlib.rs (97%) diff --git a/stdlib/src/compression/bz2.rs b/stdlib/src/bz2.rs similarity index 99% rename from stdlib/src/compression/bz2.rs rename to stdlib/src/bz2.rs index ca7d1250ce..4ae0785e47 100644 --- a/stdlib/src/compression/bz2.rs +++ b/stdlib/src/bz2.rs @@ -4,13 +4,13 @@ pub(crate) use _bz2::make_module; #[pymodule] mod _bz2 { - use crate::common::lock::PyMutex; use crate::compression::{ DecompressArgs, DecompressError, DecompressState, DecompressStatus, Decompressor, }; use crate::vm::{ VirtualMachine, builtins::{PyBytesRef, PyTypeRef}, + common::lock::PyMutex, function::{ArgBytesLike, OptionalArg}, object::{PyPayload, PyResult}, types::Constructor, diff --git a/stdlib/src/compression/mod.rs b/stdlib/src/compression.rs similarity index 95% rename from stdlib/src/compression/mod.rs rename to stdlib/src/compression.rs index fff7130022..0b65692299 100644 --- a/stdlib/src/compression/mod.rs +++ b/stdlib/src/compression.rs @@ -1,4 +1,6 @@ -// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker +// cspell:ignore chunker + +//! internal shared module for compression libraries use crate::vm::function::{ArgBytesLike, ArgSize, OptionalArg}; use crate::vm::{ @@ -6,15 +8,68 @@ use crate::vm::{ builtins::{PyBaseExceptionRef, PyBytesRef}, convert::ToPyException, }; -pub(crate) mod bz2; -#[cfg(not(any(target_os = "android", target_arch = "wasm32")))] -pub(crate) mod lzma; -pub(crate) mod zlib; pub const USE_AFTER_FINISH_ERR: &str = "Error -2: inconsistent stream state"; // TODO: don't hardcode const CHUNKSIZE: usize = u32::MAX as usize; +#[derive(FromArgs)] +pub struct DecompressArgs { + #[pyarg(positional)] + data: ArgBytesLike, + #[pyarg(any, optional)] + max_length: OptionalArg, +} + +impl DecompressArgs { + pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { + self.data.borrow_buf() + } + pub fn raw_max_length(&self) -> Option { + self.max_length.into_option().map(|ArgSize { value }| value) + } + + // negative is None + pub fn max_length(&self) -> Option { + self.max_length + .into_option() + .and_then(|ArgSize { value }| usize::try_from(value).ok()) + } +} + +pub trait Decompressor { + type Flush: DecompressFlushKind; + type Status: DecompressStatus; + type Error; + + fn total_in(&self) -> u64; + fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec, + flush: Self::Flush, + ) -> Result; + fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { + Err(err) + } +} + +pub trait DecompressStatus { + fn is_stream_end(&self) -> bool; +} + +pub trait DecompressFlushKind: Copy { + const SYNC: Self; +} + +impl DecompressFlushKind for () { + const SYNC: Self = (); +} + +pub fn flush_sync(_final_chunk: bool) -> T { + T::SYNC +} + #[derive(Clone)] pub struct Chunker<'a> { data1: &'a [u8], @@ -57,6 +112,17 @@ impl<'a> Chunker<'a> { } } +pub fn _decompress( + data: &[u8], + d: &mut D, + bufsize: usize, + max_length: Option, + calc_flush: impl Fn(bool) -> D::Flush, +) -> Result<(Vec, bool), D::Error> { + let mut data = Chunker::new(data); + _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) +} + pub fn _decompress_chunks( data: &mut Chunker<'_>, d: &mut D, @@ -207,39 +273,6 @@ impl CompressState { } } -pub trait Decompressor { - type Flush: DecompressFlushKind; - type Status: DecompressStatus; - type Error; - - fn total_in(&self) -> u64; - fn decompress_vec( - &mut self, - input: &[u8], - output: &mut Vec, - flush: Self::Flush, - ) -> Result; - fn maybe_set_dict(&mut self, err: Self::Error) -> Result<(), Self::Error> { - Err(err) - } -} - -pub trait DecompressStatus { - fn is_stream_end(&self) -> bool; -} - -pub trait DecompressFlushKind: Copy { - const SYNC: Self; -} - -impl DecompressFlushKind for () { - const SYNC: Self = (); -} - -pub fn flush_sync(_final_chunk: bool) -> T { - T::SYNC -} - #[derive(Debug)] pub struct DecompressState { decompress: D, @@ -339,25 +372,3 @@ impl ToPyException for EofError { vm.new_eof_error("End of stream already reached".to_owned()) } } - -#[derive(FromArgs)] -pub struct DecompressArgs { - #[pyarg(positional)] - data: ArgBytesLike, - #[pyarg(any, optional)] - max_length: OptionalArg, -} - -impl DecompressArgs { - pub fn data(&self) -> crate::common::borrow::BorrowedValue<'_, [u8]> { - self.data.borrow_buf() - } - pub fn raw_max_length(&self) -> Option { - self.max_length.into_option().map(|ArgSize { value }| value) - } - pub fn max_length(&self) -> Option { - self.max_length - .into_option() - .and_then(|ArgSize { value }| usize::try_from(value).ok()) - } -} diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index ed4d01f509..20dcb88232 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -11,12 +11,17 @@ pub mod array; mod binascii; mod bisect; mod cmath; -mod compression; mod contextvars; mod csv; mod dis; mod gc; +mod bz2; +mod compression; // internal module +#[cfg(not(any(target_os = "android", target_arch = "wasm32")))] +mod lzma; +mod zlib; + mod blake2; mod hashlib; mod md5; @@ -111,7 +116,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "array" => array::make_module, "binascii" => binascii::make_module, "_bisect" => bisect::make_module, - "_bz2" => compression::bz2::make_module, + "_bz2" => bz2::make_module, "cmath" => cmath::make_module, "_contextvars" => contextvars::make_module, "_csv" => csv::make_module, @@ -132,7 +137,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, "unicodedata" => unicodedata::make_module, - "zlib" => compression::zlib::make_module, + "zlib" => zlib::make_module, "_statistics" => statistics::make_module, "_suggestions" => suggestions::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, @@ -152,7 +157,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit } #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] { - "_lzma" => compression::lzma::make_module, + "_lzma" => lzma::make_module, } #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))] { diff --git a/stdlib/src/compression/lzma.rs b/stdlib/src/lzma.rs similarity index 100% rename from stdlib/src/compression/lzma.rs rename to stdlib/src/lzma.rs diff --git a/stdlib/src/compression/zlib.rs b/stdlib/src/zlib.rs similarity index 97% rename from stdlib/src/compression/zlib.rs rename to stdlib/src/zlib.rs index d4bc30389f..1100e95bfa 100644 --- a/stdlib/src/compression/zlib.rs +++ b/stdlib/src/zlib.rs @@ -5,9 +5,9 @@ pub(crate) use zlib::make_module; #[pymodule] mod zlib { use crate::compression::{ - _decompress_chunks, Chunker, CompressFlushKind, CompressState, CompressStatusKind, - Compressor, DecompressArgs, DecompressError, DecompressFlushKind, DecompressState, - DecompressStatus, Decompressor, USE_AFTER_FINISH_ERR, flush_sync, + _decompress, CompressFlushKind, CompressState, CompressStatusKind, Compressor, + DecompressArgs, DecompressError, DecompressFlushKind, DecompressState, DecompressStatus, + Decompressor, USE_AFTER_FINISH_ERR, flush_sync, }; use crate::vm::{ PyObject, PyPayload, PyResult, VirtualMachine, @@ -146,17 +146,6 @@ mod zlib { } } - fn _decompress( - data: &[u8], - d: &mut D, - bufsize: usize, - max_length: Option, - calc_flush: impl Fn(bool) -> D::Flush, - ) -> Result<(Vec, bool), D::Error> { - let mut data = Chunker::new(data); - _decompress_chunks(&mut data, d, bufsize, max_length, calc_flush) - } - #[derive(FromArgs)] struct PyFuncDecompressArgs { #[pyarg(positional)] From 06c4b151d63c91c32b66dc654f4d93a6f1a9f84a Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 7 May 2025 18:21:36 +0900 Subject: [PATCH 300/317] Add radium patch to fix CI --- Cargo.lock | 3 +-- Cargo.toml | 2 ++ common/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9e57ed825..095a3dca37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1979,8 +1979,7 @@ checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "radium" version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0b76288902db304c864a12046b73d2d895cc34a4bb8137baaeebe9978a072c" +source = "git+https://github.com/youknowone/ferrilab?branch=fix-nightly#4a301c3a223e096626a2773d1a1eed1fc4e21140" dependencies = [ "cfg-if", ] diff --git a/Cargo.toml b/Cargo.toml index b3c5b49e11..163289e8b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ opt-level = 3 lto = "thin" [patch.crates-io] +radium = { version = "1.1.0", git = "https://github.com/youknowone/ferrilab", branch = "fix-nightly" } # REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END @@ -186,6 +187,7 @@ paste = "1.0.15" proc-macro2 = "1.0.93" pymath = "0.0.2" quote = "1.0.38" +radium = "1.1" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } rustix = { version = "1.0", features = ["event"] } diff --git a/common/Cargo.toml b/common/Cargo.toml index 0649c2a509..4eab8440df 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -30,9 +30,9 @@ num-traits = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } unicode_names2 = { workspace = true } +radium = { workspace = true } lock_api = "0.4" -radium = "1.1" siphasher = "1" [target.'cfg(windows)'.dependencies] From dc75d203ffb80ec11998dec5c09c370129d35bf8 Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Wed, 7 May 2025 23:51:47 -0400 Subject: [PATCH 301/317] chore: upgrade to `ruff==0.11.8` --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e51d48953..000fecb11d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -324,11 +324,11 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - name: install ruff - run: python -m pip install ruff==0.0.291 # astral-sh/ruff#7778 + run: python -m pip install ruff==0.11.8 - name: Ensure docs generate no warnings run: cargo doc - name: run python lint - run: ruff extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source + run: ruff check extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 - name: install prettier run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH - name: check wasm code with prettier From aee4c7ac69ad2fc69d44bd3d2b8a92d9ead44b54 Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Thu, 8 May 2025 01:46:57 -0400 Subject: [PATCH 302/317] chore: migrate settings to ruff.toml (#5773) --- .github/workflows/ci.yaml | 2 +- ruff.toml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 ruff.toml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 000fecb11d..e00ad26f2f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -328,7 +328,7 @@ jobs: - name: Ensure docs generate no warnings run: cargo doc - name: run python lint - run: ruff check extra_tests wasm examples --exclude='./.*',./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 + run: ruff check - name: install prettier run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH - name: check wasm code with prettier diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000000..5e2fb8c4f5 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,20 @@ +include = [ + "examples/**/*.py", + "extra_tests/**/*.py", + "wasm/**/*.py", +] + +exclude = [ + ".*", + "Lib", + "vm/Lib", + "benches", +] + +[lint] +select = [ + "E9", # pycodestyle (error) + "F63", # pyflakes + "F7", + "F82", +] From 908386091ba3875d8a68e5b04cc9f3d6472e4b3b Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Thu, 8 May 2025 01:00:42 -0400 Subject: [PATCH 303/317] feat: implement `zlib.__version__` --- stdlib/src/zlib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 1100e95bfa..0e25f4bf23 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -31,6 +31,9 @@ mod zlib { Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_RLE, Z_SYNC_FLUSH, Z_TREES, }; + #[pyattr(name = "__version__")] + const __VERSION__: &str = "1.0"; + // we're statically linking libz-rs, so the compile-time and runtime // versions will always be the same #[pyattr(name = "ZLIB_RUNTIME_VERSION")] From b8095b84ff467559e04013780e56d7f33bc09438 Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Wed, 7 May 2025 23:27:12 -0400 Subject: [PATCH 304/317] chore: allow `uv run python -I ./whats_left.py` --- README.md | 2 +- whats_left.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e472bd9f6a..3f564aff60 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ To enhance CPython compatibility, try to increase unittest coverage by checking Another approach is to checkout the source code: builtin functions and object methods are often the simplest and easiest way to contribute. -You can also simply run `./whats_left.py` to assist in finding any unimplemented +You can also simply run `uv run python -I whats_left.py` to assist in finding any unimplemented method. ## Compiling to WebAssembly diff --git a/whats_left.py b/whats_left.py index 82df5cabe1..90200b8ff0 100755 --- a/whats_left.py +++ b/whats_left.py @@ -1,4 +1,7 @@ #!/usr/bin/env -S python3 -I +# /// script +# requires-python = ">=3.13" +# /// # This script generates Lib/snippets/whats_left_data.py with these variables defined: # expected_methods - a dictionary mapping builtin objects to their methods From 31e6cca9160429ea560729ad68ae27d8fbb7c442 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Wed, 30 Apr 2025 23:28:40 -0700 Subject: [PATCH 305/317] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 3f564aff60..5d7350bd72 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,11 @@ To build RustPython locally, first, clone the source code: git clone https://github.com/RustPython/RustPython ``` +If on windows, run the following to fix the `Lib/` directory symlink: +```bash +git config core.symlinks true +``` + Then you can change into the RustPython directory and run the demo (Note: `--release` is needed to prevent stack overflow on Windows): From 9bd7f1810ba6d9165e75a4121369015d3023816f Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sun, 4 May 2025 17:37:41 -0700 Subject: [PATCH 306/317] Update README.md Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d7350bd72..9d0e8dfc84 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ To build RustPython locally, first, clone the source code: git clone https://github.com/RustPython/RustPython ``` -If on windows, run the following to fix the `Lib/` directory symlink: +RustPython uses symlinks to manage python libraries in `Lib/`. If on windows, running the following helps: ```bash git config core.symlinks true ``` From df380bca96bc60032e1d747aa014a806e43c63d2 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 8 May 2025 21:26:28 -0700 Subject: [PATCH 307/317] lzma FORMAT_ALONE implementation (#5777) * implement init_alone * error if format is raw and there is a memlimit --- stdlib/src/lzma.rs | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/stdlib/src/lzma.rs b/stdlib/src/lzma.rs index ed8be9a695..21ba8b64c0 100644 --- a/stdlib/src/lzma.rs +++ b/stdlib/src/lzma.rs @@ -39,7 +39,7 @@ mod _lzma { use rustpython_vm::types::Constructor; use rustpython_vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; use std::fmt; - use xz2::stream::{Action, Check, Error, Filters, Status, Stream}; + use xz2::stream::{Action, Check, Error, Filters, LzmaOptions, Status, Stream}; #[cfg(windows)] type EnumVal = i32; @@ -149,6 +149,11 @@ mod _lzma { type Args = LZMADecompressorConstructorArgs; fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult { + if args.format == FORMAT_RAW && args.memlimit.is_some() { + return Err( + vm.new_value_error("Cannot specify memory limit with FORMAT_RAW".to_string()) + ); + } let memlimit = args.memlimit.unwrap_or(u64::MAX); let filters = args.filters.unwrap_or(0); let stream_result = match args.format { @@ -322,6 +327,25 @@ mod _lzma { .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?) } } + + fn init_alone( + preset: u32, + filter_specs: Option>, + vm: &VirtualMachine, + ) -> PyResult { + if let Some(_filter_specs) = filter_specs { + Err(new_lzma_error( + "TODO: RUSTPYTHON: LZMA: Alone filter specs", + vm, + )) + } else { + let options = LzmaOptions::new_preset(preset) + .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?; + let stream = Stream::new_lzma_encoder(&options) + .map_err(|_| new_lzma_error("Failed to initialize encoder", vm))?; + Ok(stream) + } + } } #[derive(FromArgs)] @@ -365,7 +389,8 @@ mod _lzma { } let stream = match args.format { FORMAT_XZ => Self::init_xz(args.check, preset, args.filters, vm)?, - // TODO: ALONE AND RAW + FORMAT_ALONE => Self::init_alone(preset, args.filters, vm)?, + // TODO: RAW _ => return Err(new_lzma_error("Invalid format", vm)), }; Ok(Self { From 4e2e0b41c646b4e1bc4650a77e5a32b8a66e1215 Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Mon, 12 May 2025 01:20:01 -0400 Subject: [PATCH 308/317] chore: add `ruff format --check` (#5774) * chore: add `ruff format --check` * fix tests --- .github/workflows/ci.yaml | 6 +- examples/atexit_example.py | 2 + examples/call_between_rust_and_python.py | 3 + examples/freeze/freeze.py | 1 - extra_tests/benchmarks/perf_fib.py | 9 +- extra_tests/custom_text_test_runner.py | 697 ++++++++---- extra_tests/jsontests.py | 11 +- extra_tests/snippets/3.1.3.4.py | 5 +- extra_tests/snippets/3.1.3.5.py | 4 +- extra_tests/snippets/builtin_abs.py | 1 - extra_tests/snippets/builtin_ascii.py | 10 +- extra_tests/snippets/builtin_bin.py | 24 +- extra_tests/snippets/builtin_bool.py | 31 +- extra_tests/snippets/builtin_bytearray.py | 223 ++-- extra_tests/snippets/builtin_bytes.py | 55 +- extra_tests/snippets/builtin_callable.py | 31 +- extra_tests/snippets/builtin_chr.py | 6 +- extra_tests/snippets/builtin_dict.py | 219 ++-- extra_tests/snippets/builtin_dict_union.py | 105 +- extra_tests/snippets/builtin_dir.py | 32 +- extra_tests/snippets/builtin_divmod.py | 6 +- extra_tests/snippets/builtin_ellipsis.py | 12 +- extra_tests/snippets/builtin_enumerate.py | 11 +- extra_tests/snippets/builtin_eval.py | 4 +- extra_tests/snippets/builtin_exceptions.py | 141 +-- extra_tests/snippets/builtin_exec.py | 31 +- extra_tests/snippets/builtin_exit.py | 2 +- extra_tests/snippets/builtin_format.py | 98 +- extra_tests/snippets/builtin_hash.py | 1 - extra_tests/snippets/builtin_hex.py | 6 +- extra_tests/snippets/builtin_isinstance.py | 15 +- extra_tests/snippets/builtin_issubclass.py | 4 +- extra_tests/snippets/builtin_len.py | 4 +- extra_tests/snippets/builtin_list.py | 497 +++++--- extra_tests/snippets/builtin_locals.py | 19 +- extra_tests/snippets/builtin_map.py | 4 +- extra_tests/snippets/builtin_mappingproxy.py | 11 +- extra_tests/snippets/builtin_max.py | 19 +- extra_tests/snippets/builtin_memoryview.py | 71 +- extra_tests/snippets/builtin_min.py | 19 +- extra_tests/snippets/builtin_none.py | 7 +- extra_tests/snippets/builtin_object.py | 5 +- extra_tests/snippets/builtin_open.py | 12 +- extra_tests/snippets/builtin_ord.py | 21 +- extra_tests/snippets/builtin_pow.py | 54 +- extra_tests/snippets/builtin_print.py | 12 +- extra_tests/snippets/builtin_property.py | 6 +- extra_tests/snippets/builtin_range.py | 44 +- extra_tests/snippets/builtin_reversed.py | 4 +- extra_tests/snippets/builtin_round.py | 6 +- extra_tests/snippets/builtin_set.py | 380 ++++--- extra_tests/snippets/builtin_slice.py | 66 +- extra_tests/snippets/builtin_str.py | 1007 +++++++++-------- extra_tests/snippets/builtin_str_encode.py | 2 + extra_tests/snippets/builtin_str_subclass.py | 2 + extra_tests/snippets/builtin_str_unicode.py | 30 +- .../snippets/builtin_str_unicode_slice.py | 36 +- extra_tests/snippets/builtin_tuple.py | 32 +- extra_tests/snippets/builtin_type.py | 436 +++---- extra_tests/snippets/builtin_type_mro.py | 11 +- extra_tests/snippets/builtin_zip.py | 10 +- extra_tests/snippets/builtins_module.py | 17 +- extra_tests/snippets/code_co_consts.py | 16 +- extra_tests/snippets/dir_main/__main__.py | 2 +- extra_tests/snippets/example_fizzbuzz.py | 1 + extra_tests/snippets/example_interactive.py | 12 +- .../snippets/forbidden_instantiation.py | 38 +- extra_tests/snippets/frozen.py | 1 + extra_tests/snippets/import.py | 47 +- extra_tests/snippets/import_file.py | 3 +- extra_tests/snippets/import_mutual1.py | 2 - extra_tests/snippets/import_mutual2.py | 1 - extra_tests/snippets/import_star.py | 2 +- extra_tests/snippets/import_target.py | 6 +- extra_tests/snippets/intro/3.1.1.1.py | 8 +- extra_tests/snippets/intro/3.1.1.3.py | 4 +- extra_tests/snippets/intro/3.1.2.1.py | 7 +- extra_tests/snippets/intro/3.1.2.10.py | 12 +- extra_tests/snippets/intro/3.1.2.3.py | 2 +- extra_tests/snippets/intro/3.1.2.5.py | 2 +- extra_tests/snippets/intro/3.1.2.6.py | 2 +- extra_tests/snippets/jit.py | 1 - extra_tests/snippets/name.py | 4 +- extra_tests/snippets/operator_arithmetic.py | 2 +- extra_tests/snippets/operator_cast.py | 24 +- extra_tests/snippets/operator_comparison.py | 41 +- extra_tests/snippets/operator_div.py | 14 +- extra_tests/snippets/operator_membership.py | 5 +- extra_tests/snippets/protocol_callable.py | 5 +- extra_tests/snippets/protocol_index_bad.py | 22 +- extra_tests/snippets/protocol_iterable.py | 22 +- extra_tests/snippets/protocol_iternext.py | 8 +- extra_tests/snippets/recursion.py | 2 + extra_tests/snippets/stdlib_abc_number.py | 2 +- extra_tests/snippets/stdlib_array.py | 34 +- extra_tests/snippets/stdlib_ast.py | 15 +- extra_tests/snippets/stdlib_collections.py | 5 +- .../snippets/stdlib_collections_deque.py | 40 +- extra_tests/snippets/stdlib_csv.py | 66 +- extra_tests/snippets/stdlib_ctypes.py | 115 +- extra_tests/snippets/stdlib_datetime.py | 40 +- extra_tests/snippets/stdlib_dir_module.py | 10 +- extra_tests/snippets/stdlib_dis.py | 27 +- extra_tests/snippets/stdlib_functools.py | 26 +- extra_tests/snippets/stdlib_hashlib.py | 43 +- extra_tests/snippets/stdlib_imp.py | 6 +- extra_tests/snippets/stdlib_io.py | 32 +- extra_tests/snippets/stdlib_io_bytesio.py | 55 +- extra_tests/snippets/stdlib_io_stringio.py | 59 +- extra_tests/snippets/stdlib_itertools.py | 241 ++-- extra_tests/snippets/stdlib_json.py | 186 +-- extra_tests/snippets/stdlib_logging.py | 10 +- extra_tests/snippets/stdlib_marshal.py | 21 +- extra_tests/snippets/stdlib_math.py | 139 ++- extra_tests/snippets/stdlib_os.py | 101 +- extra_tests/snippets/stdlib_pwd.py | 5 +- extra_tests/snippets/stdlib_random.py | 6 +- extra_tests/snippets/stdlib_re.py | 78 +- extra_tests/snippets/stdlib_signal.py | 34 +- extra_tests/snippets/stdlib_socket.py | 86 +- extra_tests/snippets/stdlib_sqlite.py | 8 +- extra_tests/snippets/stdlib_string.py | 30 +- extra_tests/snippets/stdlib_struct.py | 45 +- extra_tests/snippets/stdlib_subprocess.py | 5 + extra_tests/snippets/stdlib_sys.py | 64 +- extra_tests/snippets/stdlib_sys_getframe.py | 18 +- extra_tests/snippets/stdlib_time.py | 11 +- extra_tests/snippets/stdlib_traceback.py | 30 +- extra_tests/snippets/stdlib_types.py | 2 +- extra_tests/snippets/test_threading.py | 2 +- extra_tests/snippets/testutils.py | 48 +- extra_tests/test_snippets.py | 22 +- ruff.toml | 2 + wasm/demo/snippets/asyncbrowser.py | 1 + wasm/demo/snippets/fetch.py | 6 +- wasm/demo/snippets/import_pypi.py | 2 + wasm/demo/snippets/mandelbrot.py | 25 +- wasm/example/src/main.py | 12 +- 138 files changed, 4042 insertions(+), 2741 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e00ad26f2f..db8f9056b1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -327,8 +327,10 @@ jobs: run: python -m pip install ruff==0.11.8 - name: Ensure docs generate no warnings run: cargo doc - - name: run python lint - run: ruff check + - name: run ruff check + run: ruff check --diff + - name: run ruff format + run: ruff format --check - name: install prettier run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH - name: check wasm code with prettier diff --git a/examples/atexit_example.py b/examples/atexit_example.py index 0324d5b50e..c9c61ca567 100644 --- a/examples/atexit_example.py +++ b/examples/atexit_example.py @@ -1,7 +1,9 @@ import atexit import sys + def myexit(): sys.exit(2) + atexit.register(myexit) diff --git a/examples/call_between_rust_and_python.py b/examples/call_between_rust_and_python.py index 60335d81e9..4a52e85eac 100644 --- a/examples/call_between_rust_and_python.py +++ b/examples/call_between_rust_and_python.py @@ -1,14 +1,17 @@ from rust_py_module import RustStruct, rust_function + class PythonPerson: def __init__(self, name): self.name = name + def python_callback(): python_person = PythonPerson("Peter Python") rust_object = rust_function(42, "This is a python string", python_person) print("Printing member 'numbers' from rust struct: ", rust_object.numbers) rust_object.print_in_rust_from_python() + def take_string(string): print("Calling python function from rust with string: " + string) diff --git a/examples/freeze/freeze.py b/examples/freeze/freeze.py index c600ebf959..9cd6551966 100644 --- a/examples/freeze/freeze.py +++ b/examples/freeze/freeze.py @@ -1,4 +1,3 @@ import time print("Hello world!!!", time.time()) - diff --git a/extra_tests/benchmarks/perf_fib.py b/extra_tests/benchmarks/perf_fib.py index 89ede85179..6b999f7443 100644 --- a/extra_tests/benchmarks/perf_fib.py +++ b/extra_tests/benchmarks/perf_fib.py @@ -2,14 +2,15 @@ def fib(n): a = 1 b = 1 for _ in range(n - 1): - temp = b - b = a + b - a = temp + temp = b + b = a + b + a = temp return b + print(fib(1)) print(fib(2)) print(fib(3)) print(fib(4)) -print(fib(5)) \ No newline at end of file +print(fib(5)) diff --git a/extra_tests/custom_text_test_runner.py b/extra_tests/custom_text_test_runner.py index 8e91d7c39d..4c48e615c3 100644 --- a/extra_tests/custom_text_test_runner.py +++ b/extra_tests/custom_text_test_runner.py @@ -36,10 +36,12 @@ from unittest.runner import registerResult from functools import reduce + class TablePrinter(object): # Modified from https://github.com/agramian/table-printer, same license as above "Print a list of dicts as a table" - def __init__(self, fmt, sep='', ul=None, tl=None, bl=None): + + def __init__(self, fmt, sep="", ul=None, tl=None, bl=None): """ @param fmt: list of tuple(heading, key, width) heading: str, column label @@ -50,20 +52,44 @@ def __init__(self, fmt, sep='', ul=None, tl=None, bl=None): @param tl: string, character to draw as top line over table, or None @param bl: string, character to draw as bottom line under table, or None """ - super(TablePrinter,self).__init__() - fmt = [x + ('left',) if len(x) < 4 else x for x in fmt] - self.fmt = str(sep).join('{lb}{0}:{align}{1}{rb}'.format(key, width, lb='{', rb='}', align='<' if alignment == 'left' else '>') for heading,key,width,alignment in fmt) - self.head = {key:heading for heading,key,width,alignment in fmt} - self.ul = {key:str(ul)*width for heading,key,width,alignment in fmt} if ul else None - self.width = {key:width for heading,key,width,alignment in fmt} - self.tl = {key:str(tl)*width for heading,key,width,alignment in fmt} if tl else None - self.bl = {key:str(bl)*width for heading,key,width,alignment in fmt} if bl else None + super(TablePrinter, self).__init__() + fmt = [x + ("left",) if len(x) < 4 else x for x in fmt] + self.fmt = str(sep).join( + "{lb}{0}:{align}{1}{rb}".format( + key, width, lb="{", rb="}", align="<" if alignment == "left" else ">" + ) + for heading, key, width, alignment in fmt + ) + self.head = {key: heading for heading, key, width, alignment in fmt} + self.ul = ( + {key: str(ul) * width for heading, key, width, alignment in fmt} + if ul + else None + ) + self.width = {key: width for heading, key, width, alignment in fmt} + self.tl = ( + {key: str(tl) * width for heading, key, width, alignment in fmt} + if tl + else None + ) + self.bl = ( + {key: str(bl) * width for heading, key, width, alignment in fmt} + if bl + else None + ) def row(self, data, separation_character=False): if separation_character: - return self.fmt.format(**{ k:str(data.get(k,''))[:w] for k,w in self.width.items() }) + return self.fmt.format( + **{k: str(data.get(k, ""))[:w] for k, w in self.width.items()} + ) else: - data = { k:str(data.get(k,'')) if len(str(data.get(k,''))) <= w else '%s...' %str(data.get(k,''))[:(w-3)] for k,w in self.width.items() } + data = { + k: str(data.get(k, "")) + if len(str(data.get(k, ""))) <= w + else "%s..." % str(data.get(k, ""))[: (w - 3)] + for k, w in self.width.items() + } return self.fmt.format(**data) def __call__(self, data_list, totals=None): @@ -80,89 +106,111 @@ def __call__(self, data_list, totals=None): res.insert(len(res), _r(totals)) if self.bl: res.insert(len(res), _r(self.bl, True)) - return '\n'.join(res) + return "\n".join(res) def get_function_args(func_ref): try: - return [p for p in inspect.getargspec(func_ref).args if p != 'self'] + return [p for p in inspect.getargspec(func_ref).args if p != "self"] except: return None + def store_class_fields(class_ref, args_passed): - """ Store the passed in class fields in self - """ + """Store the passed in class fields in self""" params = get_function_args(class_ref.__init__) - for p in params: setattr(class_ref, p, args_passed[p]) + for p in params: + setattr(class_ref, p, args_passed[p]) + def sum_dict_key(d, key, cast_type=None): - """ Sum together all values matching a key given a passed dict - """ - return reduce( (lambda x, y: x + y), [eval("%s(x['%s'])" %(cast_type, key)) if cast_type else x[key] for x in d] ) + """Sum together all values matching a key given a passed dict""" + return reduce( + (lambda x, y: x + y), + [eval("%s(x['%s'])" % (cast_type, key)) if cast_type else x[key] for x in d], + ) + def case_name(name): - """ Test case name decorator to override function name. - """ + """Test case name decorator to override function name.""" + def decorator(function): - function.__dict__['test_case_name'] = name + function.__dict__["test_case_name"] = name return function + return decorator + def skip_device(name): - """ Decorator to mark a test to only run on certain devices - Takes single device name or list of names as argument + """Decorator to mark a test to only run on certain devices + Takes single device name or list of names as argument """ + def decorator(function): name_list = name if type(name) == list else [name] - function.__dict__['skip_device'] = name_list + function.__dict__["skip_device"] = name_list return function + return decorator + def _set_test_type(function, test_type): - """ Test type setter - """ - if 'test_type' in function.__dict__: - function.__dict__['test_type'].append(test_type) + """Test type setter""" + if "test_type" in function.__dict__: + function.__dict__["test_type"].append(test_type) else: - function.__dict__['test_type'] = [test_type] + function.__dict__["test_type"] = [test_type] return function + def smoke(function): - """ Test decorator to mark test as smoke type - """ - return _set_test_type(function, 'smoke') + """Test decorator to mark test as smoke type""" + return _set_test_type(function, "smoke") + def guide_discovery(function): - """ Test decorator to mark test as guide_discovery type - """ - return _set_test_type(function, 'guide_discovery') + """Test decorator to mark test as guide_discovery type""" + return _set_test_type(function, "guide_discovery") + def focus(function): - """ Test decorator to mark test as focus type to all rspec style debugging of cases - """ - return _set_test_type(function, 'focus') + """Test decorator to mark test as focus type to all rspec style debugging of cases""" + return _set_test_type(function, "focus") + class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): - if attr in ('stream', '__getstate__'): + if attr in ("stream", "__getstate__"): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write("\n") # text-mode streams translate to \r\n if needed + class CustomTextTestResult(result.TestResult): _num_formatting_chars = 150 _execution_time_significant_digits = 4 _pass_percentage_significant_digits = 2 - def __init__(self, stream, descriptions, verbosity, results_file_path, result_screenshots_dir, show_previous_results, config, test_types): + def __init__( + self, + stream, + descriptions, + verbosity, + results_file_path, + result_screenshots_dir, + show_previous_results, + config, + test_types, + ): super(CustomTextTestResult, self).__init__(stream, descriptions, verbosity) store_class_fields(self, locals()) self.show_overall_results = verbosity > 0 @@ -178,12 +226,12 @@ def __init__(self, stream, descriptions, verbosity, results_file_path, result_sc self.separator3 = "_" * CustomTextTestResult._num_formatting_chars self.separator4 = "*" * CustomTextTestResult._num_formatting_chars self.separator_failure = "!" * CustomTextTestResult._num_formatting_chars - self.separator_pre_result = '.' * CustomTextTestResult._num_formatting_chars + self.separator_pre_result = "." * CustomTextTestResult._num_formatting_chars def getDescription(self, test): doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: - return '\n'.join((str(test), doc_first_line)) + return "\n".join((str(test), doc_first_line)) else: return str(test) @@ -195,109 +243,170 @@ def startTestRun(self): self.results = None self.previous_suite_runs = [] if os.path.isfile(self.results_file_path): - with open(self.results_file_path, 'rb') as f: + with open(self.results_file_path, "rb") as f: try: self.results = json.load(f) # recreated results dict with int keys - self.results['suites'] = {int(k):v for (k,v) in list(self.results['suites'].items())} - self.suite_map = {v['name']:int(k) for (k,v) in list(self.results['suites'].items())} - self.previous_suite_runs = list(self.results['suites'].keys()) + self.results["suites"] = { + int(k): v for (k, v) in list(self.results["suites"].items()) + } + self.suite_map = { + v["name"]: int(k) + for (k, v) in list(self.results["suites"].items()) + } + self.previous_suite_runs = list(self.results["suites"].keys()) except: pass if not self.results: - self.results = {'suites': {}, - 'name': '', - 'num_passed': 0, - 'num_failed': 0, - 'num_skipped': 0, - 'num_expected_failures': 0, - 'execution_time': None} - self.suite_number = int(sorted(self.results['suites'].keys())[-1]) + 1 if len(self.results['suites']) else 0 + self.results = { + "suites": {}, + "name": "", + "num_passed": 0, + "num_failed": 0, + "num_skipped": 0, + "num_expected_failures": 0, + "execution_time": None, + } + self.suite_number = ( + int(sorted(self.results["suites"].keys())[-1]) + 1 + if len(self.results["suites"]) + else 0 + ) self.case_number = 0 self.suite_map = {} def stopTestRun(self): # if no tests or some failure occurred execution time may not have been set try: - self.results['suites'][self.suite_map[self.suite]]['execution_time'] = format(self.suite_execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) + self.results["suites"][self.suite_map[self.suite]]["execution_time"] = ( + format( + self.suite_execution_time, + ".%sf" % CustomTextTestResult._execution_time_significant_digits, + ) + ) except: pass - self.results['execution_time'] = format(self.total_execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) + self.results["execution_time"] = format( + self.total_execution_time, + ".%sf" % CustomTextTestResult._execution_time_significant_digits, + ) self.stream.writeln(self.separator3) - with open(self.results_file_path, 'w') as f: + with open(self.results_file_path, "w") as f: json.dump(self.results, f) def startTest(self, test): - suite_base_category = test.__class__.base_test_category if hasattr(test.__class__, 'base_test_category') else '' - self.next_suite = os.path.join(suite_base_category, test.__class__.name if hasattr(test.__class__, 'name') else test.__class__.__name__) + suite_base_category = ( + test.__class__.base_test_category + if hasattr(test.__class__, "base_test_category") + else "" + ) + self.next_suite = os.path.join( + suite_base_category, + test.__class__.name + if hasattr(test.__class__, "name") + else test.__class__.__name__, + ) self.case = test._testMethodName super(CustomTextTestResult, self).startTest(test) if not self.suite or self.suite != self.next_suite: if self.suite: - self.results['suites'][self.suite_map[self.suite]]['execution_time'] = format(self.suite_execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) + self.results["suites"][self.suite_map[self.suite]]["execution_time"] = ( + format( + self.suite_execution_time, + ".%sf" + % CustomTextTestResult._execution_time_significant_digits, + ) + ) self.suite_execution_time = 0 self.suite = self.next_suite if self.show_test_info: self.stream.writeln(self.separator1) - self.stream.writeln("TEST SUITE: %s" %self.suite) - self.stream.writeln("Description: %s" %self.getSuiteDescription(test)) + self.stream.writeln("TEST SUITE: %s" % self.suite) + self.stream.writeln("Description: %s" % self.getSuiteDescription(test)) try: - name_override = getattr(test, test._testMethodName).__func__.__dict__['test_case_name'] + name_override = getattr(test, test._testMethodName).__func__.__dict__[ + "test_case_name" + ] except: name_override = None self.case = name_override if name_override else self.case if self.show_test_info: # self.stream.writeln(self.separator2) - self.stream.write("CASE: %s" %self.case) - if desc := test.shortDescription(): self.stream.write(" (Description: %s)" % desc) + self.stream.write("CASE: %s" % self.case) + if desc := test.shortDescription(): + self.stream.write(" (Description: %s)" % desc) self.stream.write("... ") # self.stream.writeln(self.separator2) self.stream.flush() self.current_case_number = self.case_number if self.suite not in self.suite_map: self.suite_map[self.suite] = self.suite_number - self.results['suites'][self.suite_number] = { - 'name': self.suite, - 'class': test.__class__.__name__, - 'module': re.compile('.* \((.*)\)').match(str(test)).group(1), - 'description': self.getSuiteDescription(test), - 'cases': {}, - 'used_case_names': {}, - 'num_passed': 0, - 'num_failed': 0, - 'num_skipped': 0, - 'num_expected_failures': 0, - 'execution_time': None} + self.results["suites"][self.suite_number] = { + "name": self.suite, + "class": test.__class__.__name__, + "module": re.compile(".* \((.*)\)").match(str(test)).group(1), + "description": self.getSuiteDescription(test), + "cases": {}, + "used_case_names": {}, + "num_passed": 0, + "num_failed": 0, + "num_skipped": 0, + "num_expected_failures": 0, + "execution_time": None, + } self.suite_number += 1 self.num_cases = 0 self.num_passed = 0 self.num_failed = 0 self.num_skipped = 0 self.num_expected_failures = 0 - self.results['suites'][self.suite_map[self.suite]]['cases'][self.case_number] = { - 'name': self.case, - 'method': test._testMethodName, - 'result': None, - 'description': test.shortDescription(), - 'note': None, - 'errors': None, - 'failures': None, - 'screenshots': [], - 'new_version': 'No', - 'execution_time': None} + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.case_number + ] = { + "name": self.case, + "method": test._testMethodName, + "result": None, + "description": test.shortDescription(), + "note": None, + "errors": None, + "failures": None, + "screenshots": [], + "new_version": "No", + "execution_time": None, + } self.start_time = time.time() if self.test_types: - if ('test_type' in getattr(test, test._testMethodName).__func__.__dict__ - and set([s.lower() for s in self.test_types]) == set([s.lower() for s in getattr(test, test._testMethodName).__func__.__dict__['test_type']])): + if "test_type" in getattr( + test, test._testMethodName + ).__func__.__dict__ and set([s.lower() for s in self.test_types]) == set( + [ + s.lower() + for s in getattr(test, test._testMethodName).__func__.__dict__[ + "test_type" + ] + ] + ): pass else: - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip_why__'] = 'Test run specified to only run tests of type "%s"' %','.join(self.test_types) - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip__'] = True - if 'skip_device' in getattr(test, test._testMethodName).__func__.__dict__: - for device in getattr(test, test._testMethodName).__func__.__dict__['skip_device']: - if self.config and device.lower() in self.config['device_name'].lower(): - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip_why__'] = 'Test is marked to be skipped on %s' %device - getattr(test, test._testMethodName).__func__.__dict__['__unittest_skip__'] = True + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip_why__" + ] = 'Test run specified to only run tests of type "%s"' % ",".join( + self.test_types + ) + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip__" + ] = True + if "skip_device" in getattr(test, test._testMethodName).__func__.__dict__: + for device in getattr(test, test._testMethodName).__func__.__dict__[ + "skip_device" + ]: + if self.config and device.lower() in self.config["device_name"].lower(): + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip_why__" + ] = "Test is marked to be skipped on %s" % device + getattr(test, test._testMethodName).__func__.__dict__[ + "__unittest_skip__" + ] = True break def stopTest(self, test): @@ -307,19 +416,32 @@ def stopTest(self, test): self.total_execution_time += self.execution_time super(CustomTextTestResult, self).stopTest(test) self.num_cases += 1 - self.results['suites'][self.suite_map[self.suite]]['num_passed'] = self.num_passed - self.results['suites'][self.suite_map[self.suite]]['num_failed'] = self.num_failed - self.results['suites'][self.suite_map[self.suite]]['num_skipped'] = self.num_skipped - self.results['suites'][self.suite_map[self.suite]]['num_expected_failures'] = self.num_expected_failures - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['execution_time']= format(self.execution_time, '.%sf' %CustomTextTestResult._execution_time_significant_digits) - self.results['num_passed'] += self.num_passed - self.results['num_failed'] += self.num_failed - self.results['num_skipped'] += self.num_skipped - self.results['num_expected_failures'] += self.num_expected_failures + self.results["suites"][self.suite_map[self.suite]]["num_passed"] = ( + self.num_passed + ) + self.results["suites"][self.suite_map[self.suite]]["num_failed"] = ( + self.num_failed + ) + self.results["suites"][self.suite_map[self.suite]]["num_skipped"] = ( + self.num_skipped + ) + self.results["suites"][self.suite_map[self.suite]]["num_expected_failures"] = ( + self.num_expected_failures + ) + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["execution_time"] = format( + self.execution_time, + ".%sf" % CustomTextTestResult._execution_time_significant_digits, + ) + self.results["num_passed"] += self.num_passed + self.results["num_failed"] += self.num_failed + self.results["num_skipped"] += self.num_skipped + self.results["num_expected_failures"] += self.num_expected_failures self.case_number += 1 def print_error_string(self, err): - error_string = ''.join(traceback.format_exception(err[0], err[1], err[2])) + error_string = "".join(traceback.format_exception(err[0], err[1], err[2])) if self.show_errors: self.stream.writeln(self.separator_failure) self.stream.write(error_string) @@ -328,7 +450,9 @@ def print_error_string(self, err): def addScreenshots(self, test): for root, dirs, files in os.walk(self.result_screenshots_dir): for file in files: - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['screenshots'].append(os.path.join(root, file)) + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["screenshots"].append(os.path.join(root, file)) def addSuccess(self, test): super(CustomTextTestResult, self).addSuccess(test) @@ -336,7 +460,9 @@ def addSuccess(self, test): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("PASS") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'passed' + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "passed" self.num_passed += 1 self.addScreenshots(test) @@ -347,8 +473,12 @@ def addError(self, test, err): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("ERROR") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'error' - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['errors'] = error_string + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "error" + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["errors"] = error_string self.num_failed += 1 self.addScreenshots(test) @@ -359,8 +489,12 @@ def addFailure(self, test, err): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("FAIL") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'failed' - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['failures'] = error_string + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "failed" + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["failures"] = error_string self.num_failed += 1 self.addScreenshots(test) @@ -370,8 +504,12 @@ def addSkip(self, test, reason): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("SKIPPED {0!r}".format(reason)) self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'skipped' - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['note'] = reason + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "skipped" + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["note"] = reason self.num_skipped += 1 def addExpectedFailure(self, test, err): @@ -380,7 +518,9 @@ def addExpectedFailure(self, test, err): # self.stream.writeln(self.separator_pre_result) self.stream.writeln("EXPECTED FAILURE") self.stream.flush() - self.results['suites'][self.suite_map[self.suite]]['cases'][self.current_case_number]['result'] = 'expected_failure' + self.results["suites"][self.suite_map[self.suite]]["cases"][ + self.current_case_number + ]["result"] = "expected_failure" self.num_expected_failures += 1 self.addScreenshots(test) @@ -396,103 +536,189 @@ def addUnexpectedSuccess(self, test): def printOverallSuiteResults(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('OVERALL SUITE RESULTS') + self.stream.writeln("OVERALL SUITE RESULTS") fmt = [ - ('SUITE', 'suite', 50, 'left'), - ('CASES', 'cases', 15, 'right'), - ('PASSED', 'passed', 15, 'right'), - ('FAILED', 'failed', 15, 'right'), - ('SKIPPED', 'skipped', 15, 'right'), - ('%', 'percentage', 20, 'right'), - ('TIME (s)', 'time', 20, 'right') + ("SUITE", "suite", 50, "left"), + ("CASES", "cases", 15, "right"), + ("PASSED", "passed", 15, "right"), + ("FAILED", "failed", 15, "right"), + ("SKIPPED", "skipped", 15, "right"), + ("%", "percentage", 20, "right"), + ("TIME (s)", "time", 20, "right"), ] data = [] - for x in r: data.append({'suite': r[x]['name'], - 'cases': r[x]['num_passed'] + r[x]['num_failed'], - 'passed': r[x]['num_passed'], - 'failed': r[x]['num_failed'], - 'skipped': r[x]['num_skipped'], - 'expected_failures': r[x]['num_expected_failures'], - 'percentage': float(r[x]['num_passed'])/(r[x]['num_passed'] + r[x]['num_failed']) * 100 if (r[x]['num_passed'] + r[x]['num_failed']) > 0 else 0, - 'time': r[x]['execution_time']}) - total_suites_passed = len([x for x in data if not x['failed']]) - total_suites_passed_percentage = format(float(total_suites_passed)/len(data) * 100, '.%sf' %CustomTextTestResult._pass_percentage_significant_digits) - totals = {'suite': 'TOTALS %s/%s (%s%%) suites passed' %(total_suites_passed, len(data), total_suites_passed_percentage), - 'cases': sum_dict_key(data, 'cases'), - 'passed': sum_dict_key(data, 'passed'), - 'failed': sum_dict_key(data, 'failed'), - 'skipped': sum_dict_key(data, 'skipped'), - 'percentage': sum_dict_key(data, 'percentage')/len(data), - 'time': sum_dict_key(data, 'time', 'float')} - for x in data: operator.setitem(x, 'percentage', format(x['percentage'], '.%sf' %CustomTextTestResult._pass_percentage_significant_digits)) - totals['percentage'] = format(totals['percentage'], '.%sf' %CustomTextTestResult._pass_percentage_significant_digits) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2, bl=self.separator3)(data, totals) ) + for x in r: + data.append( + { + "suite": r[x]["name"], + "cases": r[x]["num_passed"] + r[x]["num_failed"], + "passed": r[x]["num_passed"], + "failed": r[x]["num_failed"], + "skipped": r[x]["num_skipped"], + "expected_failures": r[x]["num_expected_failures"], + "percentage": float(r[x]["num_passed"]) + / (r[x]["num_passed"] + r[x]["num_failed"]) + * 100 + if (r[x]["num_passed"] + r[x]["num_failed"]) > 0 + else 0, + "time": r[x]["execution_time"], + } + ) + total_suites_passed = len([x for x in data if not x["failed"]]) + total_suites_passed_percentage = format( + float(total_suites_passed) / len(data) * 100, + ".%sf" % CustomTextTestResult._pass_percentage_significant_digits, + ) + totals = { + "suite": "TOTALS %s/%s (%s%%) suites passed" + % (total_suites_passed, len(data), total_suites_passed_percentage), + "cases": sum_dict_key(data, "cases"), + "passed": sum_dict_key(data, "passed"), + "failed": sum_dict_key(data, "failed"), + "skipped": sum_dict_key(data, "skipped"), + "percentage": sum_dict_key(data, "percentage") / len(data), + "time": sum_dict_key(data, "time", "float"), + } + for x in data: + operator.setitem( + x, + "percentage", + format( + x["percentage"], + ".%sf" % CustomTextTestResult._pass_percentage_significant_digits, + ), + ) + totals["percentage"] = format( + totals["percentage"], + ".%sf" % CustomTextTestResult._pass_percentage_significant_digits, + ) + self.stream.writeln( + TablePrinter( + fmt, tl=self.separator1, ul=self.separator2, bl=self.separator3 + )(data, totals) + ) self.stream.writeln() def printIndividualSuiteResults(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('INDIVIDUAL SUITE RESULTS') + self.stream.writeln("INDIVIDUAL SUITE RESULTS") fmt = [ - ('CASE', 'case', 50, 'left'), - ('DESCRIPTION', 'description', 50, 'right'), - ('RESULT', 'result', 25, 'right'), - ('TIME (s)', 'time', 25, 'right') + ("CASE", "case", 50, "left"), + ("DESCRIPTION", "description", 50, "right"), + ("RESULT", "result", 25, "right"), + ("TIME (s)", "time", 25, "right"), ] for suite in r: self.stream.writeln(self.separator1) - self.stream.write('{0: <50}'.format('SUITE: %s' %r[suite]['name'])) - self.stream.writeln('{0: <100}'.format('DESCRIPTION: %s' %(r[suite]['description'] if not r[suite]['description'] or len(r[suite]['description']) <= (100 - len('DESCRIPTION: ')) - else '%s...' %r[suite]['description'][:(97 - len('DESCRIPTION: '))]))) + self.stream.write("{0: <50}".format("SUITE: %s" % r[suite]["name"])) + self.stream.writeln( + "{0: <100}".format( + "DESCRIPTION: %s" + % ( + r[suite]["description"] + if not r[suite]["description"] + or len(r[suite]["description"]) <= (100 - len("DESCRIPTION: ")) + else "%s..." + % r[suite]["description"][: (97 - len("DESCRIPTION: "))] + ) + ) + ) data = [] - cases = r[suite]['cases'] - for x in cases: data.append({'case': cases[x]['name'], - 'description': cases[x]['description'], - 'result': cases[x]['result'].upper() if cases[x]['result'] else cases[x]['result'], - 'time': cases[x]['execution_time']}) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) ) + cases = r[suite]["cases"] + for x in cases: + data.append( + { + "case": cases[x]["name"], + "description": cases[x]["description"], + "result": cases[x]["result"].upper() + if cases[x]["result"] + else cases[x]["result"], + "time": cases[x]["execution_time"], + } + ) + self.stream.writeln( + TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) + ) self.stream.writeln(self.separator3) self.stream.writeln() def printErrorsOverview(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('FAILURES AND ERRORS OVERVIEW') + self.stream.writeln("FAILURES AND ERRORS OVERVIEW") fmt = [ - ('SUITE', 'suite', 50, 'left'), - ('CASE', 'case', 50, 'left'), - ('RESULT', 'result', 50, 'right') + ("SUITE", "suite", 50, "left"), + ("CASE", "case", 50, "left"), + ("RESULT", "result", 50, "right"), ] data = [] for suite in r: - cases = {k:v for (k,v) in list(r[suite]['cases'].items()) if v['failures'] or v['errors']} - for x in cases: data.append({'suite': '%s%s' %(r[suite]['name'], ' (%s)' %r[suite]['module'] if r[suite]['class'] != r[suite]['name'] else ''), - 'case': '%s%s' %(cases[x]['name'], ' (%s)' %cases[x]['method'] if cases[x]['name'] != cases[x]['method'] else ''), - 'result': cases[x]['result'].upper()}) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) ) + cases = { + k: v + for (k, v) in list(r[suite]["cases"].items()) + if v["failures"] or v["errors"] + } + for x in cases: + data.append( + { + "suite": "%s%s" + % ( + r[suite]["name"], + " (%s)" % r[suite]["module"] + if r[suite]["class"] != r[suite]["name"] + else "", + ), + "case": "%s%s" + % ( + cases[x]["name"], + " (%s)" % cases[x]["method"] + if cases[x]["name"] != cases[x]["method"] + else "", + ), + "result": cases[x]["result"].upper(), + } + ) + self.stream.writeln( + TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) + ) self.stream.writeln(self.separator3) self.stream.writeln() def printErrorsDetail(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('FAILURES AND ERRORS DETAIL') + self.stream.writeln("FAILURES AND ERRORS DETAIL") for suite in r: - failures_and_errors = [k for (k,v) in list(r[suite]['cases'].items()) if v['failures'] or v['errors']] - #print failures_and_errors - suite_str = '%s%s' %(r[suite]['name'], ' (%s)' %r[suite]['module'] if r[suite]['class'] != r[suite]['name'] else '') + failures_and_errors = [ + k + for (k, v) in list(r[suite]["cases"].items()) + if v["failures"] or v["errors"] + ] + # print failures_and_errors + suite_str = "%s%s" % ( + r[suite]["name"], + " (%s)" % r[suite]["module"] + if r[suite]["class"] != r[suite]["name"] + else "", + ) for case in failures_and_errors: - case_ref = r[suite]['cases'][case] - case_str = '%s%s' %(case_ref['name'], ' (%s)' %case_ref['method'] if case_ref['name'] != case_ref['method'] else '') - errors = case_ref['errors'] - failures = case_ref['failures'] + case_ref = r[suite]["cases"][case] + case_str = "%s%s" % ( + case_ref["name"], + " (%s)" % case_ref["method"] + if case_ref["name"] != case_ref["method"] + else "", + ) + errors = case_ref["errors"] + failures = case_ref["failures"] self.stream.writeln(self.separator1) if errors: - self.stream.writeln('ERROR: %s [%s]' %(case_str, suite_str)) + self.stream.writeln("ERROR: %s [%s]" % (case_str, suite_str)) self.stream.writeln(self.separator2) self.stream.writeln(errors) if failures: - self.stream.writeln('FAILURE: %s [%s]' %(case_str, suite_str)) + self.stream.writeln("FAILURE: %s [%s]" % (case_str, suite_str)) self.stream.writeln(self.separator2) self.stream.writeln(failures) self.stream.writeln(self.separator3) @@ -501,52 +727,85 @@ def printErrorsDetail(self, r): def printSkippedDetail(self, r): self.stream.writeln() self.stream.writeln(self.separator4) - self.stream.writeln('SKIPPED DETAIL') + self.stream.writeln("SKIPPED DETAIL") fmt = [ - ('SUITE', 'suite', 50, 'left'), - ('CASE', 'case', 50, 'left'), - ('REASON', 'reason', 50, 'right') + ("SUITE", "suite", 50, "left"), + ("CASE", "case", 50, "left"), + ("REASON", "reason", 50, "right"), ] data = [] for suite in r: - cases = {k:v for (k,v) in list(r[suite]['cases'].items()) if v['result'] == 'skipped'} - for x in cases: data.append({'suite': '%s%s' %(r[suite]['name'], ' (%s)' %r[suite]['module'] if r[suite]['class'] != r[suite]['name'] else ''), - 'case': '%s%s' %(cases[x]['name'], ' (%s)' %cases[x]['method'] if cases[x]['name'] != cases[x]['method'] else ''), - 'reason': cases[x]['note']}) - self.stream.writeln( TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) ) + cases = { + k: v + for (k, v) in list(r[suite]["cases"].items()) + if v["result"] == "skipped" + } + for x in cases: + data.append( + { + "suite": "%s%s" + % ( + r[suite]["name"], + " (%s)" % r[suite]["module"] + if r[suite]["class"] != r[suite]["name"] + else "", + ), + "case": "%s%s" + % ( + cases[x]["name"], + " (%s)" % cases[x]["method"] + if cases[x]["name"] != cases[x]["method"] + else "", + ), + "reason": cases[x]["note"], + } + ) + self.stream.writeln( + TablePrinter(fmt, tl=self.separator1, ul=self.separator2)(data) + ) self.stream.writeln(self.separator3) self.stream.writeln() def returnCode(self): return not self.wasSuccessful() + class CustomTextTestRunner(unittest.TextTestRunner): """A test runner class that displays results in textual form. It prints out the names of tests as they are run, errors as they occur, and a summary of the results at the end of the test run. """ - def __init__(self, - stream=sys.stderr, - descriptions=True, - verbosity=1, - failfast=False, - buffer=False, - resultclass=CustomTextTestResult, - results_file_path="results.json", - result_screenshots_dir='', - show_previous_results=False, - test_name=None, - test_description=None, - config=None, - test_types=None): + def __init__( + self, + stream=sys.stderr, + descriptions=True, + verbosity=1, + failfast=False, + buffer=False, + resultclass=CustomTextTestResult, + results_file_path="results.json", + result_screenshots_dir="", + show_previous_results=False, + test_name=None, + test_description=None, + config=None, + test_types=None, + ): store_class_fields(self, locals()) self.stream = _WritelnDecorator(stream) def _makeResult(self): - return self.resultclass(self.stream, self.descriptions, self.verbosity, - self.results_file_path, self.result_screenshots_dir, self.show_previous_results, - self.config, self.test_types) + return self.resultclass( + self.stream, + self.descriptions, + self.verbosity, + self.results_file_path, + self.result_screenshots_dir, + self.show_previous_results, + self.config, + self.test_types, + ) def run(self, test): output = "" @@ -556,22 +815,26 @@ def run(self, test): result.failfast = self.failfast result.buffer = self.buffer startTime = time.time() - startTestRun = getattr(result, 'startTestRun', None) + startTestRun = getattr(result, "startTestRun", None) if startTestRun is not None: startTestRun() try: test(result) finally: - stopTestRun = getattr(result, 'stopTestRun', None) + stopTestRun = getattr(result, "stopTestRun", None) if stopTestRun is not None: stopTestRun() stopTime = time.time() timeTaken = stopTime - startTime # filter results to output if result.show_previous_results: - r = result.results['suites'] + r = result.results["suites"] else: - r = {k:v for (k,v) in list(result.results['suites'].items()) if k not in result.previous_suite_runs} + r = { + k: v + for (k, v) in list(result.results["suites"].items()) + if k not in result.previous_suite_runs + } # print results based on verbosity if result.show_all: result.printSkippedDetail(r) @@ -584,15 +847,17 @@ def run(self, test): if result.show_overall_results: result.printOverallSuiteResults(r) run = result.testsRun - self.stream.writeln("Ran %d test case%s in %.4fs" % - (run, run != 1 and "s" or "", timeTaken)) + self.stream.writeln( + "Ran %d test case%s in %.4fs" % (run, run != 1 and "s" or "", timeTaken) + ) self.stream.writeln() expectedFails = unexpectedSuccesses = skipped = 0 try: - results = map(len, (result.expectedFailures, - result.unexpectedSuccesses, - result.skipped)) + results = map( + len, + (result.expectedFailures, result.unexpectedSuccesses, result.skipped), + ) except AttributeError: pass else: diff --git a/extra_tests/jsontests.py b/extra_tests/jsontests.py index 7bc743d8d3..a54fd4234e 100644 --- a/extra_tests/jsontests.py +++ b/extra_tests/jsontests.py @@ -6,8 +6,9 @@ testnames = findtests() # idk why this fixes the hanging, if it does -testnames.remove('test_importlib') -testnames.insert(0, 'test_importlib') +testnames.remove("test_importlib") +testnames.insert(0, "test_importlib") + def loadTestsOrSkip(loader, name): try: @@ -17,12 +18,16 @@ def loadTestsOrSkip(loader, name): @unittest.skip(str(exc)) def testSkipped(self): pass + attrs = {name: testSkipped} TestClass = type("ModuleSkipped", (unittest.TestCase,), attrs) return loader.suiteClass((TestClass(name),)) + loader = unittest.defaultTestLoader -suite = loader.suiteClass([loadTestsOrSkip(loader, 'test.' + name) for name in testnames]) +suite = loader.suiteClass( + [loadTestsOrSkip(loader, "test." + name) for name in testnames] +) resultsfile = os.path.join(os.path.dirname(__file__), "cpython_tests_results.json") if os.path.exists(resultsfile): diff --git a/extra_tests/snippets/3.1.3.4.py b/extra_tests/snippets/3.1.3.4.py index 426c78b42d..f254376329 100644 --- a/extra_tests/snippets/3.1.3.4.py +++ b/extra_tests/snippets/3.1.3.4.py @@ -1,3 +1,2 @@ -l = [1,2,3] -assert [1,2,3,4,5] == (l + [4,5]) - +l = [1, 2, 3] +assert [1, 2, 3, 4, 5] == (l + [4, 5]) diff --git a/extra_tests/snippets/3.1.3.5.py b/extra_tests/snippets/3.1.3.5.py index c841430b1b..11889d936c 100644 --- a/extra_tests/snippets/3.1.3.5.py +++ b/extra_tests/snippets/3.1.3.5.py @@ -1,3 +1,3 @@ -x = [1,999,3] +x = [1, 999, 3] x[1] = 2 -assert [1,2,3] == x +assert [1, 2, 3] == x diff --git a/extra_tests/snippets/builtin_abs.py b/extra_tests/snippets/builtin_abs.py index 1b744978e5..7add4f4bcf 100644 --- a/extra_tests/snippets/builtin_abs.py +++ b/extra_tests/snippets/builtin_abs.py @@ -2,4 +2,3 @@ assert abs(7) == 7 assert abs(-3.21) == 3.21 assert abs(6.25) == 6.25 - diff --git a/extra_tests/snippets/builtin_ascii.py b/extra_tests/snippets/builtin_ascii.py index 2132723c6b..5b5b45d999 100644 --- a/extra_tests/snippets/builtin_ascii.py +++ b/extra_tests/snippets/builtin_ascii.py @@ -1,7 +1,7 @@ -assert ascii('hello world') == "'hello world'" -assert ascii('안녕 세상') == "'\\uc548\\ub155 \\uc138\\uc0c1'" -assert ascii('안녕 RustPython') == "'\\uc548\\ub155 RustPython'" -assert ascii(5) == '5' +assert ascii("hello world") == "'hello world'" +assert ascii("안녕 세상") == "'\\uc548\\ub155 \\uc138\\uc0c1'" +assert ascii("안녕 RustPython") == "'\\uc548\\ub155 RustPython'" +assert ascii(5) == "5" assert ascii(chr(0x10001)) == "'\\U00010001'" assert ascii(chr(0x9999)) == "'\\u9999'" -assert ascii(chr(0x0A)) == "'\\n'" \ No newline at end of file +assert ascii(chr(0x0A)) == "'\\n'" diff --git a/extra_tests/snippets/builtin_bin.py b/extra_tests/snippets/builtin_bin.py index 97f57b2f13..4f7a54c1b2 100644 --- a/extra_tests/snippets/builtin_bin.py +++ b/extra_tests/snippets/builtin_bin.py @@ -1,13 +1,13 @@ -assert bin(0) == '0b0' -assert bin(1) == '0b1' -assert bin(-1) == '-0b1' -assert bin(2**24) == '0b1' + '0' * 24 -assert bin(2**24-1) == '0b' + '1' * 24 -assert bin(-(2**24)) == '-0b1' + '0' * 24 -assert bin(-(2**24-1)) == '-0b' + '1' * 24 +assert bin(0) == "0b0" +assert bin(1) == "0b1" +assert bin(-1) == "-0b1" +assert bin(2**24) == "0b1" + "0" * 24 +assert bin(2**24 - 1) == "0b" + "1" * 24 +assert bin(-(2**24)) == "-0b1" + "0" * 24 +assert bin(-(2**24 - 1)) == "-0b" + "1" * 24 -a = 2 ** 65 -assert bin(a) == '0b1' + '0' * 65 -assert bin(a-1) == '0b' + '1' * 65 -assert bin(-(a)) == '-0b1' + '0' * 65 -assert bin(-(a-1)) == '-0b' + '1' * 65 +a = 2**65 +assert bin(a) == "0b1" + "0" * 65 +assert bin(a - 1) == "0b" + "1" * 65 +assert bin(-(a)) == "-0b1" + "0" * 65 +assert bin(-(a - 1)) == "-0b" + "1" * 65 diff --git a/extra_tests/snippets/builtin_bool.py b/extra_tests/snippets/builtin_bool.py index a46dbaab93..6b6b4e0e08 100644 --- a/extra_tests/snippets/builtin_bool.py +++ b/extra_tests/snippets/builtin_bool.py @@ -30,18 +30,20 @@ if not object(): raise BaseException + class Falsey: def __bool__(self): return False + assert not Falsey() -assert (True or fake) # noqa: F821 -assert (False or True) +assert True or fake # noqa: F821 +assert False or True assert not (False or False) assert ("thing" or 0) == "thing" -assert (True and True) +assert True and True assert not (False and fake) # noqa: F821 assert (True and 5) == 5 @@ -92,15 +94,17 @@ def __bool__(self): assert bool({"key": "value"}) is True assert bool([1]) is True -assert bool(set([1,2])) is True +assert bool(set([1, 2])) is True assert repr(True) == "True" + # Check __len__ work class TestMagicMethodLenZero: def __len__(self): return 0 + class TestMagicMethodLenOne: def __len__(self): return 1 @@ -118,6 +122,7 @@ def __bool__(self): def __len__(self): return 0 + class TestMagicMethodBoolFalseLenTrue: def __bool__(self): return False @@ -125,6 +130,7 @@ def __bool__(self): def __len__(self): return 1 + assert bool(TestMagicMethodBoolTrueLenFalse()) is True assert bool(TestMagicMethodBoolFalseLenTrue()) is False @@ -134,9 +140,11 @@ class TestBoolThrowError: def __bool__(self): return object() + with assert_raises(TypeError): bool(TestBoolThrowError()) + class TestLenThrowError: def __len__(self): return object() @@ -145,6 +153,7 @@ def __len__(self): with assert_raises(TypeError): bool(TestLenThrowError()) + # Verify that TypeError occurs when bad things are returned # from __bool__(). This isn't really a bool test, but # it's related. @@ -152,31 +161,45 @@ def check(o): with assert_raises(TypeError): bool(o) + class Foo(object): def __bool__(self): return self + + check(Foo()) + class Bar(object): def __bool__(self): return "Yes" + + check(Bar()) + class Baz(int): def __bool__(self): return self + + check(Baz()) + # __bool__() must return a bool not an int class Spam(int): def __bool__(self): return 1 + + check(Spam()) + class Eggs: def __len__(self): return -1 + with assert_raises(ValueError): bool(Eggs()) diff --git a/extra_tests/snippets/builtin_bytearray.py b/extra_tests/snippets/builtin_bytearray.py index 008d3d23ee..1a6993e205 100644 --- a/extra_tests/snippets/builtin_bytearray.py +++ b/extra_tests/snippets/builtin_bytearray.py @@ -40,9 +40,11 @@ ) assert repr(bytearray(b"abcd")) == "bytearray(b'abcd')" + class B(bytearray): pass + assert repr(B()) == "B(b'')" assert ( repr(B([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255])) @@ -283,9 +285,9 @@ class B(bytearray): ) == bytearray(b"jiljlkmoomkaaaa") with assert_raises(TypeError): bytearray(b"").join((b"km", "kl")) -assert bytearray(b"abc").join(( - bytearray(b"123"), bytearray(b"xyz") -)) == bytearray(b"123abcxyz") +assert bytearray(b"abc").join((bytearray(b"123"), bytearray(b"xyz"))) == bytearray( + b"123abcxyz" +) # endswith startswith @@ -372,16 +374,45 @@ class B(bytearray): assert bytearray(b"mississippi").rstrip(b"ipz") == bytearray(b"mississ") - # split -assert bytearray(b"1,2,3").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] -assert bytearray(b"1,2,3").split(bytearray(b","), maxsplit=1) == [bytearray(b"1"), bytearray(b"2,3")] -assert bytearray(b"1,2,,3,").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b""), bytearray(b"3"), bytearray(b"")] -assert bytearray(b"1 2 3").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"1,2,3").split(bytearray(b",")) == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b"3"), +] +assert bytearray(b"1,2,3").split(bytearray(b","), maxsplit=1) == [ + bytearray(b"1"), + bytearray(b"2,3"), +] +assert bytearray(b"1,2,,3,").split(bytearray(b",")) == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b""), + bytearray(b"3"), + bytearray(b""), +] +assert bytearray(b"1 2 3").split() == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b"3"), +] assert bytearray(b"1 2 3").split(maxsplit=1) == [bytearray(b"1"), bytearray(b"2 3")] -assert bytearray(b" 1 2 3 ").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] -assert bytearray(b"k\ruh\nfz e f").split() == [bytearray(b"k"), bytearray(b"uh"), bytearray(b"fz"), bytearray(b"e"), bytearray(b"f")] -assert bytearray(b"Two lines\n").split(bytearray(b"\n")) == [bytearray(b"Two lines"), bytearray(b"")] +assert bytearray(b" 1 2 3 ").split() == [ + bytearray(b"1"), + bytearray(b"2"), + bytearray(b"3"), +] +assert bytearray(b"k\ruh\nfz e f").split() == [ + bytearray(b"k"), + bytearray(b"uh"), + bytearray(b"fz"), + bytearray(b"e"), + bytearray(b"f"), +] +assert bytearray(b"Two lines\n").split(bytearray(b"\n")) == [ + bytearray(b"Two lines"), + bytearray(b""), +] assert bytearray(b"").split() == [] assert bytearray(b"").split(bytearray(b"\n")) == [bytearray(b"")] assert bytearray(b"\n").split(bytearray(b"\n")) == [bytearray(b""), bytearray(b"")] @@ -534,16 +565,21 @@ class B(bytearray): i = SPLIT_FIXTURES[n_sp] sep = None if i[1] == None else bytearray(i[1]) try: - assert bytearray(i[0]).split(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[2]] + assert bytearray(i[0]).split(sep=sep, maxsplit=i[4]) == [ + bytearray(j) for j in i[2] + ] except AssertionError: print(i[0], i[1], i[2]) print( - "Expected : ", [list(x) for x in bytearray(i[0]).split(sep=sep, maxsplit=i[4])] + "Expected : ", + [list(x) for x in bytearray(i[0]).split(sep=sep, maxsplit=i[4])], ) break try: - assert bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[3]] + assert bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [ + bytearray(j) for j in i[3] + ] except AssertionError: print(i[0], i[1], i[2]) print( @@ -557,34 +593,61 @@ class B(bytearray): # expandtabs a = bytearray(b"\x01\x03\r\x05\t8CYZ\t\x06CYZ\t\x17cba`\n\x12\x13\x14") -assert ( - a.expandtabs() == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") +assert a.expandtabs() == bytearray( + b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +) +assert a.expandtabs(5) == bytearray( + b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +) +assert bytearray(b"01\t012\t0123\t01234").expandtabs() == bytearray( + b"01 012 0123 01234" +) +assert bytearray(b"01\t012\t0123\t01234").expandtabs(4) == bytearray( + b"01 012 0123 01234" ) -assert a.expandtabs(5) == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") -assert bytearray(b"01\t012\t0123\t01234").expandtabs() == bytearray(b"01 012 0123 01234") -assert bytearray(b"01\t012\t0123\t01234").expandtabs(4) == bytearray(b"01 012 0123 01234") assert bytearray(b"123\t123").expandtabs(-5) == bytearray(b"123123") assert bytearray(b"123\t123").expandtabs(0) == bytearray(b"123123") # # partition -assert bytearray(b"123456789").partition(b"45") == ((b"123"), bytearray(b"45"), bytearray(b"6789")) -assert bytearray(b"14523456789").partition(b"45") == ((b"1"), bytearray(b"45"), bytearray(b"23456789")) +assert bytearray(b"123456789").partition(b"45") == ( + (b"123"), + bytearray(b"45"), + bytearray(b"6789"), +) +assert bytearray(b"14523456789").partition(b"45") == ( + (b"1"), + bytearray(b"45"), + bytearray(b"23456789"), +) a = bytearray(b"14523456789").partition(b"45") assert isinstance(a[1], bytearray) a = bytearray(b"14523456789").partition(memoryview(b"45")) assert isinstance(a[1], bytearray) # partition -assert bytearray(b"123456789").rpartition(bytearray(b"45")) == ((bytearray(b"123")), bytearray(b"45"), bytearray(b"6789")) -assert bytearray(b"14523456789").rpartition(bytearray(b"45")) == ((bytearray(b"14523")), bytearray(b"45"), bytearray(b"6789")) +assert bytearray(b"123456789").rpartition(bytearray(b"45")) == ( + (bytearray(b"123")), + bytearray(b"45"), + bytearray(b"6789"), +) +assert bytearray(b"14523456789").rpartition(bytearray(b"45")) == ( + (bytearray(b"14523")), + bytearray(b"45"), + bytearray(b"6789"), +) a = bytearray(b"14523456789").rpartition(b"45") assert isinstance(a[1], bytearray) a = bytearray(b"14523456789").rpartition(memoryview(b"45")) assert isinstance(a[1], bytearray) # splitlines -assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines() == [bytearray(b"ab c"), bytearray(b""), bytearray(b"de fg"), bytearray(b"kl")] +assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines() == [ + bytearray(b"ab c"), + bytearray(b""), + bytearray(b"de fg"), + bytearray(b"kl"), +] assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines(keepends=True) == [ bytearray(b"ab c\n"), bytearray(b"\n"), @@ -602,11 +665,15 @@ class B(bytearray): assert bytearray(b"42").zfill(-1) == bytearray(b"42") # replace -assert bytearray(b"123456789123").replace(b"23",b"XX") ==bytearray(b'1XX4567891XX') -assert bytearray(b"123456789123").replace(b"23",b"XX", 1) ==bytearray(b'1XX456789123') -assert bytearray(b"123456789123").replace(b"23",b"XX", 0) == bytearray(b"123456789123") -assert bytearray(b"123456789123").replace(b"23",b"XX", -1) ==bytearray(b'1XX4567891XX') -assert bytearray(b"123456789123").replace(b"23", bytearray(b"")) == bytearray(b"14567891") +assert bytearray(b"123456789123").replace(b"23", b"XX") == bytearray(b"1XX4567891XX") +assert bytearray(b"123456789123").replace(b"23", b"XX", 1) == bytearray(b"1XX456789123") +assert bytearray(b"123456789123").replace(b"23", b"XX", 0) == bytearray(b"123456789123") +assert bytearray(b"123456789123").replace(b"23", b"XX", -1) == bytearray( + b"1XX4567891XX" +) +assert bytearray(b"123456789123").replace(b"23", bytearray(b"")) == bytearray( + b"14567891" +) # clear @@ -642,25 +709,24 @@ class B(bytearray): # title assert bytearray(b"Hello world").title() == bytearray(b"Hello World") -assert ( - bytearray(b"they're bill's friends from the UK").title() - == bytearray(b"They'Re Bill'S Friends From The Uk") +assert bytearray(b"they're bill's friends from the UK").title() == bytearray( + b"They'Re Bill'S Friends From The Uk" ) # repeat by multiply -a = bytearray(b'abcd') -assert a * 0 == bytearray(b'') -assert a * -1 == bytearray(b'') -assert a * 1 == bytearray(b'abcd') -assert a * 3 == bytearray(b'abcdabcdabcd') -assert 3 * a == bytearray(b'abcdabcdabcd') - -a = bytearray(b'abcd') +a = bytearray(b"abcd") +assert a * 0 == bytearray(b"") +assert a * -1 == bytearray(b"") +assert a * 1 == bytearray(b"abcd") +assert a * 3 == bytearray(b"abcdabcdabcd") +assert 3 * a == bytearray(b"abcdabcdabcd") + +a = bytearray(b"abcd") a.__imul__(3) -assert a == bytearray(b'abcdabcdabcd') +assert a == bytearray(b"abcdabcdabcd") a.__imul__(0) -assert a == bytearray(b'') +assert a == bytearray(b"") # copy @@ -696,70 +762,89 @@ class B(bytearray): # remove -a = bytearray(b'abcdabcd') +a = bytearray(b"abcdabcd") a.remove(99) # the letter c # Only the first is removed -assert a == bytearray(b'abdabcd') +assert a == bytearray(b"abdabcd") # reverse -a = bytearray(b'hello, world') +a = bytearray(b"hello, world") a.reverse() -assert a == bytearray(b'dlrow ,olleh') +assert a == bytearray(b"dlrow ,olleh") # __setitem__ -a = bytearray(b'test') +a = bytearray(b"test") a[0] = 1 -assert a == bytearray(b'\x01est') +assert a == bytearray(b"\x01est") with assert_raises(TypeError): - a[0] = b'a' + a[0] = b"a" with assert_raises(TypeError): - a[0] = memoryview(b'a') + a[0] = memoryview(b"a") a[:2] = [0, 9] -assert a == bytearray(b'\x00\x09st') -a[1:3] = b'test' -assert a == bytearray(b'\x00testt') -a[:6] = memoryview(b'test') -assert a == bytearray(b'test') +assert a == bytearray(b"\x00\x09st") +a[1:3] = b"test" +assert a == bytearray(b"\x00testt") +a[:6] = memoryview(b"test") +assert a == bytearray(b"test") # mod -assert bytearray('rust%bpython%b', 'utf-8') % (b' ', b'!') == bytearray(b'rust python!') -assert bytearray('x=%i y=%f', 'utf-8') % (1, 2.5) == bytearray(b'x=1 y=2.500000') +assert bytearray("rust%bpython%b", "utf-8") % (b" ", b"!") == bytearray(b"rust python!") +assert bytearray("x=%i y=%f", "utf-8") % (1, 2.5) == bytearray(b"x=1 y=2.500000") # eq, ne -a = bytearray(b'hello, world') +a = bytearray(b"hello, world") b = a.copy() assert a.__ne__(b) is False -b = bytearray(b'my bytearray') +b = bytearray(b"my bytearray") assert a.__ne__(b) is True # pickle -a = bytearray(b'\xffab\x80\0\0\370\0\0') -assert pickle.dumps(a, 0) == b'c__builtin__\nbytearray\np0\n(c_codecs\nencode\np1\n(V\xffab\x80\\u0000\\u0000\xf8\\u0000\\u0000\np2\nVlatin1\np3\ntp4\nRp5\ntp6\nRp7\n.' -assert pickle.dumps(a, 1) == b'c__builtin__\nbytearray\nq\x00(c_codecs\nencode\nq\x01(X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03tq\x04Rq\x05tq\x06Rq\x07.' -assert pickle.dumps(a, 2) == b'\x80\x02c__builtin__\nbytearray\nq\x00c_codecs\nencode\nq\x01X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03\x86q\x04Rq\x05\x85q\x06Rq\x07.' -assert pickle.dumps(a, 3) == b'\x80\x03cbuiltins\nbytearray\nq\x00C\t\xffab\x80\x00\x00\xf8\x00\x00q\x01\x85q\x02Rq\x03.' -assert pickle.dumps(a, 4) == b'\x80\x04\x95*\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\tbytearray\x94\x93\x94C\t\xffab\x80\x00\x00\xf8\x00\x00\x94\x85\x94R\x94.' +a = bytearray(b"\xffab\x80\0\0\370\0\0") +assert ( + pickle.dumps(a, 0) + == b"c__builtin__\nbytearray\np0\n(c_codecs\nencode\np1\n(V\xffab\x80\\u0000\\u0000\xf8\\u0000\\u0000\np2\nVlatin1\np3\ntp4\nRp5\ntp6\nRp7\n." +) +assert ( + pickle.dumps(a, 1) + == b"c__builtin__\nbytearray\nq\x00(c_codecs\nencode\nq\x01(X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03tq\x04Rq\x05tq\x06Rq\x07." +) +assert ( + pickle.dumps(a, 2) + == b"\x80\x02c__builtin__\nbytearray\nq\x00c_codecs\nencode\nq\x01X\x0c\x00\x00\x00\xc3\xbfab\xc2\x80\x00\x00\xc3\xb8\x00\x00q\x02X\x06\x00\x00\x00latin1q\x03\x86q\x04Rq\x05\x85q\x06Rq\x07." +) +assert ( + pickle.dumps(a, 3) + == b"\x80\x03cbuiltins\nbytearray\nq\x00C\t\xffab\x80\x00\x00\xf8\x00\x00q\x01\x85q\x02Rq\x03." +) +assert ( + pickle.dumps(a, 4) + == b"\x80\x04\x95*\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\tbytearray\x94\x93\x94C\t\xffab\x80\x00\x00\xf8\x00\x00\x94\x85\x94R\x94." +) + # pickle with subclass class A(bytes): pass + a = A() a.x = 10 -a.y = A(b'123') +a.y = A(b"123") b = pickle.loads(pickle.dumps(a, 4)) assert type(a) == type(b) assert a.x == b.x assert a.y == b.y assert a == b + class B(bytearray): pass + a = B() a.x = 10 -a.y = B(b'123') +a.y = B(b"123") b = pickle.loads(pickle.dumps(a, 4)) assert type(a) == type(b) assert a.x == b.x @@ -768,4 +853,6 @@ class B(bytearray): a = bytearray() for i in range(-1, 2, 1): - assert_raises(IndexError, lambda: a[-sys.maxsize - i], _msg='bytearray index out of range') \ No newline at end of file + assert_raises( + IndexError, lambda: a[-sys.maxsize - i], _msg="bytearray index out of range" + ) diff --git a/extra_tests/snippets/builtin_bytes.py b/extra_tests/snippets/builtin_bytes.py index 2a6d0f63eb..9347fbc8fa 100644 --- a/extra_tests/snippets/builtin_bytes.py +++ b/extra_tests/snippets/builtin_bytes.py @@ -596,51 +596,56 @@ # repeat by multiply -a = b'abcd' -assert a * 0 == b'' -assert a * -1 == b'' -assert a * 1 == b'abcd' -assert a * 3 == b'abcdabcdabcd' -assert 3 * a == b'abcdabcdabcd' +a = b"abcd" +assert a * 0 == b"" +assert a * -1 == b"" +assert a * 1 == b"abcd" +assert a * 3 == b"abcdabcdabcd" +assert 3 * a == b"abcdabcdabcd" # decode -assert b'\x72\x75\x73\x74'.decode('ascii') == 'rust' -assert b'\xc2\xae\x75\x73\x74'.decode('ascii', 'replace') == '��ust' -assert b'\xc2\xae\x75\x73\x74'.decode('ascii', 'ignore') == 'ust' -assert b'\xc2\xae\x75\x73\x74'.decode('utf-8') == '®ust' -assert b'\xc2\xae\x75\x73\x74'.decode() == '®ust' -assert b'\xe4\xb8\xad\xe6\x96\x87\xe5\xad\x97'.decode('utf-8') == '中文字' +assert b"\x72\x75\x73\x74".decode("ascii") == "rust" +assert b"\xc2\xae\x75\x73\x74".decode("ascii", "replace") == "��ust" +assert b"\xc2\xae\x75\x73\x74".decode("ascii", "ignore") == "ust" +assert b"\xc2\xae\x75\x73\x74".decode("utf-8") == "®ust" +assert b"\xc2\xae\x75\x73\x74".decode() == "®ust" +assert b"\xe4\xb8\xad\xe6\x96\x87\xe5\xad\x97".decode("utf-8") == "中文字" # mod -assert b'rust%bpython%b' % (b' ', b'!') == b'rust python!' -assert b'x=%i y=%f' % (1, 2.5) == b'x=1 y=2.500000' +assert b"rust%bpython%b" % (b" ", b"!") == b"rust python!" +assert b"x=%i y=%f" % (1, 2.5) == b"x=1 y=2.500000" + # __bytes__ def test__bytes__(): - foo = b'foo\x00bar' + foo = b"foo\x00bar" assert foo.__bytes__() == foo assert type(foo.__bytes__()) == bytes + class bytes_subclass(bytes): pass - bar = bytes_subclass(b'bar\x00foo') + + bar = bytes_subclass(b"bar\x00foo") assert bar.__bytes__() == bar assert type(bar.__bytes__()) == bytes + class A: def __bytes__(self): return b"bytess" + assert bytes(A()) == b"bytess" # Issue #2125 -b = b'abc' +b = b"abc" assert bytes(b) is b # Regression to # https://github.com/RustPython/RustPython/issues/2840 -a = b'123abc!?' +a = b"123abc!?" assert id(a) == id(a) assert id(a) != id(a * -1) assert id(a) != id(a * 0) @@ -652,20 +657,24 @@ def __bytes__(self): class SubBytes(bytes): pass -b = SubBytes(b'0123abc*&') + +b = SubBytes(b"0123abc*&") assert id(b) == id(b) assert id(b) != id(b * -1) assert id(b) != id(b * 0) assert id(b) != id(b * 1) assert id(b) != id(b * 2) + class B1(bytearray): def __new__(cls, value): assert type(value) == bytes me = super().__new__(cls, value) - me.foo = 'bar' + me.foo = "bar" return me -b = B1.fromhex('a0a1a2') -assert b.foo == 'bar' -skip_if_unsupported(3,11,test__bytes__) \ No newline at end of file + +b = B1.fromhex("a0a1a2") +assert b.foo == "bar" + +skip_if_unsupported(3, 11, test__bytes__) diff --git a/extra_tests/snippets/builtin_callable.py b/extra_tests/snippets/builtin_callable.py index db554df245..52458bda94 100644 --- a/extra_tests/snippets/builtin_callable.py +++ b/extra_tests/snippets/builtin_callable.py @@ -1,25 +1,48 @@ assert not callable(1) -def f(): pass + + +def f(): + pass + + assert callable(f) assert callable(len) assert callable(lambda: 1) assert callable(int) + class C: def __init__(self): # must be defined on class self.__call__ = lambda self: 1 - def f(self): pass + + def f(self): + pass + + assert callable(C) assert not callable(C()) assert callable(C().f) + class C: - def __call__(self): pass + def __call__(self): + pass + + assert callable(C()) -class C1(C): pass + + +class C1(C): + pass + + assert callable(C1()) + + class C: __call__ = 1 + + # CPython returns true here, but fails when actually calling it assert callable(C()) diff --git a/extra_tests/snippets/builtin_chr.py b/extra_tests/snippets/builtin_chr.py index 9b95452bda..6fa56203d7 100644 --- a/extra_tests/snippets/builtin_chr.py +++ b/extra_tests/snippets/builtin_chr.py @@ -4,5 +4,7 @@ assert "é" == chr(233) assert "🤡" == chr(129313) -assert_raises(TypeError, chr, _msg='chr() takes exactly one argument (0 given)') -assert_raises(ValueError, chr, 0x110005, _msg='ValueError: chr() arg not in range(0x110000)') +assert_raises(TypeError, chr, _msg="chr() takes exactly one argument (0 given)") +assert_raises( + ValueError, chr, 0x110005, _msg="ValueError: chr() arg not in range(0x110000)" +) diff --git a/extra_tests/snippets/builtin_dict.py b/extra_tests/snippets/builtin_dict.py index 3ccea97232..abd93539a5 100644 --- a/extra_tests/snippets/builtin_dict.py +++ b/extra_tests/snippets/builtin_dict.py @@ -5,49 +5,49 @@ assert len({}) == 0 assert len({"a": "b"}) == 1 assert len({"a": "b", "b": 1}) == 2 -assert len({"a": "b", "b": 1, "a" + "b": 2*2}) == 3 +assert len({"a": "b", "b": 1, "a" + "b": 2 * 2}) == 3 d = {} -d['a'] = d +d["a"] = d assert repr(d) == "{'a': {...}}" -assert {'a': 123}.get('a') == 123 -assert {'a': 123}.get('b') == None -assert {'a': 123}.get('b', 456) == 456 +assert {"a": 123}.get("a") == 123 +assert {"a": 123}.get("b") == None +assert {"a": 123}.get("b", 456) == 456 -d = {'a': 123, 'b': 456} -assert list(reversed(d)) == ['b', 'a'] -assert list(reversed(d.keys())) == ['b', 'a'] +d = {"a": 123, "b": 456} +assert list(reversed(d)) == ["b", "a"] +assert list(reversed(d.keys())) == ["b", "a"] assert list(reversed(d.values())) == [456, 123] -assert list(reversed(d.items())) == [('b', 456), ('a', 123)] +assert list(reversed(d.items())) == [("b", 456), ("a", 123)] with assert_raises(StopIteration): dict_reversed = reversed(d) for _ in range(len(d) + 1): next(dict_reversed) -assert 'dict' in dict().__doc__ +assert "dict" in dict().__doc__ -d = {'a': 123, 'b': 456} +d = {"a": 123, "b": 456} assert 1 not in d.items() -assert 'a' not in d.items() -assert 'a', 123 not in d.items() +assert "a" not in d.items() +assert "a", 123 not in d.items() assert () not in d.items() assert (1) not in d.items() -assert ('a') not in d.items() -assert ('a', 123) in d.items() -assert ('b', 456) in d.items() -assert ('a', 123, 3) not in d.items() -assert ('a', 123, 'b', 456) not in d.items() +assert ("a") not in d.items() +assert ("a", 123) in d.items() +assert ("b", 456) in d.items() +assert ("a", 123, 3) not in d.items() +assert ("a", 123, "b", 456) not in d.items() -d = {1: 10, "a": "ABC", (3,4): 5} +d = {1: 10, "a": "ABC", (3, 4): 5} assert 1 in d.keys() assert (1) in d.keys() assert "a" in d.keys() -assert (3,4) in d.keys() +assert (3, 4) in d.keys() assert () not in d.keys() assert 10 not in d.keys() assert (1, 10) not in d.keys() assert "abc" not in d.keys() -assert ((3,4),5) not in d.keys() +assert ((3, 4), 5) not in d.keys() d1 = {"a": 1, "b": 2} d2 = {"c": 3, "d": 4} @@ -64,55 +64,55 @@ assert not d1.keys().isdisjoint(d2.keys()) -assert dict(a=2, b=3) == {'a': 2, 'b': 3} -assert dict({'a': 2, 'b': 3}, b=4) == {'a': 2, 'b': 4} -assert dict([('a', 2), ('b', 3)]) == {'a': 2, 'b': 3} +assert dict(a=2, b=3) == {"a": 2, "b": 3} +assert dict({"a": 2, "b": 3}, b=4) == {"a": 2, "b": 4} +assert dict([("a", 2), ("b", 3)]) == {"a": 2, "b": 3} assert {} == {} -assert not {'a': 2} == {} -assert not {} == {'a': 2} -assert not {'b': 2} == {'a': 2} -assert not {'a': 4} == {'a': 2} -assert {'a': 2} == {'a': 2} +assert not {"a": 2} == {} +assert not {} == {"a": 2} +assert not {"b": 2} == {"a": 2} +assert not {"a": 4} == {"a": 2} +assert {"a": 2} == {"a": 2} -nan = float('nan') -assert {'a': nan} == {'a': nan} +nan = float("nan") +assert {"a": nan} == {"a": nan} -a = {'g': 5} -b = {'a': a, 'd': 9} +a = {"g": 5} +b = {"a": a, "d": 9} c = dict(b) -c['d'] = 3 -c['a']['g'] = 2 -assert a == {'g': 2} -assert b == {'a': a, 'd': 9} +c["d"] = 3 +c["a"]["g"] = 2 +assert a == {"g": 2} +assert b == {"a": a, "d": 9} a.clear() assert len(a) == 0 -a = {'a': 5, 'b': 6} +a = {"a": 5, "b": 6} res = set() for value in a.values(): - res.add(value) -assert res == set([5,6]) + res.add(value) +assert res == set([5, 6]) count = 0 -for (key, value) in a.items(): - assert a[key] == value - count += 1 +for key, value in a.items(): + assert a[key] == value + count += 1 assert count == len(a) res = set() for key in a.keys(): - res.add(key) -assert res == set(['a','b']) + res.add(key) +assert res == set(["a", "b"]) # Deleted values are correctly skipped over: -x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} -del x['c'] +x = {"a": 1, "b": 2, "c": 3, "d": 3} +del x["c"] it = iter(x.items()) -assert ('a', 1) == next(it) -assert ('b', 2) == next(it) -assert ('d', 3) == next(it) +assert ("a", 1) == next(it) +assert ("b", 2) == next(it) +assert ("d", 3) == next(it) with assert_raises(StopIteration): next(it) @@ -121,7 +121,7 @@ assert cm.exception.args[0] == 10 # Iterating a dictionary is just its keys: -assert ['a', 'b', 'd'] == list(x) +assert ["a", "b", "d"] == list(x) # Iterating view captures dictionary when iterated. data = {1: 2, 3: 4} @@ -140,12 +140,12 @@ # But we can't add or delete items during iteration. d = {} a = iter(d.items()) -d['a'] = 2 +d["a"] = 2 b = iter(d.items()) -assert ('a', 2) == next(b) +assert ("a", 2) == next(b) with assert_raises(RuntimeError): next(a) -del d['a'] +del d["a"] with assert_raises(RuntimeError): next(b) @@ -164,21 +164,23 @@ x[(5, 6)] = 5 with assert_raises(TypeError): - x[[]] # Unhashable type. + x[[]] # Unhashable type. x["here"] = "here" assert x.get("not here", "default") == "default" assert x.get("here", "default") == "here" assert x.get("not here") == None + class LengthDict(dict): def __getitem__(self, k): return len(k) + x = LengthDict() assert type(x) == LengthDict -assert x['word'] == 4 -assert x.get('word') is None +assert x["word"] == 4 +assert x.get("word") is None assert 5 == eval("a + word", LengthDict()) @@ -189,15 +191,19 @@ def __missing__(self, k): self[k] = v return v + x = Squares() assert x[-5] == 25 + # An object that hashes to the same value always, and compares equal if any its values match. class Hashable(object): def __init__(self, *args): self.values = args + def __hash__(self): return 1 + def __eq__(self, other): for x in self.values: for y in other.values: @@ -205,39 +211,40 @@ def __eq__(self, other): return True return False + x = {} -x[Hashable(1,2)] = 8 +x[Hashable(1, 2)] = 8 -assert x[Hashable(1,2)] == 8 -assert x[Hashable(3,1)] == 8 +assert x[Hashable(1, 2)] == 8 +assert x[Hashable(3, 1)] == 8 x[Hashable(8)] = 19 -x[Hashable(19,8)] = 1 +x[Hashable(19, 8)] = 1 assert x[Hashable(8)] == 1 assert len(x) == 2 -assert list({'a': 2, 'b': 10}) == ['a', 'b'] +assert list({"a": 2, "b": 10}) == ["a", "b"] x = {} -x['a'] = 2 -x['b'] = 10 -assert list(x) == ['a', 'b'] +x["a"] = 2 +x["b"] = 10 +assert list(x) == ["a", "b"] y = x.copy() -x['c'] = 12 -assert y == {'a': 2, 'b': 10} +x["c"] = 12 +assert y == {"a": 2, "b": 10} -y.update({'c': 19, "d": -1, 'b': 12}) -assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} +y.update({"c": 19, "d": -1, "b": 12}) +assert y == {"a": 2, "b": 12, "c": 19, "d": -1} y.update(y) -assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed +assert y == {"a": 2, "b": 12, "c": 19, "d": -1} # hasn't changed # KeyError has object that used as key as an .args[0] with assert_raises(KeyError) as cm: - x['not here'] + x["not here"] assert cm.exception.args[0] == "not here" with assert_raises(KeyError) as cm: - x.pop('not here') + x.pop("not here") assert cm.exception.args[0] == "not here" with assert_raises(KeyError) as cm: @@ -247,7 +254,11 @@ def __eq__(self, other): x.pop(10) assert cm.exception.args[0] == 10 -class MyClass: pass + +class MyClass: + pass + + obj = MyClass() with assert_raises(KeyError) as cm: @@ -257,49 +268,65 @@ class MyClass: pass x.pop(obj) assert cm.exception.args[0] == obj -x = {1: 'a', '1': None} -assert x.pop(1) == 'a' -assert x.pop('1') is None +x = {1: "a", "1": None} +assert x.pop(1) == "a" +assert x.pop("1") is None assert x == {} -x = {1: 'a'} -assert (1, 'a') == x.popitem() +x = {1: "a"} +assert (1, "a") == x.popitem() assert x == {} with assert_raises(KeyError) as cm: x.popitem() -assert cm.exception.args == ('popitem(): dictionary is empty',) +assert cm.exception.args == ("popitem(): dictionary is empty",) -x = {'a': 4} -assert 4 == x.setdefault('a', 0) -assert x['a'] == 4 -assert 0 == x.setdefault('b', 0) -assert x['b'] == 0 -assert None == x.setdefault('c') -assert x['c'] is None +x = {"a": 4} +assert 4 == x.setdefault("a", 0) +assert x["a"] == 4 +assert 0 == x.setdefault("b", 0) +assert x["b"] == 0 +assert None == x.setdefault("c") +assert x["c"] is None assert {1: None, "b": None} == dict.fromkeys([1, "b"]) assert {1: 0, "b": 0} == dict.fromkeys([1, "b"], 0) -x = {'a': 1, 'b': 1, 'c': 1} -y = {'b': 2, 'c': 2, 'd': 2} -z = {'c': 3, 'd': 3, 'e': 3} +x = {"a": 1, "b": 1, "c": 1} +y = {"b": 2, "c": 2, "d": 2} +z = {"c": 3, "d": 3, "e": 3} w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} -assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} # not in cpython test suite +assert w == { + 1: 1, + "a": 1, + "b": 2, + "c": 3, + 2: 2, + "d": 3, + 3: 3, + "e": 3, + 4: 4, +} # not in cpython test suite assert str({True: True, 1.0: 1.0}) == str({True: 1.0}) + class A: def __hash__(self): return 1 + def __eq__(self, other): return isinstance(other, A) + + class B: def __hash__(self): return 1 + def __eq__(self, other): return isinstance(other, B) + s = {1: 0, A(): 1, B(): 2} assert len(s) == 3 assert s[1] == 0 @@ -307,19 +334,19 @@ def __eq__(self, other): assert s[B()] == 2 # Test dict usage in set with star expressions! -a = {'bla': 2} -b = {'c': 44, 'bla': 332, 'd': 6} -x = ['bla', 'c', 'd', 'f'] +a = {"bla": 2} +b = {"c": 44, "bla": 332, "d": 6} +x = ["bla", "c", "d", "f"] c = {*a, *b, *x} # print(c, type(c)) assert isinstance(c, set) -assert c == {'bla', 'c', 'd', 'f'} +assert c == {"bla", "c", "d", "f"} assert not {}.__ne__({}) -assert {}.__ne__({'a':'b'}) +assert {}.__ne__({"a": "b"}) assert {}.__ne__(1) == NotImplemented -it = iter({0: 1, 2: 3, 4:5, 6:7}) +it = iter({0: 1, 2: 3, 4: 5, 6: 7}) assert it.__length_hint__() == 4 next(it) assert it.__length_hint__() == 3 diff --git a/extra_tests/snippets/builtin_dict_union.py b/extra_tests/snippets/builtin_dict_union.py index f33f32a5e4..ab3fa65d37 100644 --- a/extra_tests/snippets/builtin_dict_union.py +++ b/extra_tests/snippets/builtin_dict_union.py @@ -1,78 +1,91 @@ - from testutils import assert_raises, skip_if_unsupported + def test_dunion_ior0(): - a={1:2,2:3} - b={3:4,5:6} - a|=b + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + a |= b + + assert a == {1: 2, 2: 3, 3: 4, 5: 6}, f"wrong value assigned {a=}" + assert b == {3: 4, 5: 6}, f"right hand side modified, {b=}" - assert a == {1:2,2:3,3:4,5:6}, f"wrong value assigned {a=}" - assert b == {3:4,5:6}, f"right hand side modified, {b=}" def test_dunion_or0(): - a={1:2,2:3} - b={3:4,5:6} - c=a|b + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + c = a | b - assert a == {1:2,2:3}, f"left hand side of non-assignment operator modified {a=}" - assert b == {3:4,5:6}, f"right hand side of non-assignment operator modified, {b=}" - assert c == {1:2,2:3, 3:4, 5:6}, f"unexpected result of dict union {c=}" + assert a == {1: 2, 2: 3}, f"left hand side of non-assignment operator modified {a=}" + assert b == {3: 4, 5: 6}, ( + f"right hand side of non-assignment operator modified, {b=}" + ) + assert c == {1: 2, 2: 3, 3: 4, 5: 6}, f"unexpected result of dict union {c=}" def test_dunion_or1(): - a={1:2,2:3} - b={3:4,5:6} - c=a.__or__(b) + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + c = a.__or__(b) - assert a == {1:2,2:3}, f"left hand side of non-assignment operator modified {a=}" - assert b == {3:4,5:6}, f"right hand side of non-assignment operator modified, {b=}" - assert c == {1:2,2:3, 3:4, 5:6}, f"unexpected result of dict union {c=}" + assert a == {1: 2, 2: 3}, f"left hand side of non-assignment operator modified {a=}" + assert b == {3: 4, 5: 6}, ( + f"right hand side of non-assignment operator modified, {b=}" + ) + assert c == {1: 2, 2: 3, 3: 4, 5: 6}, f"unexpected result of dict union {c=}" def test_dunion_ror0(): - a={1:2,2:3} - b={3:4,5:6} - c=b.__ror__(a) + a = {1: 2, 2: 3} + b = {3: 4, 5: 6} + c = b.__ror__(a) - assert a == {1:2,2:3}, f"left hand side of non-assignment operator modified {a=}" - assert b == {3:4,5:6}, f"right hand side of non-assignment operator modified, {b=}" - assert c == {1:2,2:3, 3:4, 5:6}, f"unexpected result of dict union {c=}" + assert a == {1: 2, 2: 3}, f"left hand side of non-assignment operator modified {a=}" + assert b == {3: 4, 5: 6}, ( + f"right hand side of non-assignment operator modified, {b=}" + ) + assert c == {1: 2, 2: 3, 3: 4, 5: 6}, f"unexpected result of dict union {c=}" def test_dunion_other_types(): def perf_test_or(other_obj): - d={1:2} + d = {1: 2} return d.__or__(other_obj) is NotImplemented def perf_test_ror(other_obj): - d={1:2} + d = {1: 2} return d.__ror__(other_obj) is NotImplemented - test_fct={'__or__':perf_test_or, '__ror__':perf_test_ror} - others=['FooBar', 42, [36], set([19]), ['aa'], None] - for tfn,tf in test_fct.items(): + test_fct = {"__or__": perf_test_or, "__ror__": perf_test_ror} + others = ["FooBar", 42, [36], set([19]), ["aa"], None] + for tfn, tf in test_fct.items(): for other in others: assert tf(other), f"Failed: dict {tfn}, accepted {other}" # __ior__() has different behavior and needs to be tested separately d = {1: 2} - assert_raises(ValueError, - lambda: d.__ior__('FooBar'), - _msg='dictionary update sequence element #0 has length 1; 2 is required') - assert_raises(TypeError, - lambda: d.__ior__(42), - _msg='\'int\' object is not iterable') - assert_raises(TypeError, - lambda: d.__ior__([36]), - _msg='cannot convert dictionary update sequence element #0 to a sequence') - assert_raises(TypeError, - lambda: d.__ior__(set([36])), - _msg='cannot convert dictionary update sequence element #0 to a sequence') - res = d.__ior__(['aa']) - assert res == {1: 2, 'a': 'a'}, f"unexpected result of dict union {res=}" - assert_raises(TypeError, - lambda: d.__ior__(None), - _msg='TypeError: \'NoneType\' object is not iterable') + assert_raises( + ValueError, + lambda: d.__ior__("FooBar"), + _msg="dictionary update sequence element #0 has length 1; 2 is required", + ) + assert_raises(TypeError, lambda: d.__ior__(42), _msg="'int' object is not iterable") + assert_raises( + TypeError, + lambda: d.__ior__([36]), + _msg="cannot convert dictionary update sequence element #0 to a sequence", + ) + assert_raises( + TypeError, + lambda: d.__ior__(set([36])), + _msg="cannot convert dictionary update sequence element #0 to a sequence", + ) + res = d.__ior__(["aa"]) + assert res == {1: 2, "a": "a"}, f"unexpected result of dict union {res=}" + assert_raises( + TypeError, + lambda: d.__ior__(None), + _msg="TypeError: 'NoneType' object is not iterable", + ) skip_if_unsupported(3, 9, test_dunion_ior0) diff --git a/extra_tests/snippets/builtin_dir.py b/extra_tests/snippets/builtin_dir.py index 3e808597c1..cd2c8c33a2 100644 --- a/extra_tests/snippets/builtin_dir.py +++ b/extra_tests/snippets/builtin_dir.py @@ -1,9 +1,11 @@ assert isinstance(dir(), list) -assert '__builtins__' in dir() +assert "__builtins__" in dir() + class A: - def test(): - pass + def test(): + pass + a = A() @@ -13,24 +15,30 @@ def test(): a.x = 3 assert "x" in dir(a), "x not in a" + class B(A): - def __dir__(self): - return ('q', 'h') + def __dir__(self): + return ("q", "h") + # Gets sorted and turned into a list -assert ['h', 'q'] == dir(B()) +assert ["h", "q"] == dir(B()) # This calls type.__dir__ so isn't changed (but inheritance works)! -assert 'test' in dir(A) +assert "test" in dir(A) + # eval() takes any mapping-like type, so dir() must support them # TODO: eval() should take any mapping as locals, not just dict-derived types class A(dict): - def __getitem__(self, x): - return dir - def keys(self): - yield 6 - yield 5 + def __getitem__(self, x): + return dir + + def keys(self): + yield 6 + yield 5 + + assert eval("dir()", {}, A()) == [5, 6] import socket diff --git a/extra_tests/snippets/builtin_divmod.py b/extra_tests/snippets/builtin_divmod.py index 5a9443afe8..f62d0f8eea 100644 --- a/extra_tests/snippets/builtin_divmod.py +++ b/extra_tests/snippets/builtin_divmod.py @@ -1,9 +1,9 @@ from testutils import assert_raises assert divmod(11, 3) == (3, 2) -assert divmod(8,11) == (0, 8) +assert divmod(8, 11) == (0, 8) assert divmod(0.873, 0.252) == (3.0, 0.11699999999999999) assert divmod(-86340, 86400) == (-1, 60) -assert_raises(ZeroDivisionError, divmod, 5, 0, _msg='divmod by zero') -assert_raises(ZeroDivisionError, divmod, 5.0, 0.0, _msg='divmod by zero') +assert_raises(ZeroDivisionError, divmod, 5, 0, _msg="divmod by zero") +assert_raises(ZeroDivisionError, divmod, 5.0, 0.0, _msg="divmod by zero") diff --git a/extra_tests/snippets/builtin_ellipsis.py b/extra_tests/snippets/builtin_ellipsis.py index 5316b9f865..cf99f3cc82 100644 --- a/extra_tests/snippets/builtin_ellipsis.py +++ b/extra_tests/snippets/builtin_ellipsis.py @@ -1,5 +1,3 @@ - - a = ... b = ... c = type(a)() # Test singleton behavior @@ -11,22 +9,22 @@ assert b is d assert d is e -assert Ellipsis.__repr__() == 'Ellipsis' -assert Ellipsis.__reduce__() == 'Ellipsis' +assert Ellipsis.__repr__() == "Ellipsis" +assert Ellipsis.__reduce__() == "Ellipsis" assert type(Ellipsis).__new__(type(Ellipsis)) == Ellipsis -assert type(Ellipsis).__reduce__(Ellipsis) == 'Ellipsis' +assert type(Ellipsis).__reduce__(Ellipsis) == "Ellipsis" try: type(Ellipsis).__new__(type(1)) except TypeError: pass else: - assert False, '`Ellipsis.__new__` should only accept `type(Ellipsis)` as argument' + assert False, "`Ellipsis.__new__` should only accept `type(Ellipsis)` as argument" try: type(Ellipsis).__reduce__(1) except TypeError: pass else: - assert False, '`Ellipsis.__reduce__` should only accept `Ellipsis` as argument' + assert False, "`Ellipsis.__reduce__` should only accept `Ellipsis` as argument" assert Ellipsis is ... Ellipsis = 2 diff --git a/extra_tests/snippets/builtin_enumerate.py b/extra_tests/snippets/builtin_enumerate.py index 35edadd1d7..0f107ea7ae 100644 --- a/extra_tests/snippets/builtin_enumerate.py +++ b/extra_tests/snippets/builtin_enumerate.py @@ -1,9 +1,14 @@ -assert list(enumerate(['a', 'b', 'c'])) == [(0, 'a'), (1, 'b'), (2, 'c')] +assert list(enumerate(["a", "b", "c"])) == [(0, "a"), (1, "b"), (2, "c")] assert type(enumerate([])) == enumerate -assert list(enumerate(['a', 'b', 'c'], -100)) == [(-100, 'a'), (-99, 'b'), (-98, 'c')] -assert list(enumerate(['a', 'b', 'c'], 2**200)) == [(2**200, 'a'), (2**200 + 1, 'b'), (2**200 + 2, 'c')] +assert list(enumerate(["a", "b", "c"], -100)) == [(-100, "a"), (-99, "b"), (-98, "c")] +assert list(enumerate(["a", "b", "c"], 2**200)) == [ + (2**200, "a"), + (2**200 + 1, "b"), + (2**200 + 2, "c"), +] + # test infinite iterator class Counter(object): diff --git a/extra_tests/snippets/builtin_eval.py b/extra_tests/snippets/builtin_eval.py index 6375bd0c1a..314abee2bb 100644 --- a/extra_tests/snippets/builtin_eval.py +++ b/extra_tests/snippets/builtin_eval.py @@ -1,4 +1,4 @@ -assert 3 == eval('1+2') +assert 3 == eval("1+2") -code = compile('5+3', 'x.py', 'eval') +code = compile("5+3", "x.py", "eval") assert eval(code) == 8 diff --git a/extra_tests/snippets/builtin_exceptions.py b/extra_tests/snippets/builtin_exceptions.py index 4bff9c0096..82aa54d632 100644 --- a/extra_tests/snippets/builtin_exceptions.py +++ b/extra_tests/snippets/builtin_exceptions.py @@ -3,41 +3,48 @@ import pickle import sys + def exceptions_eq(e1, e2): return type(e1) is type(e2) and e1.args == e2.args + def round_trip_repr(e): return exceptions_eq(e, eval(repr(e))) + # KeyError empty_exc = KeyError() -assert str(empty_exc) == '' +assert str(empty_exc) == "" assert round_trip_repr(empty_exc) assert len(empty_exc.args) == 0 assert type(empty_exc.args) == tuple -exc = KeyError('message') +exc = KeyError("message") assert str(exc) == "'message'" assert round_trip_repr(exc) assert LookupError.__str__(exc) == "message" -exc = KeyError('message', 'another message') +exc = KeyError("message", "another message") assert str(exc) == "('message', 'another message')" assert round_trip_repr(exc) -assert exc.args[0] == 'message' -assert exc.args[1] == 'another message' +assert exc.args[0] == "message" +assert exc.args[1] == "another message" + class A: def __repr__(self): - return 'A()' + return "A()" + def __str__(self): - return 'str' + return "str" + def __eq__(self, other): return type(other) is A + exc = KeyError(A()) -assert str(exc) == 'A()' +assert str(exc) == "A()" assert round_trip_repr(exc) # ImportError / ModuleNotFoundError @@ -47,33 +54,32 @@ def __eq__(self, other): assert exc.msg is None assert exc.args == () -exc = ImportError('hello') +exc = ImportError("hello") assert exc.name is None assert exc.path is None -assert exc.msg == 'hello' -assert exc.args == ('hello',) +assert exc.msg == "hello" +assert exc.args == ("hello",) -exc = ImportError('hello', name='name', path='path') -assert exc.name == 'name' -assert exc.path == 'path' -assert exc.msg == 'hello' -assert exc.args == ('hello',) +exc = ImportError("hello", name="name", path="path") +assert exc.name == "name" +assert exc.path == "path" +assert exc.msg == "hello" +assert exc.args == ("hello",) class NewException(Exception): - - def __init__(self, value): - self.value = value + def __init__(self, value): + self.value = value try: - raise NewException("test") + raise NewException("test") except NewException as e: - assert e.value == "test" + assert e.value == "test" -exc = SyntaxError('msg', 1, 2, 3, 4, 5) -assert exc.msg == 'msg' +exc = SyntaxError("msg", 1, 2, 3, 4, 5) +assert exc.msg == "msg" assert exc.filename is None assert exc.lineno is None assert exc.offset is None @@ -82,11 +88,12 @@ def __init__(self, value): # Regression to: # https://github.com/RustPython/RustPython/issues/2779 + class MyError(Exception): pass -e = MyError('message') +e = MyError("message") try: raise e from e @@ -97,23 +104,23 @@ class MyError(Exception): assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" try: - raise ValueError('test') from e + raise ValueError("test") from e except ValueError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) # ok, will print two excs assert isinstance(exc, ValueError) assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" # New case: # potential recursion on `__context__` field -e = MyError('message') +e = MyError("message") try: try: @@ -121,15 +128,15 @@ class MyError(Exception): except MyError as exc: raise e else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is None assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: try: @@ -137,15 +144,15 @@ class MyError(Exception): except MyError as exc: raise exc else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is None assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: try: @@ -153,15 +160,15 @@ class MyError(Exception): except MyError as exc: raise e from e else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: try: @@ -169,23 +176,25 @@ class MyError(Exception): except MyError as exc: raise exc from e else: - assert False, 'exception not raised' + assert False, "exception not raised" except MyError as exc: sys.excepthook(type(exc), exc, exc.__traceback__) assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" # New case: # two exception in a recursion loop + class SubError(MyError): pass -e = MyError('message') -d = SubError('sub') + +e = MyError("message") +d = SubError("sub") try: @@ -197,9 +206,9 @@ class SubError(MyError): assert exc.__cause__ is d assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" -e = MyError('message') +e = MyError("message") try: raise d from e @@ -210,20 +219,20 @@ class SubError(MyError): assert exc.__cause__ is e assert exc.__context__ is None else: - assert False, 'exception not raised' + assert False, "exception not raised" # New case: # explicit `__context__` manipulation. -e = MyError('message') +e = MyError("message") e.__context__ = e try: raise e except MyError as exc: # It was a segmentation fault before, will print info to stdout: - if platform.python_implementation() == 'RustPython': + if platform.python_implementation() == "RustPython": # For some reason `CPython` hangs on this code: sys.excepthook(type(exc), exc, exc.__traceback__) assert isinstance(exc, MyError) @@ -235,30 +244,36 @@ class SubError(MyError): # https://github.com/RustPython/RustPython/issues/2771 # `BaseException` and `Exception`: -assert BaseException.__new__.__qualname__ == 'BaseException.__new__' -assert BaseException.__init__.__qualname__ == 'BaseException.__init__' +assert BaseException.__new__.__qualname__ == "BaseException.__new__" +assert BaseException.__init__.__qualname__ == "BaseException.__init__" assert BaseException().__dict__ == {} -assert Exception.__new__.__qualname__ == 'Exception.__new__', Exception.__new__.__qualname__ -assert Exception.__init__.__qualname__ == 'Exception.__init__', Exception.__init__.__qualname__ +assert Exception.__new__.__qualname__ == "Exception.__new__", ( + Exception.__new__.__qualname__ +) +assert Exception.__init__.__qualname__ == "Exception.__init__", ( + Exception.__init__.__qualname__ +) assert Exception().__dict__ == {} # Extends `BaseException`, simple: -assert KeyboardInterrupt.__new__.__qualname__ == 'KeyboardInterrupt.__new__', KeyboardInterrupt.__new__.__qualname__ -assert KeyboardInterrupt.__init__.__qualname__ == 'KeyboardInterrupt.__init__' +assert KeyboardInterrupt.__new__.__qualname__ == "KeyboardInterrupt.__new__", ( + KeyboardInterrupt.__new__.__qualname__ +) +assert KeyboardInterrupt.__init__.__qualname__ == "KeyboardInterrupt.__init__" assert KeyboardInterrupt().__dict__ == {} # Extends `Exception`, simple: -assert TypeError.__new__.__qualname__ == 'TypeError.__new__' -assert TypeError.__init__.__qualname__ == 'TypeError.__init__' +assert TypeError.__new__.__qualname__ == "TypeError.__new__" +assert TypeError.__init__.__qualname__ == "TypeError.__init__" assert TypeError().__dict__ == {} # Extends `Exception`, complex: -assert OSError.__new__.__qualname__ == 'OSError.__new__' -assert OSError.__init__.__qualname__ == 'OSError.__init__' +assert OSError.__new__.__qualname__ == "OSError.__new__" +assert OSError.__init__.__qualname__ == "OSError.__init__" assert OSError().__dict__ == {} assert OSError.errno assert OSError.strerror @@ -299,7 +314,7 @@ class SubError(MyError): assert x.filename2 == None assert str(x) == "0" -w = OSError('foo') +w = OSError("foo") assert w.errno == None assert not sys.platform.startswith("win") or w.winerror == None assert w.strerror == None @@ -315,7 +330,7 @@ class SubError(MyError): assert x.filename2 == None assert str(x) == "foo" -w = OSError('a', 'b', 'c', 'd', 'e', 'f') +w = OSError("a", "b", "c", "d", "e", "f") assert w.errno == None assert not sys.platform.startswith("win") or w.winerror == None assert w.strerror == None @@ -332,12 +347,10 @@ class SubError(MyError): assert str(x) == "('a', 'b', 'c', 'd', 'e', 'f')" # Custom `__new__` and `__init__`: -assert ImportError.__init__.__qualname__ == 'ImportError.__init__' -assert ImportError(name='a').name == 'a' -assert ( - ModuleNotFoundError.__init__.__qualname__ == 'ModuleNotFoundError.__init__' -) -assert ModuleNotFoundError(name='a').name == 'a' +assert ImportError.__init__.__qualname__ == "ImportError.__init__" +assert ImportError(name="a").name == "a" +assert ModuleNotFoundError.__init__.__qualname__ == "ModuleNotFoundError.__init__" +assert ModuleNotFoundError(name="a").name == "a" # Check that all exceptions have string `__doc__`: diff --git a/extra_tests/snippets/builtin_exec.py b/extra_tests/snippets/builtin_exec.py index 289f878cc0..2eae90e91c 100644 --- a/extra_tests/snippets/builtin_exec.py +++ b/extra_tests/snippets/builtin_exec.py @@ -3,11 +3,11 @@ d = {} exec("def square(x):\n return x * x\n", {}, d) -assert 16 == d['square'](4) +assert 16 == d["square"](4) -exec("assert 2 == x", {}, {'x': 2}) -exec("assert 2 == x", {'x': 2}, {}) -exec("assert 4 == x", {'x': 2}, {'x': 4}) +exec("assert 2 == x", {}, {"x": 2}) +exec("assert 2 == x", {"x": 2}, {}) +exec("assert 4 == x", {"x": 2}, {"x": 4}) exec("assert max(1, 2) == 2", {}, {}) @@ -16,9 +16,11 @@ # Local environment shouldn't replace global environment: exec("assert max(1, 5, square(5)) == 25", None, {}) + # Closures aren't available if local scope is replaced: def g(): seven = "seven" + def f(): try: exec("seven", None, {}) @@ -26,7 +28,10 @@ def f(): pass else: raise NameError("seven shouldn't be in scope") + f() + + g() try: @@ -37,16 +42,16 @@ def f(): raise TypeError("exec should fail unless globals is a dict or None") g = globals() -g['x'] = 2 -exec('x += 2') +g["x"] = 2 +exec("x += 2") assert x == 4 # noqa: F821 -assert g['x'] == x # noqa: F821 +assert g["x"] == x # noqa: F821 exec("del x") -assert 'x' not in g +assert "x" not in g -assert 'g' in globals() -assert 'g' in locals() +assert "g" in globals() +assert "g" in locals() exec("assert 'g' in globals()") exec("assert 'g' in locals()") exec("assert 'g' not in globals()", {}) @@ -54,13 +59,15 @@ def f(): del g + def f(): g = 1 - assert 'g' not in globals() - assert 'g' in locals() + assert "g" not in globals() + assert "g" in locals() exec("assert 'g' not in globals()") exec("assert 'g' in locals()") exec("assert 'g' not in globals()", {}) exec("assert 'g' not in locals()", {}) + f() diff --git a/extra_tests/snippets/builtin_exit.py b/extra_tests/snippets/builtin_exit.py index f6dc387322..a61ddbc6d8 100644 --- a/extra_tests/snippets/builtin_exit.py +++ b/extra_tests/snippets/builtin_exit.py @@ -36,4 +36,4 @@ sys.exit(1) with assert_raises(SystemExit): - sys.exit("AB") \ No newline at end of file + sys.exit("AB") diff --git a/extra_tests/snippets/builtin_format.py b/extra_tests/snippets/builtin_format.py index 6a8e6077ee..ac96b6545b 100644 --- a/extra_tests/snippets/builtin_format.py +++ b/extra_tests/snippets/builtin_format.py @@ -2,37 +2,52 @@ assert format(5, "b") == "101" -assert_raises(TypeError, format, 2, 3, _msg='format called with number') +assert_raises(TypeError, format, 2, 3, _msg="format called with number") assert format({}) == "{}" -assert_raises(TypeError, format, {}, 'b', _msg='format_spec not empty for dict') +assert_raises(TypeError, format, {}, "b", _msg="format_spec not empty for dict") + class BadFormat: def __format__(self, spec): return 42 + + assert_raises(TypeError, format, BadFormat()) + def test_zero_padding(): i = 1 - assert f'{i:04d}' == '0001' + assert f"{i:04d}" == "0001" + test_zero_padding() -assert '{:,}'.format(100) == '100' -assert '{:,}'.format(1024) == '1,024' -assert '{:_}'.format(65536) == '65_536' -assert '{:_}'.format(4294967296) == '4_294_967_296' -assert f'{100:_}' == '100' -assert f'{1024:_}' == '1_024' -assert f'{65536:,}' == '65,536' -assert f'{4294967296:,}' == '4,294,967,296' -assert 'F' == "{0:{base}}".format(15, base="X") -assert f'{255:#X}' == "0XFF" +assert "{:,}".format(100) == "100" +assert "{:,}".format(1024) == "1,024" +assert "{:_}".format(65536) == "65_536" +assert "{:_}".format(4294967296) == "4_294_967_296" +assert f"{100:_}" == "100" +assert f"{1024:_}" == "1_024" +assert f"{65536:,}" == "65,536" +assert f"{4294967296:,}" == "4,294,967,296" +assert "F" == "{0:{base}}".format(15, base="X") +assert f"{255:#X}" == "0XFF" assert f"{65:c}" == "A" -assert f"{0x1f5a5:c}" == "🖥" -assert_raises(ValueError, "{:+c}".format, 1, _msg="Sign not allowed with integer format specifier 'c'") -assert_raises(ValueError, "{:#c}".format, 1, _msg="Alternate form (#) not allowed with integer format specifier 'c'") +assert f"{0x1F5A5:c}" == "🖥" +assert_raises( + ValueError, + "{:+c}".format, + 1, + _msg="Sign not allowed with integer format specifier 'c'", +) +assert_raises( + ValueError, + "{:#c}".format, + 1, + _msg="Alternate form (#) not allowed with integer format specifier 'c'", +) assert f"{256:#010x}" == "0x00000100" assert f"{256:0=#10x}" == "0x00000100" assert f"{256:0>#10x}" == "000000x100" @@ -66,14 +81,31 @@ def test_zero_padding(): assert f"{123.456:+011,}" == "+00,123.456" assert f"{1234:.3g}" == "1.23e+03" assert f"{1234567:.6G}" == "1.23457E+06" -assert f'{"🐍":4}' == "🐍 " -assert_raises(ValueError, "{:,o}".format, 1, _msg="ValueError: Cannot specify ',' with 'o'.") -assert_raises(ValueError, "{:_n}".format, 1, _msg="ValueError: Cannot specify '_' with 'n'.") -assert_raises(ValueError, "{:,o}".format, 1.0, _msg="ValueError: Cannot specify ',' with 'o'.") -assert_raises(ValueError, "{:_n}".format, 1.0, _msg="ValueError: Cannot specify '_' with 'n'.") -assert_raises(ValueError, "{:,}".format, "abc", _msg="ValueError: Cannot specify ',' with 's'.") -assert_raises(ValueError, "{:,x}".format, "abc", _msg="ValueError: Cannot specify ',' with 'x'.") -assert_raises(OverflowError, "{:c}".format, 0x110000, _msg="OverflowError: %c arg not in range(0x110000)") +assert f"{'🐍':4}" == "🐍 " +assert_raises( + ValueError, "{:,o}".format, 1, _msg="ValueError: Cannot specify ',' with 'o'." +) +assert_raises( + ValueError, "{:_n}".format, 1, _msg="ValueError: Cannot specify '_' with 'n'." +) +assert_raises( + ValueError, "{:,o}".format, 1.0, _msg="ValueError: Cannot specify ',' with 'o'." +) +assert_raises( + ValueError, "{:_n}".format, 1.0, _msg="ValueError: Cannot specify '_' with 'n'." +) +assert_raises( + ValueError, "{:,}".format, "abc", _msg="ValueError: Cannot specify ',' with 's'." +) +assert_raises( + ValueError, "{:,x}".format, "abc", _msg="ValueError: Cannot specify ',' with 'x'." +) +assert_raises( + OverflowError, + "{:c}".format, + 0x110000, + _msg="OverflowError: %c arg not in range(0x110000)", +) assert f"{3:f}" == "3.000000" assert f"{3.1415:.0f}" == "3" assert f"{3.1415:.1f}" == "3.1" @@ -115,14 +147,14 @@ def test_zero_padding(): assert f"{3.1415:#.4e}" == "3.1415e+00" assert f"{3.1415:#.5e}" == "3.14150e+00" assert f"{3.1415:#.5E}" == "3.14150E+00" -assert f"{3.1415:.0%}" == '314%' -assert f"{3.1415:.1%}" == '314.2%' -assert f"{3.1415:.2%}" == '314.15%' -assert f"{3.1415:.3%}" == '314.150%' -assert f"{3.1415:#.0%}" == '314.%' -assert f"{3.1415:#.1%}" == '314.2%' -assert f"{3.1415:#.2%}" == '314.15%' -assert f"{3.1415:#.3%}" == '314.150%' +assert f"{3.1415:.0%}" == "314%" +assert f"{3.1415:.1%}" == "314.2%" +assert f"{3.1415:.2%}" == "314.15%" +assert f"{3.1415:.3%}" == "314.150%" +assert f"{3.1415:#.0%}" == "314.%" +assert f"{3.1415:#.1%}" == "314.2%" +assert f"{3.1415:#.2%}" == "314.15%" +assert f"{3.1415:#.3%}" == "314.150%" assert f"{3.1415:.0}" == "3e+00" assert f"{3.1415:.1}" == "3e+00" assert f"{3.1415:.2}" == "3.1" @@ -137,5 +169,5 @@ def test_zero_padding(): # test issue 4558 x = 123456789012345678901234567890 for i in range(0, 30): - format(x, ',') + format(x, ",") x = x // 10 diff --git a/extra_tests/snippets/builtin_hash.py b/extra_tests/snippets/builtin_hash.py index bd98199db9..96ccc46ba8 100644 --- a/extra_tests/snippets/builtin_hash.py +++ b/extra_tests/snippets/builtin_hash.py @@ -1,4 +1,3 @@ - from testutils import assert_raises diff --git a/extra_tests/snippets/builtin_hex.py b/extra_tests/snippets/builtin_hex.py index fac5e09c22..740817bc42 100644 --- a/extra_tests/snippets/builtin_hex.py +++ b/extra_tests/snippets/builtin_hex.py @@ -1,6 +1,6 @@ from testutils import assert_raises -assert hex(16) == '0x10' -assert hex(-16) == '-0x10' +assert hex(16) == "0x10" +assert hex(-16) == "-0x10" -assert_raises(TypeError, hex, {}, _msg='ord() called with dict') +assert_raises(TypeError, hex, {}, _msg="ord() called with dict") diff --git a/extra_tests/snippets/builtin_isinstance.py b/extra_tests/snippets/builtin_isinstance.py index c02f331d25..866c83f7cc 100644 --- a/extra_tests/snippets/builtin_isinstance.py +++ b/extra_tests/snippets/builtin_isinstance.py @@ -1,4 +1,3 @@ - class Regular: pass @@ -41,14 +40,17 @@ class AlwaysInstanceOf(metaclass=MCAlwaysInstanceOf): assert isinstance(Regular(), AlwaysInstanceOf) assert isinstance(1, AlwaysInstanceOf) + class GenericInstance: def __instancecheck__(self, _): return True + assert isinstance(Regular(), GenericInstance()) assert isinstance([], GenericInstance()) assert isinstance(1, GenericInstance()) + class MCReturnInt(type): def __instancecheck__(self, instance): return 3 @@ -60,4 +62,13 @@ class ReturnInt(metaclass=MCReturnInt): assert isinstance("a", ReturnInt) is True -assert isinstance(1, ((int, float,), str)) +assert isinstance( + 1, + ( + ( + int, + float, + ), + str, + ), +) diff --git a/extra_tests/snippets/builtin_issubclass.py b/extra_tests/snippets/builtin_issubclass.py index 7f1d87abb1..7c047515d4 100644 --- a/extra_tests/snippets/builtin_issubclass.py +++ b/extra_tests/snippets/builtin_issubclass.py @@ -1,4 +1,3 @@ - class A: pass @@ -49,14 +48,17 @@ class InheritedAlwaysSubClass(AlwaysSubClass): assert issubclass(InheritedAlwaysSubClass, AlwaysSubClass) assert issubclass(AlwaysSubClass, InheritedAlwaysSubClass) + class GenericInstance: def __subclasscheck__(self, _): return True + assert issubclass(A, GenericInstance()) assert issubclass(list, GenericInstance()) assert issubclass([], GenericInstance()) + class MCAVirtualSubClass(type): def __subclasscheck__(self, subclass): return subclass is A diff --git a/extra_tests/snippets/builtin_len.py b/extra_tests/snippets/builtin_len.py index 4872f20c42..4190e31698 100644 --- a/extra_tests/snippets/builtin_len.py +++ b/extra_tests/snippets/builtin_len.py @@ -1,2 +1,2 @@ -assert 3 == len([1,2,3]) -assert 2 == len((1,2)) +assert 3 == len([1, 2, 3]) +assert 2 == len((1, 2)) diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index b5c08796ba..cb02228c9f 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -12,38 +12,85 @@ assert y == [2, 1, 2, 3, 1, 2, 3] a = [] -a.extend((1,2,3,4)) +a.extend((1, 2, 3, 4)) assert a == [1, 2, 3, 4] -a.extend('abcdefg') -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g'] +a.extend("abcdefg") +assert a == [1, 2, 3, 4, "a", "b", "c", "d", "e", "f", "g"] a.extend(range(10)) -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, +] a = [] -a.extend({1,2,3,4}) +a.extend({1, 2, 3, 4}) assert a == [1, 2, 3, 4] -a.extend({'a': 1, 'b': 2, 'z': 51}) -assert a == [1, 2, 3, 4, 'a', 'b', 'z'] +a.extend({"a": 1, "b": 2, "z": 51}) +assert a == [1, 2, 3, 4, "a", "b", "z"] + class Iter: def __iter__(self): yield 12 yield 28 + a.extend(Iter()) -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28] +assert a == [1, 2, 3, 4, "a", "b", "z", 12, 28] + +a.extend(bytes(b"hello world")) +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "z", + 12, + 28, + 104, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, +] -a.extend(bytes(b'hello world')) -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100] class Next: def __next__(self): yield 12 yield 28 + assert_raises(TypeError, lambda: [].extend(3)) assert_raises(TypeError, lambda: [].extend(slice(0, 10, 1))) @@ -56,12 +103,12 @@ def __next__(self): assert x == [1, 2, 3] * 2 # index() -assert ['a', 'b', 'c'].index('b') == 1 +assert ["a", "b", "c"].index("b") == 1 assert [5, 6, 7].index(7) == 2 -assert_raises(ValueError, lambda: ['a', 'b', 'c'].index('z')) +assert_raises(ValueError, lambda: ["a", "b", "c"].index("z")) -x = [[1,0,-3], 'a', 1] -y = [[3,2,1], 'z', 2] +x = [[1, 0, -3], "a", 1] +y = [[3, 2, 1], "z", 2] assert x < y, "list __lt__ failed" x = [5, 13, 31] @@ -73,9 +120,12 @@ def __next__(self): assert x.pop() == 2 assert x == [0, 1] + def test_pop(lst, idx, value, new_lst): assert lst.pop(idx) == value assert lst == new_lst + + test_pop([0, 1, 2], -1, 2, [0, 1]) test_pop([0, 1, 2], 0, 0, [1, 2]) test_pop([0, 1, 2], 1, 1, [0, 2]) @@ -91,23 +141,23 @@ def test_pop(lst, idx, value, new_lst): assert repr(recursive) == "[[...]]" # insert() -x = ['a', 'b', 'c'] -x.insert(0, 'z') # insert is in-place, no return value -assert x == ['z', 'a', 'b', 'c'] +x = ["a", "b", "c"] +x.insert(0, "z") # insert is in-place, no return value +assert x == ["z", "a", "b", "c"] -x = ['a', 'b', 'c'] -x.insert(100, 'z') -assert x == ['a', 'b', 'c', 'z'] +x = ["a", "b", "c"] +x.insert(100, "z") +assert x == ["a", "b", "c", "z"] -x = ['a', 'b', 'c'] -x.insert(-1, 'z') -assert x == ['a', 'b', 'z', 'c'] +x = ["a", "b", "c"] +x.insert(-1, "z") +assert x == ["a", "b", "z", "c"] -x = ['a', 'b', 'c'] -x.insert(-100, 'z') -assert x == ['z', 'a', 'b', 'c'] +x = ["a", "b", "c"] +x.insert(-100, "z") +assert x == ["z", "a", "b", "c"] -assert_raises(OverflowError, lambda: x.insert(100000000000000000000, 'z')) +assert_raises(OverflowError, lambda: x.insert(100000000000000000000, "z")) x = [[], 2, {}] y = x.copy() @@ -123,7 +173,7 @@ def test_pop(lst, idx, value, new_lst): assert len(a) == 2 assert not 1 in a -assert_raises(ValueError, lambda: a.remove(10), _msg='Remove not exist element') +assert_raises(ValueError, lambda: a.remove(10), _msg="Remove not exist element") foo = bar = [1] foo += [2] @@ -138,10 +188,12 @@ def test_pop(lst, idx, value, new_lst): x.remove(x) assert x not in x + class Foo(object): def __eq__(self, x): return False + foo = Foo() foo1 = Foo() x = [1, foo, 2, foo, []] @@ -173,17 +225,17 @@ def __eq__(self, x): assert [foo] == [foo] for size in [1, 2, 3, 4, 5, 8, 10, 100, 1000]: - lst = list(range(size)) - orig = lst[:] - lst.sort() - assert lst == orig - assert sorted(lst) == orig - assert_raises(ZeroDivisionError, sorted, lst, key=lambda x: 1/x) - lst.reverse() - assert sorted(lst) == orig - assert sorted(lst, reverse=True) == lst - assert sorted(lst, key=lambda x: -x) == lst - assert sorted(lst, key=lambda x: -x, reverse=True) == orig + lst = list(range(size)) + orig = lst[:] + lst.sort() + assert lst == orig + assert sorted(lst) == orig + assert_raises(ZeroDivisionError, sorted, lst, key=lambda x: 1 / x) + lst.reverse() + assert sorted(lst) == orig + assert sorted(lst, reverse=True) == lst + assert sorted(lst, key=lambda x: -x) == lst + assert sorted(lst, key=lambda x: -x, reverse=True) == orig assert sorted([(1, 2, 3), (0, 3, 6)]) == [(0, 3, 6), (1, 2, 3)] assert sorted([(1, 2, 3), (0, 3, 6)], key=lambda x: x[0]) == [(0, 3, 6), (1, 2, 3)] @@ -191,34 +243,52 @@ def __eq__(self, x): assert sorted([(1, 2), (), (5,)], key=len) == [(), (5,), (1, 2)] lst = [3, 1, 5, 2, 4] + + class C: - def __init__(self, x): self.x = x - def __lt__(self, other): return self.x < other.x + def __init__(self, x): + self.x = x + + def __lt__(self, other): + return self.x < other.x + + lst.sort(key=C) assert lst == [1, 2, 3, 4, 5] lst = [3, 1, 5, 2, 4] + + class C: - def __init__(self, x): self.x = x - def __gt__(self, other): return self.x > other.x + def __init__(self, x): + self.x = x + + def __gt__(self, other): + return self.x > other.x + + lst.sort(key=C) assert lst == [1, 2, 3, 4, 5] lst = [5, 1, 2, 3, 4] + + def f(x): lst.append(1) return x -assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" + + +assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" assert lst == [1, 2, 3, 4, 5] # __delitem__ -x = ['a', 'b', 'c'] +x = ["a", "b", "c"] del x[0] -assert x == ['b', 'c'] +assert x == ["b", "c"] -x = ['a', 'b', 'c'] +x = ["a", "b", "c"] del x[-1] -assert x == ['a', 'b'] +assert x == ["a", "b"] x = y = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] del x[2:14:3] @@ -232,24 +302,30 @@ def f(x): x = list(range(12)) del x[10:2:-2] -assert x == [0,1,2,3,5,7,9,11] +assert x == [0, 1, 2, 3, 5, 7, 9, 11] + def bad_del_1(): - del ['a', 'b']['a'] + del ["a", "b"]["a"] + + assert_raises(TypeError, bad_del_1) + def bad_del_2(): - del ['a', 'b'][2] + del ["a", "b"][2] + + assert_raises(IndexError, bad_del_2) # __setitem__ # simple index x = [1, 2, 3, 4, 5] -x[0] = 'a' -assert x == ['a', 2, 3, 4, 5] -x[-1] = 'b' -assert x == ['a', 2, 3, 4, 'b'] +x[0] = "a" +assert x == ["a", 2, 3, 4, 5] +x[-1] = "b" +assert x == ["a", 2, 3, 4, "b"] # make sure refrences are assigned correctly y = [] x[1] = y @@ -257,14 +333,17 @@ def bad_del_2(): assert x[1] == y assert x[1] == [100] -#index bounds + +# index bounds def set_index_out_of_bounds_high(): - x = [0, 1, 2, 3, 4] - x[5] = 'a' + x = [0, 1, 2, 3, 4] + x[5] = "a" + def set_index_out_of_bounds_low(): - x = [0, 1, 2, 3, 4] - x[-6] = 'a' + x = [0, 1, 2, 3, 4] + x[-6] = "a" + assert_raises(IndexError, set_index_out_of_bounds_high) assert_raises(IndexError, set_index_out_of_bounds_low) @@ -275,20 +354,20 @@ def set_index_out_of_bounds_low(): y = a[:] assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # replace whole list -x[:] = ['a', 'b', 'c'] -y[::1] = ['a', 'b', 'c'] -assert x == ['a', 'b', 'c'] +x[:] = ["a", "b", "c"] +y[::1] = ["a", "b", "c"] +assert x == ["a", "b", "c"] assert x == y # splice list start x = a[:] y = a[:] z = a[:] zz = a[:] -x[:1] = ['a', 'b', 'c'] -y[0:1] = ['a', 'b', 'c'] -z[:1:1] = ['a', 'b', 'c'] -zz[0:1:1] = ['a', 'b', 'c'] -assert x == ['a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x[:1] = ["a", "b", "c"] +y[0:1] = ["a", "b", "c"] +z[:1:1] = ["a", "b", "c"] +zz[0:1:1] = ["a", "b", "c"] +assert x == ["a", "b", "c", 1, 2, 3, 4, 5, 6, 7, 8, 9] assert x == y assert x == z assert x == zz @@ -297,11 +376,11 @@ def set_index_out_of_bounds_low(): y = a[:] z = a[:] zz = a[:] -x[5:] = ['a', 'b', 'c'] -y[5::1] = ['a', 'b', 'c'] -z[5:10] = ['a', 'b', 'c'] -zz[5:10:1] = ['a', 'b', 'c'] -assert x == [0, 1, 2, 3, 4, 'a', 'b', 'c'] +x[5:] = ["a", "b", "c"] +y[5::1] = ["a", "b", "c"] +z[5:10] = ["a", "b", "c"] +zz[5:10:1] = ["a", "b", "c"] +assert x == [0, 1, 2, 3, 4, "a", "b", "c"] assert x == y assert x == z assert x == zz @@ -310,11 +389,11 @@ def set_index_out_of_bounds_low(): y = a[:] z = a[:] zz = a[:] -x[1:1] = ['a', 'b', 'c'] -y[1:0] = ['a', 'b', 'c'] -z[1:1:1] = ['a', 'b', 'c'] -zz[1:0:1] = ['a', 'b', 'c'] -assert x == [0, 'a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x[1:1] = ["a", "b", "c"] +y[1:0] = ["a", "b", "c"] +z[1:1:1] = ["a", "b", "c"] +zz[1:0:1] = ["a", "b", "c"] +assert x == [0, "a", "b", "c", 1, 2, 3, 4, 5, 6, 7, 8, 9] assert x == y assert x == z assert x == zz @@ -323,24 +402,24 @@ def set_index_out_of_bounds_low(): y = a[:] z = a[:] zz = a[:] -x[-1:-1] = ['a', 'b', 'c'] -y[-1:9] = ['a', 'b', 'c'] -z[-1:-1:1] = ['a', 'b', 'c'] -zz[-1:9:1] = ['a', 'b', 'c'] -assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c', 9] +x[-1:-1] = ["a", "b", "c"] +y[-1:9] = ["a", "b", "c"] +z[-1:-1:1] = ["a", "b", "c"] +zz[-1:9:1] = ["a", "b", "c"] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, "a", "b", "c", 9] assert x == y assert x == z assert x == zz # splice mid x = a[:] y = a[:] -x[3:5] = ['a', 'b', 'c', 'd', 'e'] -y[3:5:1] = ['a', 'b', 'c', 'd', 'e'] -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 5, 6, 7, 8, 9] +x[3:5] = ["a", "b", "c", "d", "e"] +y[3:5:1] = ["a", "b", "c", "d", "e"] +assert x == [0, 1, 2, "a", "b", "c", "d", "e", 5, 6, 7, 8, 9] assert x == y x = a[:] -x[3:5] = ['a'] -assert x == [0, 1, 2, 'a', 5, 6, 7, 8, 9] +x[3:5] = ["a"] +assert x == [0, 1, 2, "a", 5, 6, 7, 8, 9] # assign empty to non stepped empty slice does nothing x = a[:] y = a[:] @@ -359,84 +438,93 @@ def set_index_out_of_bounds_low(): yy = [] x = a[:] y = a[:] -x[3:5] = ['a', 'b', 'c', 'd', yy] -y[3:5:1] = ['a', 'b', 'c', 'd', yy] -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [], 5, 6, 7, 8, 9] +x[3:5] = ["a", "b", "c", "d", yy] +y[3:5:1] = ["a", "b", "c", "d", yy] +assert x == [0, 1, 2, "a", "b", "c", "d", [], 5, 6, 7, 8, 9] assert x == y yy.append(100) -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [100], 5, 6, 7, 8, 9] +assert x == [0, 1, 2, "a", "b", "c", "d", [100], 5, 6, 7, 8, 9] assert x == y assert x[7] == yy assert x[7] == [100] assert y[7] == yy assert y[7] == [100] + # no zero step def no_zero_step_set(): - x = [1, 2, 3, 4, 5] - x[0:4:0] = [11, 12, 13, 14, 15] + x = [1, 2, 3, 4, 5] + x[0:4:0] = [11, 12, 13, 14, 15] + + assert_raises(ValueError, no_zero_step_set) # stepped slice index # forward slice x = a[:] -x[2:8:2] = ['a', 'b', 'c'] -assert x == [0, 1, 'a', 3, 'b', 5, 'c', 7, 8, 9] +x[2:8:2] = ["a", "b", "c"] +assert x == [0, 1, "a", 3, "b", 5, "c", 7, 8, 9] x = a[:] y = a[:] z = a[:] zz = a[:] -c = ['a', 'b', 'c', 'd', 'e'] +c = ["a", "b", "c", "d", "e"] x[::2] = c y[-10::2] = c z[0:10:2] = c -zz[-13:13:2] = c # slice indexes will be truncated to bounds -assert x == ['a', 1, 'b', 3, 'c', 5, 'd', 7, 'e', 9] +zz[-13:13:2] = c # slice indexes will be truncated to bounds +assert x == ["a", 1, "b", 3, "c", 5, "d", 7, "e", 9] assert x == y assert x == z assert x == zz # backward slice x = a[:] -x[8:2:-2] = ['a', 'b', 'c'] -assert x == [0, 1, 2, 3, 'c', 5, 'b', 7, 'a', 9] +x[8:2:-2] = ["a", "b", "c"] +assert x == [0, 1, 2, 3, "c", 5, "b", 7, "a", 9] x = a[:] y = a[:] z = a[:] zz = a[:] -c = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] +c = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] x[::-1] = c y[9:-11:-1] = c z[9::-1] = c -zz[11:-13:-1] = c # slice indexes will be truncated to bounds -assert x == ['j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'] +zz[11:-13:-1] = c # slice indexes will be truncated to bounds +assert x == ["j", "i", "h", "g", "f", "e", "d", "c", "b", "a"] assert x == y assert x == z assert x == zz # step size bigger than len x = a[:] -x[::200] = ['a'] -assert x == ['a', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x[::200] = ["a"] +assert x == ["a", 1, 2, 3, 4, 5, 6, 7, 8, 9] x = a[:] -x[5::200] = ['a'] -assert x == [0, 1, 2, 3, 4, 'a', 6, 7, 8, 9] +x[5::200] = ["a"] +assert x == [0, 1, 2, 3, 4, "a", 6, 7, 8, 9] + # bad stepped slices def stepped_slice_assign_too_big(): - x = [0, 1, 2, 3, 4] - x[::2] = ['a', 'b', 'c', 'd'] + x = [0, 1, 2, 3, 4] + x[::2] = ["a", "b", "c", "d"] + assert_raises(ValueError, stepped_slice_assign_too_big) + def stepped_slice_assign_too_small(): - x = [0, 1, 2, 3, 4] - x[::2] = ['a', 'b'] + x = [0, 1, 2, 3, 4] + x[::2] = ["a", "b"] + assert_raises(ValueError, stepped_slice_assign_too_small) + # must assign iter t0 slice def must_assign_iter_to_slice(): - x = [0, 1, 2, 3, 4] - x[::2] = 42 + x = [0, 1, 2, 3, 4] + x[::2] = 42 + assert_raises(TypeError, must_assign_iter_to_slice) @@ -446,74 +534,87 @@ def must_assign_iter_to_slice(): # string x = a[:] x[3:8] = "abcdefghi" -assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 8, 9] +assert x == [0, 1, 2, "a", "b", "c", "d", "e", "f", "g", "h", "i", 8, 9] # tuple x = a[:] x[3:8] = (11, 12, 13, 14, 15) assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] + # class # __next__ class CIterNext: - def __init__(self, sec=(1, 2, 3)): - self.sec = sec - self.index = 0 - def __iter__(self): - return self - def __next__(self): - if self.index >= len(self.sec): - raise StopIteration - v = self.sec[self.index] - self.index += 1 - return v + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + self.index = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.index >= len(self.sec): + raise StopIteration + v = self.sec[self.index] + self.index += 1 + return v + x = list(range(10)) x[3:8] = CIterNext() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + # __iter__ yield class CIter: - def __init__(self, sec=(1, 2, 3)): - self.sec = sec - def __iter__(self): - for n in self.sec: - yield n + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + + def __iter__(self): + for n in self.sec: + yield n + x = list(range(10)) x[3:8] = CIter() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + # __getitem but no __iter__ sequence class CGetItem: - def __init__(self, sec=(1, 2, 3)): - self.sec = sec - def __getitem__(self, sub): - return self.sec[sub] + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + + def __getitem__(self, sub): + return self.sec[sub] + x = list(range(10)) x[3:8] = CGetItem() assert x == [0, 1, 2, 1, 2, 3, 8, 9] + # iter raises error class CIterError: - def __iter__(self): - for i in range(10): - if i > 5: - raise RuntimeError - yield i + def __iter__(self): + for i in range(10): + if i > 5: + raise RuntimeError + yield i + def bad_iter_assign(): - x = list(range(10)) - x[3:8] = CIterError() + x = list(range(10)) + x[3:8] = CIterError() + assert_raises(RuntimeError, bad_iter_assign) # slice assign when step or stop is -1 a = list(range(10)) x = a[:] -x[-1:-5:-1] = ['a', 'b', 'c', 'd'] -assert x == [0, 1, 2, 3, 4, 5, 'd', 'c', 'b', 'a'] +x[-1:-5:-1] = ["a", "b", "c", "d"] +assert x == [0, 1, 2, 3, 4, 5, "d", "c", "b", "a"] x = a[:] x[-5:-1:-1] = [] assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -557,17 +658,17 @@ def bad_iter_assign(): assert not [0, 0] > [0, 0] assert not [0, 0] < [0, 0] -assert not [float('nan'), float('nan')] <= [float('nan'), 1] -assert not [float('nan'), float('nan')] <= [float('nan'), float('nan')] -assert not [float('nan'), float('nan')] >= [float('nan'), float('nan')] -assert not [float('nan'), float('nan')] < [float('nan'), float('nan')] -assert not [float('nan'), float('nan')] > [float('nan'), float('nan')] +assert not [float("nan"), float("nan")] <= [float("nan"), 1] +assert not [float("nan"), float("nan")] <= [float("nan"), float("nan")] +assert not [float("nan"), float("nan")] >= [float("nan"), float("nan")] +assert not [float("nan"), float("nan")] < [float("nan"), float("nan")] +assert not [float("nan"), float("nan")] > [float("nan"), float("nan")] -assert [float('inf'), float('inf')] >= [float('inf'), 1] -assert [float('inf'), float('inf')] <= [float('inf'), float('inf')] -assert [float('inf'), float('inf')] >= [float('inf'), float('inf')] -assert not [float('inf'), float('inf')] < [float('inf'), float('inf')] -assert not [float('inf'), float('inf')] > [float('inf'), float('inf')] +assert [float("inf"), float("inf")] >= [float("inf"), 1] +assert [float("inf"), float("inf")] <= [float("inf"), float("inf")] +assert [float("inf"), float("inf")] >= [float("inf"), float("inf")] +assert not [float("inf"), float("inf")] < [float("inf"), float("inf")] +assert not [float("inf"), float("inf")] > [float("inf"), float("inf")] # list __iadd__ a = [] @@ -575,62 +676,111 @@ def bad_iter_assign(): assert a == [1, 2, 3] a = [] -a += (1,2,3,4) +a += (1, 2, 3, 4) assert a == [1, 2, 3, 4] -a += 'abcdefg' -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g'] +a += "abcdefg" +assert a == [1, 2, 3, 4, "a", "b", "c", "d", "e", "f", "g"] a += range(10) -assert a == [1, 2, 3, 4, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "c", + "d", + "e", + "f", + "g", + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, +] a = [] -a += {1,2,3,4} +a += {1, 2, 3, 4} assert a == [1, 2, 3, 4] -a += {'a': 1, 'b': 2, 'z': 51} -assert a == [1, 2, 3, 4, 'a', 'b', 'z'] +a += {"a": 1, "b": 2, "z": 51} +assert a == [1, 2, 3, 4, "a", "b", "z"] + class Iter: def __iter__(self): yield 12 yield 28 + a += Iter() -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28] +assert a == [1, 2, 3, 4, "a", "b", "z", 12, 28] + +a += bytes(b"hello world") +assert a == [ + 1, + 2, + 3, + 4, + "a", + "b", + "z", + 12, + 28, + 104, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, +] -a += bytes(b'hello world') -assert a == [1, 2, 3, 4, 'a', 'b', 'z', 12, 28, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100] class Next: def __next__(self): yield 12 yield 28 + def iadd_int(): a = [] a += 3 + def iadd_slice(): a = [] a += slice(0, 10, 1) + assert_raises(TypeError, iadd_int) assert_raises(TypeError, iadd_slice) -it = iter([1,2,3,4]) +it = iter([1, 2, 3, 4]) assert it.__length_hint__() == 4 assert next(it) == 1 assert it.__length_hint__() == 3 -assert list(it) == [2,3,4] +assert list(it) == [2, 3, 4] assert it.__length_hint__() == 0 -it = reversed([1,2,3,4]) +it = reversed([1, 2, 3, 4]) assert it.__length_hint__() == 4 assert next(it) == 4 assert it.__length_hint__() == 3 -assert list(it) == [3,2,1] +assert list(it) == [3, 2, 1] assert it.__length_hint__() == 0 a = [*[1, 2], 3, *[4, 5]] @@ -648,21 +798,25 @@ def iadd_slice(): class C(base): def __iter__(self): a.append(2) + def inner(): yield 3 a.append(4) + return inner() a = [1] b = [*a, *C(), *a.copy()] assert b == [1, 3, 1, 2, 4] + # Test for list entering daedlock or not (https://github.com/RustPython/RustPython/pull/2933) class MutatingCompare: def __eq__(self, other): self.list.pop() return True + m = MutatingCompare() l = [1, 2, 3, m, 4] @@ -675,18 +829,21 @@ def __eq__(self, other): l = [1, 2, 3, m, 4] m.list = l -l.remove(4) -assert_raises(ValueError, lambda: l.index(4)) # element 4 must not be in the list +l.remove(4) +assert_raises(ValueError, lambda: l.index(4)) # element 4 must not be in the list + # Test no panic occurred when list elements was deleted in __eq__ class rewrite_list_eq(list): pass + class poc: def __eq__(self, other): list1.clear() return self + list1 = rewrite_list_eq([poc()]) list1.remove(list1) assert list1 == [] diff --git a/extra_tests/snippets/builtin_locals.py b/extra_tests/snippets/builtin_locals.py index 6f3fd847c4..a10cfa389c 100644 --- a/extra_tests/snippets/builtin_locals.py +++ b/extra_tests/snippets/builtin_locals.py @@ -1,19 +1,18 @@ - a = 5 b = 6 loc = locals() -assert loc['a'] == 5 -assert loc['b'] == 6 +assert loc["a"] == 5 +assert loc["b"] == 6 -def f(): - c = 4 - a = 7 - loc = locals() +def f(): + c = 4 + a = 7 - assert loc['a'] == 4 - assert loc['c'] == 7 - assert not 'b' in loc + loc = locals() + assert loc["a"] == 4 + assert loc["c"] == 7 + assert not "b" in loc diff --git a/extra_tests/snippets/builtin_map.py b/extra_tests/snippets/builtin_map.py index 0de8d2c597..559d108e38 100644 --- a/extra_tests/snippets/builtin_map.py +++ b/extra_tests/snippets/builtin_map.py @@ -1,5 +1,5 @@ a = list(map(str, [1, 2, 3])) -assert a == ['1', '2', '3'] +assert a == ["1", "2", "3"] b = list(map(lambda x, y: x + y, [1, 2, 4], [3, 5])) @@ -20,7 +20,7 @@ def __iter__(self): return self -it = map(lambda x: x+1, Counter()) +it = map(lambda x: x + 1, Counter()) assert next(it) == 2 assert next(it) == 3 diff --git a/extra_tests/snippets/builtin_mappingproxy.py b/extra_tests/snippets/builtin_mappingproxy.py index cfba56a8df..fdd653c408 100644 --- a/extra_tests/snippets/builtin_mappingproxy.py +++ b/extra_tests/snippets/builtin_mappingproxy.py @@ -1,5 +1,6 @@ from testutils import assert_raises + class A(dict): def a(): pass @@ -8,16 +9,16 @@ def b(): pass -assert A.__dict__['a'] == A.a +assert A.__dict__["a"] == A.a with assert_raises(KeyError) as cm: - A.__dict__['not here'] + A.__dict__["not here"] assert cm.exception.args[0] == "not here" -assert 'b' in A.__dict__ -assert 'c' not in A.__dict__ +assert "b" in A.__dict__ +assert "c" not in A.__dict__ -assert '__dict__' in A.__dict__ +assert "__dict__" in A.__dict__ assert A.__dict__.get("not here", "default") == "default" assert A.__dict__.get("a", "default") is A.a diff --git a/extra_tests/snippets/builtin_max.py b/extra_tests/snippets/builtin_max.py index cb62123656..fbb0626768 100644 --- a/extra_tests/snippets/builtin_max.py +++ b/extra_tests/snippets/builtin_max.py @@ -3,17 +3,22 @@ # simple values assert max(0, 0) == 0 assert max(1, 0) == 1 -assert max(1., 0.) == 1. +assert max(1.0, 0.0) == 1.0 assert max(-1, 0) == 0 assert max(1, 2, 3) == 3 # iterables assert max([1, 2, 3]) == 3 assert max((1, 2, 3)) == 3 -assert max({ - "a": 0, - "b": 1, -}) == "b" +assert ( + max( + { + "a": 0, + "b": 1, + } + ) + == "b" +) assert max([1, 2], default=0) == 2 assert max([], default=0) == 0 assert_raises(ValueError, max, []) @@ -30,7 +35,7 @@ # custom class -class MyComparable(): +class MyComparable: nb = 0 def __init__(self): @@ -47,7 +52,7 @@ def __gt__(self, other): assert max([first, second]) == second -class MyNotComparable(): +class MyNotComparable: pass diff --git a/extra_tests/snippets/builtin_memoryview.py b/extra_tests/snippets/builtin_memoryview.py index 81cd5015c1..f206056ebf 100644 --- a/extra_tests/snippets/builtin_memoryview.py +++ b/extra_tests/snippets/builtin_memoryview.py @@ -10,48 +10,52 @@ assert hash(obj) == hash(a) -class A(array.array): - ... -class B(bytes): - ... +class A(array.array): ... -class C(): - ... -memoryview(bytearray('abcde', encoding='utf-8')) -memoryview(array.array('i', [1, 2, 3])) -memoryview(A('b', [0])) -memoryview(B('abcde', encoding='utf-8')) +class B(bytes): ... + + +class C: ... + + +memoryview(bytearray("abcde", encoding="utf-8")) +memoryview(array.array("i", [1, 2, 3])) +memoryview(A("b", [0])) +memoryview(B("abcde", encoding="utf-8")) assert_raises(TypeError, lambda: memoryview([1, 2, 3])) assert_raises(TypeError, lambda: memoryview((1, 2, 3))) assert_raises(TypeError, lambda: memoryview({})) -assert_raises(TypeError, lambda: memoryview('string')) +assert_raises(TypeError, lambda: memoryview("string")) assert_raises(TypeError, lambda: memoryview(C())) + def test_slice(): - b = b'123456789' + b = b"123456789" m = memoryview(b) m2 = memoryview(b) assert m == m assert m == m2 - assert m.tobytes() == b'123456789' + assert m.tobytes() == b"123456789" assert m == b - assert m[::2].tobytes() == b'13579' - assert m[::2] == b'13579' - assert m[1::2].tobytes() == b'2468' - assert m[::2][1:].tobytes() == b'3579' - assert m[::2][1:-1].tobytes() == b'357' - assert m[::2][::2].tobytes() == b'159' - assert m[::2][1::2].tobytes() == b'37' - assert m[::-1].tobytes() == b'987654321' - assert m[::-2].tobytes() == b'97531' + assert m[::2].tobytes() == b"13579" + assert m[::2] == b"13579" + assert m[1::2].tobytes() == b"2468" + assert m[::2][1:].tobytes() == b"3579" + assert m[::2][1:-1].tobytes() == b"357" + assert m[::2][::2].tobytes() == b"159" + assert m[::2][1::2].tobytes() == b"37" + assert m[::-1].tobytes() == b"987654321" + assert m[::-2].tobytes() == b"97531" + test_slice() + def test_resizable(): - b = bytearray(b'123') + b = bytearray(b"123") b.append(4) m = memoryview(b) assert_raises(BufferError, lambda: b.append(5)) @@ -68,18 +72,21 @@ def test_resizable(): m4.release() b.append(7) + test_resizable() + def test_delitem(): - a = b'abc' + a = b"abc" b = memoryview(a) - assert_raises(TypeError, lambda : b.__delitem__()) - assert_raises(TypeError, lambda : b.__delitem__(0)) - assert_raises(TypeError, lambda : b.__delitem__(10)) - a = bytearray(b'abc') + assert_raises(TypeError, lambda: b.__delitem__()) + assert_raises(TypeError, lambda: b.__delitem__(0)) + assert_raises(TypeError, lambda: b.__delitem__(10)) + a = bytearray(b"abc") b = memoryview(a) - assert_raises(TypeError, lambda : b.__delitem__()) - assert_raises(TypeError, lambda : b.__delitem__(1)) - assert_raises(TypeError, lambda : b.__delitem__(12)) + assert_raises(TypeError, lambda: b.__delitem__()) + assert_raises(TypeError, lambda: b.__delitem__(1)) + assert_raises(TypeError, lambda: b.__delitem__(12)) + -test_delitem() \ No newline at end of file +test_delitem() diff --git a/extra_tests/snippets/builtin_min.py b/extra_tests/snippets/builtin_min.py index 50ebc91f54..fc8eebba2c 100644 --- a/extra_tests/snippets/builtin_min.py +++ b/extra_tests/snippets/builtin_min.py @@ -3,17 +3,22 @@ # simple values assert min(0, 0) == 0 assert min(1, 0) == 0 -assert min(1., 0.) == 0. +assert min(1.0, 0.0) == 0.0 assert min(-1, 0) == -1 assert min(1, 2, 3) == 1 # iterables assert min([1, 2, 3]) == 1 assert min((1, 2, 3)) == 1 -assert min({ - "a": 0, - "b": 1, -}) == "a" +assert ( + min( + { + "a": 0, + "b": 1, + } + ) + == "a" +) assert min([1, 2], default=0) == 1 assert min([], default=0) == 0 @@ -31,7 +36,7 @@ # custom class -class MyComparable(): +class MyComparable: nb = 0 def __init__(self): @@ -48,7 +53,7 @@ def __gt__(self, other): assert min([first, second]) == first -class MyNotComparable(): +class MyNotComparable: pass diff --git a/extra_tests/snippets/builtin_none.py b/extra_tests/snippets/builtin_none.py index d605f1d742..230a7229e0 100644 --- a/extra_tests/snippets/builtin_none.py +++ b/extra_tests/snippets/builtin_none.py @@ -4,19 +4,22 @@ x = None assert x is y + def none(): pass + def none2(): return None + assert none() is none() assert none() is x assert none() is none2() -assert str(None) == 'None' -assert repr(None) == 'None' +assert str(None) == "None" +assert repr(None) == "None" assert type(None)() is None assert None.__eq__(3) is NotImplemented diff --git a/extra_tests/snippets/builtin_object.py b/extra_tests/snippets/builtin_object.py index 0cb6f31d66..64486e1673 100644 --- a/extra_tests/snippets/builtin_object.py +++ b/extra_tests/snippets/builtin_object.py @@ -1,6 +1,7 @@ class MyObject: pass + assert not MyObject() == MyObject() assert MyObject() != MyObject() myobj = MyObject() @@ -21,8 +22,8 @@ class MyObject: assert obj.__eq__(obj) is True assert obj.__ne__(obj) is False -assert not hasattr(obj, 'a') -obj.__dict__ = {'a': 1} +assert not hasattr(obj, "a") +obj.__dict__ = {"a": 1} assert obj.a == 1 # Value inside the formatter goes through a different path of resolution. diff --git a/extra_tests/snippets/builtin_open.py b/extra_tests/snippets/builtin_open.py index f2c783f2a5..99dd337414 100644 --- a/extra_tests/snippets/builtin_open.py +++ b/extra_tests/snippets/builtin_open.py @@ -1,19 +1,19 @@ from testutils import assert_raises -fd = open('README.md') -assert 'RustPython' in fd.read() +fd = open("README.md") +assert "RustPython" in fd.read() -assert_raises(FileNotFoundError, open, 'DoesNotExist') +assert_raises(FileNotFoundError, open, "DoesNotExist") # Use open as a context manager -with open('README.md', 'rt') as fp: +with open("README.md", "rt") as fp: contents = fp.read() assert type(contents) == str, "type is " + str(type(contents)) -with open('README.md', 'r') as fp: +with open("README.md", "r") as fp: contents = fp.read() assert type(contents) == str, "type is " + str(type(contents)) -with open('README.md', 'rb') as fp: +with open("README.md", "rb") as fp: contents = fp.read() assert type(contents) == bytes, "type is " + str(type(contents)) diff --git a/extra_tests/snippets/builtin_ord.py b/extra_tests/snippets/builtin_ord.py index 271728b84a..e451e078c3 100644 --- a/extra_tests/snippets/builtin_ord.py +++ b/extra_tests/snippets/builtin_ord.py @@ -3,11 +3,18 @@ assert ord("a") == 97 assert ord("é") == 233 assert ord("🤡") == 129313 -assert ord(b'a') == 97 -assert ord(bytearray(b'a')) == 97 +assert ord(b"a") == 97 +assert ord(bytearray(b"a")) == 97 -assert_raises(TypeError, ord, _msg='ord() is called with no argument') -assert_raises(TypeError, ord, "", _msg='ord() is called with an empty string') -assert_raises(TypeError, ord, "ab", _msg='ord() is called with more than one character') -assert_raises(TypeError, ord, b"ab", _msg='ord() expected a character, but string of length 2 found') -assert_raises(TypeError, ord, 1, _msg='ord() expected a string, bytes or bytearray, but found int') +assert_raises(TypeError, ord, _msg="ord() is called with no argument") +assert_raises(TypeError, ord, "", _msg="ord() is called with an empty string") +assert_raises(TypeError, ord, "ab", _msg="ord() is called with more than one character") +assert_raises( + TypeError, + ord, + b"ab", + _msg="ord() expected a character, but string of length 2 found", +) +assert_raises( + TypeError, ord, 1, _msg="ord() expected a string, bytes or bytearray, but found int" +) diff --git a/extra_tests/snippets/builtin_pow.py b/extra_tests/snippets/builtin_pow.py index d31e5dd713..c769914231 100644 --- a/extra_tests/snippets/builtin_pow.py +++ b/extra_tests/snippets/builtin_pow.py @@ -12,7 +12,7 @@ assert pow(2.0, 1) == 2.0 assert pow(0, 10**1000) == 0 assert pow(1, 10**1000) == 1 -assert pow(-1, 10**1000+1) == -1 +assert pow(-1, 10**1000 + 1) == -1 assert pow(-1, 10**1000) == 1 assert pow(2, 4, 5) == 1 @@ -59,7 +59,7 @@ def powtest(type): assert_raises(ZeroDivisionError, pow, zero, exp) il, ih = -20, 20 - jl, jh = -5, 5 + jl, jh = -5, 5 kl, kh = -10, 10 asseq = assert_equal if type == float: @@ -76,10 +76,7 @@ def powtest(type): if type == float or j < 0: assert_raises(TypeError, pow, type(i), j, k) continue - asseq( - pow(type(i), j, k), - pow(type(i), j) % type(k) - ) + asseq(pow(type(i), j, k), pow(type(i), j) % type(k)) def test_powint(): @@ -92,40 +89,35 @@ def test_powfloat(): def test_other(): # Other tests-- not very systematic - assert_equal(pow(3,3) % 8, pow(3,3,8)) - assert_equal(pow(3,3) % -8, pow(3,3,-8)) - assert_equal(pow(3,2) % -2, pow(3,2,-2)) - assert_equal(pow(-3,3) % 8, pow(-3,3,8)) - assert_equal(pow(-3,3) % -8, pow(-3,3,-8)) - assert_equal(pow(5,2) % -8, pow(5,2,-8)) - - assert_equal(pow(3,3) % 8, pow(3,3,8)) - assert_equal(pow(3,3) % -8, pow(3,3,-8)) - assert_equal(pow(3,2) % -2, pow(3,2,-2)) - assert_equal(pow(-3,3) % 8, pow(-3,3,8)) - assert_equal(pow(-3,3) % -8, pow(-3,3,-8)) - assert_equal(pow(5,2) % -8, pow(5,2,-8)) + assert_equal(pow(3, 3) % 8, pow(3, 3, 8)) + assert_equal(pow(3, 3) % -8, pow(3, 3, -8)) + assert_equal(pow(3, 2) % -2, pow(3, 2, -2)) + assert_equal(pow(-3, 3) % 8, pow(-3, 3, 8)) + assert_equal(pow(-3, 3) % -8, pow(-3, 3, -8)) + assert_equal(pow(5, 2) % -8, pow(5, 2, -8)) + + assert_equal(pow(3, 3) % 8, pow(3, 3, 8)) + assert_equal(pow(3, 3) % -8, pow(3, 3, -8)) + assert_equal(pow(3, 2) % -2, pow(3, 2, -2)) + assert_equal(pow(-3, 3) % 8, pow(-3, 3, 8)) + assert_equal(pow(-3, 3) % -8, pow(-3, 3, -8)) + assert_equal(pow(5, 2) % -8, pow(5, 2, -8)) for i in range(-10, 11): for j in range(0, 6): for k in range(-7, 11): if j >= 0 and k != 0: - assert_equal( - pow(i,j) % k, - pow(i,j,k) - ) + assert_equal(pow(i, j) % k, pow(i, j, k)) if j >= 0 and k != 0: - assert_equal( - pow(int(i),j) % k, - pow(int(i),j,k) - ) + assert_equal(pow(int(i), j) % k, pow(int(i), j, k)) def test_bug643260(): class TestRpow: def __rpow__(self, other): return None - None ** TestRpow() # Won't fail when __rpow__ invoked. SF bug #643260. + + None ** TestRpow() # Won't fail when __rpow__ invoked. SF bug #643260. def test_bug705231(): @@ -141,15 +133,15 @@ def test_bug705231(): for b in range(-10, 11): eq(pow(a, float(b)), b & 1 and -1.0 or 1.0) for n in range(0, 100): - fiveto = float(5 ** n) + fiveto = float(5**n) # For small n, fiveto will be odd. Eventually we run out of # mantissa bits, though, and thereafer fiveto will be even. expected = fiveto % 2.0 and -1.0 or 1.0 eq(pow(a, fiveto), expected) eq(pow(a, -fiveto), expected) - eq(expected, 1.0) # else we didn't push fiveto to evenness + eq(expected, 1.0) # else we didn't push fiveto to evenness -tests = [f for name, f in locals().items() if name.startswith('test_')] +tests = [f for name, f in locals().items() if name.startswith("test_")] for f in tests: f() diff --git a/extra_tests/snippets/builtin_print.py b/extra_tests/snippets/builtin_print.py index db53c80edb..b9f3b8cc17 100644 --- a/extra_tests/snippets/builtin_print.py +++ b/extra_tests/snippets/builtin_print.py @@ -3,14 +3,14 @@ print(2 + 3) -assert_raises(TypeError, print, 'test', end=4, _msg='wrong type passed to end') -assert_raises(TypeError, print, 'test', sep=['a'], _msg='wrong type passed to sep') +assert_raises(TypeError, print, "test", end=4, _msg="wrong type passed to end") +assert_raises(TypeError, print, "test", sep=["a"], _msg="wrong type passed to sep") try: - print('test', end=None, sep=None, flush=None) + print("test", end=None, sep=None, flush=None) except: - assert False, 'Expected None passed to end, sep, and flush to not raise errors' + assert False, "Expected None passed to end, sep, and flush to not raise errors" buf = io.StringIO() -print('hello, world', file=buf) -assert buf.getvalue() == 'hello, world\n', buf.getvalue() +print("hello, world", file=buf) +assert buf.getvalue() == "hello, world\n", buf.getvalue() diff --git a/extra_tests/snippets/builtin_property.py b/extra_tests/snippets/builtin_property.py index 2a97c99b3b..de64e52622 100644 --- a/extra_tests/snippets/builtin_property.py +++ b/extra_tests/snippets/builtin_property.py @@ -62,8 +62,8 @@ def foo(self): assert p.__doc__ is None # Test property instance __doc__ attribute: -p.__doc__ = '222' -assert p.__doc__ == '222' +p.__doc__ = "222" +assert p.__doc__ == "222" p1 = property("a", "b", "c") @@ -83,5 +83,5 @@ def foo(self): assert p1.__get__(None, object) is p1 # assert p1.__doc__ is 'a'.__doc__ -p2 = property('a', doc='pdoc') +p2 = property("a", doc="pdoc") # assert p2.__doc__ == 'pdoc' diff --git a/extra_tests/snippets/builtin_range.py b/extra_tests/snippets/builtin_range.py index 9f8f03b63c..6bfb99f453 100644 --- a/extra_tests/snippets/builtin_range.py +++ b/extra_tests/snippets/builtin_range.py @@ -1,11 +1,11 @@ from testutils import assert_raises -assert range(2**63+1)[2**63] == 9223372036854775808 +assert range(2**63 + 1)[2**63] == 9223372036854775808 # len tests -assert len(range(10, 5)) == 0, 'Range with no elements should have length = 0' -assert len(range(10, 5, -2)) == 3, 'Expected length 3, for elements: 10, 8, 6' -assert len(range(5, 10, 2)) == 3, 'Expected length 3, for elements: 5, 7, 9' +assert len(range(10, 5)) == 0, "Range with no elements should have length = 0" +assert len(range(10, 5, -2)) == 3, "Expected length 3, for elements: 10, 8, 6" +assert len(range(5, 10, 2)) == 3, "Expected length 3, for elements: 5, 7, 9" # index tests assert range(10).index(6) == 6 @@ -13,18 +13,18 @@ assert range(4, 10, 2).index(6) == 1 assert range(10, 4, -2).index(8) == 1 -assert_raises(ValueError, lambda: range(10).index(-1), _msg='out of bounds') -assert_raises(ValueError, lambda: range(10).index(10), _msg='out of bounds') -assert_raises(ValueError, lambda: range(4, 10, 2).index(5), _msg='out of step') -assert_raises(ValueError, lambda: range(10).index('foo'), _msg='not an int') -assert_raises(ValueError, lambda: range(1, 10, 0), _msg='step is zero') +assert_raises(ValueError, lambda: range(10).index(-1), _msg="out of bounds") +assert_raises(ValueError, lambda: range(10).index(10), _msg="out of bounds") +assert_raises(ValueError, lambda: range(4, 10, 2).index(5), _msg="out of step") +assert_raises(ValueError, lambda: range(10).index("foo"), _msg="not an int") +assert_raises(ValueError, lambda: range(1, 10, 0), _msg="step is zero") # get tests assert range(10)[0] == 0 assert range(10)[9] == 9 assert range(10, 0, -1)[0] == 10 assert range(10, 0, -1)[9] == 1 -assert_raises(IndexError, lambda: range(10)[10], _msg='out of bound') +assert_raises(IndexError, lambda: range(10)[10], _msg="out of bound") # slice tests assert range(10)[0:3] == range(3) @@ -34,13 +34,17 @@ assert range(10, 100, 3)[4:1000:5] == range(22, 100, 15) assert range(10)[:] == range(10) assert range(10, 0, -2)[0:5:2] == range(10, 0, -4) -assert range(10)[10:11] == range(10,10) +assert range(10)[10:11] == range(10, 10) assert range(0, 10, -1)[::-1] == range(1, 1) assert range(0, 10)[::-1] == range(9, -1, -1) assert range(0, -10)[::-1] == range(-1, -1, -1) assert range(0, -10)[::-1][::-1] == range(0, 0) -assert_raises(ValueError, lambda: range(0, 10)[::0], _msg='slice step cannot be zero') -assert_raises(TypeError, lambda: range(0, 10)['a':], _msg='slice indices must be integers or None or have an __index__ method') +assert_raises(ValueError, lambda: range(0, 10)[::0], _msg="slice step cannot be zero") +assert_raises( + TypeError, + lambda: range(0, 10)["a":], + _msg="slice indices must be integers or None or have an __index__ method", +) # count tests assert range(10).count(2) == 1 @@ -64,22 +68,22 @@ assert range(10).__ne__(range(0, 11, 1)) is True assert range(0, 10, 3).__eq__(range(0, 11, 3)) is True assert range(0, 10, 3).__ne__(range(0, 11, 3)) is False -#__lt__ +# __lt__ assert range(1, 2, 3).__lt__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__lt__(range(1, 2)) == NotImplemented assert range(2).__lt__(range(0, 2)) == NotImplemented -#__gt__ +# __gt__ assert range(1, 2, 3).__gt__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__gt__(range(1, 2)) == NotImplemented assert range(2).__gt__(range(0, 2)) == NotImplemented -#__le__ +# __le__ assert range(1, 2, 3).__le__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__le__(range(1, 2)) == NotImplemented assert range(2).__le__(range(0, 2)) == NotImplemented -#__ge__ +# __ge__ assert range(1, 2, 3).__ge__(range(1, 2, 3)) == NotImplemented assert range(1, 2, 1).__ge__(range(1, 2)) == NotImplemented assert range(2).__ge__(range(0, 2)) == NotImplemented @@ -101,12 +105,12 @@ assert -1 not in range(10) assert 9 not in range(10, 4, -2) assert 4 not in range(10, 4, -2) -assert 'foo' not in range(10) +assert "foo" not in range(10) # __reversed__ assert list(reversed(range(5))) == [4, 3, 2, 1, 0] assert list(reversed(range(5, 0, -1))) == [1, 2, 3, 4, 5] -assert list(reversed(range(1,10,5))) == [6, 1] +assert list(reversed(range(1, 10, 5))) == [6, 1] # __reduce__ assert range(10).__reduce__()[0] == range @@ -120,7 +124,7 @@ # negative index assert range(10)[-1] == 9 -assert_raises(IndexError, lambda: range(10)[-11], _msg='out of bound') +assert_raises(IndexError, lambda: range(10)[-11], _msg="out of bound") assert range(10)[-2:4] == range(8, 4) assert range(10)[-6:-2] == range(4, 8) assert range(50, 0, -2)[-5] == 10 diff --git a/extra_tests/snippets/builtin_reversed.py b/extra_tests/snippets/builtin_reversed.py index 261b5c3263..0ec2f2828b 100644 --- a/extra_tests/snippets/builtin_reversed.py +++ b/extra_tests/snippets/builtin_reversed.py @@ -1,4 +1,4 @@ assert list(reversed(range(5))) == [4, 3, 2, 1, 0] -l = [5,4,3,2,1] -assert list(reversed(l)) == [1,2,3,4,5] +l = [5, 4, 3, 2, 1] +assert list(reversed(l)) == [1, 2, 3, 4, 5] diff --git a/extra_tests/snippets/builtin_round.py b/extra_tests/snippets/builtin_round.py index b4b070c4cc..99c4ed1d27 100644 --- a/extra_tests/snippets/builtin_round.py +++ b/extra_tests/snippets/builtin_round.py @@ -8,11 +8,11 @@ assert round(-1.5) == -2 # ValueError: cannot convert float NaN to integer -assert_raises(ValueError, round, float('nan')) +assert_raises(ValueError, round, float("nan")) # OverflowError: cannot convert float infinity to integer -assert_raises(OverflowError, round, float('inf')) +assert_raises(OverflowError, round, float("inf")) # OverflowError: cannot convert float infinity to integer -assert_raises(OverflowError, round, -float('inf')) +assert_raises(OverflowError, round, -float("inf")) assert round(0) == 0 assert isinstance(round(0), int) diff --git a/extra_tests/snippets/builtin_set.py b/extra_tests/snippets/builtin_set.py index 79f0602aea..1b2f6ff096 100644 --- a/extra_tests/snippets/builtin_set.py +++ b/extra_tests/snippets/builtin_set.py @@ -1,38 +1,54 @@ from testutils import assert_raises -assert set([1,2]) == set([1,2]) -assert not set([1,2,3]) == set([1,2]) +assert set([1, 2]) == set([1, 2]) +assert not set([1, 2, 3]) == set([1, 2]) -assert set([1,2,3]) >= set([1,2]) -assert set([1,2]) >= set([1,2]) -assert not set([1,3]) >= set([1,2]) +assert set([1, 2, 3]) >= set([1, 2]) +assert set([1, 2]) >= set([1, 2]) +assert not set([1, 3]) >= set([1, 2]) -assert set([1,2,3]).issuperset(set([1,2])) -assert set([1,2]).issuperset(set([1,2])) -assert not set([1,3]).issuperset(set([1,2])) +assert set([1, 2, 3]).issuperset(set([1, 2])) +assert set([1, 2]).issuperset(set([1, 2])) +assert not set([1, 3]).issuperset(set([1, 2])) -assert set([1,2,3]) > set([1,2]) -assert not set([1,2]) > set([1,2]) -assert not set([1,3]) > set([1,2]) +assert set([1, 2, 3]) > set([1, 2]) +assert not set([1, 2]) > set([1, 2]) +assert not set([1, 3]) > set([1, 2]) -assert set([1,2]) <= set([1,2,3]) -assert set([1,2]) <= set([1,2]) -assert not set([1,3]) <= set([1,2]) +assert set([1, 2]) <= set([1, 2, 3]) +assert set([1, 2]) <= set([1, 2]) +assert not set([1, 3]) <= set([1, 2]) -assert set([1,2]).issubset(set([1,2,3])) -assert set([1,2]).issubset(set([1,2])) -assert not set([1,3]).issubset(set([1,2])) +assert set([1, 2]).issubset(set([1, 2, 3])) +assert set([1, 2]).issubset(set([1, 2])) +assert not set([1, 3]).issubset(set([1, 2])) -assert set([1,2]) < set([1,2,3]) -assert not set([1,2]) < set([1,2]) -assert not set([1,3]) < set([1,2]) +assert set([1, 2]) < set([1, 2, 3]) +assert not set([1, 2]) < set([1, 2]) +assert not set([1, 3]) < set([1, 2]) assert (set() == []) is False assert set().__eq__([]) == NotImplemented -assert_raises(TypeError, lambda: set() < [], _msg="'<' not supported between instances of 'set' and 'list'") -assert_raises(TypeError, lambda: set() <= [], _msg="'<=' not supported between instances of 'set' and 'list'") -assert_raises(TypeError, lambda: set() > [], _msg="'>' not supported between instances of 'set' and 'list'") -assert_raises(TypeError, lambda: set() >= [], _msg="'>=' not supported between instances of 'set' and 'list'") +assert_raises( + TypeError, + lambda: set() < [], + _msg="'<' not supported between instances of 'set' and 'list'", +) +assert_raises( + TypeError, + lambda: set() <= [], + _msg="'<=' not supported between instances of 'set' and 'list'", +) +assert_raises( + TypeError, + lambda: set() > [], + _msg="'>' not supported between instances of 'set' and 'list'", +) +assert_raises( + TypeError, + lambda: set() >= [], + _msg="'>=' not supported between instances of 'set' and 'list'", +) assert set().issuperset([]) assert set().issubset([]) assert not set().issuperset([1, 2, 3]) @@ -47,6 +63,7 @@ assert_raises(TypeError, set().issuperset, 3, _msg="'int' object is not iterable") assert_raises(TypeError, set().issubset, 3, _msg="'int' object is not iterable") + class Hashable(object): def __init__(self, obj): self.obj = obj @@ -57,6 +74,7 @@ def __repr__(self): def __hash__(self): return id(self) + assert repr(set()) == "set()" assert repr(set([1, 2, 3])) == "{1, 2, 3}" @@ -64,9 +82,11 @@ def __hash__(self): recursive.add(Hashable(recursive)) assert repr(recursive) == "{set(...)}" + class S(set): pass + assert repr(S()) == "S()" assert repr(S([1, 2, 3])) == "S({1, 2, 3})" @@ -79,44 +99,44 @@ class S(set): a.clear() assert len(a) == 0 -assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5]) -assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5]) -assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) +assert set([1, 2, 3]).union(set([4, 5])) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]).union(set([1, 2, 3, 4, 5])) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]).union([1, 2, 3, 4, 5]) == set([1, 2, 3, 4, 5]) -assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) -assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) -assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) +assert set([1, 2, 3]) | set([4, 5]) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) | set([1, 2, 3, 4, 5]) == set([1, 2, 3, 4, 5]) +assert_raises(TypeError, lambda: set([1, 2, 3]) | [1, 2, 3, 4, 5]) -assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) -assert set([1,2,3]).intersection(set([5,6])) == set([]) -assert set([1,2,3]).intersection([1,2]) == set([1,2]) +assert set([1, 2, 3]).intersection(set([1, 2])) == set([1, 2]) +assert set([1, 2, 3]).intersection(set([5, 6])) == set([]) +assert set([1, 2, 3]).intersection([1, 2]) == set([1, 2]) -assert set([1,2,3]) & set([4,5]) == set([]) -assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3]) -assert_raises(TypeError, lambda: set([1,2,3]) & [1,2,3,4,5]) +assert set([1, 2, 3]) & set([4, 5]) == set([]) +assert set([1, 2, 3]) & set([1, 2, 3, 4, 5]) == set([1, 2, 3]) +assert_raises(TypeError, lambda: set([1, 2, 3]) & [1, 2, 3, 4, 5]) -assert set([1,2,3]).difference(set([1,2])) == set([3]) -assert set([1,2,3]).difference(set([5,6])) == set([1,2,3]) -assert set([1,2,3]).difference([1,2]) == set([3]) +assert set([1, 2, 3]).difference(set([1, 2])) == set([3]) +assert set([1, 2, 3]).difference(set([5, 6])) == set([1, 2, 3]) +assert set([1, 2, 3]).difference([1, 2]) == set([3]) -assert set([1,2,3]) - set([4,5]) == set([1,2,3]) -assert set([1,2,3]) - set([1,2,3,4,5]) == set([]) -assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) +assert set([1, 2, 3]) - set([4, 5]) == set([1, 2, 3]) +assert set([1, 2, 3]) - set([1, 2, 3, 4, 5]) == set([]) +assert_raises(TypeError, lambda: set([1, 2, 3]) - [1, 2, 3, 4, 5]) -assert set([1,2]).__sub__(set([2,3])) == set([1]) -assert set([1,2]).__rsub__(set([2,3])) == set([3]) +assert set([1, 2]).__sub__(set([2, 3])) == set([1]) +assert set([1, 2]).__rsub__(set([2, 3])) == set([3]) -assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) -assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) -assert set([1,2,3]).symmetric_difference([1,2]) == set([3]) +assert set([1, 2, 3]).symmetric_difference(set([1, 2])) == set([3]) +assert set([1, 2, 3]).symmetric_difference(set([5, 6])) == set([1, 2, 3, 5, 6]) +assert set([1, 2, 3]).symmetric_difference([1, 2]) == set([3]) -assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) -assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) -assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) +assert set([1, 2, 3]) ^ set([4, 5]) == set([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) ^ set([1, 2, 3, 4, 5]) == set([4, 5]) +assert_raises(TypeError, lambda: set([1, 2, 3]) ^ [1, 2, 3, 4, 5]) -assert set([1,2,3]).isdisjoint(set([5,6])) == True -assert set([1,2,3]).isdisjoint(set([2,5,6])) == False -assert set([1,2,3]).isdisjoint([5,6]) == True +assert set([1, 2, 3]).isdisjoint(set([5, 6])) == True +assert set([1, 2, 3]).isdisjoint(set([2, 5, 6])) == False +assert set([1, 2, 3]).isdisjoint([5, 6]) == True assert_raises(TypeError, lambda: set() & []) assert_raises(TypeError, lambda: set() | []) @@ -132,7 +152,7 @@ class S(set): assert not 1 in a assert a.discard(42) is None -a = set([1,2,3]) +a = set([1, 2, 3]) b = a.copy() assert len(a) == 3 assert len(b) == 3 @@ -140,71 +160,71 @@ class S(set): assert len(a) == 3 assert len(b) == 0 -a = set([1,2]) +a = set([1, 2]) b = a.pop() -assert b in [1,2] +assert b in [1, 2] c = a.pop() -assert (c in [1,2] and c != b) +assert c in [1, 2] and c != b assert_raises(KeyError, lambda: a.pop()) -a = set([1,2,3]) -a.update([3,4,5]) -assert a == set([1,2,3,4,5]) +a = set([1, 2, 3]) +a.update([3, 4, 5]) +assert a == set([1, 2, 3, 4, 5]) assert_raises(TypeError, lambda: a.update(1)) -a = set([1,2,3]) +a = set([1, 2, 3]) b = set() for e in a: - assert e == 1 or e == 2 or e == 3 - b.add(e) + assert e == 1 or e == 2 or e == 3 + b.add(e) assert a == b -a = set([1,2,3]) -a |= set([3,4,5]) -assert a == set([1,2,3,4,5]) +a = set([1, 2, 3]) +a |= set([3, 4, 5]) +assert a == set([1, 2, 3, 4, 5]) with assert_raises(TypeError): - a |= 1 + a |= 1 with assert_raises(TypeError): - a |= [1,2,3] + a |= [1, 2, 3] -a = set([1,2,3]) -a.intersection_update([2,3,4,5]) -assert a == set([2,3]) +a = set([1, 2, 3]) +a.intersection_update([2, 3, 4, 5]) +assert a == set([2, 3]) assert_raises(TypeError, lambda: a.intersection_update(1)) -a = set([1,2,3]) -a &= set([2,3,4,5]) -assert a == set([2,3]) +a = set([1, 2, 3]) +a &= set([2, 3, 4, 5]) +assert a == set([2, 3]) with assert_raises(TypeError): - a &= 1 + a &= 1 with assert_raises(TypeError): - a &= [1,2,3] + a &= [1, 2, 3] -a = set([1,2,3]) -a.difference_update([3,4,5]) -assert a == set([1,2]) +a = set([1, 2, 3]) +a.difference_update([3, 4, 5]) +assert a == set([1, 2]) assert_raises(TypeError, lambda: a.difference_update(1)) -a = set([1,2,3]) -a -= set([3,4,5]) -assert a == set([1,2]) +a = set([1, 2, 3]) +a -= set([3, 4, 5]) +assert a == set([1, 2]) with assert_raises(TypeError): - a -= 1 + a -= 1 with assert_raises(TypeError): - a -= [1,2,3] + a -= [1, 2, 3] -a = set([1,2,3]) -a.symmetric_difference_update([3,4,5]) -assert a == set([1,2,4,5]) +a = set([1, 2, 3]) +a.symmetric_difference_update([3, 4, 5]) +assert a == set([1, 2, 4, 5]) assert_raises(TypeError, lambda: a.difference_update(1)) -a = set([1,2,3]) -a ^= set([3,4,5]) -assert a == set([1,2,4,5]) +a = set([1, 2, 3]) +a ^= set([3, 4, 5]) +assert a == set([1, 2, 4, 5]) with assert_raises(TypeError): - a ^= 1 + a ^= 1 with assert_raises(TypeError): - a ^= [1,2,3] + a ^= [1, 2, 3] a = set([1, 2, 3]) i = iter(a) @@ -218,142 +238,154 @@ class S(set): # frozen set -assert frozenset([1,2]) == frozenset([1,2]) -assert not frozenset([1,2,3]) == frozenset([1,2]) +assert frozenset([1, 2]) == frozenset([1, 2]) +assert not frozenset([1, 2, 3]) == frozenset([1, 2]) -assert frozenset([1,2,3]) >= frozenset([1,2]) -assert frozenset([1,2]) >= frozenset([1,2]) -assert not frozenset([1,3]) >= frozenset([1,2]) +assert frozenset([1, 2, 3]) >= frozenset([1, 2]) +assert frozenset([1, 2]) >= frozenset([1, 2]) +assert not frozenset([1, 3]) >= frozenset([1, 2]) -assert frozenset([1,2,3]).issuperset(frozenset([1,2])) -assert frozenset([1,2]).issuperset(frozenset([1,2])) -assert not frozenset([1,3]).issuperset(frozenset([1,2])) +assert frozenset([1, 2, 3]).issuperset(frozenset([1, 2])) +assert frozenset([1, 2]).issuperset(frozenset([1, 2])) +assert not frozenset([1, 3]).issuperset(frozenset([1, 2])) -assert frozenset([1,2,3]) > frozenset([1,2]) -assert not frozenset([1,2]) > frozenset([1,2]) -assert not frozenset([1,3]) > frozenset([1,2]) +assert frozenset([1, 2, 3]) > frozenset([1, 2]) +assert not frozenset([1, 2]) > frozenset([1, 2]) +assert not frozenset([1, 3]) > frozenset([1, 2]) -assert frozenset([1,2]) <= frozenset([1,2,3]) -assert frozenset([1,2]) <= frozenset([1,2]) -assert not frozenset([1,3]) <= frozenset([1,2]) +assert frozenset([1, 2]) <= frozenset([1, 2, 3]) +assert frozenset([1, 2]) <= frozenset([1, 2]) +assert not frozenset([1, 3]) <= frozenset([1, 2]) -assert frozenset([1,2]).issubset(frozenset([1,2,3])) -assert frozenset([1,2]).issubset(frozenset([1,2])) -assert not frozenset([1,3]).issubset(frozenset([1,2])) +assert frozenset([1, 2]).issubset(frozenset([1, 2, 3])) +assert frozenset([1, 2]).issubset(frozenset([1, 2])) +assert not frozenset([1, 3]).issubset(frozenset([1, 2])) -assert frozenset([1,2]) < frozenset([1,2,3]) -assert not frozenset([1,2]) < frozenset([1,2]) -assert not frozenset([1,3]) < frozenset([1,2]) +assert frozenset([1, 2]) < frozenset([1, 2, 3]) +assert not frozenset([1, 2]) < frozenset([1, 2]) +assert not frozenset([1, 3]) < frozenset([1, 2]) a = frozenset([1, 2, 3]) assert len(a) == 3 b = a.copy() assert b == a -assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) +assert frozenset([1, 2, 3]).union(frozenset([4, 5])) == frozenset([1, 2, 3, 4, 5]) +assert frozenset([1, 2, 3]).union(frozenset([1, 2, 3, 4, 5])) == frozenset( + [1, 2, 3, 4, 5] +) +assert frozenset([1, 2, 3]).union([1, 2, 3, 4, 5]) == frozenset([1, 2, 3, 4, 5]) -assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) +assert frozenset([1, 2, 3]) | frozenset([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert frozenset([1, 2, 3]) | frozenset([1, 2, 3, 4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) | [1, 2, 3, 4, 5]) -assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) -assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) -assert frozenset([1,2,3]).intersection([1,2]) == frozenset([1,2]) +assert frozenset([1, 2, 3]).intersection(frozenset([1, 2])) == frozenset([1, 2]) +assert frozenset([1, 2, 3]).intersection(frozenset([5, 6])) == frozenset([]) +assert frozenset([1, 2, 3]).intersection([1, 2]) == frozenset([1, 2]) -assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) -assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) & [1,2,3,4,5]) +assert frozenset([1, 2, 3]) & frozenset([4, 5]) == frozenset([]) +assert frozenset([1, 2, 3]) & frozenset([1, 2, 3, 4, 5]) == frozenset([1, 2, 3]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) & [1, 2, 3, 4, 5]) -assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) -assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) -assert frozenset([1,2,3]).difference([1,2]) == frozenset([3]) +assert frozenset([1, 2, 3]).difference(frozenset([1, 2])) == frozenset([3]) +assert frozenset([1, 2, 3]).difference(frozenset([5, 6])) == frozenset([1, 2, 3]) +assert frozenset([1, 2, 3]).difference([1, 2]) == frozenset([3]) -assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) -assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) +assert frozenset([1, 2, 3]) - frozenset([4, 5]) == frozenset([1, 2, 3]) +assert frozenset([1, 2, 3]) - frozenset([1, 2, 3, 4, 5]) == frozenset([]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) - [1, 2, 3, 4, 5]) -assert frozenset([1,2]).__sub__(frozenset([2,3])) == frozenset([1]) -assert frozenset([1,2]).__rsub__(frozenset([2,3])) == frozenset([3]) +assert frozenset([1, 2]).__sub__(frozenset([2, 3])) == frozenset([1]) +assert frozenset([1, 2]).__rsub__(frozenset([2, 3])) == frozenset([3]) -assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) -assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) -assert frozenset([1,2,3]).symmetric_difference([1,2]) == frozenset([3]) +assert frozenset([1, 2, 3]).symmetric_difference(frozenset([1, 2])) == frozenset([3]) +assert frozenset([1, 2, 3]).symmetric_difference(frozenset([5, 6])) == frozenset( + [1, 2, 3, 5, 6] +) +assert frozenset([1, 2, 3]).symmetric_difference([1, 2]) == frozenset([3]) -assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) -assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) -assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) +assert frozenset([1, 2, 3]) ^ frozenset([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert frozenset([1, 2, 3]) ^ frozenset([1, 2, 3, 4, 5]) == frozenset([4, 5]) +assert_raises(TypeError, lambda: frozenset([1, 2, 3]) ^ [1, 2, 3, 4, 5]) -assert frozenset([1,2,3]).isdisjoint(frozenset([5,6])) == True -assert frozenset([1,2,3]).isdisjoint(frozenset([2,5,6])) == False -assert frozenset([1,2,3]).isdisjoint([5,6]) == True +assert frozenset([1, 2, 3]).isdisjoint(frozenset([5, 6])) == True +assert frozenset([1, 2, 3]).isdisjoint(frozenset([2, 5, 6])) == False +assert frozenset([1, 2, 3]).isdisjoint([5, 6]) == True assert_raises(TypeError, frozenset, [[]]) -a = frozenset([1,2,3]) +a = frozenset([1, 2, 3]) b = set() for e in a: - assert e == 1 or e == 2 or e == 3 - b.add(e) + assert e == 1 or e == 2 or e == 3 + b.add(e) assert a == b # set and frozen set -assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) -assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) +assert frozenset([1, 2, 3]).union(set([4, 5])) == frozenset([1, 2, 3, 4, 5]) +assert set([1, 2, 3]).union(frozenset([4, 5])) == set([1, 2, 3, 4, 5]) + +assert frozenset([1, 2, 3]) | set([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) | frozenset([4, 5]) == set([1, 2, 3, 4, 5]) -assert frozenset([1,2,3]) | set([4,5]) == frozenset([1,2,3,4,5]) -assert set([1,2,3]) | frozenset([4,5]) == set([1,2,3,4,5]) +assert frozenset([1, 2, 3]).intersection(set([5, 6])) == frozenset([]) +assert set([1, 2, 3]).intersection(frozenset([5, 6])) == set([]) -assert frozenset([1,2,3]).intersection(set([5,6])) == frozenset([]) -assert set([1,2,3]).intersection(frozenset([5,6])) == set([]) +assert frozenset([1, 2, 3]) & set([1, 2, 3, 4, 5]) == frozenset([1, 2, 3]) +assert set([1, 2, 3]) & frozenset([1, 2, 3, 4, 5]) == set([1, 2, 3]) -assert frozenset([1,2,3]) & set([1,2,3,4,5]) == frozenset([1,2,3]) -assert set([1,2,3]) & frozenset([1,2,3,4,5]) == set([1,2,3]) +assert frozenset([1, 2, 3]).difference(set([5, 6])) == frozenset([1, 2, 3]) +assert set([1, 2, 3]).difference(frozenset([5, 6])) == set([1, 2, 3]) -assert frozenset([1,2,3]).difference(set([5,6])) == frozenset([1,2,3]) -assert set([1,2,3]).difference(frozenset([5,6])) == set([1,2,3]) +assert frozenset([1, 2, 3]) - set([4, 5]) == frozenset([1, 2, 3]) +assert set([1, 2, 3]) - frozenset([4, 5]) == frozenset([1, 2, 3]) -assert frozenset([1,2,3]) - set([4,5]) == frozenset([1,2,3]) -assert set([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) +assert frozenset([1, 2]).__sub__(set([2, 3])) == frozenset([1]) +assert frozenset([1, 2]).__rsub__(set([2, 3])) == set([3]) +assert set([1, 2]).__sub__(frozenset([2, 3])) == set([1]) +assert set([1, 2]).__rsub__(frozenset([2, 3])) == frozenset([3]) -assert frozenset([1,2]).__sub__(set([2,3])) == frozenset([1]) -assert frozenset([1,2]).__rsub__(set([2,3])) == set([3]) -assert set([1,2]).__sub__(frozenset([2,3])) == set([1]) -assert set([1,2]).__rsub__(frozenset([2,3])) == frozenset([3]) +assert frozenset([1, 2, 3]).symmetric_difference(set([1, 2])) == frozenset([3]) +assert set([1, 2, 3]).symmetric_difference(frozenset([1, 2])) == set([3]) -assert frozenset([1,2,3]).symmetric_difference(set([1,2])) == frozenset([3]) -assert set([1,2,3]).symmetric_difference(frozenset([1,2])) == set([3]) +assert frozenset([1, 2, 3]) ^ set([4, 5]) == frozenset([1, 2, 3, 4, 5]) +assert set([1, 2, 3]) ^ frozenset([4, 5]) == set([1, 2, 3, 4, 5]) -assert frozenset([1,2,3]) ^ set([4,5]) == frozenset([1,2,3,4,5]) -assert set([1,2,3]) ^ frozenset([4,5]) == set([1,2,3,4,5]) class A: def __hash__(self): return 1 + + class B: def __hash__(self): return 1 + s = {1, A(), B()} assert len(s) == 3 s = {True} s.add(1.0) -assert str(s) == '{True}' +assert str(s) == "{True}" + class EqObject: def __init__(self, eq): self.eq = eq + def __eq__(self, other): return self.eq + def __hash__(self): return bool(self.eq) -assert 'x' == (EqObject('x') == EqObject('x')) -s = {EqObject('x')} -assert EqObject('x') in s -assert '[]' == (EqObject('[]') == EqObject('[]')) + +assert "x" == (EqObject("x") == EqObject("x")) +s = {EqObject("x")} +assert EqObject("x") in s +assert "[]" == (EqObject("[]") == EqObject("[]")) s = {EqObject([])} assert EqObject([]) not in s x = object() @@ -372,8 +404,8 @@ def __hash__(self): assert frozenset().__ne__(1) == NotImplemented empty_set = set() -non_empty_set = set([1,2,3]) -set_from_literal = {1,2,3} +non_empty_set = set([1, 2, 3]) +set_from_literal = {1, 2, 3} assert 1 in non_empty_set assert 4 not in non_empty_set @@ -382,8 +414,8 @@ def __hash__(self): assert 4 not in set_from_literal # TODO: Assert that empty aruguments raises exception. -non_empty_set.add('a') -assert 'a' in non_empty_set +non_empty_set.add("a") +assert "a" in non_empty_set # TODO: Assert that empty arguments, or item not in set raises exception. non_empty_set.remove(1) @@ -394,8 +426,10 @@ def __hash__(self): assert repr(frozenset()) == "frozenset()" assert repr(frozenset([1, 2, 3])) == "frozenset({1, 2, 3})" + class FS(frozenset): pass + assert repr(FS()) == "FS()" assert repr(FS([1, 2, 3])) == "FS({1, 2, 3})" diff --git a/extra_tests/snippets/builtin_slice.py b/extra_tests/snippets/builtin_slice.py index b5c3a8ceb4..9a5c1bc78d 100644 --- a/extra_tests/snippets/builtin_slice.py +++ b/extra_tests/snippets/builtin_slice.py @@ -10,16 +10,16 @@ assert a.stop == 10 assert a.step == 1 -assert slice(10).__repr__() == 'slice(None, 10, None)' -assert slice(None).__repr__() == 'slice(None, None, None)' -assert slice(0, 10, 13).__repr__() == 'slice(0, 10, 13)' -assert slice('0', 1.1, 2+3j).__repr__() == "slice('0', 1.1, (2+3j))" +assert slice(10).__repr__() == "slice(None, 10, None)" +assert slice(None).__repr__() == "slice(None, None, None)" +assert slice(0, 10, 13).__repr__() == "slice(0, 10, 13)" +assert slice("0", 1.1, 2 + 3j).__repr__() == "slice('0', 1.1, (2+3j))" assert slice(10) == slice(10) assert slice(-1) != slice(1) assert slice(0, 10, 3) != slice(0, 11, 3) -assert slice(0, None, 3) != slice(0, 'a', 3) -assert slice(0, 'a', 3) == slice(0, 'a', 3) +assert slice(0, None, 3) != slice(0, "a", 3) +assert slice(0, "a", 3) == slice(0, "a", 3) assert slice(0, 0, 0).__eq__(slice(0, 0, 0)) assert not slice(0, 0, 1).__eq__(slice(0, 0, 0)) @@ -65,29 +65,29 @@ assert not slice(0, 0, 0) > slice(0, 0, 0) assert not slice(0, 0, 0) < slice(0, 0, 0) -assert not slice(0, float('nan'), float('nan')) <= slice(0, float('nan'), 1) -assert not slice(0, float('nan'), float('nan')) <= slice(0, float('nan'), float('nan')) -assert not slice(0, float('nan'), float('nan')) >= slice(0, float('nan'), float('nan')) -assert not slice(0, float('nan'), float('nan')) < slice(0, float('nan'), float('nan')) -assert not slice(0, float('nan'), float('nan')) > slice(0, float('nan'), float('nan')) +assert not slice(0, float("nan"), float("nan")) <= slice(0, float("nan"), 1) +assert not slice(0, float("nan"), float("nan")) <= slice(0, float("nan"), float("nan")) +assert not slice(0, float("nan"), float("nan")) >= slice(0, float("nan"), float("nan")) +assert not slice(0, float("nan"), float("nan")) < slice(0, float("nan"), float("nan")) +assert not slice(0, float("nan"), float("nan")) > slice(0, float("nan"), float("nan")) -assert slice(0, float('inf'), float('inf')) >= slice(0, float('inf'), 1) -assert slice(0, float('inf'), float('inf')) <= slice(0, float('inf'), float('inf')) -assert slice(0, float('inf'), float('inf')) >= slice(0, float('inf'), float('inf')) -assert not slice(0, float('inf'), float('inf')) < slice(0, float('inf'), float('inf')) -assert not slice(0, float('inf'), float('inf')) > slice(0, float('inf'), float('inf')) +assert slice(0, float("inf"), float("inf")) >= slice(0, float("inf"), 1) +assert slice(0, float("inf"), float("inf")) <= slice(0, float("inf"), float("inf")) +assert slice(0, float("inf"), float("inf")) >= slice(0, float("inf"), float("inf")) +assert not slice(0, float("inf"), float("inf")) < slice(0, float("inf"), float("inf")) +assert not slice(0, float("inf"), float("inf")) > slice(0, float("inf"), float("inf")) assert_raises(TypeError, lambda: slice(0) < 3) assert_raises(TypeError, lambda: slice(0) > 3) assert_raises(TypeError, lambda: slice(0) <= 3) assert_raises(TypeError, lambda: slice(0) >= 3) -assert slice(None ).indices(10) == (0, 10, 1) -assert slice(None, None, 2).indices(10) == (0, 10, 2) -assert slice(1, None, 2).indices(10) == (1, 10, 2) -assert slice(None, None, -1).indices(10) == (9, -1, -1) -assert slice(None, None, -2).indices(10) == (9, -1, -2) -assert slice(3, None, -2).indices(10) == (3, -1, -2) +assert slice(None).indices(10) == (0, 10, 1) +assert slice(None, None, 2).indices(10) == (0, 10, 2) +assert slice(1, None, 2).indices(10) == (1, 10, 2) +assert slice(None, None, -1).indices(10) == (9, -1, -1) +assert slice(None, None, -2).indices(10) == (9, -1, -2) +assert slice(3, None, -2).indices(10) == (3, -1, -2) # issue 3004 tests assert slice(None, -9).indices(10) == (0, 1, 1) @@ -103,21 +103,17 @@ assert slice(None, 9, -1).indices(10) == (9, 9, -1) assert slice(None, 10, -1).indices(10) == (9, 9, -1) -assert \ - slice(-100, 100).indices(10) == \ - slice(None ).indices(10) +assert slice(-100, 100).indices(10) == slice(None).indices(10) -assert \ - slice(100, -100, -1).indices(10) == \ - slice(None, None, -1).indices(10) +assert slice(100, -100, -1).indices(10) == slice(None, None, -1).indices(10) -assert slice(-100, 100, 2).indices(10) == (0, 10, 2) +assert slice(-100, 100, 2).indices(10) == (0, 10, 2) try: - slice(None, None, 0) - assert "zero step" == "throws an exception" + slice(None, None, 0) + assert "zero step" == "throws an exception" except: - pass + pass a = [] b = [1, 2] @@ -167,8 +163,8 @@ def __index__(self): return self.x -assert c[CustomIndex(1):CustomIndex(3)] == [1, 2] -assert d[CustomIndex(1):CustomIndex(3)] == "23" +assert c[CustomIndex(1) : CustomIndex(3)] == [1, 2] +assert d[CustomIndex(1) : CustomIndex(3)] == "23" def test_all_slices(): @@ -176,7 +172,7 @@ def test_all_slices(): test all possible slices except big number """ - mod = __import__('cpython_generated_slices') + mod = __import__("cpython_generated_slices") ll = mod.LL start = mod.START diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index cd7133e355..1b9a7bde1a 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -1,14 +1,17 @@ from testutils import assert_raises, AssertRaises, skip_if_unsupported assert "".__eq__(1) == NotImplemented -assert "a" == 'a' +assert "a" == "a" assert """a""" == "a" assert len(""" " "" " "" """) == 11 -assert "\"" == '"' -assert "\"" == """\"""" +assert '"' == '"' +assert '"' == """\"""" -assert "\n" == """ +assert ( + "\n" + == """ """ +) assert len(""" " \" """) == 5 assert len("é") == 1 @@ -30,7 +33,7 @@ assert repr("a") == "'a'" assert repr("can't") == '"can\'t"' assert repr('"won\'t"') == "'\"won\\'t\"'" -assert repr('\n\t') == "'\\n\\t'" +assert repr("\n\t") == "'\\n\\t'" assert str(["a", "b", "can't"]) == "['a', 'b', \"can't\"]" @@ -42,22 +45,22 @@ assert 0 * "x" == "" assert -1 * "x" == "" -assert_raises(OverflowError, lambda: 'xy' * 234234234234234234234234234234) - -a = 'Hallo' -assert a.lower() == 'hallo' -assert a.upper() == 'HALLO' -assert a.startswith('H') -assert a.startswith(('H', 1)) -assert a.startswith(('A', 'H')) -assert not a.startswith('f') -assert not a.startswith(('A', 'f')) -assert a.endswith('llo') -assert a.endswith(('lo', 1)) -assert a.endswith(('A', 'lo')) -assert not a.endswith('on') -assert not a.endswith(('A', 'll')) -assert a.zfill(8) == '000Hallo' +assert_raises(OverflowError, lambda: "xy" * 234234234234234234234234234234) + +a = "Hallo" +assert a.lower() == "hallo" +assert a.upper() == "HALLO" +assert a.startswith("H") +assert a.startswith(("H", 1)) +assert a.startswith(("A", "H")) +assert not a.startswith("f") +assert not a.startswith(("A", "f")) +assert a.endswith("llo") +assert a.endswith(("lo", 1)) +assert a.endswith(("A", "lo")) +assert not a.endswith("on") +assert not a.endswith(("A", "ll")) +assert a.zfill(8) == "000Hallo" assert a.isalnum() assert not a.isdigit() assert not a.isdecimal() @@ -65,34 +68,34 @@ assert a.istitle() assert a.isalpha() -s = '1 2 3' -assert s.split(' ', 1) == ['1', '2 3'] -assert s.rsplit(' ', 1) == ['1 2', '3'] - -b = ' hallo ' -assert b.strip() == 'hallo' -assert b.lstrip() == 'hallo ' -assert b.rstrip() == ' hallo' - -s = '^*RustPython*^' -assert s.strip('^*') == 'RustPython' -assert s.lstrip('^*') == 'RustPython*^' -assert s.rstrip('^*') == '^*RustPython' - -s = 'RustPython' -assert s.ljust(8) == 'RustPython' -assert s.rjust(8) == 'RustPython' -assert s.ljust(12) == 'RustPython ' -assert s.rjust(12) == ' RustPython' -assert s.ljust(12, '_') == 'RustPython__' -assert s.rjust(12, '_') == '__RustPython' +s = "1 2 3" +assert s.split(" ", 1) == ["1", "2 3"] +assert s.rsplit(" ", 1) == ["1 2", "3"] + +b = " hallo " +assert b.strip() == "hallo" +assert b.lstrip() == "hallo " +assert b.rstrip() == " hallo" + +s = "^*RustPython*^" +assert s.strip("^*") == "RustPython" +assert s.lstrip("^*") == "RustPython*^" +assert s.rstrip("^*") == "^*RustPython" + +s = "RustPython" +assert s.ljust(8) == "RustPython" +assert s.rjust(8) == "RustPython" +assert s.ljust(12) == "RustPython " +assert s.rjust(12) == " RustPython" +assert s.ljust(12, "_") == "RustPython__" +assert s.rjust(12, "_") == "__RustPython" # The fill character must be exactly one character long -assert_raises(TypeError, lambda: s.ljust(12, '__')) -assert_raises(TypeError, lambda: s.rjust(12, '__')) +assert_raises(TypeError, lambda: s.ljust(12, "__")) +assert_raises(TypeError, lambda: s.rjust(12, "__")) -c = 'hallo' -assert c.capitalize() == 'Hallo' -assert c.center(11, '-') == '---hallo---' +c = "hallo" +assert c.capitalize() == "Hallo" +assert c.center(11, "-") == "---hallo---" assert ["koki".center(i, "|") for i in range(3, 10)] == [ "koki", "koki", @@ -118,110 +121,157 @@ # requires CPython 3.7, and the CI currently runs with 3.6 # assert c.isascii() -assert c.index('a') == 1 -assert c.rindex('l') == 3 -assert c.find('h') == 0 -assert c.rfind('x') == -1 +assert c.index("a") == 1 +assert c.rindex("l") == 3 +assert c.find("h") == 0 +assert c.rfind("x") == -1 assert c.islower() -assert c.title() == 'Hallo' -assert c.count('l') == 2 - -assert 'aaa'.count('a') == 3 -assert 'aaa'.count('a', 1) == 2 -assert 'aaa'.count('a', 1, 2) == 1 -assert 'aaa'.count('a', 2, 2) == 0 -assert 'aaa'.count('a', 2, 1) == 0 - -assert '___a__'.find('a') == 3 -assert '___a__'.find('a', -10) == 3 -assert '___a__'.find('a', -3) == 3 -assert '___a__'.find('a', -2) == -1 -assert '___a__'.find('a', -1) == -1 -assert '___a__'.find('a', 0) == 3 -assert '___a__'.find('a', 3) == 3 -assert '___a__'.find('a', 4) == -1 -assert '___a__'.find('a', 10) == -1 -assert '___a__'.rfind('a', 3) == 3 -assert '___a__'.index('a', 3) == 3 - -assert '___a__'.find('a', 0, -10) == -1 -assert '___a__'.find('a', 0, -3) == -1 -assert '___a__'.find('a', 0, -2) == 3 -assert '___a__'.find('a', 0, -1) == 3 -assert '___a__'.find('a', 0, 0) == -1 -assert '___a__'.find('a', 0, 3) == -1 -assert '___a__'.find('a', 0, 4) == 3 -assert '___a__'.find('a', 0, 10) == 3 - -assert '___a__'.find('a', 3, 3) == -1 -assert '___a__'.find('a', 3, 4) == 3 -assert '___a__'.find('a', 4, 3) == -1 - -assert 'abcd'.startswith('b', 1) -assert 'abcd'.startswith(('b', 'z'), 1) -assert not 'abcd'.startswith('b', -4) -assert 'abcd'.startswith('b', -3) - -assert not 'abcd'.startswith('b', 3, 3) -assert 'abcd'.startswith('', 3, 3) -assert not 'abcd'.startswith('', 4, 3) - -assert ' '.isspace() -assert 'hello\nhallo\nHallo'.splitlines() == ['hello', 'hallo', 'Hallo'] -assert 'hello\nhallo\nHallo\n'.splitlines() == ['hello', 'hallo', 'Hallo'] -assert 'hello\nhallo\nHallo'.splitlines(keepends=True) == ['hello\n', 'hallo\n', 'Hallo'] -assert 'hello\nhallo\nHallo\n'.splitlines(keepends=True) == ['hello\n', 'hallo\n', 'Hallo\n'] -assert 'hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.'.splitlines() == ['hello', 'hallo', 'Hallo', 'HELLO', 'hoho', 'haha', 'another', 'yetanother', 'last', '.'] -assert 'hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.'.splitlines(keepends=True) == ['hello\x0b', 'hallo\x0c', 'Hallo\x1c', 'HELLO\x1d', 'hoho\x1e', 'haha\x85', 'another\u2028', 'yetanother\u2029', 'last\r\n', '.'] -assert 'abc\t12345\txyz'.expandtabs() == 'abc 12345 xyz' -assert '-'.join(['1', '2', '3']) == '1-2-3' -assert 'HALLO'.isupper() -assert "hello, my name is".partition("my ") == ('hello, ', 'my ', 'name is') -assert "hello".partition("is") == ('hello', '', '') -assert "hello, my name is".rpartition("is") == ('hello, my name ', 'is', '') -assert "hello".rpartition("is") == ('', '', 'hello') -assert not ''.isdecimal() -assert '123'.isdecimal() -assert not '\u00B2'.isdecimal() - -assert not ''.isidentifier() -assert 'python'.isidentifier() -assert '_'.isidentifier() -assert '유니코드'.isidentifier() -assert not '😂'.isidentifier() -assert not '123'.isidentifier() +assert c.title() == "Hallo" +assert c.count("l") == 2 + +assert "aaa".count("a") == 3 +assert "aaa".count("a", 1) == 2 +assert "aaa".count("a", 1, 2) == 1 +assert "aaa".count("a", 2, 2) == 0 +assert "aaa".count("a", 2, 1) == 0 + +assert "___a__".find("a") == 3 +assert "___a__".find("a", -10) == 3 +assert "___a__".find("a", -3) == 3 +assert "___a__".find("a", -2) == -1 +assert "___a__".find("a", -1) == -1 +assert "___a__".find("a", 0) == 3 +assert "___a__".find("a", 3) == 3 +assert "___a__".find("a", 4) == -1 +assert "___a__".find("a", 10) == -1 +assert "___a__".rfind("a", 3) == 3 +assert "___a__".index("a", 3) == 3 + +assert "___a__".find("a", 0, -10) == -1 +assert "___a__".find("a", 0, -3) == -1 +assert "___a__".find("a", 0, -2) == 3 +assert "___a__".find("a", 0, -1) == 3 +assert "___a__".find("a", 0, 0) == -1 +assert "___a__".find("a", 0, 3) == -1 +assert "___a__".find("a", 0, 4) == 3 +assert "___a__".find("a", 0, 10) == 3 + +assert "___a__".find("a", 3, 3) == -1 +assert "___a__".find("a", 3, 4) == 3 +assert "___a__".find("a", 4, 3) == -1 + +assert "abcd".startswith("b", 1) +assert "abcd".startswith(("b", "z"), 1) +assert not "abcd".startswith("b", -4) +assert "abcd".startswith("b", -3) + +assert not "abcd".startswith("b", 3, 3) +assert "abcd".startswith("", 3, 3) +assert not "abcd".startswith("", 4, 3) + +assert " ".isspace() +assert "hello\nhallo\nHallo".splitlines() == ["hello", "hallo", "Hallo"] +assert "hello\nhallo\nHallo\n".splitlines() == ["hello", "hallo", "Hallo"] +assert "hello\nhallo\nHallo".splitlines(keepends=True) == [ + "hello\n", + "hallo\n", + "Hallo", +] +assert "hello\nhallo\nHallo\n".splitlines(keepends=True) == [ + "hello\n", + "hallo\n", + "Hallo\n", +] +assert ( + "hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.".splitlines() + == [ + "hello", + "hallo", + "Hallo", + "HELLO", + "hoho", + "haha", + "another", + "yetanother", + "last", + ".", + ] +) +assert ( + "hello\vhallo\x0cHallo\x1cHELLO\x1dhoho\x1ehaha\x85another\u2028yetanother\u2029last\r\n.".splitlines( + keepends=True + ) + == [ + "hello\x0b", + "hallo\x0c", + "Hallo\x1c", + "HELLO\x1d", + "hoho\x1e", + "haha\x85", + "another\u2028", + "yetanother\u2029", + "last\r\n", + ".", + ] +) +assert "abc\t12345\txyz".expandtabs() == "abc 12345 xyz" +assert "-".join(["1", "2", "3"]) == "1-2-3" +assert "HALLO".isupper() +assert "hello, my name is".partition("my ") == ("hello, ", "my ", "name is") +assert "hello".partition("is") == ("hello", "", "") +assert "hello, my name is".rpartition("is") == ("hello, my name ", "is", "") +assert "hello".rpartition("is") == ("", "", "hello") +assert not "".isdecimal() +assert "123".isdecimal() +assert not "\u00b2".isdecimal() + +assert not "".isidentifier() +assert "python".isidentifier() +assert "_".isidentifier() +assert "유니코드".isidentifier() +assert not "😂".isidentifier() +assert not "123".isidentifier() # String Formatting assert "{} {}".format(1, 2) == "1 2" assert "{0} {1}".format(2, 3) == "2 3" assert "--{:s>4}--".format(1) == "--sss1--" assert "{keyword} {0}".format(1, keyword=2) == "2 1" -assert "repr() shows quotes: {!r}; str() doesn't: {!s}".format( - 'test1', 'test2' -) == "repr() shows quotes: 'test1'; str() doesn't: test2", 'Output: {!r}, {!s}'.format('test1', 'test2') +assert ( + "repr() shows quotes: {!r}; str() doesn't: {!s}".format("test1", "test2") + == "repr() shows quotes: 'test1'; str() doesn't: test2" +), "Output: {!r}, {!s}".format("test1", "test2") class Foo: def __str__(self): - return 'str(Foo)' + return "str(Foo)" def __repr__(self): - return 'repr(Foo)' + return "repr(Foo)" f = Foo() -assert "{} {!s} {!r} {!a}".format(f, f, f, f) == 'str(Foo) str(Foo) repr(Foo) repr(Foo)' -assert "{foo} {foo!s} {foo!r} {foo!a}".format(foo=f) == 'str(Foo) str(Foo) repr(Foo) repr(Foo)' +assert "{} {!s} {!r} {!a}".format(f, f, f, f) == "str(Foo) str(Foo) repr(Foo) repr(Foo)" +assert ( + "{foo} {foo!s} {foo!r} {foo!a}".format(foo=f) + == "str(Foo) str(Foo) repr(Foo) repr(Foo)" +) # assert '{} {!r} {:10} {!r:10} {foo!r:10} {foo!r} {foo}'.format('txt1', 'txt2', 'txt3', 'txt4', 'txt5', foo='bar') # Printf-style String formatting assert "%d %d" % (1, 2) == "1 2" -assert "%*c " % (3, '❤') == " ❤ " -assert "%(first)s %(second)s" % {'second': 'World!', 'first': "Hello,"} == "Hello, World!" -assert "%(key())s" % {'key()': 'aaa'} +assert "%*c " % (3, "❤") == " ❤ " +assert ( + "%(first)s %(second)s" % {"second": "World!", "first": "Hello,"} == "Hello, World!" +) +assert "%(key())s" % {"key()": "aaa"} assert "%s %a %r" % (f, f, f) == "str(Foo) repr(Foo) repr(Foo)" -assert "repr() shows quotes: %r; str() doesn't: %s" % ("test1", "test2") == "repr() shows quotes: 'test1'; str() doesn't: test2" +assert ( + "repr() shows quotes: %r; str() doesn't: %s" % ("test1", "test2") + == "repr() shows quotes: 'test1'; str() doesn't: test2" +) assert "%f" % (1.2345) == "1.234500" assert "%+f" % (1.2345) == "+1.234500" assert "% f" % (1.2345) == " 1.234500" @@ -229,111 +279,132 @@ def __repr__(self): assert "%f" % (1.23456789012) == "1.234568" assert "%f" % (123) == "123.000000" assert "%f" % (-123) == "-123.000000" -assert "%e" % 1 == '1.000000e+00' -assert "%e" % 0 == '0.000000e+00' -assert "%e" % 0.1 == '1.000000e-01' -assert "%e" % 10 == '1.000000e+01' -assert "%.10e" % 1.2345678901234567890 == '1.2345678901e+00' -assert '%e' % float('nan') == 'nan' -assert '%e' % float('-nan') == 'nan' -assert '%E' % float('nan') == 'NAN' -assert '%e' % float('inf') == 'inf' -assert '%e' % float('-inf') == '-inf' -assert '%E' % float('inf') == 'INF' -assert "%g" % 123456.78901234567890 == '123457' -assert "%.0g" % 123456.78901234567890 == '1e+05' -assert "%.1g" % 123456.78901234567890 == '1e+05' -assert "%.2g" % 123456.78901234567890 == '1.2e+05' -assert "%g" % 1234567.8901234567890 == '1.23457e+06' -assert "%.0g" % 1234567.8901234567890 == '1e+06' -assert "%.1g" % 1234567.8901234567890 == '1e+06' -assert "%.2g" % 1234567.8901234567890 == '1.2e+06' -assert "%.3g" % 1234567.8901234567890 == '1.23e+06' -assert "%.5g" % 1234567.8901234567890 == '1.2346e+06' -assert "%.6g" % 1234567.8901234567890 == '1.23457e+06' -assert "%.7g" % 1234567.8901234567890 == '1234568' -assert "%.8g" % 1234567.8901234567890 == '1234567.9' -assert "%G" % 123456.78901234567890 == '123457' -assert "%.0G" % 123456.78901234567890 == '1E+05' -assert "%.1G" % 123456.78901234567890 == '1E+05' -assert "%.2G" % 123456.78901234567890 == '1.2E+05' -assert "%G" % 1234567.8901234567890 == '1.23457E+06' -assert "%.0G" % 1234567.8901234567890 == '1E+06' -assert "%.1G" % 1234567.8901234567890 == '1E+06' -assert "%.2G" % 1234567.8901234567890 == '1.2E+06' -assert "%.3G" % 1234567.8901234567890 == '1.23E+06' -assert "%.5G" % 1234567.8901234567890 == '1.2346E+06' -assert "%.6G" % 1234567.8901234567890 == '1.23457E+06' -assert "%.7G" % 1234567.8901234567890 == '1234568' -assert "%.8G" % 1234567.8901234567890 == '1234567.9' -assert '%g' % 0.12345678901234567890 == '0.123457' -assert '%g' % 0.12345678901234567890e-1 == '0.0123457' -assert '%g' % 0.12345678901234567890e-2 == '0.00123457' -assert '%g' % 0.12345678901234567890e-3 == '0.000123457' -assert '%g' % 0.12345678901234567890e-4 == '1.23457e-05' -assert '%g' % 0.12345678901234567890e-5 == '1.23457e-06' -assert '%.6g' % 0.12345678901234567890e-5 == '1.23457e-06' -assert '%.10g' % 0.12345678901234567890e-5 == '1.23456789e-06' -assert '%.20g' % 0.12345678901234567890e-5 == '1.2345678901234567384e-06' -assert '%G' % 0.12345678901234567890 == '0.123457' -assert '%G' % 0.12345678901234567890E-1 == '0.0123457' -assert '%G' % 0.12345678901234567890E-2 == '0.00123457' -assert '%G' % 0.12345678901234567890E-3 == '0.000123457' -assert '%G' % 0.12345678901234567890E-4 == '1.23457E-05' -assert '%G' % 0.12345678901234567890E-5 == '1.23457E-06' -assert '%.6G' % 0.12345678901234567890E-5 == '1.23457E-06' -assert '%.10G' % 0.12345678901234567890E-5 == '1.23456789E-06' -assert '%.20G' % 0.12345678901234567890E-5 == '1.2345678901234567384E-06' -assert '%g' % float('nan') == 'nan' -assert '%g' % float('-nan') == 'nan' -assert '%G' % float('nan') == 'NAN' -assert '%g' % float('inf') == 'inf' -assert '%g' % float('-inf') == '-inf' -assert '%G' % float('inf') == 'INF' -assert "%.0g" % 1.020e-13 == '1e-13' -assert "%.0g" % 1.020e-13 == '1e-13' -assert "%.1g" % 1.020e-13 == '1e-13' -assert "%.2g" % 1.020e-13 == '1e-13' -assert "%.3g" % 1.020e-13 == '1.02e-13' -assert "%.4g" % 1.020e-13 == '1.02e-13' -assert "%.5g" % 1.020e-13 == '1.02e-13' -assert "%.6g" % 1.020e-13 == '1.02e-13' -assert "%.7g" % 1.020e-13 == '1.02e-13' -assert "%g" % 1.020e-13 == '1.02e-13' -assert "%g" % 1.020e-4 == '0.000102' - -assert_raises(TypeError, lambda: "My name is %s and I'm %(age)d years old" % ("Foo", 25), _msg='format requires a mapping') -assert_raises(TypeError, lambda: "My name is %(name)s" % "Foo", _msg='format requires a mapping') -assert_raises(ValueError, lambda: "This %(food}s is great!" % {"food": "cookie"}, _msg='incomplete format key') -assert_raises(ValueError, lambda: "My name is %" % "Foo", _msg='incomplete format') - -assert 'a' < 'b' -assert 'a' <= 'b' -assert 'a' <= 'a' -assert 'z' > 'b' -assert 'z' >= 'b' -assert 'a' >= 'a' +assert "%e" % 1 == "1.000000e+00" +assert "%e" % 0 == "0.000000e+00" +assert "%e" % 0.1 == "1.000000e-01" +assert "%e" % 10 == "1.000000e+01" +assert "%.10e" % 1.2345678901234567890 == "1.2345678901e+00" +assert "%e" % float("nan") == "nan" +assert "%e" % float("-nan") == "nan" +assert "%E" % float("nan") == "NAN" +assert "%e" % float("inf") == "inf" +assert "%e" % float("-inf") == "-inf" +assert "%E" % float("inf") == "INF" +assert "%g" % 123456.78901234567890 == "123457" +assert "%.0g" % 123456.78901234567890 == "1e+05" +assert "%.1g" % 123456.78901234567890 == "1e+05" +assert "%.2g" % 123456.78901234567890 == "1.2e+05" +assert "%g" % 1234567.8901234567890 == "1.23457e+06" +assert "%.0g" % 1234567.8901234567890 == "1e+06" +assert "%.1g" % 1234567.8901234567890 == "1e+06" +assert "%.2g" % 1234567.8901234567890 == "1.2e+06" +assert "%.3g" % 1234567.8901234567890 == "1.23e+06" +assert "%.5g" % 1234567.8901234567890 == "1.2346e+06" +assert "%.6g" % 1234567.8901234567890 == "1.23457e+06" +assert "%.7g" % 1234567.8901234567890 == "1234568" +assert "%.8g" % 1234567.8901234567890 == "1234567.9" +assert "%G" % 123456.78901234567890 == "123457" +assert "%.0G" % 123456.78901234567890 == "1E+05" +assert "%.1G" % 123456.78901234567890 == "1E+05" +assert "%.2G" % 123456.78901234567890 == "1.2E+05" +assert "%G" % 1234567.8901234567890 == "1.23457E+06" +assert "%.0G" % 1234567.8901234567890 == "1E+06" +assert "%.1G" % 1234567.8901234567890 == "1E+06" +assert "%.2G" % 1234567.8901234567890 == "1.2E+06" +assert "%.3G" % 1234567.8901234567890 == "1.23E+06" +assert "%.5G" % 1234567.8901234567890 == "1.2346E+06" +assert "%.6G" % 1234567.8901234567890 == "1.23457E+06" +assert "%.7G" % 1234567.8901234567890 == "1234568" +assert "%.8G" % 1234567.8901234567890 == "1234567.9" +assert "%g" % 0.12345678901234567890 == "0.123457" +assert "%g" % 0.12345678901234567890e-1 == "0.0123457" +assert "%g" % 0.12345678901234567890e-2 == "0.00123457" +assert "%g" % 0.12345678901234567890e-3 == "0.000123457" +assert "%g" % 0.12345678901234567890e-4 == "1.23457e-05" +assert "%g" % 0.12345678901234567890e-5 == "1.23457e-06" +assert "%.6g" % 0.12345678901234567890e-5 == "1.23457e-06" +assert "%.10g" % 0.12345678901234567890e-5 == "1.23456789e-06" +assert "%.20g" % 0.12345678901234567890e-5 == "1.2345678901234567384e-06" +assert "%G" % 0.12345678901234567890 == "0.123457" +assert "%G" % 0.12345678901234567890e-1 == "0.0123457" +assert "%G" % 0.12345678901234567890e-2 == "0.00123457" +assert "%G" % 0.12345678901234567890e-3 == "0.000123457" +assert "%G" % 0.12345678901234567890e-4 == "1.23457E-05" +assert "%G" % 0.12345678901234567890e-5 == "1.23457E-06" +assert "%.6G" % 0.12345678901234567890e-5 == "1.23457E-06" +assert "%.10G" % 0.12345678901234567890e-5 == "1.23456789E-06" +assert "%.20G" % 0.12345678901234567890e-5 == "1.2345678901234567384E-06" +assert "%g" % float("nan") == "nan" +assert "%g" % float("-nan") == "nan" +assert "%G" % float("nan") == "NAN" +assert "%g" % float("inf") == "inf" +assert "%g" % float("-inf") == "-inf" +assert "%G" % float("inf") == "INF" +assert "%.0g" % 1.020e-13 == "1e-13" +assert "%.0g" % 1.020e-13 == "1e-13" +assert "%.1g" % 1.020e-13 == "1e-13" +assert "%.2g" % 1.020e-13 == "1e-13" +assert "%.3g" % 1.020e-13 == "1.02e-13" +assert "%.4g" % 1.020e-13 == "1.02e-13" +assert "%.5g" % 1.020e-13 == "1.02e-13" +assert "%.6g" % 1.020e-13 == "1.02e-13" +assert "%.7g" % 1.020e-13 == "1.02e-13" +assert "%g" % 1.020e-13 == "1.02e-13" +assert "%g" % 1.020e-4 == "0.000102" + +assert_raises( + TypeError, + lambda: "My name is %s and I'm %(age)d years old" % ("Foo", 25), + _msg="format requires a mapping", +) +assert_raises( + TypeError, lambda: "My name is %(name)s" % "Foo", _msg="format requires a mapping" +) +assert_raises( + ValueError, + lambda: "This %(food}s is great!" % {"food": "cookie"}, + _msg="incomplete format key", +) +assert_raises(ValueError, lambda: "My name is %" % "Foo", _msg="incomplete format") + +assert "a" < "b" +assert "a" <= "b" +assert "a" <= "a" +assert "z" > "b" +assert "z" >= "b" +assert "a" >= "a" # str.translate -assert "abc".translate({97: '🎅', 98: None, 99: "xd"}) == "🎅xd" +assert "abc".translate({97: "🎅", 98: None, 99: "xd"}) == "🎅xd" # str.maketrans assert str.maketrans({"a": "abc", "b": None, "c": 33}) == {97: "abc", 98: None, 99: 33} -assert str.maketrans("hello", "world", "rust") == {104: 119, 101: 111, 108: 108, 111: 100, 114: None, 117: None, 115: None, 116: None} +assert str.maketrans("hello", "world", "rust") == { + 104: 119, + 101: 111, + 108: 108, + 111: 100, + 114: None, + 117: None, + 115: None, + 116: None, +} + def try_mutate_str(): - word = "word" - word[0] = 'x' + word = "word" + word[0] = "x" + assert_raises(TypeError, try_mutate_str) -ss = ['Hello', '안녕', '👋'] -bs = [b'Hello', b'\xec\x95\x88\xeb\x85\x95', b'\xf0\x9f\x91\x8b'] +ss = ["Hello", "안녕", "👋"] +bs = [b"Hello", b"\xec\x95\x88\xeb\x85\x95", b"\xf0\x9f\x91\x8b"] for s, b in zip(ss, bs): assert s.encode() == b -for s, b, e in zip(ss, bs, ['u8', 'U8', 'utf-8', 'UTF-8', 'utf_8']): +for s, b, e in zip(ss, bs, ["u8", "U8", "utf-8", "UTF-8", "utf_8"]): assert s.encode(e) == b # assert s.encode(encoding=e) == b @@ -349,9 +420,9 @@ def try_mutate_str(): assert "\u0037" == "7" assert "\u0040" == "@" assert "\u0041" == "A" -assert "\u00BE" == "¾" +assert "\u00be" == "¾" assert "\u9487" == "钇" -assert "\U0001F609" == "😉" +assert "\U0001f609" == "😉" # test str iter iterable_str = "12345678😉" @@ -383,16 +454,16 @@ def try_mutate_str(): assert next(str_iter_reversed, None) == None assert_raises(StopIteration, next, str_iter_reversed) -assert str.__rmod__('%i', 30) == NotImplemented -assert_raises(TypeError, lambda: str.__rmod__(30, '%i')) +assert str.__rmod__("%i", 30) == NotImplemented +assert_raises(TypeError, lambda: str.__rmod__(30, "%i")) # test str index -index_str = 'Rust Python' +index_str = "Rust Python" -assert index_str[0] == 'R' -assert index_str[-1] == 'n' +assert index_str[0] == "R" +assert index_str[-1] == "n" -assert_raises(TypeError, lambda: index_str['a']) +assert_raises(TypeError, lambda: index_str["a"]) assert chr(9).__repr__() == "'\\t'" assert chr(99).__repr__() == "'c'" @@ -424,321 +495,330 @@ def try_mutate_str(): # >>> '{x} {y}'.format_map({'x': 1, 'y': 2}) # '1 2' -assert '{x} {y}'.format_map({'x': 1, 'y': 2}) == '1 2' +assert "{x} {y}".format_map({"x": 1, "y": 2}) == "1 2" # >>> '{x:04d}'.format_map({'x': 1}) # '0001' -assert '{x:04d}'.format_map({'x': 1}) == '0001' +assert "{x:04d}".format_map({"x": 1}) == "0001" # >>> '{x} {y}'.format_map('foo') # Traceback (most recent call last): # File "", line 1, in # TypeError: string indices must be integers with AssertRaises(TypeError, None): - '{x} {y}'.format_map('foo') + "{x} {y}".format_map("foo") # >>> '{x} {y}'.format_map(['foo']) # Traceback (most recent call last): # File "", line 1, in # TypeError: list indices must be integers or slices, not str with AssertRaises(TypeError, None): - '{x} {y}'.format_map(['foo']) + "{x} {y}".format_map(["foo"]) # >>> '{x} {y}'.format_map() # Traceback (most recent call last): # File "", line 1, in # TypeError: format_map() takes exactly one argument (0 given) -with AssertRaises(TypeError, msg='TypeError: format_map() takes exactly one argument (0 given)'): - '{x} {y}'.format_map(), +with AssertRaises( + TypeError, msg="TypeError: format_map() takes exactly one argument (0 given)" +): + ("{x} {y}".format_map(),) # >>> '{x} {y}'.format_map('foo', 'bar') # Traceback (most recent call last): # File "", line 1, in # TypeError: format_map() takes exactly one argument (2 given) -with AssertRaises(TypeError, msg='TypeError: format_map() takes exactly one argument (2 given)'): - '{x} {y}'.format_map('foo', 'bar') +with AssertRaises( + TypeError, msg="TypeError: format_map() takes exactly one argument (2 given)" +): + "{x} {y}".format_map("foo", "bar") # >>> '{x} {y}'.format_map({'x': 1}) # Traceback (most recent call last): # File "", line 1, in # KeyError: 'y' with AssertRaises(KeyError, msg="KeyError: 'y'"): - '{x} {y}'.format_map({'x': 1}) + "{x} {y}".format_map({"x": 1}) # >>> '{x} {y}'.format_map({'x': 1, 'z': 2}) # Traceback (most recent call last): # File "", line 1, in # KeyError: 'y' with AssertRaises(KeyError, msg="KeyError: 'y'"): - '{x} {y}'.format_map({'x': 1, 'z': 2}) + "{x} {y}".format_map({"x": 1, "z": 2}) # >>> '{{literal}}'.format_map('foo') # '{literal}' -assert '{{literal}}'.format_map('foo') == '{literal}' +assert "{{literal}}".format_map("foo") == "{literal}" # test formatting float values -assert f'{5:f}' == '5.000000' -assert f'{-5:f}' == '-5.000000' -assert f'{5.0:f}' == '5.000000' -assert f'{-5.0:f}' == '-5.000000' -assert f'{5:.2f}' == '5.00' -assert f'{5.0:.2f}' == '5.00' -assert f'{-5:.2f}' == '-5.00' -assert f'{-5.0:.2f}' == '-5.00' -assert f'{5.0:04f}' == '5.000000' -assert f'{5.1234:+f}' == '+5.123400' -assert f'{5.1234: f}' == ' 5.123400' -assert f'{5.1234:-f}' == '5.123400' -assert f'{-5.1234:-f}' == '-5.123400' -assert f'{1.0:+}' == '+1.0' -assert f'--{1.0:f>4}--' == '--f1.0--' -assert f'--{1.0:f<4}--' == '--1.0f--' -assert f'--{1.0:d^4}--' == '--1.0d--' -assert f'--{1.0:d^5}--' == '--d1.0d--' -assert f'--{1.1:f>6}--' == '--fff1.1--' -assert '{}'.format(float('nan')) == 'nan' -assert '{:f}'.format(float('nan')) == 'nan' -assert '{:f}'.format(float('-nan')) == 'nan' -assert '{:F}'.format(float('nan')) == 'NAN' -assert '{}'.format(float('inf')) == 'inf' -assert '{:f}'.format(float('inf')) == 'inf' -assert '{:f}'.format(float('-inf')) == '-inf' -assert '{:F}'.format(float('inf')) == 'INF' -assert f'{1234567890.1234:,.2f}' == '1,234,567,890.12' -assert f'{1234567890.1234:_.2f}' == '1_234_567_890.12' +assert f"{5:f}" == "5.000000" +assert f"{-5:f}" == "-5.000000" +assert f"{5.0:f}" == "5.000000" +assert f"{-5.0:f}" == "-5.000000" +assert f"{5:.2f}" == "5.00" +assert f"{5.0:.2f}" == "5.00" +assert f"{-5:.2f}" == "-5.00" +assert f"{-5.0:.2f}" == "-5.00" +assert f"{5.0:04f}" == "5.000000" +assert f"{5.1234:+f}" == "+5.123400" +assert f"{5.1234: f}" == " 5.123400" +assert f"{5.1234:-f}" == "5.123400" +assert f"{-5.1234:-f}" == "-5.123400" +assert f"{1.0:+}" == "+1.0" +assert f"--{1.0:f>4}--" == "--f1.0--" +assert f"--{1.0:f<4}--" == "--1.0f--" +assert f"--{1.0:d^4}--" == "--1.0d--" +assert f"--{1.0:d^5}--" == "--d1.0d--" +assert f"--{1.1:f>6}--" == "--fff1.1--" +assert "{}".format(float("nan")) == "nan" +assert "{:f}".format(float("nan")) == "nan" +assert "{:f}".format(float("-nan")) == "nan" +assert "{:F}".format(float("nan")) == "NAN" +assert "{}".format(float("inf")) == "inf" +assert "{:f}".format(float("inf")) == "inf" +assert "{:f}".format(float("-inf")) == "-inf" +assert "{:F}".format(float("inf")) == "INF" +assert f"{1234567890.1234:,.2f}" == "1,234,567,890.12" +assert f"{1234567890.1234:_.2f}" == "1_234_567_890.12" with AssertRaises(ValueError, msg="Unknown format code 'd' for object of type 'float'"): - f'{5.0:04d}' + f"{5.0:04d}" # Test % formatting -assert f'{10:%}' == '1000.000000%' -assert f'{10.0:%}' == '1000.000000%' -assert f'{10.0:.2%}' == '1000.00%' -assert f'{10.0:.8%}' == '1000.00000000%' -assert f'{-10:%}' == '-1000.000000%' -assert f'{-10.0:%}' == '-1000.000000%' -assert f'{-10.0:.2%}' == '-1000.00%' -assert f'{-10.0:.8%}' == '-1000.00000000%' -assert '{:%}'.format(float('nan')) == 'nan%' -assert '{:.2%}'.format(float('nan')) == 'nan%' -assert '{:%}'.format(float('inf')) == 'inf%' -assert '{:.2%}'.format(float('inf')) == 'inf%' -with AssertRaises(ValueError, msg='Invalid format specifier'): - f'{10.0:%3}' +assert f"{10:%}" == "1000.000000%" +assert f"{10.0:%}" == "1000.000000%" +assert f"{10.0:.2%}" == "1000.00%" +assert f"{10.0:.8%}" == "1000.00000000%" +assert f"{-10:%}" == "-1000.000000%" +assert f"{-10.0:%}" == "-1000.000000%" +assert f"{-10.0:.2%}" == "-1000.00%" +assert f"{-10.0:.8%}" == "-1000.00000000%" +assert "{:%}".format(float("nan")) == "nan%" +assert "{:.2%}".format(float("nan")) == "nan%" +assert "{:%}".format(float("inf")) == "inf%" +assert "{:.2%}".format(float("inf")) == "inf%" +with AssertRaises(ValueError, msg="Invalid format specifier"): + f"{10.0:%3}" # Test e & E formatting -assert '{:e}'.format(10) == '1.000000e+01' -assert '{:.2e}'.format(11) == '1.10e+01' -assert '{:e}'.format(10.0) == '1.000000e+01' -assert '{:e}'.format(-10.0) == '-1.000000e+01' -assert '{:.2e}'.format(10.0) == '1.00e+01' -assert '{:.2e}'.format(-10.0) == '-1.00e+01' -assert '{:.2e}'.format(10.1) == '1.01e+01' -assert '{:.2e}'.format(-10.1) == '-1.01e+01' -assert '{:.2e}'.format(10.001) == '1.00e+01' -assert '{:.4e}'.format(100.234) == '1.0023e+02' -assert '{:.5e}'.format(100.234) == '1.00234e+02' -assert '{:.2E}'.format(10.0) == '1.00E+01' -assert '{:.2E}'.format(-10.0) == '-1.00E+01' -assert '{:e}'.format(float('nan')) == 'nan' -assert '{:e}'.format(float('-nan')) == 'nan' -assert '{:E}'.format(float('nan')) == 'NAN' -assert '{:e}'.format(float('inf')) == 'inf' -assert '{:e}'.format(float('-inf')) == '-inf' -assert '{:E}'.format(float('inf')) == 'INF' +assert "{:e}".format(10) == "1.000000e+01" +assert "{:.2e}".format(11) == "1.10e+01" +assert "{:e}".format(10.0) == "1.000000e+01" +assert "{:e}".format(-10.0) == "-1.000000e+01" +assert "{:.2e}".format(10.0) == "1.00e+01" +assert "{:.2e}".format(-10.0) == "-1.00e+01" +assert "{:.2e}".format(10.1) == "1.01e+01" +assert "{:.2e}".format(-10.1) == "-1.01e+01" +assert "{:.2e}".format(10.001) == "1.00e+01" +assert "{:.4e}".format(100.234) == "1.0023e+02" +assert "{:.5e}".format(100.234) == "1.00234e+02" +assert "{:.2E}".format(10.0) == "1.00E+01" +assert "{:.2E}".format(-10.0) == "-1.00E+01" +assert "{:e}".format(float("nan")) == "nan" +assert "{:e}".format(float("-nan")) == "nan" +assert "{:E}".format(float("nan")) == "NAN" +assert "{:e}".format(float("inf")) == "inf" +assert "{:e}".format(float("-inf")) == "-inf" +assert "{:E}".format(float("inf")) == "INF" # Test g & G formatting -assert '{:g}'.format(10.0) == '10' -assert '{:g}'.format(100000.0) == '100000' -assert '{:g}'.format(123456.78901234567890) == '123457' -assert '{:.0g}'.format(123456.78901234567890) == '1e+05' -assert '{:.1g}'.format(123456.78901234567890) == '1e+05' -assert '{:.2g}'.format(123456.78901234567890) == '1.2e+05' -assert '{:g}'.format(1234567.8901234567890) == '1.23457e+06' -assert '{:.0g}'.format(1234567.8901234567890) == '1e+06' -assert '{:.1g}'.format(1234567.8901234567890) == '1e+06' -assert '{:.2g}'.format(1234567.8901234567890) == '1.2e+06' -assert '{:.3g}'.format(1234567.8901234567890) == '1.23e+06' -assert '{:.5g}'.format(1234567.8901234567890) == '1.2346e+06' -assert '{:.6g}'.format(1234567.8901234567890) == '1.23457e+06' -assert '{:.7g}'.format(1234567.8901234567890) == '1234568' -assert '{:.8g}'.format(1234567.8901234567890) == '1234567.9' -assert '{:G}'.format(123456.78901234567890) == '123457' -assert '{:.0G}'.format(123456.78901234567890) == '1E+05' -assert '{:.1G}'.format(123456.78901234567890) == '1E+05' -assert '{:.2G}'.format(123456.78901234567890) == '1.2E+05' -assert '{:G}'.format(1234567.8901234567890) == '1.23457E+06' -assert '{:.0G}'.format(1234567.8901234567890) == '1E+06' -assert '{:.1G}'.format(1234567.8901234567890) == '1E+06' -assert '{:.2G}'.format(1234567.8901234567890) == '1.2E+06' -assert '{:.3G}'.format(1234567.8901234567890) == '1.23E+06' -assert '{:.5G}'.format(1234567.8901234567890) == '1.2346E+06' -assert '{:.6G}'.format(1234567.8901234567890) == '1.23457E+06' -assert '{:.7G}'.format(1234567.8901234567890) == '1234568' -assert '{:.8G}'.format(1234567.8901234567890) == '1234567.9' -assert '{:g}'.format(0.12345678901234567890) == '0.123457' -assert '{:g}'.format(0.12345678901234567890e-1) == '0.0123457' -assert '{:g}'.format(0.12345678901234567890e-2) == '0.00123457' -assert '{:g}'.format(0.12345678901234567890e-3) == '0.000123457' -assert '{:g}'.format(0.12345678901234567890e-4) == '1.23457e-05' -assert '{:g}'.format(0.12345678901234567890e-5) == '1.23457e-06' -assert '{:.6g}'.format(0.12345678901234567890e-5) == '1.23457e-06' -assert '{:.10g}'.format(0.12345678901234567890e-5) == '1.23456789e-06' -assert '{:.20g}'.format(0.12345678901234567890e-5) == '1.2345678901234567384e-06' -assert '{:G}'.format(0.12345678901234567890) == '0.123457' -assert '{:G}'.format(0.12345678901234567890E-1) == '0.0123457' -assert '{:G}'.format(0.12345678901234567890E-2) == '0.00123457' -assert '{:G}'.format(0.12345678901234567890E-3) == '0.000123457' -assert '{:G}'.format(0.12345678901234567890E-4) == '1.23457E-05' -assert '{:G}'.format(0.12345678901234567890E-5) == '1.23457E-06' -assert '{:.6G}'.format(0.12345678901234567890E-5) == '1.23457E-06' -assert '{:.10G}'.format(0.12345678901234567890E-5) == '1.23456789E-06' -assert '{:.20G}'.format(0.12345678901234567890E-5) == '1.2345678901234567384E-06' -assert '{:g}'.format(float('nan')) == 'nan' -assert '{:g}'.format(float('-nan')) == 'nan' -assert '{:G}'.format(float('nan')) == 'NAN' -assert '{:g}'.format(float('inf')) == 'inf' -assert '{:g}'.format(float('-inf')) == '-inf' -assert '{:G}'.format(float('inf')) == 'INF' -assert '{:.0g}'.format(1.020e-13) == '1e-13' -assert '{:.0g}'.format(1.020e-13) == '1e-13' -assert '{:.1g}'.format(1.020e-13) == '1e-13' -assert '{:.2g}'.format(1.020e-13) == '1e-13' -assert '{:.3g}'.format(1.020e-13) == '1.02e-13' -assert '{:.4g}'.format(1.020e-13) == '1.02e-13' -assert '{:.5g}'.format(1.020e-13) == '1.02e-13' -assert '{:.6g}'.format(1.020e-13) == '1.02e-13' -assert '{:.7g}'.format(1.020e-13) == '1.02e-13' -assert '{:g}'.format(1.020e-13) == '1.02e-13' -assert "{:g}".format(1.020e-4) == '0.000102' +assert "{:g}".format(10.0) == "10" +assert "{:g}".format(100000.0) == "100000" +assert "{:g}".format(123456.78901234567890) == "123457" +assert "{:.0g}".format(123456.78901234567890) == "1e+05" +assert "{:.1g}".format(123456.78901234567890) == "1e+05" +assert "{:.2g}".format(123456.78901234567890) == "1.2e+05" +assert "{:g}".format(1234567.8901234567890) == "1.23457e+06" +assert "{:.0g}".format(1234567.8901234567890) == "1e+06" +assert "{:.1g}".format(1234567.8901234567890) == "1e+06" +assert "{:.2g}".format(1234567.8901234567890) == "1.2e+06" +assert "{:.3g}".format(1234567.8901234567890) == "1.23e+06" +assert "{:.5g}".format(1234567.8901234567890) == "1.2346e+06" +assert "{:.6g}".format(1234567.8901234567890) == "1.23457e+06" +assert "{:.7g}".format(1234567.8901234567890) == "1234568" +assert "{:.8g}".format(1234567.8901234567890) == "1234567.9" +assert "{:G}".format(123456.78901234567890) == "123457" +assert "{:.0G}".format(123456.78901234567890) == "1E+05" +assert "{:.1G}".format(123456.78901234567890) == "1E+05" +assert "{:.2G}".format(123456.78901234567890) == "1.2E+05" +assert "{:G}".format(1234567.8901234567890) == "1.23457E+06" +assert "{:.0G}".format(1234567.8901234567890) == "1E+06" +assert "{:.1G}".format(1234567.8901234567890) == "1E+06" +assert "{:.2G}".format(1234567.8901234567890) == "1.2E+06" +assert "{:.3G}".format(1234567.8901234567890) == "1.23E+06" +assert "{:.5G}".format(1234567.8901234567890) == "1.2346E+06" +assert "{:.6G}".format(1234567.8901234567890) == "1.23457E+06" +assert "{:.7G}".format(1234567.8901234567890) == "1234568" +assert "{:.8G}".format(1234567.8901234567890) == "1234567.9" +assert "{:g}".format(0.12345678901234567890) == "0.123457" +assert "{:g}".format(0.12345678901234567890e-1) == "0.0123457" +assert "{:g}".format(0.12345678901234567890e-2) == "0.00123457" +assert "{:g}".format(0.12345678901234567890e-3) == "0.000123457" +assert "{:g}".format(0.12345678901234567890e-4) == "1.23457e-05" +assert "{:g}".format(0.12345678901234567890e-5) == "1.23457e-06" +assert "{:.6g}".format(0.12345678901234567890e-5) == "1.23457e-06" +assert "{:.10g}".format(0.12345678901234567890e-5) == "1.23456789e-06" +assert "{:.20g}".format(0.12345678901234567890e-5) == "1.2345678901234567384e-06" +assert "{:G}".format(0.12345678901234567890) == "0.123457" +assert "{:G}".format(0.12345678901234567890e-1) == "0.0123457" +assert "{:G}".format(0.12345678901234567890e-2) == "0.00123457" +assert "{:G}".format(0.12345678901234567890e-3) == "0.000123457" +assert "{:G}".format(0.12345678901234567890e-4) == "1.23457E-05" +assert "{:G}".format(0.12345678901234567890e-5) == "1.23457E-06" +assert "{:.6G}".format(0.12345678901234567890e-5) == "1.23457E-06" +assert "{:.10G}".format(0.12345678901234567890e-5) == "1.23456789E-06" +assert "{:.20G}".format(0.12345678901234567890e-5) == "1.2345678901234567384E-06" +assert "{:g}".format(float("nan")) == "nan" +assert "{:g}".format(float("-nan")) == "nan" +assert "{:G}".format(float("nan")) == "NAN" +assert "{:g}".format(float("inf")) == "inf" +assert "{:g}".format(float("-inf")) == "-inf" +assert "{:G}".format(float("inf")) == "INF" +assert "{:.0g}".format(1.020e-13) == "1e-13" +assert "{:.0g}".format(1.020e-13) == "1e-13" +assert "{:.1g}".format(1.020e-13) == "1e-13" +assert "{:.2g}".format(1.020e-13) == "1e-13" +assert "{:.3g}".format(1.020e-13) == "1.02e-13" +assert "{:.4g}".format(1.020e-13) == "1.02e-13" +assert "{:.5g}".format(1.020e-13) == "1.02e-13" +assert "{:.6g}".format(1.020e-13) == "1.02e-13" +assert "{:.7g}".format(1.020e-13) == "1.02e-13" +assert "{:g}".format(1.020e-13) == "1.02e-13" +assert "{:g}".format(1.020e-4) == "0.000102" # Test n & N formatting -assert '{:n}'.format(999999.1234) == '999999' -assert '{:n}'.format(9999.1234) == '9999.12' -assert '{:n}'.format(-1000000.1234) == '-1e+06' -assert '{:n}'.format(1000000.1234) == '1e+06' -assert '{:.1n}'.format(1000000.1234) == '1e+06' -assert '{:.2n}'.format(1000000.1234) == '1e+06' -assert '{:.3n}'.format(1000000.1234) == '1e+06' -assert '{:.4n}'.format(1000000.1234) == '1e+06' -assert '{:.5n}'.format(1000000.1234) == '1e+06' -assert '{:.6n}'.format(1000000.1234) == '1e+06' -assert '{:.7n}'.format(1000000.1234) == '1000000' -assert '{:.8n}'.format(1000000.1234) == '1000000.1' -assert '{:.10n}'.format(1000000.1234) == '1000000.123' -assert '{:.11n}'.format(1000000.1234) == '1000000.1234' -assert '{:.11n}'.format(-1000000.1234) == '-1000000.1234' -assert '{:0n}'.format(-1000000.1234) == '-1e+06' -assert '{:n}'.format(-1000000.1234) == '-1e+06' -assert '{:-1n}'.format(-1000000.1234) == '-1e+06' +assert "{:n}".format(999999.1234) == "999999" +assert "{:n}".format(9999.1234) == "9999.12" +assert "{:n}".format(-1000000.1234) == "-1e+06" +assert "{:n}".format(1000000.1234) == "1e+06" +assert "{:.1n}".format(1000000.1234) == "1e+06" +assert "{:.2n}".format(1000000.1234) == "1e+06" +assert "{:.3n}".format(1000000.1234) == "1e+06" +assert "{:.4n}".format(1000000.1234) == "1e+06" +assert "{:.5n}".format(1000000.1234) == "1e+06" +assert "{:.6n}".format(1000000.1234) == "1e+06" +assert "{:.7n}".format(1000000.1234) == "1000000" +assert "{:.8n}".format(1000000.1234) == "1000000.1" +assert "{:.10n}".format(1000000.1234) == "1000000.123" +assert "{:.11n}".format(1000000.1234) == "1000000.1234" +assert "{:.11n}".format(-1000000.1234) == "-1000000.1234" +assert "{:0n}".format(-1000000.1234) == "-1e+06" +assert "{:n}".format(-1000000.1234) == "-1e+06" +assert "{:-1n}".format(-1000000.1234) == "-1e+06" with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:N}'.format(999999.1234) + "{:N}".format(999999.1234) with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:.1N}'.format(1000000.1234) + "{:.1N}".format(1000000.1234) with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:0N}'.format(-1000000.1234) + "{:0N}".format(-1000000.1234) with AssertRaises(ValueError, msg="Unknown format code 'N' for object of type 'float'"): - '{:-1N}'.format(-1000000.1234) + "{:-1N}".format(-1000000.1234) + # remove*fix test def test_removeprefix(): - s = 'foobarfoo' - s_ref='foobarfoo' - assert s.removeprefix('f') == s_ref[1:] - assert s.removeprefix('fo') == s_ref[2:] - assert s.removeprefix('foo') == s_ref[3:] - - assert s.removeprefix('') == s_ref - assert s.removeprefix('bar') == s_ref - assert s.removeprefix('lol') == s_ref - assert s.removeprefix('_foo') == s_ref - assert s.removeprefix('-foo') == s_ref - assert s.removeprefix('afoo') == s_ref - assert s.removeprefix('*foo') == s_ref - - assert s==s_ref, 'undefined test fail' - - s_uc = '😱foobarfoo🖖' - s_ref_uc = '😱foobarfoo🖖' - assert s_uc.removeprefix('😱') == s_ref_uc[1:] - assert s_uc.removeprefix('😱fo') == s_ref_uc[3:] - assert s_uc.removeprefix('😱foo') == s_ref_uc[4:] - - assert s_uc.removeprefix('🖖') == s_ref_uc - assert s_uc.removeprefix('foo') == s_ref_uc - assert s_uc.removeprefix(' ') == s_ref_uc - assert s_uc.removeprefix('_😱') == s_ref_uc - assert s_uc.removeprefix(' 😱') == s_ref_uc - assert s_uc.removeprefix('-😱') == s_ref_uc - assert s_uc.removeprefix('#😱') == s_ref_uc + s = "foobarfoo" + s_ref = "foobarfoo" + assert s.removeprefix("f") == s_ref[1:] + assert s.removeprefix("fo") == s_ref[2:] + assert s.removeprefix("foo") == s_ref[3:] + + assert s.removeprefix("") == s_ref + assert s.removeprefix("bar") == s_ref + assert s.removeprefix("lol") == s_ref + assert s.removeprefix("_foo") == s_ref + assert s.removeprefix("-foo") == s_ref + assert s.removeprefix("afoo") == s_ref + assert s.removeprefix("*foo") == s_ref + + assert s == s_ref, "undefined test fail" + + s_uc = "😱foobarfoo🖖" + s_ref_uc = "😱foobarfoo🖖" + assert s_uc.removeprefix("😱") == s_ref_uc[1:] + assert s_uc.removeprefix("😱fo") == s_ref_uc[3:] + assert s_uc.removeprefix("😱foo") == s_ref_uc[4:] + + assert s_uc.removeprefix("🖖") == s_ref_uc + assert s_uc.removeprefix("foo") == s_ref_uc + assert s_uc.removeprefix(" ") == s_ref_uc + assert s_uc.removeprefix("_😱") == s_ref_uc + assert s_uc.removeprefix(" 😱") == s_ref_uc + assert s_uc.removeprefix("-😱") == s_ref_uc + assert s_uc.removeprefix("#😱") == s_ref_uc + def test_removeprefix_types(): - s='0123456' - s_ref='0123456' - others=[0,['012']] - found=False + s = "0123456" + s_ref = "0123456" + others = [0, ["012"]] + found = False for o in others: try: s.removeprefix(o) except: - found=True + found = True + + assert found, f"Removeprefix accepts other type: {type(o)}: {o=}" - assert found, f'Removeprefix accepts other type: {type(o)}: {o=}' def test_removesuffix(): - s='foobarfoo' - s_ref='foobarfoo' - assert s.removesuffix('o') == s_ref[:-1] - assert s.removesuffix('oo') == s_ref[:-2] - assert s.removesuffix('foo') == s_ref[:-3] - - assert s.removesuffix('') == s_ref - assert s.removesuffix('bar') == s_ref - assert s.removesuffix('lol') == s_ref - assert s.removesuffix('foo_') == s_ref - assert s.removesuffix('foo-') == s_ref - assert s.removesuffix('foo*') == s_ref - assert s.removesuffix('fooa') == s_ref - - assert s==s_ref, 'undefined test fail' - - s_uc = '😱foobarfoo🖖' - s_ref_uc = '😱foobarfoo🖖' - assert s_uc.removesuffix('🖖') == s_ref_uc[:-1] - assert s_uc.removesuffix('oo🖖') == s_ref_uc[:-3] - assert s_uc.removesuffix('foo🖖') == s_ref_uc[:-4] - - assert s_uc.removesuffix('😱') == s_ref_uc - assert s_uc.removesuffix('foo') == s_ref_uc - assert s_uc.removesuffix(' ') == s_ref_uc - assert s_uc.removesuffix('🖖_') == s_ref_uc - assert s_uc.removesuffix('🖖 ') == s_ref_uc - assert s_uc.removesuffix('🖖-') == s_ref_uc - assert s_uc.removesuffix('🖖#') == s_ref_uc + s = "foobarfoo" + s_ref = "foobarfoo" + assert s.removesuffix("o") == s_ref[:-1] + assert s.removesuffix("oo") == s_ref[:-2] + assert s.removesuffix("foo") == s_ref[:-3] + + assert s.removesuffix("") == s_ref + assert s.removesuffix("bar") == s_ref + assert s.removesuffix("lol") == s_ref + assert s.removesuffix("foo_") == s_ref + assert s.removesuffix("foo-") == s_ref + assert s.removesuffix("foo*") == s_ref + assert s.removesuffix("fooa") == s_ref + + assert s == s_ref, "undefined test fail" + + s_uc = "😱foobarfoo🖖" + s_ref_uc = "😱foobarfoo🖖" + assert s_uc.removesuffix("🖖") == s_ref_uc[:-1] + assert s_uc.removesuffix("oo🖖") == s_ref_uc[:-3] + assert s_uc.removesuffix("foo🖖") == s_ref_uc[:-4] + + assert s_uc.removesuffix("😱") == s_ref_uc + assert s_uc.removesuffix("foo") == s_ref_uc + assert s_uc.removesuffix(" ") == s_ref_uc + assert s_uc.removesuffix("🖖_") == s_ref_uc + assert s_uc.removesuffix("🖖 ") == s_ref_uc + assert s_uc.removesuffix("🖖-") == s_ref_uc + assert s_uc.removesuffix("🖖#") == s_ref_uc + def test_removesuffix_types(): - s='0123456' - s_ref='0123456' - others=[0,6,['6']] - found=False + s = "0123456" + s_ref = "0123456" + others = [0, 6, ["6"]] + found = False for o in others: try: s.removesuffix(o) except: - found=True + found = True - assert found, f'Removesuffix accepts other type: {type(o)}: {o=}' + assert found, f"Removesuffix accepts other type: {type(o)}: {o=}" -skip_if_unsupported(3,9,test_removeprefix) -skip_if_unsupported(3,9,test_removeprefix_types) -skip_if_unsupported(3,9,test_removesuffix) -skip_if_unsupported(3,9,test_removesuffix_types) + +skip_if_unsupported(3, 9, test_removeprefix) +skip_if_unsupported(3, 9, test_removeprefix_types) +skip_if_unsupported(3, 9, test_removesuffix) +skip_if_unsupported(3, 9, test_removesuffix_types) # Regression to # https://github.com/RustPython/RustPython/issues/2840 -a = 'abc123()' +a = "abc123()" assert id(a) == id(a) assert id(a) != id(a * -1) @@ -751,7 +831,8 @@ def test_removesuffix_types(): class MyString(str): pass -b = MyString('0123abc*&') + +b = MyString("0123abc*&") assert id(b) == id(b) assert id(b) != id(b * -1) assert id(b) != id(b * 0) diff --git a/extra_tests/snippets/builtin_str_encode.py b/extra_tests/snippets/builtin_str_encode.py index 790e156e6f..156a83e205 100644 --- a/extra_tests/snippets/builtin_str_encode.py +++ b/extra_tests/snippets/builtin_str_encode.py @@ -10,11 +10,13 @@ assert_raises(UnicodeEncodeError, "¿como estás?".encode, "ascii") + def round_trip(s, encoding="utf-8"): encoded = s.encode(encoding) decoded = encoded.decode(encoding) assert s == decoded + round_trip("👺♦ 𝐚Şđƒ ☆☝") round_trip("☢🐣 ᖇ𝓤𝕊тⓟ𝕐𝕥卄σ𝔫 ♬👣") round_trip("💀👌 ק𝔂tℍⓞ𝓷 ３ 🔥👤") diff --git a/extra_tests/snippets/builtin_str_subclass.py b/extra_tests/snippets/builtin_str_subclass.py index 3ec266d5c3..73e23615c4 100644 --- a/extra_tests/snippets/builtin_str_subclass.py +++ b/extra_tests/snippets/builtin_str_subclass.py @@ -3,6 +3,7 @@ x = "An interesting piece of text" assert x is str(x) + class Stringy(str): def __new__(cls, value=""): return str.__new__(cls, value) @@ -10,6 +11,7 @@ def __new__(cls, value=""): def __init__(self, value): self.x = "substr" + y = Stringy(1) assert type(y) is Stringy, "Type of Stringy should be stringy" assert type(str(y)) is str, "Str of a str-subtype should be a str." diff --git a/extra_tests/snippets/builtin_str_unicode.py b/extra_tests/snippets/builtin_str_unicode.py index 8858cf9bfd..ca4a99199c 100644 --- a/extra_tests/snippets/builtin_str_unicode.py +++ b/extra_tests/snippets/builtin_str_unicode.py @@ -1,29 +1,29 @@ - # Test the unicode support! 👋 -ᚴ=2 +ᚴ = 2 -assert ᚴ*8 == 16 +assert ᚴ * 8 == 16 -ᚴ="👋" +ᚴ = "👋" -c = ᚴ*3 +c = ᚴ * 3 -assert c == '👋👋👋' +assert c == "👋👋👋" import unicodedata -assert unicodedata.category('a') == 'Ll' -assert unicodedata.category('A') == 'Lu' -assert unicodedata.name('a') == 'LATIN SMALL LETTER A' -assert unicodedata.lookup('LATIN SMALL LETTER A') == 'a' -assert unicodedata.bidirectional('a') == 'L' -assert unicodedata.east_asian_width('\u231a') == 'W' -assert unicodedata.normalize('NFC', 'bla') == 'bla' + +assert unicodedata.category("a") == "Ll" +assert unicodedata.category("A") == "Lu" +assert unicodedata.name("a") == "LATIN SMALL LETTER A" +assert unicodedata.lookup("LATIN SMALL LETTER A") == "a" +assert unicodedata.bidirectional("a") == "L" +assert unicodedata.east_asian_width("\u231a") == "W" +assert unicodedata.normalize("NFC", "bla") == "bla" # testing unicodedata.ucd_3_2_0 for idna -assert "abcСĤ".encode("idna") == b'xn--abc-7sa390b' -assert "abc䄣IJ".encode("idna") == b'xn--abcij-zb5f' +assert "abcСĤ".encode("idna") == b"xn--abc-7sa390b" +assert "abc䄣IJ".encode("idna") == b"xn--abcij-zb5f" # from CPython tests assert "python.org".encode("idna") == b"python.org" diff --git a/extra_tests/snippets/builtin_str_unicode_slice.py b/extra_tests/snippets/builtin_str_unicode_slice.py index c6ce88d549..252f84b1c7 100644 --- a/extra_tests/snippets/builtin_str_unicode_slice.py +++ b/extra_tests/snippets/builtin_str_unicode_slice.py @@ -1,13 +1,14 @@ def test_slice_bounds(s): # End out of range assert s[0:100] == s - assert s[0:-100] == '' + assert s[0:-100] == "" # Start out of range - assert s[100:1] == '' + assert s[100:1] == "" # Out of range both sides # This is the behaviour in cpython # assert s[-100:100] == s + def expect_index_error(s, index): try: s[index] @@ -16,6 +17,7 @@ def expect_index_error(s, index): else: assert False + unicode_str = "∀∂" assert unicode_str[0] == "∀" assert unicode_str[1] == "∂" @@ -35,25 +37,25 @@ def expect_index_error(s, index): hebrew_text = "בְּרֵאשִׁית, בָּרָא אֱלֹהִים, אֵת הַשָּׁמַיִם, וְאֵת הָאָרֶץ" assert len(hebrew_text) == 60 assert len(hebrew_text[:]) == 60 -assert hebrew_text[0] == 'ב' -assert hebrew_text[1] == 'ְ' -assert hebrew_text[2] == 'ּ' -assert hebrew_text[3] == 'ר' -assert hebrew_text[4] == 'ֵ' -assert hebrew_text[5] == 'א' -assert hebrew_text[6] == 'ש' -assert hebrew_text[5:10] == 'אשִׁי' +assert hebrew_text[0] == "ב" +assert hebrew_text[1] == "ְ" +assert hebrew_text[2] == "ּ" +assert hebrew_text[3] == "ר" +assert hebrew_text[4] == "ֵ" +assert hebrew_text[5] == "א" +assert hebrew_text[6] == "ש" +assert hebrew_text[5:10] == "אשִׁי" assert len(hebrew_text[5:10]) == 5 -assert hebrew_text[-20:50] == 'מַיִם, וְא' +assert hebrew_text[-20:50] == "מַיִם, וְא" assert len(hebrew_text[-20:50]) == 10 -assert hebrew_text[:-30:1] == 'בְּרֵאשִׁית, בָּרָא אֱלֹהִים, ' +assert hebrew_text[:-30:1] == "בְּרֵאשִׁית, בָּרָא אֱלֹהִים, " assert len(hebrew_text[:-30:1]) == 30 -assert hebrew_text[10:-30] == 'ת, בָּרָא אֱלֹהִים, ' +assert hebrew_text[10:-30] == "ת, בָּרָא אֱלֹהִים, " assert len(hebrew_text[10:-30]) == 20 -assert hebrew_text[10:30:3] == 'תבר לִ,' +assert hebrew_text[10:30:3] == "תבר לִ," assert len(hebrew_text[10:30:3]) == 7 -assert hebrew_text[10:30:-3] == '' -assert hebrew_text[30:10:-3] == 'אםהֱאּ ' +assert hebrew_text[10:30:-3] == "" +assert hebrew_text[30:10:-3] == "אםהֱאּ " assert len(hebrew_text[30:10:-3]) == 7 -assert hebrew_text[30:10:-1] == 'א ,םיִהֹלֱא אָרָּב ,' +assert hebrew_text[30:10:-1] == "א ,םיִהֹלֱא אָרָּב ," assert len(hebrew_text[30:10:-1]) == 20 diff --git a/extra_tests/snippets/builtin_tuple.py b/extra_tests/snippets/builtin_tuple.py index fd59e90609..fc2f8d5bb7 100644 --- a/extra_tests/snippets/builtin_tuple.py +++ b/extra_tests/snippets/builtin_tuple.py @@ -1,8 +1,8 @@ from testutils import assert_raises -assert (1,2) == (1,2) +assert (1, 2) == (1, 2) -x = (1,2) +x = (1, 2) assert x[0] == 1 y = (1,) @@ -19,7 +19,7 @@ assert x > y, "tuple __gt__ failed" -b = (1,2,3) +b = (1, 2, 3) assert b.index(2) == 1 recursive_list = [] @@ -30,15 +30,17 @@ assert (None, "", 1).index(1) == 2 assert 1 in (None, "", 1) + class Foo(object): def __eq__(self, x): return False + foo = Foo() assert (foo,) == (foo,) a = (1, 2, 3) -a += 1, +a += (1,) assert a == (1, 2, 3, 1) b = (55, *a) @@ -80,14 +82,14 @@ def __eq__(self, x): assert not (0, 0) > (0, 0) assert not (0, 0) < (0, 0) -assert not (float('nan'), float('nan')) <= (float('nan'), 1) -assert not (float('nan'), float('nan')) <= (float('nan'), float('nan')) -assert not (float('nan'), float('nan')) >= (float('nan'), float('nan')) -assert not (float('nan'), float('nan')) < (float('nan'), float('nan')) -assert not (float('nan'), float('nan')) > (float('nan'), float('nan')) - -assert (float('inf'), float('inf')) >= (float('inf'), 1) -assert (float('inf'), float('inf')) <= (float('inf'), float('inf')) -assert (float('inf'), float('inf')) >= (float('inf'), float('inf')) -assert not (float('inf'), float('inf')) < (float('inf'), float('inf')) -assert not (float('inf'), float('inf')) > (float('inf'), float('inf')) +assert not (float("nan"), float("nan")) <= (float("nan"), 1) +assert not (float("nan"), float("nan")) <= (float("nan"), float("nan")) +assert not (float("nan"), float("nan")) >= (float("nan"), float("nan")) +assert not (float("nan"), float("nan")) < (float("nan"), float("nan")) +assert not (float("nan"), float("nan")) > (float("nan"), float("nan")) + +assert (float("inf"), float("inf")) >= (float("inf"), 1) +assert (float("inf"), float("inf")) <= (float("inf"), float("inf")) +assert (float("inf"), float("inf")) >= (float("inf"), float("inf")) +assert not (float("inf"), float("inf")) < (float("inf"), float("inf")) +assert not (float("inf"), float("inf")) > (float("inf"), float("inf")) diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index 9f30b9b0ed..923028f2cd 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -13,9 +13,9 @@ print("abc") # print(u"abc") # Structural below -print((1, 2)) # Tuple can be any length, but fixed after declared -x = (1,2) -print(x[0]) # Tuple can be any length, but fixed after declared +print((1, 2)) # Tuple can be any length, but fixed after declared +x = (1, 2) +print(x[0]) # Tuple can be any length, but fixed after declared print([1, 2, 3]) # print({"first":1,"second":2}) @@ -52,25 +52,25 @@ a = complex(2, 4) assert type(a) is complex assert type(a + a) is complex -assert repr(a) == '(2+4j)' +assert repr(a) == "(2+4j)" a = 10j -assert repr(a) == '10j' +assert repr(a) == "10j" a = 1 assert a.conjugate() == a a = 12345 -b = a*a*a*a*a*a*a*a +b = a * a * a * a * a * a * a * a assert b.bit_length() == 109 -assert type.__module__ == 'builtins' -assert type.__qualname__ == 'type' -assert type.__name__ == 'type' +assert type.__module__ == "builtins" +assert type.__qualname__ == "type" +assert type.__name__ == "type" assert isinstance(type.__doc__, str) -assert object.__qualname__ == 'object' -assert int.__qualname__ == 'int' +assert object.__qualname__ == "object" +assert int.__qualname__ == "int" class A(type): @@ -78,8 +78,8 @@ class A(type): class B(type): - __module__ = 'b' - __qualname__ = 'BB' + __module__ = "b" + __qualname__ = "BB" class C: @@ -87,23 +87,23 @@ class C: class D: - __module__ = 'd' - __qualname__ = 'DD' - - -assert A.__module__ == '__main__' -assert A.__qualname__ == 'A' -assert B.__module__ == 'b' -assert B.__qualname__ == 'BB' -assert C.__module__ == '__main__' -assert C.__qualname__ == 'C' -assert D.__module__ == 'd' -assert D.__qualname__ == 'DD' - -A.__qualname__ = 'AA' -B.__qualname__ = 'b' -assert A.__qualname__ == 'AA' -assert B.__qualname__ == 'b' + __module__ = "d" + __qualname__ = "DD" + + +assert A.__module__ == "__main__" +assert A.__qualname__ == "A" +assert B.__module__ == "b" +assert B.__qualname__ == "BB" +assert C.__module__ == "__main__" +assert C.__qualname__ == "C" +assert D.__module__ == "d" +assert D.__qualname__ == "DD" + +A.__qualname__ = "AA" +B.__qualname__ = "b" +assert A.__qualname__ == "AA" +assert B.__qualname__ == "b" with assert_raises(TypeError): del D.__qualname__ with assert_raises(TypeError): @@ -114,7 +114,8 @@ class D: from testutils import assert_raises import platform -if platform.python_implementation() == 'RustPython': + +if platform.python_implementation() == "RustPython": gc = None else: import gc @@ -123,13 +124,13 @@ class D: assert type(object) is type assert type(object()) is object -new_type = type('New', (object,), {}) +new_type = type("New", (object,), {}) assert type(new_type) is type assert type(new_type()) is new_type -metaclass = type('MCl', (type,), {}) -cls = metaclass('Cls', (object,), {}) +metaclass = type("MCl", (type,), {}) +cls = metaclass("Cls", (object,), {}) inst = cls() assert type(inst) is cls @@ -154,10 +155,22 @@ class D: assert not issubclass(type, (int, float)) assert issubclass(type, (int, type)) -class A: pass -class B(A): pass -class C(A): pass -class D(B, C): pass + +class A: + pass + + +class B(A): + pass + + +class C(A): + pass + + +class D(B, C): + pass + assert A.__subclasses__() == [B, C] assert B.__subclasses__() == [D] @@ -174,7 +187,7 @@ class D(B, C): pass if gc: # gc sweep is needed here for CPython... gc.collect() - # ...while RustPython doesn't have `gc` yet. + # ...while RustPython doesn't have `gc` yet. if gc: # D.__new__ is a method bound to the D type, so just deleting D @@ -185,43 +198,53 @@ class D(B, C): pass assert type in object.__subclasses__() -assert cls.__name__ == 'Cls' +assert cls.__name__ == "Cls" # mro assert int.mro() == [int, object] assert bool.mro() == [bool, int, object] assert object.mro() == [object] + class A: pass + class B(A): pass + assert A.mro() == [A, object] assert B.mro() == [B, A, object] + class AA: pass + class BB(AA): pass + class C(B, BB): pass + assert C.mro() == [C, B, A, BB, AA, object] -assert type(Exception.args).__name__ == 'getset_descriptor' +assert type(Exception.args).__name__ == "getset_descriptor" assert type(None).__bool__(None) is False + class A: pass + class B: pass + a = A() a.__class__ = B assert isinstance(a, B) @@ -234,30 +257,33 @@ class B: # Regression to # https://github.com/RustPython/RustPython/issues/2310 import builtins -assert builtins.iter.__class__.__module__ == 'builtins' -assert builtins.iter.__class__.__qualname__ == 'builtin_function_or_method' -assert iter.__class__.__module__ == 'builtins' -assert iter.__class__.__qualname__ == 'builtin_function_or_method' -assert type(iter).__module__ == 'builtins' -assert type(iter).__qualname__ == 'builtin_function_or_method' +assert builtins.iter.__class__.__module__ == "builtins" +assert builtins.iter.__class__.__qualname__ == "builtin_function_or_method" + +assert iter.__class__.__module__ == "builtins" +assert iter.__class__.__qualname__ == "builtin_function_or_method" +assert type(iter).__module__ == "builtins" +assert type(iter).__qualname__ == "builtin_function_or_method" # Regression to # https://github.com/RustPython/RustPython/issues/2767 # Marked as `#[pymethod]`: -assert str.replace.__qualname__ == 'str.replace' -assert str().replace.__qualname__ == 'str.replace' -assert int.to_bytes.__qualname__ == 'int.to_bytes' -assert int().to_bytes.__qualname__ == 'int.to_bytes' +assert str.replace.__qualname__ == "str.replace" +assert str().replace.__qualname__ == "str.replace" +assert int.to_bytes.__qualname__ == "int.to_bytes" +assert int().to_bytes.__qualname__ == "int.to_bytes" # Marked as `#[pyclassmethod]`: -assert dict.fromkeys.__qualname__ == 'dict.fromkeys' -assert object.__init_subclass__.__qualname__ == 'object.__init_subclass__' +assert dict.fromkeys.__qualname__ == "dict.fromkeys" +assert object.__init_subclass__.__qualname__ == "object.__init_subclass__" # Dynamic with `#[extend_class]`: -assert bytearray.maketrans.__qualname__ == 'bytearray.maketrans', bytearray.maketrans.__qualname__ +assert bytearray.maketrans.__qualname__ == "bytearray.maketrans", ( + bytearray.maketrans.__qualname__ +) # Third-party: @@ -285,47 +311,48 @@ def c(cls): def s(): pass -assert MyTypeWithMethod.method.__name__ == 'method' -assert MyTypeWithMethod().method.__name__ == 'method' -assert MyTypeWithMethod.clsmethod.__name__ == 'clsmethod' -assert MyTypeWithMethod().clsmethod.__name__ == 'clsmethod' -assert MyTypeWithMethod.stmethod.__name__ == 'stmethod' -assert MyTypeWithMethod().stmethod.__name__ == 'stmethod' - -assert MyTypeWithMethod.method.__qualname__ == 'MyTypeWithMethod.method' -assert MyTypeWithMethod().method.__qualname__ == 'MyTypeWithMethod.method' -assert MyTypeWithMethod.clsmethod.__qualname__ == 'MyTypeWithMethod.clsmethod' -assert MyTypeWithMethod().clsmethod.__qualname__ == 'MyTypeWithMethod.clsmethod' -assert MyTypeWithMethod.stmethod.__qualname__ == 'MyTypeWithMethod.stmethod' -assert MyTypeWithMethod().stmethod.__qualname__ == 'MyTypeWithMethod.stmethod' - -assert MyTypeWithMethod.N.m.__name__ == 'm' -assert MyTypeWithMethod().N.m.__name__ == 'm' -assert MyTypeWithMethod.N.c.__name__ == 'c' -assert MyTypeWithMethod().N.c.__name__ == 'c' -assert MyTypeWithMethod.N.s.__name__ == 's' -assert MyTypeWithMethod().N.s.__name__ == 's' - -assert MyTypeWithMethod.N.m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod().N.m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod.N.c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod().N.c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod.N.s.__qualname__ == 'MyTypeWithMethod.N.s' -assert MyTypeWithMethod().N.s.__qualname__ == 'MyTypeWithMethod.N.s' - -assert MyTypeWithMethod.N().m.__name__ == 'm' -assert MyTypeWithMethod().N().m.__name__ == 'm' -assert MyTypeWithMethod.N().c.__name__ == 'c' -assert MyTypeWithMethod().N().c.__name__ == 'c' -assert MyTypeWithMethod.N().s.__name__ == 's' -assert MyTypeWithMethod().N.s.__name__ == 's' - -assert MyTypeWithMethod.N().m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod().N().m.__qualname__ == 'MyTypeWithMethod.N.m' -assert MyTypeWithMethod.N().c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod().N().c.__qualname__ == 'MyTypeWithMethod.N.c' -assert MyTypeWithMethod.N().s.__qualname__ == 'MyTypeWithMethod.N.s' -assert MyTypeWithMethod().N().s.__qualname__ == 'MyTypeWithMethod.N.s' + +assert MyTypeWithMethod.method.__name__ == "method" +assert MyTypeWithMethod().method.__name__ == "method" +assert MyTypeWithMethod.clsmethod.__name__ == "clsmethod" +assert MyTypeWithMethod().clsmethod.__name__ == "clsmethod" +assert MyTypeWithMethod.stmethod.__name__ == "stmethod" +assert MyTypeWithMethod().stmethod.__name__ == "stmethod" + +assert MyTypeWithMethod.method.__qualname__ == "MyTypeWithMethod.method" +assert MyTypeWithMethod().method.__qualname__ == "MyTypeWithMethod.method" +assert MyTypeWithMethod.clsmethod.__qualname__ == "MyTypeWithMethod.clsmethod" +assert MyTypeWithMethod().clsmethod.__qualname__ == "MyTypeWithMethod.clsmethod" +assert MyTypeWithMethod.stmethod.__qualname__ == "MyTypeWithMethod.stmethod" +assert MyTypeWithMethod().stmethod.__qualname__ == "MyTypeWithMethod.stmethod" + +assert MyTypeWithMethod.N.m.__name__ == "m" +assert MyTypeWithMethod().N.m.__name__ == "m" +assert MyTypeWithMethod.N.c.__name__ == "c" +assert MyTypeWithMethod().N.c.__name__ == "c" +assert MyTypeWithMethod.N.s.__name__ == "s" +assert MyTypeWithMethod().N.s.__name__ == "s" + +assert MyTypeWithMethod.N.m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod().N.m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod.N.c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod().N.c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod.N.s.__qualname__ == "MyTypeWithMethod.N.s" +assert MyTypeWithMethod().N.s.__qualname__ == "MyTypeWithMethod.N.s" + +assert MyTypeWithMethod.N().m.__name__ == "m" +assert MyTypeWithMethod().N().m.__name__ == "m" +assert MyTypeWithMethod.N().c.__name__ == "c" +assert MyTypeWithMethod().N().c.__name__ == "c" +assert MyTypeWithMethod.N().s.__name__ == "s" +assert MyTypeWithMethod().N.s.__name__ == "s" + +assert MyTypeWithMethod.N().m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod().N().m.__qualname__ == "MyTypeWithMethod.N.m" +assert MyTypeWithMethod.N().c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod().N().c.__qualname__ == "MyTypeWithMethod.N.c" +assert MyTypeWithMethod.N().s.__qualname__ == "MyTypeWithMethod.N.s" +assert MyTypeWithMethod().N().s.__qualname__ == "MyTypeWithMethod.N.s" # Regresesion to @@ -339,26 +366,27 @@ def s(): # Regression to # https://github.com/RustPython/RustPython/issues/2788 -assert iter.__qualname__ == iter.__name__ == 'iter' -assert max.__qualname__ == max.__name__ == 'max' -assert min.__qualname__ == min.__name__ == 'min' +assert iter.__qualname__ == iter.__name__ == "iter" +assert max.__qualname__ == max.__name__ == "max" +assert min.__qualname__ == min.__name__ == "min" def custom_func(): pass -assert custom_func.__qualname__ == 'custom_func' + +assert custom_func.__qualname__ == "custom_func" # Regression to # https://github.com/RustPython/RustPython/issues/2786 -assert object.__new__.__name__ == '__new__' -assert object.__new__.__qualname__ == 'object.__new__' -assert object.__subclasshook__.__name__ == '__subclasshook__' -assert object.__subclasshook__.__qualname__ == 'object.__subclasshook__' -assert type.__new__.__name__ == '__new__' -assert type.__new__.__qualname__ == 'type.__new__' +assert object.__new__.__name__ == "__new__" +assert object.__new__.__qualname__ == "object.__new__" +assert object.__subclasshook__.__name__ == "__subclasshook__" +assert object.__subclasshook__.__qualname__ == "object.__subclasshook__" +assert type.__new__.__name__ == "__new__" +assert type.__new__.__qualname__ == "type.__new__" class AQ: @@ -414,75 +442,76 @@ def three_cls(cls): def three_st(): pass -assert AQ.one.__name__ == 'one' -assert AQ().one.__name__ == 'one' -assert AQ.one_cls.__name__ == 'one_cls' -assert AQ().one_cls.__name__ == 'one_cls' -assert AQ.one_st.__name__ == 'one_st' -assert AQ().one_st.__name__ == 'one_st' - -assert AQ.one.__qualname__ == 'AQ.one' -assert AQ().one.__qualname__ == 'AQ.one' -assert AQ.one_cls.__qualname__ == 'AQ.one_cls' -assert AQ().one_cls.__qualname__ == 'AQ.one_cls' -assert AQ.one_st.__qualname__ == 'AQ.one_st' -assert AQ().one_st.__qualname__ == 'AQ.one_st' - -assert AQ.two.__name__ == 'two' -assert AQ().two.__name__ == 'two' -assert AQ.two_cls.__name__ == 'two_cls' -assert AQ().two_cls.__name__ == 'two_cls' -assert AQ.two_st.__name__ == 'two_st' -assert AQ().two_st.__name__ == 'two_st' - -assert AQ.two.__qualname__ == 'AQ.two' -assert AQ().two.__qualname__ == 'AQ.two' -assert AQ.two_cls.__qualname__ == 'AQ.two_cls' -assert AQ().two_cls.__qualname__ == 'AQ.two_cls' -assert AQ.two_st.__qualname__ == 'AQ.two_st' -assert AQ().two_st.__qualname__ == 'AQ.two_st' - -assert BQ.one.__name__ == 'one' -assert BQ().one.__name__ == 'one' -assert BQ.one_cls.__name__ == 'one_cls' -assert BQ().one_cls.__name__ == 'one_cls' -assert BQ.one_st.__name__ == 'one_st' -assert BQ().one_st.__name__ == 'one_st' - -assert BQ.one.__qualname__ == 'BQ.one' -assert BQ().one.__qualname__ == 'BQ.one' -assert BQ.one_cls.__qualname__ == 'BQ.one_cls' -assert BQ().one_cls.__qualname__ == 'BQ.one_cls' -assert BQ.one_st.__qualname__ == 'BQ.one_st' -assert BQ().one_st.__qualname__ == 'BQ.one_st' - -assert BQ.two.__name__ == 'two' -assert BQ().two.__name__ == 'two' -assert BQ.two_cls.__name__ == 'two_cls' -assert BQ().two_cls.__name__ == 'two_cls' -assert BQ.two_st.__name__ == 'two_st' -assert BQ().two_st.__name__ == 'two_st' - -assert BQ.two.__qualname__ == 'AQ.two' -assert BQ().two.__qualname__ == 'AQ.two' -assert BQ.two_cls.__qualname__ == 'AQ.two_cls' -assert BQ().two_cls.__qualname__ == 'AQ.two_cls' -assert BQ.two_st.__qualname__ == 'AQ.two_st' -assert BQ().two_st.__qualname__ == 'AQ.two_st' - -assert BQ.three.__name__ == 'three' -assert BQ().three.__name__ == 'three' -assert BQ.three_cls.__name__ == 'three_cls' -assert BQ().three_cls.__name__ == 'three_cls' -assert BQ.three_st.__name__ == 'three_st' -assert BQ().three_st.__name__ == 'three_st' - -assert BQ.three.__qualname__ == 'BQ.three' -assert BQ().three.__qualname__ == 'BQ.three' -assert BQ.three_cls.__qualname__ == 'BQ.three_cls' -assert BQ().three_cls.__qualname__ == 'BQ.three_cls' -assert BQ.three_st.__qualname__ == 'BQ.three_st' -assert BQ().three_st.__qualname__ == 'BQ.three_st' + +assert AQ.one.__name__ == "one" +assert AQ().one.__name__ == "one" +assert AQ.one_cls.__name__ == "one_cls" +assert AQ().one_cls.__name__ == "one_cls" +assert AQ.one_st.__name__ == "one_st" +assert AQ().one_st.__name__ == "one_st" + +assert AQ.one.__qualname__ == "AQ.one" +assert AQ().one.__qualname__ == "AQ.one" +assert AQ.one_cls.__qualname__ == "AQ.one_cls" +assert AQ().one_cls.__qualname__ == "AQ.one_cls" +assert AQ.one_st.__qualname__ == "AQ.one_st" +assert AQ().one_st.__qualname__ == "AQ.one_st" + +assert AQ.two.__name__ == "two" +assert AQ().two.__name__ == "two" +assert AQ.two_cls.__name__ == "two_cls" +assert AQ().two_cls.__name__ == "two_cls" +assert AQ.two_st.__name__ == "two_st" +assert AQ().two_st.__name__ == "two_st" + +assert AQ.two.__qualname__ == "AQ.two" +assert AQ().two.__qualname__ == "AQ.two" +assert AQ.two_cls.__qualname__ == "AQ.two_cls" +assert AQ().two_cls.__qualname__ == "AQ.two_cls" +assert AQ.two_st.__qualname__ == "AQ.two_st" +assert AQ().two_st.__qualname__ == "AQ.two_st" + +assert BQ.one.__name__ == "one" +assert BQ().one.__name__ == "one" +assert BQ.one_cls.__name__ == "one_cls" +assert BQ().one_cls.__name__ == "one_cls" +assert BQ.one_st.__name__ == "one_st" +assert BQ().one_st.__name__ == "one_st" + +assert BQ.one.__qualname__ == "BQ.one" +assert BQ().one.__qualname__ == "BQ.one" +assert BQ.one_cls.__qualname__ == "BQ.one_cls" +assert BQ().one_cls.__qualname__ == "BQ.one_cls" +assert BQ.one_st.__qualname__ == "BQ.one_st" +assert BQ().one_st.__qualname__ == "BQ.one_st" + +assert BQ.two.__name__ == "two" +assert BQ().two.__name__ == "two" +assert BQ.two_cls.__name__ == "two_cls" +assert BQ().two_cls.__name__ == "two_cls" +assert BQ.two_st.__name__ == "two_st" +assert BQ().two_st.__name__ == "two_st" + +assert BQ.two.__qualname__ == "AQ.two" +assert BQ().two.__qualname__ == "AQ.two" +assert BQ.two_cls.__qualname__ == "AQ.two_cls" +assert BQ().two_cls.__qualname__ == "AQ.two_cls" +assert BQ.two_st.__qualname__ == "AQ.two_st" +assert BQ().two_st.__qualname__ == "AQ.two_st" + +assert BQ.three.__name__ == "three" +assert BQ().three.__name__ == "three" +assert BQ.three_cls.__name__ == "three_cls" +assert BQ().three_cls.__name__ == "three_cls" +assert BQ.three_st.__name__ == "three_st" +assert BQ().three_st.__name__ == "three_st" + +assert BQ.three.__qualname__ == "BQ.three" +assert BQ().three.__qualname__ == "BQ.three" +assert BQ.three_cls.__qualname__ == "BQ.three_cls" +assert BQ().three_cls.__qualname__ == "BQ.three_cls" +assert BQ.three_st.__qualname__ == "BQ.three_st" +assert BQ().three_st.__qualname__ == "BQ.three_st" class ClassWithNew: @@ -494,73 +523,74 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls, *args, **kwargs) -assert ClassWithNew.__new__.__qualname__ == 'ClassWithNew.__new__' -assert ClassWithNew().__new__.__qualname__ == 'ClassWithNew.__new__' -assert ClassWithNew.__new__.__name__ == '__new__' -assert ClassWithNew().__new__.__name__ == '__new__' +assert ClassWithNew.__new__.__qualname__ == "ClassWithNew.__new__" +assert ClassWithNew().__new__.__qualname__ == "ClassWithNew.__new__" +assert ClassWithNew.__new__.__name__ == "__new__" +assert ClassWithNew().__new__.__name__ == "__new__" -assert ClassWithNew.N.__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew().N.__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew.N.__new__.__name__ == '__new__' -assert ClassWithNew().N.__new__.__name__ == '__new__' -assert ClassWithNew.N().__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew().N().__new__.__qualname__ == 'ClassWithNew.N.__new__' -assert ClassWithNew.N().__new__.__name__ == '__new__' -assert ClassWithNew().N().__new__.__name__ == '__new__' +assert ClassWithNew.N.__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew().N.__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew.N.__new__.__name__ == "__new__" +assert ClassWithNew().N.__new__.__name__ == "__new__" +assert ClassWithNew.N().__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew().N().__new__.__qualname__ == "ClassWithNew.N.__new__" +assert ClassWithNew.N().__new__.__name__ == "__new__" +assert ClassWithNew().N().__new__.__name__ == "__new__" # Regression to: # https://github.com/RustPython/RustPython/issues/2762 assert type.__prepare__() == {} -assert type.__prepare__('name') == {} -assert type.__prepare__('name', object) == {} -assert type.__prepare__('name', (bytes, str)) == {} +assert type.__prepare__("name") == {} +assert type.__prepare__("name", object) == {} +assert type.__prepare__("name", (bytes, str)) == {} assert type.__prepare__(a=1, b=2) == {} -assert type.__prepare__('name', (object, int), kw=True) == {} +assert type.__prepare__("name", (object, int), kw=True) == {} # Previously we needed `name` to be `str`: assert type.__prepare__(1) == {} assert int.__prepare__() == {} -assert int.__prepare__('name', (object, int), kw=True) == {} +assert int.__prepare__("name", (object, int), kw=True) == {} # Regression to # https://github.com/RustPython/RustPython/issues/2790 # `#[pyproperty]` -assert BaseException.args.__qualname__ == 'BaseException.args' +assert BaseException.args.__qualname__ == "BaseException.args" # class extension without `#[pyproperty]` override -assert Exception.args.__qualname__ == 'BaseException.args' +assert Exception.args.__qualname__ == "BaseException.args" # dynamic with `.new_readonly_getset` -assert SyntaxError.msg.__qualname__ == 'SyntaxError.msg' +assert SyntaxError.msg.__qualname__ == "SyntaxError.msg" # Regression to # https://github.com/RustPython/RustPython/issues/2794 -assert type.__subclasshook__.__qualname__ == 'type.__subclasshook__' -assert object.__subclasshook__.__qualname__ == 'object.__subclasshook__' +assert type.__subclasshook__.__qualname__ == "type.__subclasshook__" +assert object.__subclasshook__.__qualname__ == "object.__subclasshook__" # Regression to # https://github.com/RustPython/RustPython/issues/2776 -assert repr(BQ.one).startswith(' 3 < 4 assert not 1 > 2 < 3 < 4 + def test_type_error(x, y): assert_raises(TypeError, lambda: x < y) assert_raises(TypeError, lambda: x <= y) assert_raises(TypeError, lambda: x > y) assert_raises(TypeError, lambda: x >= y) + test_type_error([], 0) test_type_error((), 0) @@ -34,6 +36,7 @@ def test_type_error(x, y): # floats that cannot be converted to big ints shouldn’t crash the vm import math + assert not (10**500 == math.inf) assert not (math.inf == 10**500) assert not (10**500 == math.nan) @@ -41,23 +44,23 @@ def test_type_error(x, y): # comparisons # floats with worse than integer precision -assert 2.**54 > 2**54 - 1 -assert 2.**54 < 2**54 + 1 -assert 2.**54 >= 2**54 - 1 -assert 2.**54 <= 2**54 + 1 -assert 2.**54 == 2**54 -assert not 2.**54 == 2**54 + 1 +assert 2.0**54 > 2**54 - 1 +assert 2.0**54 < 2**54 + 1 +assert 2.0**54 >= 2**54 - 1 +assert 2.0**54 <= 2**54 + 1 +assert 2.0**54 == 2**54 +assert not 2.0**54 == 2**54 + 1 # inverse operands -assert 2**54 - 1 < 2.**54 -assert 2**54 + 1 > 2.**54 -assert 2**54 - 1 <= 2.**54 -assert 2**54 + 1 >= 2.**54 -assert 2**54 == 2.**54 -assert not 2**54 + 1 == 2.**54 +assert 2**54 - 1 < 2.0**54 +assert 2**54 + 1 > 2.0**54 +assert 2**54 - 1 <= 2.0**54 +assert 2**54 + 1 >= 2.0**54 +assert 2**54 == 2.0**54 +assert not 2**54 + 1 == 2.0**54 -assert not 2.**54 < 2**54 - 1 -assert not 2.**54 > 2**54 + 1 +assert not 2.0**54 < 2**54 - 1 +assert not 2.0**54 > 2**54 + 1 # sub-int numbers assert 1.3 > 1 @@ -68,17 +71,17 @@ def test_type_error(x, y): assert -0.3 <= 0 # int out of float range comparisons -assert 10**500 > 2.**54 -assert -10**500 < -0.12 +assert 10**500 > 2.0**54 +assert -(10**500) < -0.12 # infinity and NaN comparisons assert math.inf > 10**500 assert math.inf >= 10**500 assert not math.inf < 10**500 -assert -math.inf < -10*500 -assert -math.inf <= -10*500 -assert not -math.inf > -10*500 +assert -math.inf < -10 * 500 +assert -math.inf <= -10 * 500 +assert not -math.inf > -10 * 500 assert not math.nan > 123 assert not math.nan < 123 diff --git a/extra_tests/snippets/operator_div.py b/extra_tests/snippets/operator_div.py index 8520a877c5..e99533cbd5 100644 --- a/extra_tests/snippets/operator_div.py +++ b/extra_tests/snippets/operator_div.py @@ -2,7 +2,7 @@ assert_raises(ZeroDivisionError, lambda: 5 / 0) assert_raises(ZeroDivisionError, lambda: 5 / -0.0) -assert_raises(ZeroDivisionError, lambda: 5 / (2-2)) +assert_raises(ZeroDivisionError, lambda: 5 / (2 - 2)) assert_raises(ZeroDivisionError, lambda: 5 % 0) assert_raises(ZeroDivisionError, lambda: 5 // 0) assert_raises(ZeroDivisionError, lambda: 5.3 // (-0.0)) @@ -18,12 +18,16 @@ res = 10**3000 / (10**2998 + 5 * 10**2996) assert 95.238095 <= res <= 95.238096 -assert 10**500 / (2*10**(500-308)) == 5e307 -assert 10**500 / (10**(500-308)) == 1e308 -assert_raises(OverflowError, lambda: 10**500 / (10**(500-309)), _msg='too big result') +assert 10**500 / (2 * 10 ** (500 - 308)) == 5e307 +assert 10**500 / (10 ** (500 - 308)) == 1e308 +assert_raises( + OverflowError, lambda: 10**500 / (10 ** (500 - 309)), _msg="too big result" +) # a bit more than f64::MAX = 1.7976931348623157e+308_f64 assert (2 * 10**308) / 2 == 1e308 # when dividing too big int by a float, the operation should fail -assert_raises(OverflowError, lambda: (2 * 10**308) / 2.0, _msg='division of big int by float') +assert_raises( + OverflowError, lambda: (2 * 10**308) / 2.0, _msg="division of big int by float" +) diff --git a/extra_tests/snippets/operator_membership.py b/extra_tests/snippets/operator_membership.py index 2987c3c0fe..07065e2244 100644 --- a/extra_tests/snippets/operator_membership.py +++ b/extra_tests/snippets/operator_membership.py @@ -46,15 +46,16 @@ assert 1 in range(0, 2) assert 3 not in range(0, 2) + # test __contains__ in user objects -class MyNotContainingClass(): +class MyNotContainingClass: pass assert_raises(TypeError, lambda: 1 in MyNotContainingClass()) -class MyContainingClass(): +class MyContainingClass: def __init__(self, value): self.value = value diff --git a/extra_tests/snippets/protocol_callable.py b/extra_tests/snippets/protocol_callable.py index c549ef468e..1df0e71793 100644 --- a/extra_tests/snippets/protocol_callable.py +++ b/extra_tests/snippets/protocol_callable.py @@ -1,4 +1,4 @@ -class Callable(): +class Callable: def __init__(self): self.count = 0 @@ -6,13 +6,16 @@ def __call__(self): self.count += 1 return self.count + c = Callable() assert 1 == c() assert 2 == c() + class Inherited(Callable): pass + i = Inherited() assert 1 == i() diff --git a/extra_tests/snippets/protocol_index_bad.py b/extra_tests/snippets/protocol_index_bad.py index af71f2e689..d4ac003c85 100644 --- a/extra_tests/snippets/protocol_index_bad.py +++ b/extra_tests/snippets/protocol_index_bad.py @@ -1,5 +1,6 @@ -""" Test that indexing ops don't hang when an object with a mutating +"""Test that indexing ops don't hang when an object with a mutating __index__ is used.""" + from testutils import assert_raises from array import array @@ -15,18 +16,19 @@ def __index__(self): def run_setslice(): with assert_raises(IndexError): e[BadIndex()] = 42 - e[BadIndex():0:-1] = e - e[0:BadIndex():1] = e - e[0:10:BadIndex()] = e + e[BadIndex() : 0 : -1] = e + e[0 : BadIndex() : 1] = e + e[0 : 10 : BadIndex()] = e def run_delslice(): - del e[BadIndex():0:-1] - del e[0:BadIndex():1] - del e[0:10:BadIndex()] + del e[BadIndex() : 0 : -1] + del e[0 : BadIndex() : 1] + del e[0 : 10 : BadIndex()] + -# Check types -instances = [list(), bytearray(), array('b')] +# Check types +instances = [list(), bytearray(), array("b")] for e in instances: run_setslice() - run_delslice() \ No newline at end of file + run_delslice() diff --git a/extra_tests/snippets/protocol_iterable.py b/extra_tests/snippets/protocol_iterable.py index 7158296c38..7f32504d0a 100644 --- a/extra_tests/snippets/protocol_iterable.py +++ b/extra_tests/snippets/protocol_iterable.py @@ -1,5 +1,6 @@ from testutils import assert_raises + def test_container(x): assert 3 in x assert 4 not in x @@ -10,27 +11,40 @@ def test_container(x): lst.extend(x) assert lst == [0, 1, 2, 3] + class C: def __iter__(self): return iter([0, 1, 2, 3]) + + test_container(C()) + class C: def __getitem__(self, x): - return (0, 1, 2, 3)[x] # raises IndexError on x==4 + return (0, 1, 2, 3)[x] # raises IndexError on x==4 + + test_container(C()) + class C: def __getitem__(self, x): if x > 3: raise StopIteration return x + + test_container(C()) -class C: pass + +class C: + pass + + assert_raises(TypeError, lambda: 5 in C()) assert_raises(TypeError, iter, C) -it = iter([1,2,3,4,5]) +it = iter([1, 2, 3, 4, 5]) call_it = iter(lambda: next(it), 4) -assert list(call_it) == [1,2,3] +assert list(call_it) == [1, 2, 3] diff --git a/extra_tests/snippets/protocol_iternext.py b/extra_tests/snippets/protocol_iternext.py index b2b30961c0..0eff9cff4e 100644 --- a/extra_tests/snippets/protocol_iternext.py +++ b/extra_tests/snippets/protocol_iternext.py @@ -1,5 +1,3 @@ - - ls = [1, 2, 3] i = iter(ls) @@ -7,10 +5,10 @@ assert i.__next__() == 2 assert next(i) == 3 -assert next(i, 'w00t') == 'w00t' +assert next(i, "w00t") == "w00t" -s = '你好' +s = "你好" i = iter(s) i.__setstate__(1) -assert i.__next__() == '好' +assert i.__next__() == "好" assert i.__reduce__()[2] == 2 diff --git a/extra_tests/snippets/recursion.py b/extra_tests/snippets/recursion.py index f2a8d4e11d..2d3b2205d6 100644 --- a/extra_tests/snippets/recursion.py +++ b/extra_tests/snippets/recursion.py @@ -1,8 +1,10 @@ from testutils import assert_raises + class Foo(object): pass + Foo.__repr__ = Foo.__str__ foo = Foo() diff --git a/extra_tests/snippets/stdlib_abc_number.py b/extra_tests/snippets/stdlib_abc_number.py index c6aee97ec8..2c1e81c1f8 100644 --- a/extra_tests/snippets/stdlib_abc_number.py +++ b/extra_tests/snippets/stdlib_abc_number.py @@ -71,4 +71,4 @@ class A(int): assert 1_2.3_4e0_0 == 12.34 with assert_raises(SyntaxError): - eval('1__2') + eval("1__2") diff --git a/extra_tests/snippets/stdlib_array.py b/extra_tests/snippets/stdlib_array.py index 6c4af54597..a31b2f8e42 100644 --- a/extra_tests/snippets/stdlib_array.py +++ b/extra_tests/snippets/stdlib_array.py @@ -23,6 +23,7 @@ b = array("B", [3, 2, 1, 0]) assert a.__ne__(b) is True + def test_float_with_integer_input(): f = array("f", [0, 1, 2.0, 3.0]) f.append(4) @@ -33,10 +34,11 @@ def test_float_with_integer_input(): f[0] = -2 assert f == array("f", [-2, 0, 2, 3, 4]) + test_float_with_integer_input() # slice assignment step overflow behaviour test -T = 'I' +T = "I" a = array(T, range(10)) b = array(T, [100]) a[::9999999999] = b @@ -57,9 +59,10 @@ def test_float_with_integer_input(): del a[0:0:-9999999999] assert a == array(T, [1, 2, 3, 4, 5, 6, 7, 8]) + def test_float_with_nan(): - f = float('nan') - a = array('f') + f = float("nan") + a = array("f") a.append(f) assert not (a == a) assert a != a @@ -68,30 +71,35 @@ def test_float_with_nan(): assert not (a > a) assert not (a >= a) + test_float_with_nan() + def test_different_type_cmp(): - a = array('i', [-1, -2, -3, -4]) - b = array('I', [1, 2, 3, 4]) - c = array('f', [1, 2, 3, 4]) + a = array("i", [-1, -2, -3, -4]) + b = array("I", [1, 2, 3, 4]) + c = array("f", [1, 2, 3, 4]) assert a < b assert b > a assert b == c assert a < c assert c > a + test_different_type_cmp() + def test_array_frombytes(): - a = array('b', [-1, -2]) + a = array("b", [-1, -2]) b = bytearray(a.tobytes()) - c = array('b', b) + c = array("b", b) assert a == c + test_array_frombytes() # test that indexing on an empty array doesn't panic -a = array('b') +a = array("b") with assert_raises(IndexError): a[0] with assert_raises(IndexError): @@ -99,21 +107,21 @@ def test_array_frombytes(): with assert_raises(IndexError): del a[42] -test_str = '🌉abc🌐def🌉🌐' -u = array('u', test_str) +test_str = "🌉abc🌐def🌉🌐" +u = array("u", test_str) # skip as 2 bytes character environment with CPython is failing the test if u.itemsize >= 4: assert u.__reduce_ex__(1)[1][1] == list(test_str) assert loads(dumps(u, 1)) == loads(dumps(u, 3)) # test array name -a = array('b', []) +a = array("b", []) assert str(a.__class__.__name__) == "array" # test arrayiterator name i = iter(a) assert str(i.__class__.__name__) == "arrayiterator" # teset array.__contains__ -a = array('B', [0]) +a = array("B", [0]) assert a.__contains__(0) assert not a.__contains__(1) diff --git a/extra_tests/snippets/stdlib_ast.py b/extra_tests/snippets/stdlib_ast.py index 08c1b3b76e..dc626506fa 100644 --- a/extra_tests/snippets/stdlib_ast.py +++ b/extra_tests/snippets/stdlib_ast.py @@ -1,5 +1,5 @@ - import ast + print(ast) source = """ @@ -11,30 +11,29 @@ def foo(): print(n) print(n.body) print(n.body[0].name) -assert n.body[0].name == 'foo' +assert n.body[0].name == "foo" foo = n.body[0] assert foo.lineno == 2 print(foo.body) assert len(foo.body) == 2 print(foo.body[0]) print(foo.body[0].value.func.id) -assert foo.body[0].value.func.id == 'print' +assert foo.body[0].value.func.id == "print" assert foo.body[0].lineno == 3 assert foo.body[1].lineno == 4 n = ast.parse("3 < 4 > 5\n") assert n.body[0].value.left.value == 3 -assert 'Lt' in str(n.body[0].value.ops[0]) -assert 'Gt' in str(n.body[0].value.ops[1]) +assert "Lt" in str(n.body[0].value.ops[0]) +assert "Gt" in str(n.body[0].value.ops[1]) assert n.body[0].value.comparators[0].value == 4 assert n.body[0].value.comparators[1].value == 5 -n = ast.parse('from ... import a\n') +n = ast.parse("from ... import a\n") print(n) i = n.body[0] assert i.level == 3 assert i.module is None -assert i.names[0].name == 'a' +assert i.names[0].name == "a" assert i.names[0].asname is None - diff --git a/extra_tests/snippets/stdlib_collections.py b/extra_tests/snippets/stdlib_collections.py index 641a6e2a25..8fd7cb6a88 100644 --- a/extra_tests/snippets/stdlib_collections.py +++ b/extra_tests/snippets/stdlib_collections.py @@ -50,9 +50,10 @@ class BadRepr: def __repr__(self): self.d.pop() - return '' + return "" + b = BadRepr() d = deque([1, b, 2]) b.d = d -repr(d) \ No newline at end of file +repr(d) diff --git a/extra_tests/snippets/stdlib_collections_deque.py b/extra_tests/snippets/stdlib_collections_deque.py index 44498633bf..86e566f418 100644 --- a/extra_tests/snippets/stdlib_collections_deque.py +++ b/extra_tests/snippets/stdlib_collections_deque.py @@ -5,9 +5,9 @@ def test_deque_iterator__new__(): klass = type(iter(deque())) - s = 'abcd' + s = "abcd" d = klass(deque(s)) - assert (list(d) == list(s)) + assert list(d) == list(s) test_deque_iterator__new__() @@ -17,22 +17,22 @@ def test_deque_iterator__new__positional_index(): klass = type(iter(deque())) # index between 0 and len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s)): d = klass(deque(s), i) - assert (list(d) == list(s)[i:]) + assert list(d) == list(s)[i:] # negative index - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 0): d = klass(deque(s), i) - assert (list(d) == list(s)) + assert list(d) == list(s) # index ge len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s), 400): d = klass(deque(s), i) - assert (list(d) == list()) + assert list(d) == list() test_deque_iterator__new__positional_index() @@ -41,10 +41,10 @@ def test_deque_iterator__new__positional_index(): def test_deque_iterator__new__not_using_keyword_index(): klass = type(iter(deque())) - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 400): d = klass(deque(s), index=i) - assert (list(d) == list(s)) + assert list(d) == list(s) test_deque_iterator__new__not_using_keyword_index() @@ -54,22 +54,22 @@ def test_deque_reverse_iterator__new__positional_index(): klass = type(reversed(deque())) # index between 0 and len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s)): d = klass(deque(s), i) - assert (list(d) == list(reversed(s))[i:]) + assert list(d) == list(reversed(s))[i:] # negative index - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 0): d = klass(deque(s), i) - assert (list(d) == list(reversed(s))) + assert list(d) == list(reversed(s)) # index ge len - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(len(s), 400): d = klass(deque(s), i) - assert (list(d) == list()) + assert list(d) == list() test_deque_reverse_iterator__new__positional_index() @@ -78,10 +78,10 @@ def test_deque_reverse_iterator__new__positional_index(): def test_deque_reverse_iterator__new__not_using_keyword_index(): klass = type(reversed(deque())) - for s in ('abcd', range(200)): + for s in ("abcd", range(200)): for i in range(-100, 400): d = klass(deque(s), index=i) - assert (list(d) == list(reversed(s))) + assert list(d) == list(reversed(s)) test_deque_reverse_iterator__new__not_using_keyword_index() @@ -89,11 +89,13 @@ def test_deque_reverse_iterator__new__not_using_keyword_index(): assert repr(deque()) == "deque([])" assert repr(deque([1, 2, 3])) == "deque([1, 2, 3])" + class D(deque): pass + assert repr(D()) == "D([])" assert repr(D([1, 2, 3])) == "D([1, 2, 3])" -assert_raises(ValueError, lambda: deque().index(10,0,10000000000000000000000000)) \ No newline at end of file +assert_raises(ValueError, lambda: deque().index(10, 0, 10000000000000000000000000)) diff --git a/extra_tests/snippets/stdlib_csv.py b/extra_tests/snippets/stdlib_csv.py index 6ba66d30f7..f762c58010 100644 --- a/extra_tests/snippets/stdlib_csv.py +++ b/extra_tests/snippets/stdlib_csv.py @@ -2,44 +2,48 @@ import csv -for row in csv.reader(['one,two,three']): - [one, two, three] = row - assert one == 'one' - assert two == 'two' - assert three == 'three' +for row in csv.reader(["one,two,three"]): + [one, two, three] = row + assert one == "one" + assert two == "two" + assert three == "three" + def f(): - iter = ['one,two,three', 'four,five,six'] - reader = csv.reader(iter) + iter = ["one,two,three", "four,five,six"] + reader = csv.reader(iter) + + [one, two, three] = next(reader) + [four, five, six] = next(reader) - [one,two,three] = next(reader) - [four,five,six] = next(reader) + assert one == "one" + assert two == "two" + assert three == "three" + assert four == "four" + assert five == "five" + assert six == "six" - assert one == 'one' - assert two == 'two' - assert three == 'three' - assert four == 'four' - assert five == 'five' - assert six == 'six' f() + def test_delim(): - iter = ['one|two|three', 'four|five|six'] - reader = csv.reader(iter, delimiter='|') - - [one,two,three] = next(reader) - [four,five,six] = next(reader) - - assert one == 'one' - assert two == 'two' - assert three == 'three' - assert four == 'four' - assert five == 'five' - assert six == 'six' - - with assert_raises(TypeError): - iter = ['one,,two,,three'] - csv.reader(iter, delimiter=',,') + iter = ["one|two|three", "four|five|six"] + reader = csv.reader(iter, delimiter="|") + + [one, two, three] = next(reader) + [four, five, six] = next(reader) + + assert one == "one" + assert two == "two" + assert three == "three" + assert four == "four" + assert five == "five" + assert six == "six" + + with assert_raises(TypeError): + iter = ["one,,two,,three"] + csv.reader(iter, delimiter=",,") + test_delim() diff --git a/extra_tests/snippets/stdlib_ctypes.py b/extra_tests/snippets/stdlib_ctypes.py index 95ee9900fb..32ed17d19f 100644 --- a/extra_tests/snippets/stdlib_ctypes.py +++ b/extra_tests/snippets/stdlib_ctypes.py @@ -9,8 +9,8 @@ from struct import calcsize as _calcsize -assert Array.__class__.__name__ == 'PyCArrayType' -assert Array.__base__.__name__ == '_CData' +assert Array.__class__.__name__ == "PyCArrayType" +assert Array.__base__.__name__ == "_CData" DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": @@ -19,13 +19,16 @@ # libraries. OS X 10.3 is Darwin 7, so we check for # that. - if int(_os.uname().release.split('.')[0]) < 8: + if int(_os.uname().release.split(".")[0]) < 8: DEFAULT_MODE = RTLD_GLOBAL -from _ctypes import FUNCFLAG_CDECL as _FUNCFLAG_CDECL, \ - FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, \ - FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \ - FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR +from _ctypes import ( + FUNCFLAG_CDECL as _FUNCFLAG_CDECL, + FUNCFLAG_PYTHONAPI as _FUNCFLAG_PYTHONAPI, + FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, + FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR, +) + def create_string_buffer(init, size=None): """create_string_buffer(aBytes) -> character array @@ -34,7 +37,7 @@ def create_string_buffer(init, size=None): """ if isinstance(init, bytes): if size is None: - size = len(init)+1 + size = len(init) + 1 _sys.audit("ctypes.create_string_buffer", init, size) buftype = c_char.__mul__(size) print(type(c_char.__mul__(size))) @@ -50,32 +53,47 @@ def create_string_buffer(init, size=None): return buf raise TypeError(init) + def _check_size(typ, typecode=None): # Check if sizeof(ctypes_type) against struct.calcsize. This # should protect somewhat against a misconfigured libffi. from struct import calcsize + if typecode is None: # Most _type_ codes are the same as used in struct typecode = typ._type_ actual, required = sizeof(typ), calcsize(typecode) if actual != required: - raise SystemError("sizeof(%s) wrong: %d instead of %d" % \ - (typ, actual, required)) + raise SystemError( + "sizeof(%s) wrong: %d instead of %d" % (typ, actual, required) + ) + class c_short(_SimpleCData): _type_ = "h" + + _check_size(c_short) + class c_ushort(_SimpleCData): _type_ = "H" + + _check_size(c_ushort) + class c_long(_SimpleCData): _type_ = "l" + + _check_size(c_long) + class c_ulong(_SimpleCData): _type_ = "L" + + _check_size(c_ulong) if _calcsize("i") == _calcsize("l"): @@ -83,24 +101,36 @@ class c_ulong(_SimpleCData): c_int = c_long c_uint = c_ulong else: + class c_int(_SimpleCData): _type_ = "i" + _check_size(c_int) class c_uint(_SimpleCData): _type_ = "I" + _check_size(c_uint) + class c_float(_SimpleCData): _type_ = "f" + + _check_size(c_float) + class c_double(_SimpleCData): _type_ = "d" + + _check_size(c_double) + class c_longdouble(_SimpleCData): _type_ = "g" + + if sizeof(c_longdouble) == sizeof(c_double): c_longdouble = c_double @@ -109,54 +139,76 @@ class c_longdouble(_SimpleCData): c_longlong = c_long c_ulonglong = c_ulong else: + class c_longlong(_SimpleCData): _type_ = "q" + _check_size(c_longlong) class c_ulonglong(_SimpleCData): _type_ = "Q" + ## def from_param(cls, val): ## return ('d', float(val), val) ## from_param = classmethod(from_param) _check_size(c_ulonglong) + class c_ubyte(_SimpleCData): _type_ = "B" + + c_ubyte.__ctype_le__ = c_ubyte.__ctype_be__ = c_ubyte # backward compatibility: ##c_uchar = c_ubyte _check_size(c_ubyte) + class c_byte(_SimpleCData): _type_ = "b" + + c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte _check_size(c_byte) + class c_char(_SimpleCData): _type_ = "c" + + c_char.__ctype_le__ = c_char.__ctype_be__ = c_char _check_size(c_char) + class c_char_p(_SimpleCData): _type_ = "z" + def __repr__(self): return "%s(%s)" % (self.__class__.__name__, c_void_p.from_buffer(self).value) + + _check_size(c_char_p, "P") + class c_void_p(_SimpleCData): _type_ = "P" -c_voidp = c_void_p # backwards compatibility (to a bug) + + +c_voidp = c_void_p # backwards compatibility (to a bug) _check_size(c_void_p) + class c_bool(_SimpleCData): _type_ = "?" + + _check_size(c_bool) i = c_int(42) f = c_float(3.14) # s = create_string_buffer(b'\000' * 32) assert i.value == 42 -assert abs(f.value - 3.14) < 1e-06 +assert abs(f.value - 3.14) < 1e-06 if _os.name == "nt": from _ctypes import LoadLibrary as _dlopen @@ -164,6 +216,7 @@ class c_bool(_SimpleCData): elif _os.name == "posix": from _ctypes import dlopen as _dlopen + class CDLL(object): """An instance of this class represents a loaded dll/shared library, exporting functions using the standard C calling @@ -178,17 +231,23 @@ class CDLL(object): Calling the functions releases the Python GIL during the call and reacquires it afterwards. """ + _func_flags_ = _FUNCFLAG_CDECL _func_restype_ = c_int # default values for repr - _name = '' + _name = "" _handle = 0 _FuncPtr = None - def __init__(self, name, mode=DEFAULT_MODE, handle=None, - use_errno=False, - use_last_error=False, - winmode=None): + def __init__( + self, + name, + mode=DEFAULT_MODE, + handle=None, + use_errno=False, + use_last_error=False, + winmode=None, + ): self._name = name flags = self._func_flags_ if use_errno: @@ -202,20 +261,22 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, Otherwise, name is presented to dlopen() as a file argument. """ if name and name.endswith(")") and ".a(" in name: - mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) + mode |= _os.RTLD_MEMBER | _os.RTLD_NOW if _os.name == "nt": if winmode is not None: mode = winmode else: import nt + mode = 4096 - if '/' in name or '\\' in name: + if "/" in name or "\\" in name: self._name = nt._getfullpathname(self._name) mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR class _FuncPtr(_CFuncPtr): _flags_ = flags _restype_ = self._func_restype_ + self._FuncPtr = _FuncPtr if handle is None: @@ -224,13 +285,15 @@ class _FuncPtr(_CFuncPtr): self._handle = handle def __repr__(self): - return "<%s '%s', handle %x at %#x>" % \ - (self.__class__.__name__, self._name, - (self._handle & (_sys.maxsize*2 + 1)), - id(self) & (_sys.maxsize*2 + 1)) + return "<%s '%s', handle %x at %#x>" % ( + self.__class__.__name__, + self._name, + (self._handle & (_sys.maxsize * 2 + 1)), + id(self) & (_sys.maxsize * 2 + 1), + ) def __getattr__(self, name): - if name.startswith('__') and name.endswith('__'): + if name.startswith("__") and name.endswith("__"): raise AttributeError(name) func = self.__getitem__(name) setattr(self, name, func) @@ -242,12 +305,13 @@ def __getitem__(self, name_or_ordinal): func.__name__ = name_or_ordinal return func + class LibraryLoader(object): def __init__(self, dlltype): self._dlltype = dlltype def __getattr__(self, name): - if name[0] == '_': + if name[0] == "_": raise AttributeError(name) try: dll = self._dlltype(name) @@ -264,6 +328,7 @@ def LoadLibrary(self, name): __class_getitem__ = classmethod(_types.GenericAlias) + cdll = LibraryLoader(CDLL) test_byte_array = create_string_buffer(b"Hello, World!\n") diff --git a/extra_tests/snippets/stdlib_datetime.py b/extra_tests/snippets/stdlib_datetime.py index cd1f27733b..60e8049401 100644 --- a/extra_tests/snippets/stdlib_datetime.py +++ b/extra_tests/snippets/stdlib_datetime.py @@ -19,8 +19,13 @@ import time as _time from testutils import ( - assert_raises, assert_equal, assert_true, assert_false, assert_isinstance, - assert_in) + assert_raises, + assert_equal, + assert_true, + assert_false, + assert_isinstance, + assert_in, +) # An arbitrary collection of objects of non-datetime types, for testing # mixed-type comparisons. @@ -40,7 +45,7 @@ assert_equal(datetime_module.MINYEAR, 1) assert_equal(datetime_module.MAXYEAR, 9999) -if hasattr(datetime_module, '_divide_and_round'): +if hasattr(datetime_module, "_divide_and_round"): # def test_divide_and_round(self): dar = datetime_module._divide_and_round @@ -68,8 +73,8 @@ ############################################################################# # tzinfo tests -class FixedOffset(tzinfo): +class FixedOffset(tzinfo): def __init__(self, offset, name, dstoffset=42): if isinstance(offset, int): offset = timedelta(minutes=offset) @@ -78,24 +83,30 @@ def __init__(self, offset, name, dstoffset=42): self.__offset = offset self.__name = name self.__dstoffset = dstoffset + def __repr__(self): return self.__name.lower() + def utcoffset(self, dt): return self.__offset + def tzname(self, dt): return self.__name + def dst(self, dt): return self.__dstoffset -class PicklableFixedOffset(FixedOffset): +class PicklableFixedOffset(FixedOffset): def __init__(self, offset=None, name=None, dstoffset=None): FixedOffset.__init__(self, offset, name, dstoffset) + class _TZInfo(tzinfo): def utcoffset(self, datetime_module): return random.random() + # class TestTZInfo(unittest.TestCase): # def test_refcnt_crash_bug_22044(self): @@ -117,11 +128,14 @@ def utcoffset(self, datetime_module): with assert_raises(NotImplementedError): useless.dst(dt) + # def test_subclass_must_override(self): class NotEnough(tzinfo): def __init__(self, offset, name): self.__offset = offset self.__name = name + + assert_true(issubclass(NotEnough, tzinfo)) ne = NotEnough(3, "NotByALongShot") assert_isinstance(ne, tzinfo) @@ -138,14 +152,14 @@ def __init__(self, offset, name): # XXX: bug #1302 # def test_normal(self): -#fo = FixedOffset(3, "Three") -#assert_isinstance(fo, tzinfo) -#for dt in datetime.now(), None: +# fo = FixedOffset(3, "Three") +# assert_isinstance(fo, tzinfo) +# for dt in datetime.now(), None: # assert_equal(fo.utcoffset(dt), timedelta(minutes=3)) # assert_equal(fo.tzname(dt), "Three") # assert_equal(fo.dst(dt), timedelta(minutes=42)) -''' +""" class TestTimeZone(unittest.TestCase): def setUp(self): @@ -277,17 +291,17 @@ def test_deepcopy(self): tz = timezone.utc tz_copy = copy.deepcopy(tz) self.assertIs(tz_copy, tz) -''' +""" ############################################################################# # Base class for testing a particular aspect of timedelta, time, date and # datetime comparisons. # class HarmlessMixedComparison: - # Test that __eq__ and __ne__ don't complain for mixed-type comparisons. +# Test that __eq__ and __ne__ don't complain for mixed-type comparisons. - # Subclasses must define 'theclass', and theclass(1, 1, 1) must be a - # legit constructor. +# Subclasses must define 'theclass', and theclass(1, 1, 1) must be a +# legit constructor. for theclass in timedelta, date, time: # def test_harmless_mixed_comparison(self): diff --git a/extra_tests/snippets/stdlib_dir_module.py b/extra_tests/snippets/stdlib_dir_module.py index 560fb02bf6..a8ab233f37 100644 --- a/extra_tests/snippets/stdlib_dir_module.py +++ b/extra_tests/snippets/stdlib_dir_module.py @@ -1,13 +1,14 @@ from testutils import assert_equal import dir_module + assert dir_module.value == 5 assert dir_module.value2 == 7 try: dir_module.unknown_attr except AttributeError as e: - assert 'dir_module' in str(e) + assert "dir_module" in str(e) else: assert False @@ -15,7 +16,7 @@ try: dir_module.unknown_attr except AttributeError as e: - assert 'dir_module' not in str(e) + assert "dir_module" not in str(e) else: assert False @@ -23,9 +24,10 @@ try: dir_module.unknown_attr except AttributeError as e: - assert 'dir_module' not in str(e) + assert "dir_module" not in str(e) else: assert False from dir_module import dir_module_inner -assert dir_module_inner.__name__ == 'dir_module.dir_module_inner' + +assert dir_module_inner.__name__ == "dir_module.dir_module_inner" diff --git a/extra_tests/snippets/stdlib_dis.py b/extra_tests/snippets/stdlib_dis.py index e9951ef402..42296168f8 100644 --- a/extra_tests/snippets/stdlib_dis.py +++ b/extra_tests/snippets/stdlib_dis.py @@ -2,37 +2,53 @@ dis.dis(compile("5 + x + 5 or 2", "", "eval")) print("\n") -dis.dis(compile(""" +dis.dis( + compile( + """ def f(x): return 1 -""", "", "exec")) +""", + "", + "exec", + ) +) print("\n") -dis.dis(compile(""" +dis.dis( + compile( + """ if a: 1 or 2 elif x == 'hello': 3 else: 4 -""", "", "exec")) +""", + "", + "exec", + ) +) print("\n") dis.dis(compile("f(x=1, y=2)", "", "eval")) print("\n") + def f(): with g(): # noqa: F821 try: for a in {1: 4, 2: 5}: yield [True and False or True, []] except Exception: - raise not ValueError({1 for i in [1,2,3]}) + raise not ValueError({1 for i in [1, 2, 3]}) + dis.dis(f) + class A(object): def f(): x += 1 # noqa: F821 pass + def g(): for i in range(5): if i: @@ -40,5 +56,6 @@ def g(): else: break + print("A.f\n") dis.dis(A.f) diff --git a/extra_tests/snippets/stdlib_functools.py b/extra_tests/snippets/stdlib_functools.py index 0bdafcb3b8..3d323bfbad 100644 --- a/extra_tests/snippets/stdlib_functools.py +++ b/extra_tests/snippets/stdlib_functools.py @@ -1,6 +1,7 @@ from functools import reduce from testutils import assert_raises + class Squares: def __init__(self, max): self.max = max @@ -10,21 +11,24 @@ def __len__(self): return len(self.sofar) def __getitem__(self, i): - if not 0 <= i < self.max: raise IndexError + if not 0 <= i < self.max: + raise IndexError n = len(self.sofar) while n <= i: - self.sofar.append(n*n) + self.sofar.append(n * n) n += 1 return self.sofar[i] + def add(a, b): return a + b -assert reduce(add, ['a', 'b', 'c']) == 'abc' -assert reduce(add, ['a', 'b', 'c'], str(42)) == '42abc' -assert reduce(add, [['a', 'c'], [], ['d', 'w']], []) == ['a','c','d','w'] -assert reduce(add, [['a', 'c'], [], ['d', 'w']], []) == ['a','c','d','w'] -assert reduce(lambda x, y: x*y, range(2, 21), 1) == 2432902008176640000 + +assert reduce(add, ["a", "b", "c"]) == "abc" +assert reduce(add, ["a", "b", "c"], str(42)) == "42abc" +assert reduce(add, [["a", "c"], [], ["d", "w"]], []) == ["a", "c", "d", "w"] +assert reduce(add, [["a", "c"], [], ["d", "w"]], []) == ["a", "c", "d", "w"] +assert reduce(lambda x, y: x * y, range(2, 21), 1) == 2432902008176640000 assert reduce(add, Squares(10)) == 285 assert reduce(add, Squares(10), 0) == 285 assert reduce(add, Squares(0), 0) == 0 @@ -40,32 +44,40 @@ def add(a, b): with assert_raises(TypeError): reduce(42, 42, 42) + class TestFailingIter: def __iter__(self): raise RuntimeError + with assert_raises(RuntimeError): reduce(add, TestFailingIter()) assert reduce(add, [], None) == None assert reduce(add, [], 42) == 42 + class BadSeq: def __getitem__(self, index): raise ValueError + + with assert_raises(ValueError): reduce(42, BadSeq()) + # Test reduce()'s use of iterators. class SequenceClass: def __init__(self, n): self.n = n + def __getitem__(self, i): if 0 <= i < self.n: return i else: raise IndexError + assert reduce(add, SequenceClass(5)) == 10 assert reduce(add, SequenceClass(5), 42) == 52 with assert_raises(TypeError): diff --git a/extra_tests/snippets/stdlib_hashlib.py b/extra_tests/snippets/stdlib_hashlib.py index 811e3d27b6..c5feb709e1 100644 --- a/extra_tests/snippets/stdlib_hashlib.py +++ b/extra_tests/snippets/stdlib_hashlib.py @@ -1,39 +1,50 @@ - import hashlib # print(hashlib.md5) h = hashlib.md5() -h.update(b'a') -g = hashlib.md5(b'a') -assert h.name == g.name == 'md5' +h.update(b"a") +g = hashlib.md5(b"a") +assert h.name == g.name == "md5" print(h.hexdigest()) print(g.hexdigest()) -assert h.hexdigest() == g.hexdigest() == '0cc175b9c0f1b6a831c399e269772661' +assert h.hexdigest() == g.hexdigest() == "0cc175b9c0f1b6a831c399e269772661" assert h.digest_size == g.digest_size == 16 h = hashlib.sha256() -h.update(b'a') -g = hashlib.sha256(b'a') -assert h.name == g.name == 'sha256' +h.update(b"a") +g = hashlib.sha256(b"a") +assert h.name == g.name == "sha256" assert h.digest_size == g.digest_size == 32 print(h.hexdigest()) print(g.hexdigest()) -assert h.hexdigest() == g.hexdigest() == 'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb' +assert ( + h.hexdigest() + == g.hexdigest() + == "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb" +) h = hashlib.sha512() -g = hashlib.sha512(b'a') -assert h.name == g.name == 'sha512' -h.update(b'a') +g = hashlib.sha512(b"a") +assert h.name == g.name == "sha512" +h.update(b"a") print(h.hexdigest()) print(g.hexdigest()) -assert h.hexdigest() == g.hexdigest() == '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' +assert ( + h.hexdigest() + == g.hexdigest() + == "1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75" +) h = hashlib.new("blake2s", b"fubar") print(h.hexdigest()) -assert h.hexdigest() == 'a0e1ad0c123c9c65e8ef850db2ce4b5cef2c35b06527c615b0154353574d0415' -h.update(b'bla') +assert ( + h.hexdigest() == "a0e1ad0c123c9c65e8ef850db2ce4b5cef2c35b06527c615b0154353574d0415" +) +h.update(b"bla") print(h.hexdigest()) -assert h.hexdigest() == '25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81' +assert ( + h.hexdigest() == "25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81" +) diff --git a/extra_tests/snippets/stdlib_imp.py b/extra_tests/snippets/stdlib_imp.py index bd28e95f3d..835b50d617 100644 --- a/extra_tests/snippets/stdlib_imp.py +++ b/extra_tests/snippets/stdlib_imp.py @@ -8,9 +8,11 @@ assert _imp.is_frozen("__hello__") == True assert _imp.is_frozen("math") == False + class FakeSpec: - def __init__(self, name): - self.name = name + def __init__(self, name): + self.name = name + A = FakeSpec("time") diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index 3dda7278c0..011bb051c2 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -2,30 +2,30 @@ import os from testutils import assert_raises -fi = FileIO('README.md') +fi = FileIO("README.md") assert fi.seekable() bb = BufferedReader(fi) assert bb.seekable() result = bb.read() -assert len(result) <= 8*1024 +assert len(result) <= 8 * 1024 assert len(result) >= 0 assert isinstance(result, bytes) -with FileIO('README.md') as fio: - res = fio.read() - assert len(result) <= 8*1024 - assert len(result) >= 0 - assert isinstance(result, bytes) +with FileIO("README.md") as fio: + res = fio.read() + assert len(result) <= 8 * 1024 + assert len(result) >= 0 + assert isinstance(result, bytes) -fd = os.open('README.md', os.O_RDONLY) +fd = os.open("README.md", os.O_RDONLY) with FileIO(fd) as fio: - res2 = fio.read() - assert res == res2 + res2 = fio.read() + assert res == res2 -fi = FileIO('README.md') +fi = FileIO("README.md") fi.read() fi.close() assert fi.closefd @@ -34,8 +34,8 @@ with assert_raises(ValueError): fi.read() -with FileIO('README.md') as fio: - nres = fio.read(1) - assert len(nres) == 1 - nres = fio.read(2) - assert len(nres) == 2 +with FileIO("README.md") as fio: + nres = fio.read(1) + assert len(nres) == 1 + nres = fio.read(2) + assert len(nres) == 2 diff --git a/extra_tests/snippets/stdlib_io_bytesio.py b/extra_tests/snippets/stdlib_io_bytesio.py index 5714448764..d6ed298a0a 100644 --- a/extra_tests/snippets/stdlib_io_bytesio.py +++ b/extra_tests/snippets/stdlib_io_bytesio.py @@ -1,8 +1,8 @@ - from io import BytesIO + def test_01(): - bytes_string = b'Test String 1' + bytes_string = b"Test String 1" f = BytesIO() f.write(bytes_string) @@ -10,45 +10,49 @@ def test_01(): assert f.tell() == len(bytes_string) assert f.getvalue() == bytes_string + def test_02(): - bytes_string = b'Test String 2' + bytes_string = b"Test String 2" f = BytesIO(bytes_string) assert f.read() == bytes_string - assert f.read() == b'' + assert f.read() == b"" + def test_03(): """ - Tests that the read method (integer arg) - returns the expected value + Tests that the read method (integer arg) + returns the expected value """ - string = b'Test String 3' + string = b"Test String 3" f = BytesIO(string) - assert f.read(1) == b'T' - assert f.read(1) == b'e' - assert f.read(1) == b's' - assert f.read(1) == b't' + assert f.read(1) == b"T" + assert f.read(1) == b"e" + assert f.read(1) == b"s" + assert f.read(1) == b"t" + def test_04(): """ - Tests that the read method increments the - cursor position and the seek method moves - the cursor to the appropriate position + Tests that the read method increments the + cursor position and the seek method moves + the cursor to the appropriate position """ - string = b'Test String 4' + string = b"Test String 4" f = BytesIO(string) - assert f.read(4) == b'Test' + assert f.read(4) == b"Test" assert f.tell() == 4 assert f.seek(0) == 0 - assert f.read(4) == b'Test' + assert f.read(4) == b"Test" + def test_05(): """ - Tests that the write method accpets bytearray + Tests that the write method accpets bytearray """ - bytes_string = b'Test String 5' + bytes_string = b"Test String 5" f = BytesIO() f.write(bytearray(bytes_string)) @@ -58,16 +62,16 @@ def test_05(): def test_06(): """ - Tests readline + Tests readline """ - bytes_string = b'Test String 6\nnew line is here\nfinished' + bytes_string = b"Test String 6\nnew line is here\nfinished" f = BytesIO(bytes_string) - assert f.readline() == b'Test String 6\n' - assert f.readline() == b'new line is here\n' - assert f.readline() == b'finished' - assert f.readline() == b'' + assert f.readline() == b"Test String 6\n" + assert f.readline() == b"new line is here\n" + assert f.readline() == b"finished" + assert f.readline() == b"" if __name__ == "__main__": @@ -77,4 +81,3 @@ def test_06(): test_04() test_05() test_06() - diff --git a/extra_tests/snippets/stdlib_io_stringio.py b/extra_tests/snippets/stdlib_io_stringio.py index 828f0506ed..5419eef2bb 100644 --- a/extra_tests/snippets/stdlib_io_stringio.py +++ b/extra_tests/snippets/stdlib_io_stringio.py @@ -1,68 +1,73 @@ - from io import StringIO + def test_01(): """ - Test that the constructor and getvalue - method return expected values + Test that the constructor and getvalue + method return expected values """ - string = 'Test String 1' + string = "Test String 1" f = StringIO() f.write(string) assert f.tell() == len(string) assert f.getvalue() == string + def test_02(): """ - Test that the read method (no arg) - results the expected value + Test that the read method (no arg) + results the expected value """ - string = 'Test String 2' + string = "Test String 2" f = StringIO(string) assert f.read() == string - assert f.read() == '' + assert f.read() == "" + def test_03(): """ - Tests that the read method (integer arg) - returns the expected value + Tests that the read method (integer arg) + returns the expected value """ - string = 'Test String 3' + string = "Test String 3" f = StringIO(string) - assert f.read(1) == 'T' - assert f.read(1) == 'e' - assert f.read(1) == 's' - assert f.read(1) == 't' + assert f.read(1) == "T" + assert f.read(1) == "e" + assert f.read(1) == "s" + assert f.read(1) == "t" + def test_04(): """ - Tests that the read method increments the - cursor position and the seek method moves - the cursor to the appropriate position + Tests that the read method increments the + cursor position and the seek method moves + the cursor to the appropriate position """ - string = 'Test String 4' + string = "Test String 4" f = StringIO(string) - assert f.read(4) == 'Test' + assert f.read(4) == "Test" assert f.tell() == 4 assert f.seek(0) == 0 - assert f.read(4) == 'Test' + assert f.read(4) == "Test" + def test_05(): """ - Tests readline + Tests readline """ - string = 'Test String 6\nnew line is here\nfinished' + string = "Test String 6\nnew line is here\nfinished" f = StringIO(string) - assert f.readline() == 'Test String 6\n' - assert f.readline() == 'new line is here\n' - assert f.readline() == 'finished' - assert f.readline() == '' + assert f.readline() == "Test String 6\n" + assert f.readline() == "new line is here\n" + assert f.readline() == "finished" + assert f.readline() == "" + if __name__ == "__main__": test_01() diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index 58684f611d..724e9f1d5f 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -12,13 +12,13 @@ assert list(chain([], "", b"", ())) == [] assert list(chain([1, 2, 3, 4])) == [1, 2, 3, 4] -assert list(chain("ab", "cd", (), 'e')) == ['a', 'b', 'c', 'd', 'e'] +assert list(chain("ab", "cd", (), "e")) == ["a", "b", "c", "d", "e"] with assert_raises(TypeError): list(chain(1)) x = chain("ab", 1) -assert next(x) == 'a' -assert next(x) == 'b' +assert next(x) == "a" +assert next(x) == "b" with assert_raises(TypeError): next(x) @@ -37,17 +37,17 @@ list(chain(1)) args = ["abc", "def"] -assert list(chain.from_iterable(args)) == ['a', 'b', 'c', 'd', 'e', 'f'] +assert list(chain.from_iterable(args)) == ["a", "b", "c", "d", "e", "f"] args = [[], "", b"", ()] assert list(chain.from_iterable(args)) == [] -args = ["ab", "cd", (), 'e'] -assert list(chain.from_iterable(args)) == ['a', 'b', 'c', 'd', 'e'] +args = ["ab", "cd", (), "e"] +assert list(chain.from_iterable(args)) == ["a", "b", "c", "d", "e"] x = chain.from_iterable(["ab", 1]) -assert next(x) == 'a' -assert next(x) == 'b' +assert next(x) == "a" +assert next(x) == "b" with assert_raises(TypeError): next(x) @@ -174,16 +174,17 @@ # itertools.starmap tests starmap = itertools.starmap -assert list(starmap(pow, zip(range(3), range(1,7)))) == [0**1, 1**2, 2**3] +assert list(starmap(pow, zip(range(3), range(1, 7)))) == [0**1, 1**2, 2**3] assert list(starmap(pow, [])) == [] -assert list(starmap(pow, [iter([4,5])])) == [4**5] +assert list(starmap(pow, [iter([4, 5])])) == [4**5] with assert_raises(TypeError): starmap(pow) # itertools.takewhile tests def underten(x): - return x<10 + return x < 10 + from itertools import takewhile as tw @@ -227,12 +228,30 @@ def underten(x): next(t) it = tw(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert pickle.dumps(it, 0) == b'citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb.' -assert pickle.dumps(it, 1) == b'citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' +assert ( + pickle.dumps(it, 0) + == b"citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb." +) +assert ( + pickle.dumps(it, 1) + == b"citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b." +) +assert ( + pickle.dumps(it, 2) + == b"\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." +) +assert ( + pickle.dumps(it, 3) + == b"\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." +) +assert ( + pickle.dumps(it, 4) + == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." +) +assert ( + pickle.dumps(it, 5) + == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." +) next(it) next(it) next(it) @@ -240,12 +259,30 @@ def underten(x): next(it) except StopIteration: pass -assert pickle.dumps(it, 0) == b'citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb.' -assert pickle.dumps(it, 1) == b'citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' +assert ( + pickle.dumps(it, 0) + == b"citertools\ntakewhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb." +) +assert ( + pickle.dumps(it, 1) + == b"citertools\ntakewhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b." +) +assert ( + pickle.dumps(it, 2) + == b"\x80\x02citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." +) +assert ( + pickle.dumps(it, 3) + == b"\x80\x03citertools\ntakewhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." +) +assert ( + pickle.dumps(it, 4) + == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." +) +assert ( + pickle.dumps(it, 5) + == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\ttakewhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." +) for proto in range(pickle.HIGHEST_PROTOCOL + 1): try: next(pickle.loads(pickle.dumps(it, proto))) @@ -254,16 +291,18 @@ def underten(x): pass - # itertools.islice tests + def assert_matches_seq(it, seq): assert list(it) == list(seq) + def test_islice_pickle(it): for p in range(pickle.HIGHEST_PROTOCOL + 1): it == pickle.loads(pickle.dumps(it, p)) + i = itertools.islice it = i([1, 2, 3, 4, 5], 3) @@ -295,7 +334,7 @@ def test_islice_pickle(it): test_islice_pickle(it) # itertools.filterfalse -it = itertools.filterfalse(lambda x: x%2, range(10)) +it = itertools.filterfalse(lambda x: x % 2, range(10)) assert 0 == next(it) assert 2 == next(it) assert 4 == next(it) @@ -314,7 +353,7 @@ def test_islice_pickle(it): # itertools.dropwhile -it = itertools.dropwhile(lambda x: x<5, [1,4,6,4,1]) +it = itertools.dropwhile(lambda x: x < 5, [1, 4, 6, 4, 1]) assert 6 == next(it) assert 4 == next(it) assert 1 == next(it) @@ -322,19 +361,55 @@ def test_islice_pickle(it): next(it) it = itertools.dropwhile(underten, [1, 3, 5, 20, 2, 4, 6, 8]) -assert pickle.dumps(it, 0) == b'citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb.' -assert pickle.dumps(it, 1) == b'citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b.' +assert ( + pickle.dumps(it, 0) + == b"citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI0\nbtp6\nRp7\nI0\nb." +) +assert ( + pickle.dumps(it, 1) + == b"citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x00btq\x06Rq\x07K\x00b." +) +assert ( + pickle.dumps(it, 2) + == b"\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." +) +assert ( + pickle.dumps(it, 3) + == b"\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x00b\x86q\x06Rq\x07K\x00b." +) +assert ( + pickle.dumps(it, 4) + == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." +) +assert ( + pickle.dumps(it, 5) + == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x00b\x86\x94R\x94K\x00b." +) next(it) -assert pickle.dumps(it, 0) == b'citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb.' -assert pickle.dumps(it, 1) == b'citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 2) == b'\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 3) == b'\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b.' -assert pickle.dumps(it, 4) == b'\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' -assert pickle.dumps(it, 5) == b'\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b.' +assert ( + pickle.dumps(it, 0) + == b"citertools\ndropwhile\np0\n(c__main__\nunderten\np1\nc__builtin__\niter\np2\n((lp3\nI1\naI3\naI5\naI20\naI2\naI4\naI6\naI8\natp4\nRp5\nI4\nbtp6\nRp7\nI1\nb." +) +assert ( + pickle.dumps(it, 1) + == b"citertools\ndropwhile\nq\x00(c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02(]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08etq\x04Rq\x05K\x04btq\x06Rq\x07K\x01b." +) +assert ( + pickle.dumps(it, 2) + == b"\x80\x02citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01c__builtin__\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." +) +assert ( + pickle.dumps(it, 3) + == b"\x80\x03citertools\ndropwhile\nq\x00c__main__\nunderten\nq\x01cbuiltins\niter\nq\x02]q\x03(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85q\x04Rq\x05K\x04b\x86q\x06Rq\x07K\x01b." +) +assert ( + pickle.dumps(it, 4) + == b"\x80\x04\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." +) +assert ( + pickle.dumps(it, 5) + == b"\x80\x05\x95i\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x94\x8c\tdropwhile\x94\x93\x94\x8c\x08__main__\x94\x8c\x08underten\x94\x93\x94\x8c\x08builtins\x94\x8c\x04iter\x94\x93\x94]\x94(K\x01K\x03K\x05K\x14K\x02K\x04K\x06K\x08e\x85\x94R\x94K\x04b\x86\x94R\x94K\x01b." +) for proto in range(pickle.HIGHEST_PROTOCOL + 1): assert next(pickle.loads(pickle.dumps(it, proto))) == 2 @@ -352,7 +427,7 @@ def test_islice_pickle(it): with assert_raises(StopIteration): next(it) -it = itertools.accumulate([3, 2, 4, 1, 0, 5, 8], lambda a, v: a*v) +it = itertools.accumulate([3, 2, 4, 1, 0, 5, 8], lambda a, v: a * v) assert 3 == next(it) assert 6 == next(it) assert 24 == next(it) @@ -364,12 +439,12 @@ def test_islice_pickle(it): next(it) # itertools.compress -assert list(itertools.compress("ABCDEF", [1,0,1,0,1,1])) == list("ACEF") -assert list(itertools.compress("ABCDEF", [0,0,0,0,0,0])) == list("") -assert list(itertools.compress("ABCDEF", [1,1,1,1,1,1])) == list("ABCDEF") -assert list(itertools.compress("ABCDEF", [1,0,1])) == list("AC") -assert list(itertools.compress("ABC", [0,1,1,1,1,1])) == list("BC") -assert list(itertools.compress("ABCDEF", [True,False,"t","",1,9])) == list("ACEF") +assert list(itertools.compress("ABCDEF", [1, 0, 1, 0, 1, 1])) == list("ACEF") +assert list(itertools.compress("ABCDEF", [0, 0, 0, 0, 0, 0])) == list("") +assert list(itertools.compress("ABCDEF", [1, 1, 1, 1, 1, 1])) == list("ABCDEF") +assert list(itertools.compress("ABCDEF", [1, 0, 1])) == list("AC") +assert list(itertools.compress("ABC", [0, 1, 1, 1, 1, 1])) == list("BC") +assert list(itertools.compress("ABCDEF", [True, False, "t", "", 1, 9])) == list("ACEF") # itertools.tee @@ -384,7 +459,7 @@ def test_islice_pickle(it): t = itertools.tee(range(1000)) assert list(t[0]) == list(t[1]) == list(range(1000)) -t = itertools.tee([1,22,333], 3) +t = itertools.tee([1, 22, 333], 3) assert len(t) == 3 assert 1 == next(t[0]) assert 1 == next(t[1]) @@ -402,29 +477,29 @@ def test_islice_pickle(it): with assert_raises(StopIteration): next(t[1]) -t0, t1 = itertools.tee([1,2,3]) +t0, t1 = itertools.tee([1, 2, 3]) tc = t0.__copy__() -assert list(t0) == [1,2,3] -assert list(t1) == [1,2,3] -assert list(tc) == [1,2,3] +assert list(t0) == [1, 2, 3] +assert list(t1) == [1, 2, 3] +assert list(tc) == [1, 2, 3] -t0, t1 = itertools.tee([1,2,3]) +t0, t1 = itertools.tee([1, 2, 3]) assert 1 == next(t0) # advance index of t0 by 1 before __copy__() t0c = t0.__copy__() t1c = t1.__copy__() -assert list(t0) == [2,3] -assert list(t0c) == [2,3] -assert list(t1) == [1,2,3] -assert list(t1c) == [1,2,3] +assert list(t0) == [2, 3] +assert list(t0c) == [2, 3] +assert list(t1) == [1, 2, 3] +assert list(t1c) == [1, 2, 3] -t0, t1 = itertools.tee([1,2,3]) +t0, t1 = itertools.tee([1, 2, 3]) t2, t3 = itertools.tee(t0) -assert list(t1) == [1,2,3] -assert list(t2) == [1,2,3] -assert list(t3) == [1,2,3] +assert list(t1) == [1, 2, 3] +assert list(t2) == [1, 2, 3] +assert list(t3) == [1, 2, 3] -t = itertools.tee([1,2,3]) -assert list(t[0]) == [1,2,3] +t = itertools.tee([1, 2, 3]) +assert list(t[0]) == [1, 2, 3] assert list(t[0]) == [] # itertools.product @@ -530,22 +605,36 @@ def test_islice_pickle(it): # itertools.zip_longest tests zl = itertools.zip_longest -assert list(zl(['a', 'b', 'c'], range(3), [9, 8, 7])) \ - == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7)] -assert list(zl(['a', 'b', 'c'], range(3), [9, 8, 7, 99])) \ - == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7), (None, None, 99)] -assert list(zl(['a', 'b', 'c'], range(3), [9, 8, 7, 99], fillvalue='d')) \ - == [('a', 0, 9), ('b', 1, 8), ('c', 2, 7), ('d', 'd', 99)] - -assert list(zl(['a', 'b', 'c'])) == [('a',), ('b',), ('c',)] +assert list(zl(["a", "b", "c"], range(3), [9, 8, 7])) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), +] +assert list(zl(["a", "b", "c"], range(3), [9, 8, 7, 99])) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), + (None, None, 99), +] +assert list(zl(["a", "b", "c"], range(3), [9, 8, 7, 99], fillvalue="d")) == [ + ("a", 0, 9), + ("b", 1, 8), + ("c", 2, 7), + ("d", "d", 99), +] + +assert list(zl(["a", "b", "c"])) == [("a",), ("b",), ("c",)] assert list(zl()) == [] -assert list(zl(*zl(['a', 'b', 'c'], range(1, 4)))) \ - == [('a', 'b', 'c'), (1, 2, 3)] -assert list(zl(*zl(['a', 'b', 'c'], range(1, 5)))) \ - == [('a', 'b', 'c', None), (1, 2, 3, 4)] -assert list(zl(*zl(['a', 'b', 'c'], range(1, 5), fillvalue=100))) \ - == [('a', 'b', 'c', 100), (1, 2, 3, 4)] +assert list(zl(*zl(["a", "b", "c"], range(1, 4)))) == [("a", "b", "c"), (1, 2, 3)] +assert list(zl(*zl(["a", "b", "c"], range(1, 5)))) == [ + ("a", "b", "c", None), + (1, 2, 3, 4), +] +assert list(zl(*zl(["a", "b", "c"], range(1, 5), fillvalue=100))) == [ + ("a", "b", "c", 100), + (1, 2, 3, 4), +] # test infinite iterator @@ -565,7 +654,7 @@ def __iter__(self): assert next(it) == (1, 4) assert next(it) == (2, 5) -it = zl([1,2], [3]) +it = zl([1, 2], [3]) assert next(it) == (1, 3) assert next(it) == (2, None) with assert_raises(StopIteration): diff --git a/extra_tests/snippets/stdlib_json.py b/extra_tests/snippets/stdlib_json.py index a91f3bd817..ca35c2c2b2 100644 --- a/extra_tests/snippets/stdlib_json.py +++ b/extra_tests/snippets/stdlib_json.py @@ -2,22 +2,26 @@ import json from io import StringIO, BytesIO + def round_trip_test(obj): # serde_json and Python's json module produce slightly differently spaced # output; direct string comparison can't pass on both so we use this as a # proxy return obj == json.loads(json.dumps(obj)) + def json_dump(obj): f = StringIO() json.dump(obj, f) f.seek(0) return f.getvalue() + def json_load(obj): f = StringIO(obj) if isinstance(obj, str) else BytesIO(bytes(obj)) return json.load(f) + assert '"string"' == json.dumps("string") assert '"string"' == json_dump("string") @@ -33,41 +37,41 @@ def json_load(obj): assert "false" == json.dumps(False) assert "false" == json_dump(False) -assert 'null' == json.dumps(None) -assert 'null' == json_dump(None) +assert "null" == json.dumps(None) +assert "null" == json_dump(None) -assert '[]' == json.dumps([]) -assert '[]' == json_dump([]) +assert "[]" == json.dumps([]) +assert "[]" == json_dump([]) -assert '[1]' == json.dumps([1]) -assert '[1]' == json_dump([1]) +assert "[1]" == json.dumps([1]) +assert "[1]" == json_dump([1]) -assert '[[1]]' == json.dumps([[1]]) -assert '[[1]]' == json_dump([[1]]) +assert "[[1]]" == json.dumps([[1]]) +assert "[[1]]" == json_dump([[1]]) assert round_trip_test([1, "string", 1.0, True]) -assert '[]' == json.dumps(()) -assert '[]' == json_dump(()) +assert "[]" == json.dumps(()) +assert "[]" == json_dump(()) -assert '[1]' == json.dumps((1,)) -assert '[1]' == json_dump((1,)) +assert "[1]" == json.dumps((1,)) +assert "[1]" == json_dump((1,)) -assert '[[1]]' == json.dumps(((1,),)) -assert '[[1]]' == json_dump(((1,),)) +assert "[[1]]" == json.dumps(((1,),)) +assert "[[1]]" == json_dump(((1,),)) # tuples don't round-trip through json assert [1, "string", 1.0, True] == json.loads(json.dumps((1, "string", 1.0, True))) -assert '{}' == json.dumps({}) -assert '{}' == json_dump({}) -assert round_trip_test({'a': 'b'}) +assert "{}" == json.dumps({}) +assert "{}" == json_dump({}) +assert round_trip_test({"a": "b"}) # should reject non-str keys in jsons assert_raises(json.JSONDecodeError, lambda: json.loads('{3: "abc"}')) assert_raises(json.JSONDecodeError, lambda: json_load('{3: "abc"}')) # should serialize non-str keys as strings -assert json.dumps({'3': 'abc'}) == json.dumps({3: 'abc'}) +assert json.dumps({"3": "abc"}) == json.dumps({3: "abc"}) assert 1 == json.loads("1") assert 1 == json.loads(b"1") @@ -104,51 +108,60 @@ def json_load(obj): assert "str" == json_load(b'"str"') assert "str" == json_load(bytearray(b'"str"')) -assert True is json.loads('true') -assert True is json.loads(b'true') -assert True is json.loads(bytearray(b'true')) -assert True is json_load('true') -assert True is json_load(b'true') -assert True is json_load(bytearray(b'true')) - -assert False is json.loads('false') -assert False is json.loads(b'false') -assert False is json.loads(bytearray(b'false')) -assert False is json_load('false') -assert False is json_load(b'false') -assert False is json_load(bytearray(b'false')) - -assert None is json.loads('null') -assert None is json.loads(b'null') -assert None is json.loads(bytearray(b'null')) -assert None is json_load('null') -assert None is json_load(b'null') -assert None is json_load(bytearray(b'null')) - -assert [] == json.loads('[]') -assert [] == json.loads(b'[]') -assert [] == json.loads(bytearray(b'[]')) -assert [] == json_load('[]') -assert [] == json_load(b'[]') -assert [] == json_load(bytearray(b'[]')) - -assert ['a'] == json.loads('["a"]') -assert ['a'] == json.loads(b'["a"]') -assert ['a'] == json.loads(bytearray(b'["a"]')) -assert ['a'] == json_load('["a"]') -assert ['a'] == json_load(b'["a"]') -assert ['a'] == json_load(bytearray(b'["a"]')) - -assert [['a'], 'b'] == json.loads('[["a"], "b"]') -assert [['a'], 'b'] == json.loads(b'[["a"], "b"]') -assert [['a'], 'b'] == json.loads(bytearray(b'[["a"], "b"]')) -assert [['a'], 'b'] == json_load('[["a"], "b"]') -assert [['a'], 'b'] == json_load(b'[["a"], "b"]') -assert [['a'], 'b'] == json_load(bytearray(b'[["a"], "b"]')) - -class String(str): pass -class Bytes(bytes): pass -class ByteArray(bytearray): pass +assert True is json.loads("true") +assert True is json.loads(b"true") +assert True is json.loads(bytearray(b"true")) +assert True is json_load("true") +assert True is json_load(b"true") +assert True is json_load(bytearray(b"true")) + +assert False is json.loads("false") +assert False is json.loads(b"false") +assert False is json.loads(bytearray(b"false")) +assert False is json_load("false") +assert False is json_load(b"false") +assert False is json_load(bytearray(b"false")) + +assert None is json.loads("null") +assert None is json.loads(b"null") +assert None is json.loads(bytearray(b"null")) +assert None is json_load("null") +assert None is json_load(b"null") +assert None is json_load(bytearray(b"null")) + +assert [] == json.loads("[]") +assert [] == json.loads(b"[]") +assert [] == json.loads(bytearray(b"[]")) +assert [] == json_load("[]") +assert [] == json_load(b"[]") +assert [] == json_load(bytearray(b"[]")) + +assert ["a"] == json.loads('["a"]') +assert ["a"] == json.loads(b'["a"]') +assert ["a"] == json.loads(bytearray(b'["a"]')) +assert ["a"] == json_load('["a"]') +assert ["a"] == json_load(b'["a"]') +assert ["a"] == json_load(bytearray(b'["a"]')) + +assert [["a"], "b"] == json.loads('[["a"], "b"]') +assert [["a"], "b"] == json.loads(b'[["a"], "b"]') +assert [["a"], "b"] == json.loads(bytearray(b'[["a"], "b"]')) +assert [["a"], "b"] == json_load('[["a"], "b"]') +assert [["a"], "b"] == json_load(b'[["a"], "b"]') +assert [["a"], "b"] == json_load(bytearray(b'[["a"], "b"]')) + + +class String(str): + pass + + +class Bytes(bytes): + pass + + +class ByteArray(bytearray): + pass + assert "string" == json.loads(String('"string"')) assert "string" == json.loads(Bytes(b'"string"')) @@ -160,29 +173,46 @@ class ByteArray(bytearray): pass assert '"string"' == json.dumps(String("string")) assert '"string"' == json_dump(String("string")) -class Int(int): pass -class Float(float): pass -assert '1' == json.dumps(Int(1)) -assert '1' == json_dump(Int(1)) +class Int(int): + pass + + +class Float(float): + pass + + +assert "1" == json.dumps(Int(1)) +assert "1" == json_dump(Int(1)) + +assert "0.5" == json.dumps(Float(0.5)) +assert "0.5" == json_dump(Float(0.5)) + + +class List(list): + pass + + +class Tuple(tuple): + pass + -assert '0.5' == json.dumps(Float(0.5)) -assert '0.5' == json_dump(Float(0.5)) +class Dict(dict): + pass -class List(list): pass -class Tuple(tuple): pass -class Dict(dict): pass -assert '[1]' == json.dumps(List([1])) -assert '[1]' == json_dump(List([1])) +assert "[1]" == json.dumps(List([1])) +assert "[1]" == json_dump(List([1])) -assert json.dumps((1, "string", 1.0, True)) == json.dumps(Tuple((1, "string", 1.0, True))) +assert json.dumps((1, "string", 1.0, True)) == json.dumps( + Tuple((1, "string", 1.0, True)) +) assert json_dump((1, "string", 1.0, True)) == json_dump(Tuple((1, "string", 1.0, True))) -assert json.dumps({'a': 'b'}) == json.dumps(Dict({'a': 'b'})) -assert json_dump({'a': 'b'}) == json_dump(Dict({'a': 'b'})) +assert json.dumps({"a": "b"}) == json.dumps(Dict({"a": "b"})) +assert json_dump({"a": "b"}) == json_dump(Dict({"a": "b"})) i = 7**500 assert json.dumps(i) == str(i) -assert json.decoder.scanstring('✨x"', 1) == ('x', 3) +assert json.decoder.scanstring('✨x"', 1) == ("x", 3) diff --git a/extra_tests/snippets/stdlib_logging.py b/extra_tests/snippets/stdlib_logging.py index 0356404624..18c26f0fb6 100644 --- a/extra_tests/snippets/stdlib_logging.py +++ b/extra_tests/snippets/stdlib_logging.py @@ -1,4 +1,3 @@ - import io import sys @@ -7,12 +6,11 @@ import logging -logging.error('WOOT') -logging.warning('WARN') +logging.error("WOOT") +logging.warning("WARN") res = f.getvalue() -assert 'WOOT' in res -assert 'WARN' in res +assert "WOOT" in res +assert "WARN" in res print(res) - diff --git a/extra_tests/snippets/stdlib_marshal.py b/extra_tests/snippets/stdlib_marshal.py index 8ad11c3cc6..c5fb1e533e 100644 --- a/extra_tests/snippets/stdlib_marshal.py +++ b/extra_tests/snippets/stdlib_marshal.py @@ -1,11 +1,12 @@ import unittest import marshal + class MarshalTests(unittest.TestCase): """ Testing the (incomplete) marshal module. """ - + def dump_then_load(self, data): return marshal.loads(marshal.dumps(data)) @@ -34,7 +35,7 @@ def test_marshal_str(self): def test_marshal_list(self): self._test_marshal([]) self._test_marshal([1, "hello", 1.0]) - self._test_marshal([[0], ['a','b']]) + self._test_marshal([[0], ["a", "b"]]) def test_marshal_tuple(self): self._test_marshal(()) @@ -42,31 +43,31 @@ def test_marshal_tuple(self): def test_marshal_dict(self): self._test_marshal({}) - self._test_marshal({'a':1, 1:'a'}) - self._test_marshal({'a':{'b':2}, 'c':[0.0, 4.0, 6, 9]}) - + self._test_marshal({"a": 1, 1: "a"}) + self._test_marshal({"a": {"b": 2}, "c": [0.0, 4.0, 6, 9]}) + def test_marshal_set(self): self._test_marshal(set()) self._test_marshal({1, 2, 3}) - self._test_marshal({1, 'a', 'b'}) + self._test_marshal({1, "a", "b"}) def test_marshal_frozen_set(self): self._test_marshal(frozenset()) self._test_marshal(frozenset({1, 2, 3})) - self._test_marshal(frozenset({1, 'a', 'b'})) + self._test_marshal(frozenset({1, "a", "b"})) def test_marshal_bytearray(self): self.assertEqual( self.dump_then_load(bytearray([])), - bytearray(b''), + bytearray(b""), ) self.assertEqual( self.dump_then_load(bytearray([1, 2])), - bytearray(b'\x01\x02'), + bytearray(b"\x01\x02"), ) def test_roundtrip(self): - orig = compile("1 + 1", "", 'eval') + orig = compile("1 + 1", "", "eval") dumped = marshal.dumps(orig) loaded = marshal.loads(dumped) diff --git a/extra_tests/snippets/stdlib_math.py b/extra_tests/snippets/stdlib_math.py index 94d8c7347c..090de710ed 100644 --- a/extra_tests/snippets/stdlib_math.py +++ b/extra_tests/snippets/stdlib_math.py @@ -1,9 +1,9 @@ import math from testutils import assert_raises, skip_if_unsupported -NAN = float('nan') -INF = float('inf') -NINF = float('-inf') +NAN = float("nan") +INF = float("inf") +NINF = float("-inf") # assert(math.exp(2) == math.exp(2.0)) # assert(math.exp(True) == math.exp(1.0)) @@ -46,6 +46,7 @@ def float_ceil_exists(): assert isinstance(math.ceil(3.3), int) assert isinstance(math.floor(4.4), int) + class A(object): def __trunc__(self): return 2 @@ -56,10 +57,12 @@ def __ceil__(self): def __floor__(self): return 4 + assert math.trunc(A()) == 2 assert math.ceil(A()) == 3 assert math.floor(A()) == 4 + class A(object): def __trunc__(self): return 2.2 @@ -70,23 +73,26 @@ def __ceil__(self): def __floor__(self): return 4.4 + assert math.trunc(A()) == 2.2 assert math.ceil(A()) == 3.3 assert math.floor(A()) == 4.4 + class A(object): def __trunc__(self): - return 'trunc' + return "trunc" def __ceil__(self): - return 'ceil' + return "ceil" def __floor__(self): - return 'floor' + return "floor" + -assert math.trunc(A()) == 'trunc' -assert math.ceil(A()) == 'ceil' -assert math.floor(A()) == 'floor' +assert math.trunc(A()) == "trunc" +assert math.ceil(A()) == "ceil" +assert math.floor(A()) == "floor" with assert_raises(TypeError): math.trunc(object()) @@ -97,44 +103,54 @@ def __floor__(self): isclose = math.isclose + def assertIsClose(a, b, *args, **kwargs): assert isclose(a, b, *args, **kwargs) == True, "%s and %s should be close!" % (a, b) + def assertIsNotClose(a, b, *args, **kwargs): - assert isclose(a, b, *args, **kwargs) == False, "%s and %s should not be close!" % (a, b) + assert isclose(a, b, *args, **kwargs) == False, "%s and %s should not be close!" % ( + a, + b, + ) + def assertAllClose(examples, *args, **kwargs): for a, b in examples: assertIsClose(a, b, *args, **kwargs) + def assertAllNotClose(examples, *args, **kwargs): for a, b in examples: assertIsNotClose(a, b, *args, **kwargs) + # test_negative_tolerances: ValueError should be raised if either tolerance is less than zero assert_raises(ValueError, lambda: isclose(1, 1, rel_tol=-1e-100)) assert_raises(ValueError, lambda: isclose(1, 1, rel_tol=1e-100, abs_tol=-1e10)) # test_identical: identical values must test as close -identical_examples = [(2.0, 2.0), - (0.1e200, 0.1e200), - (1.123e-300, 1.123e-300), - (12345, 12345.0), - (0.0, -0.0), - (345678, 345678)] +identical_examples = [ + (2.0, 2.0), + (0.1e200, 0.1e200), + (1.123e-300, 1.123e-300), + (12345, 12345.0), + (0.0, -0.0), + (345678, 345678), +] assertAllClose(identical_examples, rel_tol=0.0, abs_tol=0.0) # test_eight_decimal_places: examples that are close to 1e-8, but not 1e-9 -eight_decimal_places_examples = [(1e8, 1e8 + 1), - (-1e-8, -1.000000009e-8), - (1.12345678, 1.12345679)] +eight_decimal_places_examples = [ + (1e8, 1e8 + 1), + (-1e-8, -1.000000009e-8), + (1.12345678, 1.12345679), +] assertAllClose(eight_decimal_places_examples, rel_tol=1e-08) assertAllNotClose(eight_decimal_places_examples, rel_tol=1e-09) # test_near_zero: values close to zero -near_zero_examples = [(1e-9, 0.0), - (-1e-9, 0.0), - (-1e-150, 0.0)] +near_zero_examples = [(1e-9, 0.0), (-1e-9, 0.0), (-1e-150, 0.0)] # these should not be close to any rel_tol assertAllNotClose(near_zero_examples, rel_tol=0.9) # these should be close to abs_tol=1e-8 @@ -147,35 +163,36 @@ def assertAllNotClose(examples, *args, **kwargs): assertIsClose(NINF, NINF, abs_tol=0.0) # test_inf_ninf_nan(self): these should never be close (following IEEE 754 rules for equality) -not_close_examples = [(NAN, NAN), - (NAN, 1e-100), - (1e-100, NAN), - (INF, NAN), - (NAN, INF), - (INF, NINF), - (INF, 1.0), - (1.0, INF), - (INF, 1e308), - (1e308, INF)] +not_close_examples = [ + (NAN, NAN), + (NAN, 1e-100), + (1e-100, NAN), + (INF, NAN), + (NAN, INF), + (INF, NINF), + (INF, 1.0), + (1.0, INF), + (INF, 1e308), + (1e308, INF), +] # use largest reasonable tolerance assertAllNotClose(not_close_examples, abs_tol=0.999999999999999) # test_zero_tolerance: test with zero tolerance -zero_tolerance_close_examples = [(1.0, 1.0), - (-3.4, -3.4), - (-1e-300, -1e-300)] +zero_tolerance_close_examples = [(1.0, 1.0), (-3.4, -3.4), (-1e-300, -1e-300)] assertAllClose(zero_tolerance_close_examples, rel_tol=0.0) -zero_tolerance_not_close_examples = [(1.0, 1.000000000000001), - (0.99999999999999, 1.0), - (1.0e200, .999999999999999e200)] +zero_tolerance_not_close_examples = [ + (1.0, 1.000000000000001), + (0.99999999999999, 1.0), + (1.0e200, 0.999999999999999e200), +] assertAllNotClose(zero_tolerance_not_close_examples, rel_tol=0.0) # test_asymmetry: test the asymmetry example from PEP 485 assertAllClose([(9, 10), (10, 9)], rel_tol=0.1) # test_integers: test with integer values -integer_examples = [(100000001, 100000000), - (123456789, 123456788)] +integer_examples = [(100000001, 100000000), (123456789, 123456788)] assertAllClose(integer_examples, rel_tol=1e-8) assertAllNotClose(integer_examples, rel_tol=1e-9) @@ -184,26 +201,26 @@ def assertAllNotClose(examples, *args, **kwargs): # test_fractions: test with Fraction values assert math.copysign(1, 42) == 1.0 -assert math.copysign(0., 42) == 0.0 -assert math.copysign(1., -42) == -1.0 -assert math.copysign(3, 0.) == 3.0 -assert math.copysign(4., -0.) == -4.0 +assert math.copysign(0.0, 42) == 0.0 +assert math.copysign(1.0, -42) == -1.0 +assert math.copysign(3, 0.0) == 3.0 +assert math.copysign(4.0, -0.0) == -4.0 assert_raises(TypeError, math.copysign) # copysign should let us distinguish signs of zeros -assert math.copysign(1., 0.) == 1. -assert math.copysign(1., -0.) == -1. -assert math.copysign(INF, 0.) == INF -assert math.copysign(INF, -0.) == NINF -assert math.copysign(NINF, 0.) == INF -assert math.copysign(NINF, -0.) == NINF +assert math.copysign(1.0, 0.0) == 1.0 +assert math.copysign(1.0, -0.0) == -1.0 +assert math.copysign(INF, 0.0) == INF +assert math.copysign(INF, -0.0) == NINF +assert math.copysign(NINF, 0.0) == INF +assert math.copysign(NINF, -0.0) == NINF # and of infinities -assert math.copysign(1., INF) == 1. -assert math.copysign(1., NINF) == -1. +assert math.copysign(1.0, INF) == 1.0 +assert math.copysign(1.0, NINF) == -1.0 assert math.copysign(INF, INF) == INF assert math.copysign(INF, NINF) == NINF assert math.copysign(NINF, INF) == INF assert math.copysign(NINF, NINF) == NINF -assert math.isnan(math.copysign(NAN, 1.)) +assert math.isnan(math.copysign(NAN, 1.0)) assert math.isnan(math.copysign(NAN, INF)) assert math.isnan(math.copysign(NAN, NINF)) assert math.isnan(math.copysign(NAN, NAN)) @@ -212,7 +229,7 @@ def assertAllNotClose(examples, *args, **kwargs): # given platform. assert math.isinf(math.copysign(INF, NAN)) # similarly, copysign(2., NAN) could be 2. or -2. -assert abs(math.copysign(2., NAN)) == 2. +assert abs(math.copysign(2.0, NAN)) == 2.0 assert str(math.frexp(0.0)) == str((+0.0, 0)) assert str(math.frexp(-0.0)) == str((-0.0, 0)) @@ -248,7 +265,7 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.factorial(20) == 2432902008176640000 assert_raises(ValueError, lambda: math.factorial(-1)) -if hasattr(math, 'nextafter'): +if hasattr(math, "nextafter"): try: assert math.nextafter(4503599627370496.0, -INF) == 4503599627370495.5 assert math.nextafter(4503599627370496.0, INF) == 4503599627370497.0 @@ -278,13 +295,13 @@ def assertAllNotClose(examples, *args, **kwargs): assert math.fmod(-10, 1) == -0.0 assert math.fmod(-10, 0.5) == -0.0 assert math.fmod(-10, 1.5) == -1.0 -assert math.isnan(math.fmod(NAN, 1.)) == True -assert math.isnan(math.fmod(1., NAN)) == True +assert math.isnan(math.fmod(NAN, 1.0)) == True +assert math.isnan(math.fmod(1.0, NAN)) == True assert math.isnan(math.fmod(NAN, NAN)) == True -assert_raises(ValueError, lambda: math.fmod(1., 0.)) -assert_raises(ValueError, lambda: math.fmod(INF, 1.)) -assert_raises(ValueError, lambda: math.fmod(NINF, 1.)) -assert_raises(ValueError, lambda: math.fmod(INF, 0.)) +assert_raises(ValueError, lambda: math.fmod(1.0, 0.0)) +assert_raises(ValueError, lambda: math.fmod(INF, 1.0)) +assert_raises(ValueError, lambda: math.fmod(NINF, 1.0)) +assert_raises(ValueError, lambda: math.fmod(INF, 0.0)) assert math.fmod(3.0, INF) == 3.0 assert math.fmod(-3.0, INF) == -3.0 assert math.fmod(3.0, NINF) == 3.0 diff --git a/extra_tests/snippets/stdlib_os.py b/extra_tests/snippets/stdlib_os.py index ab40582fd8..f5d26030a4 100644 --- a/extra_tests/snippets/stdlib_os.py +++ b/extra_tests/snippets/stdlib_os.py @@ -5,25 +5,24 @@ from testutils import assert_raises -assert os.name == 'posix' or os.name == 'nt' +assert os.name == "posix" or os.name == "nt" -fd = os.open('README.md', os.O_RDONLY) +fd = os.open("README.md", os.O_RDONLY) assert fd > 0 os.close(fd) assert_raises(OSError, lambda: os.read(fd, 10)) -assert_raises(FileNotFoundError, - lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY)) -assert_raises(FileNotFoundError, - lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY)) -assert_raises(FileNotFoundError, - lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2')) +assert_raises(FileNotFoundError, lambda: os.open("DOES_NOT_EXIST", os.O_RDONLY)) +assert_raises(FileNotFoundError, lambda: os.open("DOES_NOT_EXIST", os.O_WRONLY)) +assert_raises( + FileNotFoundError, lambda: os.rename("DOES_NOT_EXIST", "DOES_NOT_EXIST 2") +) # sendfile only supports in_fd as non-socket on linux and solaris if hasattr(os, "sendfile") and sys.platform.startswith("linux"): - src_fd = os.open('README.md', os.O_RDONLY) - dest_fd = os.open('destination.md', os.O_RDWR | os.O_CREAT) - src_len = os.stat('README.md').st_size + src_fd = os.open("README.md", os.O_RDONLY) + dest_fd = os.open("destination.md", os.O_RDWR | os.O_CREAT) + src_len = os.stat("README.md").st_size bytes_sent = os.sendfile(dest_fd, src_fd, 0, src_len) assert src_len == bytes_sent @@ -32,10 +31,10 @@ assert os.read(src_fd, src_len) == os.read(dest_fd, bytes_sent) os.close(src_fd) os.close(dest_fd) - os.remove('destination.md') + os.remove("destination.md") try: - os.open('DOES_NOT_EXIST', 0) + os.open("DOES_NOT_EXIST", 0) except OSError as err: assert err.errno == 2 @@ -81,15 +80,14 @@ assert_raises(TypeError, lambda: os.fspath([1, 2, 3])) -class TestWithTempDir(): +class TestWithTempDir: def __enter__(self): if os.name == "nt": base_folder = os.environ["TEMP"] else: base_folder = "/tmp" - name = os.path.join(base_folder, - "rustpython_test_os_" + str(int(time.time()))) + name = os.path.join(base_folder, "rustpython_test_os_" + str(int(time.time()))) while os.path.isdir(name): name = name + "_" @@ -102,7 +100,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): pass -class TestWithTempCurrentDir(): +class TestWithTempCurrentDir: def __enter__(self): self.prev_cwd = os.getcwd() @@ -130,8 +128,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert os.write(fd, CONTENT3) == len(CONTENT3) os.close(fd) - assert_raises(FileExistsError, - lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) + assert_raises( + FileExistsError, lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + ) fd = os.open(fname, os.O_RDONLY) assert os.read(fd, len(CONTENT2)) == CONTENT2 @@ -150,7 +149,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert not os.isatty(fd) # TODO: get os.lseek working on windows - if os.name != 'nt': + if os.name != "nt": fd = os.open(fname3, 0) assert os.read(fd, len(CONTENT2)) == CONTENT2 assert os.read(fd, len(CONTENT3)) == CONTENT3 @@ -201,8 +200,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if dir_entry.is_symlink(): symlinks.add(dir_entry.name) - assert names == set( - [FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) + assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) assert dirs == set([FOLDER, SYMLINK_FOLDER]) assert dirs_no_symlink == set([FOLDER]) @@ -270,23 +268,26 @@ def __exit__(self, exc_type, exc_val, exc_tb): os.stat(fname).st_mode == os.stat(symlink_file).st_mode os.stat(fname, follow_symlinks=False).st_ino == os.stat( - symlink_file, follow_symlinks=False).st_ino + symlink_file, follow_symlinks=False + ).st_ino os.stat(fname, follow_symlinks=False).st_mode == os.stat( - symlink_file, follow_symlinks=False).st_mode + symlink_file, follow_symlinks=False + ).st_mode # os.chmod if os.name != "nt": os.chmod(fname, 0o666) - assert oct(os.stat(fname).st_mode) == '0o100666' + assert oct(os.stat(fname).st_mode) == "0o100666" -# os.chown + # os.chown if os.name != "nt": # setup root_in_posix = False - if hasattr(os, 'geteuid'): - root_in_posix = (os.geteuid() == 0) + if hasattr(os, "geteuid"): + root_in_posix = os.geteuid() == 0 try: import pwd + all_users = [u.pw_uid for u in pwd.getpwall()] except (ImportError, AttributeError): all_users = [] @@ -299,10 +300,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): if not root_in_posix and len(all_users) > 1: uid_1, uid_2 = all_users[:2] gid = os.stat(fname1).st_gid - assert_raises(PermissionError, - lambda: os.chown(fname1, uid_1, gid)) - assert_raises(PermissionError, - lambda: os.chown(fname1, uid_2, gid)) + assert_raises(PermissionError, lambda: os.chown(fname1, uid_1, gid)) + assert_raises(PermissionError, lambda: os.chown(fname1, uid_2, gid)) # test chown with root perm and file name if root_in_posix and len(all_users) > 1: @@ -327,7 +326,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert uid == uid_2 # test gid change - if hasattr(os, 'getgroups'): + if hasattr(os, "getgroups"): groups = os.getgroups() if len(groups) > 1: gid_1, gid_2 = groups[:2] @@ -434,7 +433,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): os.close(wfd) # os.pipe2 -if sys.platform.startswith('linux') or sys.platform.startswith('freebsd'): +if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): rfd, wfd = os.pipe2(0) try: os.write(wfd, CONTENT2) @@ -460,11 +459,11 @@ def __exit__(self, exc_type, exc_val, exc_tb): with TestWithTempDir() as tmpdir: for i in range(0, 4): - file_name = os.path.join(tmpdir, 'file' + str(i)) - with open(file_name, 'w') as f: - f.write('test') + file_name = os.path.join(tmpdir, "file" + str(i)) + with open(file_name, "w") as f: + f.write("test") - expected_files = ['file0', 'file1', 'file2', 'file3'] + expected_files = ["file0", "file1", "file2", "file3"] dir_iter = os.scandir(tmpdir) collected_files = [dir_entry.name for dir_entry in dir_iter] @@ -476,13 +475,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): dir_iter.close() - expected_files_bytes = [(file.encode(), os.path.join(tmpdir, - file).encode()) - for file in expected_files] + expected_files_bytes = [ + (file.encode(), os.path.join(tmpdir, file).encode()) for file in expected_files + ] dir_iter_bytes = os.scandir(tmpdir.encode()) - collected_files_bytes = [(dir_entry.name, dir_entry.path) - for dir_entry in dir_iter_bytes] + collected_files_bytes = [ + (dir_entry.name, dir_entry.path) for dir_entry in dir_iter_bytes + ] assert set(collected_files_bytes) == set(expected_files_bytes) @@ -492,8 +492,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert set(collected_files) == set(expected_files) collected_files = os.listdir(tmpdir.encode()) - assert set(collected_files) == set( - [file.encode() for file in expected_files]) + assert set(collected_files) == set([file.encode() for file in expected_files]) with TestWithTempCurrentDir(): os.chdir(tmpdir) @@ -502,20 +501,20 @@ def __exit__(self, exc_type, exc_val, exc_tb): assert set(collected_files) == set(expected_files) # system() -if os.name in ('posix', 'nt'): - assert os.system('echo test') == 0 - assert os.system('&') != 0 +if os.name in ("posix", "nt"): + assert os.system("echo test") == 0 + assert os.system("&") != 0 for arg in [None, 1, 1.0, TabError]: assert_raises(TypeError, os.system, arg) # Testing for os.pathconf_names -if not sys.platform.startswith('win'): +if not sys.platform.startswith("win"): assert len(os.pathconf_names) > 0 - assert 'PC_NAME_MAX' in os.pathconf_names + assert "PC_NAME_MAX" in os.pathconf_names for option, index in os.pathconf_names.items(): if sys.platform == "darwin": # TODO: check why it fails if option in ["PC_MAX_CANON", "PC_MAX_INPUT", "PC_VDISABLE"]: continue - assert os.pathconf('/', index) == os.pathconf('/', option) + assert os.pathconf("/", index) == os.pathconf("/", option) diff --git a/extra_tests/snippets/stdlib_pwd.py b/extra_tests/snippets/stdlib_pwd.py index 6ef3a64d02..2a44aed32c 100644 --- a/extra_tests/snippets/stdlib_pwd.py +++ b/extra_tests/snippets/stdlib_pwd.py @@ -1,4 +1,5 @@ import sys + # windows doesn't support pwd if sys.platform.startswith("win"): exit(0) @@ -7,6 +8,6 @@ import pwd with assert_raises(KeyError): - fake_name = 'fake_user' + fake_name = "fake_user" while pwd.getpwnam(fake_name): - fake_name += '1' + fake_name += "1" diff --git a/extra_tests/snippets/stdlib_random.py b/extra_tests/snippets/stdlib_random.py index 969b09d339..60fc9a3097 100644 --- a/extra_tests/snippets/stdlib_random.py +++ b/extra_tests/snippets/stdlib_random.py @@ -15,8 +15,8 @@ assert random.choice(left) == 5 # random.choices -expected = ['red', 'green', 'red', 'black', 'black', 'red'] -result = random.choices(['red', 'black', 'green'], [18, 18, 2], k=6) +expected = ["red", "green", "red", "black", "black", "red"] +result = random.choices(["red", "black", "green"], [18, 18, 2], k=6) assert expected == result # random.sample @@ -30,7 +30,7 @@ assert len(zero_size_buf) == 0 non_zero_buf = random.randbytes(4) assert type(non_zero_buf) is bytes -assert list(non_zero_buf) == list(b'\xb9\x7fi\xf7') +assert list(non_zero_buf) == list(b"\xb9\x7fi\xf7") # TODO : random.random(), random.uniform(), random.triangular(), # random.betavariate, random.expovariate, random.gammavariate, diff --git a/extra_tests/snippets/stdlib_re.py b/extra_tests/snippets/stdlib_re.py index 17ecdba7f6..89d729f2b2 100644 --- a/extra_tests/snippets/stdlib_re.py +++ b/extra_tests/snippets/stdlib_re.py @@ -1,8 +1,7 @@ - import re haystack = "Hello world" -needle = 'ello' +needle = "ello" mo = re.search(needle, haystack) print(mo) @@ -12,62 +11,67 @@ assert mo.start() == 1 assert mo.end() == 5 -assert re.escape('python.exe') == 'python\\.exe' +assert re.escape("python.exe") == "python\\.exe" -p = re.compile('ab') -s = p.sub('x', 'abcabca') +p = re.compile("ab") +s = p.sub("x", "abcabca") # print(s) -assert s == 'xcxca' +assert s == "xcxca" -idpattern = r'([_a-z][_a-z0-9]*)' +idpattern = r"([_a-z][_a-z0-9]*)" -mo = re.search(idpattern, '7382 _boe0+2') -assert mo.group(0) == '_boe0' +mo = re.search(idpattern, "7382 _boe0+2") +assert mo.group(0) == "_boe0" # tes op range -assert re.compile('[a-z]').match('a').span() == (0, 1) -assert re.compile('[a-z]').fullmatch('z').span() == (0, 1) +assert re.compile("[a-z]").match("a").span() == (0, 1) +assert re.compile("[a-z]").fullmatch("z").span() == (0, 1) # test op charset -assert re.compile('[_a-z0-9]*').match('_09az').group() == '_09az' +assert re.compile("[_a-z0-9]*").match("_09az").group() == "_09az" # test op bigcharset -assert re.compile('[你好a-z]*').match('a好z你?').group() == 'a好z你' -assert re.compile('[你好a-z]+').search('1232321 a好z你 !!?').group() == 'a好z你' +assert re.compile("[你好a-z]*").match("a好z你?").group() == "a好z你" +assert re.compile("[你好a-z]+").search("1232321 a好z你 !!?").group() == "a好z你" # test op repeat one -assert re.compile('a*').match('aaa').span() == (0, 3) -assert re.compile('abcd*').match('abcdddd').group() == 'abcdddd' -assert re.compile('abcd*').match('abc').group() == 'abc' -assert re.compile('abcd*e').match('abce').group() == 'abce' -assert re.compile('abcd*e+').match('abcddeee').group() == 'abcddeee' -assert re.compile('abcd+').match('abcddd').group() == 'abcddd' +assert re.compile("a*").match("aaa").span() == (0, 3) +assert re.compile("abcd*").match("abcdddd").group() == "abcdddd" +assert re.compile("abcd*").match("abc").group() == "abc" +assert re.compile("abcd*e").match("abce").group() == "abce" +assert re.compile("abcd*e+").match("abcddeee").group() == "abcddeee" +assert re.compile("abcd+").match("abcddd").group() == "abcddd" # test op mark -assert re.compile('(a)b').match('ab').group(0, 1) == ('ab', 'a') -assert re.compile('a(b)(cd)').match('abcd').group(0, 1, 2) == ('abcd', 'b', 'cd') +assert re.compile("(a)b").match("ab").group(0, 1) == ("ab", "a") +assert re.compile("a(b)(cd)").match("abcd").group(0, 1, 2) == ("abcd", "b", "cd") # test op repeat -assert re.compile('(ab)+').match('abab') -assert re.compile('(a)(b)(cd)*').match('abcdcdcd').group(0, 1, 2, 3) == ('abcdcdcd', 'a', 'b', 'cd') -assert re.compile('ab()+cd').match('abcd').group() == 'abcd' -assert re.compile('(a)+').match('aaa').groups() == ('a',) -assert re.compile('(a+)').match('aaa').groups() == ('aaa',) +assert re.compile("(ab)+").match("abab") +assert re.compile("(a)(b)(cd)*").match("abcdcdcd").group(0, 1, 2, 3) == ( + "abcdcdcd", + "a", + "b", + "cd", +) +assert re.compile("ab()+cd").match("abcd").group() == "abcd" +assert re.compile("(a)+").match("aaa").groups() == ("a",) +assert re.compile("(a+)").match("aaa").groups() == ("aaa",) # test Match object method -assert re.compile('(a)(bc)').match('abc')[1] == 'a' -assert re.compile('a(b)(?Pc)d').match('abcd').groupdict() == {'a': 'c'} +assert re.compile("(a)(bc)").match("abc")[1] == "a" +assert re.compile("a(b)(?Pc)d").match("abcd").groupdict() == {"a": "c"} # test op branch -assert re.compile(r'((?=\d|\.\d)(?P\d*)|a)').match('123.2132').group() == '123' +assert re.compile(r"((?=\d|\.\d)(?P\d*)|a)").match("123.2132").group() == "123" -assert re.sub(r'^\s*', 'X', 'test') == 'Xtest' +assert re.sub(r"^\s*", "X", "test") == "Xtest" -assert re.match(r'\babc\b', 'abc').group() == 'abc' +assert re.match(r"\babc\b", "abc").group() == "abc" -urlpattern = re.compile('//([^/#?]*)(.*)', re.DOTALL) -url = '//www.example.org:80/foo/bar/baz.html' -assert urlpattern.match(url).group(1) == 'www.example.org:80' +urlpattern = re.compile("//([^/#?]*)(.*)", re.DOTALL) +url = "//www.example.org:80/foo/bar/baz.html" +assert urlpattern.match(url).group(1) == "www.example.org:80" -assert re.compile('(?:\w+(?:\s|/(?!>))*)*').match('a /bb />ccc').group() == 'a /bb ' -assert re.compile('(?:(1)?)*').match('111').group() == '111' \ No newline at end of file +assert re.compile("(?:\w+(?:\s|/(?!>))*)*").match("a /bb />ccc").group() == "a /bb " +assert re.compile("(?:(1)?)*").match("111").group() == "111" diff --git a/extra_tests/snippets/stdlib_signal.py b/extra_tests/snippets/stdlib_signal.py index eb4a25f90d..0abfd7cb71 100644 --- a/extra_tests/snippets/stdlib_signal.py +++ b/extra_tests/snippets/stdlib_signal.py @@ -7,11 +7,12 @@ signals = [] + def handler(signum, frame): - signals.append(signum) + signals.append(signum) -signal.signal(signal.SIGILL, signal.SIG_IGN); +signal.signal(signal.SIGILL, signal.SIG_IGN) assert signal.getsignal(signal.SIGILL) is signal.SIG_IGN old_signal = signal.signal(signal.SIGILL, signal.SIG_DFL) @@ -21,24 +22,21 @@ def handler(signum, frame): # unix if "win" not in sys.platform: - signal.signal(signal.SIGALRM, handler) - assert signal.getsignal(signal.SIGALRM) is handler - - signal.alarm(1) - time.sleep(2.0) - assert signals == [signal.SIGALRM] - - signal.signal(signal.SIGALRM, signal.SIG_IGN) - signal.alarm(1) - time.sleep(2.0) - - assert signals == [signal.SIGALRM] + signal.signal(signal.SIGALRM, handler) + assert signal.getsignal(signal.SIGALRM) is handler - signal.signal(signal.SIGALRM, handler) - signal.alarm(1) - time.sleep(2.0) + signal.alarm(1) + time.sleep(2.0) + assert signals == [signal.SIGALRM] - assert signals == [signal.SIGALRM, signal.SIGALRM] + signal.signal(signal.SIGALRM, signal.SIG_IGN) + signal.alarm(1) + time.sleep(2.0) + assert signals == [signal.SIGALRM] + signal.signal(signal.SIGALRM, handler) + signal.alarm(1) + time.sleep(2.0) + assert signals == [signal.SIGALRM, signal.SIGALRM] diff --git a/extra_tests/snippets/stdlib_socket.py b/extra_tests/snippets/stdlib_socket.py index bbedb794ba..199ff9fe47 100644 --- a/extra_tests/snippets/stdlib_socket.py +++ b/extra_tests/snippets/stdlib_socket.py @@ -5,8 +5,8 @@ assert _socket.socket == _socket.SocketType -MESSAGE_A = b'aaaa' -MESSAGE_B= b'bbbbb' +MESSAGE_A = b"aaaa" +MESSAGE_B = b"bbbbb" # TCP @@ -26,9 +26,9 @@ assert recv_a == MESSAGE_A assert recv_b == MESSAGE_B -fd = open('README.md', 'rb') +fd = open("README.md", "rb") connector.sendfile(fd) -recv_readme = connection.recv(os.stat('README.md').st_size) +recv_readme = connection.recv(os.stat("README.md").st_size) # need this because sendfile leaves the cursor at the end of the file fd.seek(0) assert recv_readme == fd.read() @@ -36,14 +36,14 @@ # fileno if os.name == "posix": - connector_fd = connector.fileno() - connection_fd = connection.fileno() - os.write(connector_fd, MESSAGE_A) - connection.send(MESSAGE_B) - recv_a = connection.recv(len(MESSAGE_A)) - recv_b = os.read(connector_fd, (len(MESSAGE_B))) - assert recv_a == MESSAGE_A - assert recv_b == MESSAGE_B + connector_fd = connector.fileno() + connection_fd = connection.fileno() + os.write(connector_fd, MESSAGE_A) + connection.send(MESSAGE_B) + recv_a = connection.recv(len(MESSAGE_A)) + recv_b = os.read(connector_fd, (len(MESSAGE_B))) + assert recv_a == MESSAGE_A + assert recv_b == MESSAGE_B connection.close() connector.close() @@ -51,30 +51,30 @@ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) with assert_raises(TypeError): - s.connect(("127.0.0.1", 8888, 8888)) + s.connect(("127.0.0.1", 8888, 8888)) with assert_raises(OSError): - # Lets hope nobody is listening on port 1 - s.connect(("127.0.0.1", 1)) + # Lets hope nobody is listening on port 1 + s.connect(("127.0.0.1", 1)) with assert_raises(TypeError): - s.bind(("127.0.0.1", 8888, 8888)) + s.bind(("127.0.0.1", 8888, 8888)) with assert_raises(OSError): - # Lets hope nobody run this test on machine with ip 1.2.3.4 - s.bind(("1.2.3.4", 8888)) + # Lets hope nobody run this test on machine with ip 1.2.3.4 + s.bind(("1.2.3.4", 8888)) with assert_raises(TypeError): - s.bind((888, 8888)) + s.bind((888, 8888)) s.close() s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.bind(("127.0.0.1", 0)) with assert_raises(OSError): - s.recv(100) + s.recv(100) with assert_raises(OSError): - s.send(MESSAGE_A) + s.send(MESSAGE_A) s.close() @@ -117,48 +117,48 @@ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) with assert_raises(OSError): - s.bind(("1.2.3.4", 888)) + s.bind(("1.2.3.4", 888)) s.close() ### Errors with assert_raises(OSError): - socket.socket(100, socket.SOCK_STREAM) + socket.socket(100, socket.SOCK_STREAM) with assert_raises(OSError): - socket.socket(socket.AF_INET, 1000) + socket.socket(socket.AF_INET, 1000) with assert_raises(OSError): - socket.inet_aton("test") + socket.inet_aton("test") with assert_raises(OverflowError): - socket.htonl(-1) + socket.htonl(-1) -assert socket.htonl(0)==0 -assert socket.htonl(10)==167772160 +assert socket.htonl(0) == 0 +assert socket.htonl(10) == 167772160 -assert socket.inet_aton("127.0.0.1")==b"\x7f\x00\x00\x01" -assert socket.inet_aton("255.255.255.255")==b"\xff\xff\xff\xff" +assert socket.inet_aton("127.0.0.1") == b"\x7f\x00\x00\x01" +assert socket.inet_aton("255.255.255.255") == b"\xff\xff\xff\xff" -assert socket.inet_ntoa(b"\x7f\x00\x00\x01")=="127.0.0.1" -assert socket.inet_ntoa(b"\xff\xff\xff\xff")=="255.255.255.255" +assert socket.inet_ntoa(b"\x7f\x00\x00\x01") == "127.0.0.1" +assert socket.inet_ntoa(b"\xff\xff\xff\xff") == "255.255.255.255" with assert_raises(OSError): - socket.inet_ntoa(b"\xff\xff\xff\xff\xff") + socket.inet_ntoa(b"\xff\xff\xff\xff\xff") with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - pass + pass with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen(1) - connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - connector.connect(("127.0.0.1", listener.getsockname()[1])) - (connection, addr) = listener.accept() - connection.settimeout(1.0) - with assert_raises(OSError): # TODO: check that it raises a socket.timeout - # testing that it doesn't work with the timeout; that it stops blocking eventually - connection.recv(len(MESSAGE_A)) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connector.connect(("127.0.0.1", listener.getsockname()[1])) + (connection, addr) = listener.accept() + connection.settimeout(1.0) + with assert_raises(OSError): # TODO: check that it raises a socket.timeout + # testing that it doesn't work with the timeout; that it stops blocking eventually + connection.recv(len(MESSAGE_A)) for exc, expected_name in [ (socket.gaierror, "gaierror"), diff --git a/extra_tests/snippets/stdlib_sqlite.py b/extra_tests/snippets/stdlib_sqlite.py index 8ec5416fe2..f2e02b48cf 100644 --- a/extra_tests/snippets/stdlib_sqlite.py +++ b/extra_tests/snippets/stdlib_sqlite.py @@ -18,6 +18,7 @@ INSERT INTO foo(key) VALUES (11); """) + class AggrSum: def __init__(self): self.val = 0.0 @@ -28,6 +29,7 @@ def step(self, val): def finalize(self): return self.val + cx.create_aggregate("mysum", 1, AggrSum) cur.execute("select mysum(key) from foo") assert cur.fetchone()[0] == 28.0 @@ -35,15 +37,19 @@ def finalize(self): # toobig = 2**64 # cur.execute("insert into foo(key) values (?)", (toobig,)) + class AggrText: def __init__(self): self.txt = "" + def step(self, txt): txt = str(txt) self.txt = self.txt + txt + def finalize(self): return self.txt + cx.create_aggregate("aggtxt", 1, AggrText) cur.execute("select aggtxt(key) from foo") -assert cur.fetchone()[0] == '341011' \ No newline at end of file +assert cur.fetchone()[0] == "341011" diff --git a/extra_tests/snippets/stdlib_string.py b/extra_tests/snippets/stdlib_string.py index 9151d2f593..ae544f3289 100644 --- a/extra_tests/snippets/stdlib_string.py +++ b/extra_tests/snippets/stdlib_string.py @@ -1,22 +1,26 @@ import string -assert string.ascii_letters == 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' -assert string.ascii_lowercase == 'abcdefghijklmnopqrstuvwxyz' -assert string.ascii_uppercase == 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' -assert string.digits == '0123456789' -assert string.hexdigits == '0123456789abcdefABCDEF' -assert string.octdigits == '01234567' -assert string.punctuation == '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' -assert string.whitespace == ' \t\n\r\x0b\x0c', string.whitespace -assert string.printable == '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c' +assert string.ascii_letters == "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +assert string.ascii_lowercase == "abcdefghijklmnopqrstuvwxyz" +assert string.ascii_uppercase == "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +assert string.digits == "0123456789" +assert string.hexdigits == "0123456789abcdefABCDEF" +assert string.octdigits == "01234567" +assert string.punctuation == "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" +assert string.whitespace == " \t\n\r\x0b\x0c", string.whitespace +assert ( + string.printable + == "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c" +) -assert string.capwords('bla bla', ' ') == 'Bla Bla' +assert string.capwords("bla bla", " ") == "Bla Bla" from string import Template -s = Template('$who likes $what') -r = s.substitute(who='tim', what='kung pow') -assert r == 'tim likes kung pow' + +s = Template("$who likes $what") +r = s.substitute(who="tim", what="kung pow") +assert r == "tim likes kung pow" from string import Formatter diff --git a/extra_tests/snippets/stdlib_struct.py b/extra_tests/snippets/stdlib_struct.py index 83154c8100..1e08d0a223 100644 --- a/extra_tests/snippets/stdlib_struct.py +++ b/extra_tests/snippets/stdlib_struct.py @@ -1,51 +1,51 @@ - from testutils import assert_raises import struct -data = struct.pack('IH', 14, 12) +data = struct.pack("IH", 14, 12) assert data == bytes([14, 0, 0, 0, 12, 0]) -v1, v2 = struct.unpack('IH', data) +v1, v2 = struct.unpack("IH", data) assert v1 == 14 assert v2 == 12 -data = struct.pack('IH', 14, 12) +data = struct.pack(">IH", 14, 12) assert data == bytes([0, 0, 0, 14, 0, 12]) -v1, v2 = struct.unpack('>IH', data) +v1, v2 = struct.unpack(">IH", data) assert v1 == 14 assert v2 == 12 -data = struct.pack('3B', 65, 66, 67) +data = struct.pack("3B", 65, 66, 67) assert data == bytes([65, 66, 67]) -v1, v2, v3 = struct.unpack('3B', data) +v1, v2, v3 = struct.unpack("3B", data) assert v1 == 65 assert v2 == 66 assert v3 == 67 with assert_raises(Exception): - data = struct.pack('B0B', 65, 66) + data = struct.pack("B0B", 65, 66) with assert_raises(Exception): - data = struct.pack('B2B', 65, 66) + data = struct.pack("B2B", 65, 66) -data = struct.pack('B1B', 65, 66) +data = struct.pack("B1B", 65, 66) with assert_raises(Exception): - struct.pack(' 0: demo(x - 1) + sys.settrace(trc) demo(5) sys.settrace(None) @@ -53,7 +61,7 @@ def demo(x): assert sys.exc_info() == (None, None, None) try: - 1/0 + 1 / 0 except ZeroDivisionError as exc: exc_info = sys.exc_info() assert exc_info[0] == type(exc) == ZeroDivisionError @@ -62,10 +70,12 @@ def demo(x): # Recursion: + def recursive_call(n): if n > 0: recursive_call(n - 1) + sys.setrecursionlimit(200) assert sys.getrecursionlimit() == 200 @@ -74,11 +84,25 @@ def recursive_call(n): if sys.platform.startswith("win"): winver = sys.getwindowsversion() - print(f'winver: {winver} {winver.platform_version}') + print(f"winver: {winver} {winver.platform_version}") # the biggest value of wSuiteMask (https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-osversioninfoexa#members). - all_masks = 0x00000004 | 0x00000400 | 0x00004000 | 0x00000080 | 0x00000002 | 0x00000040 | 0x00000200 | \ - 0x00000100 | 0x00000001 | 0x00000020 | 0x00002000 | 0x00000010 | 0x00008000 | 0x00020000 + all_masks = ( + 0x00000004 + | 0x00000400 + | 0x00004000 + | 0x00000080 + | 0x00000002 + | 0x00000040 + | 0x00000200 + | 0x00000100 + | 0x00000001 + | 0x00000020 + | 0x00002000 + | 0x00000010 + | 0x00008000 + | 0x00020000 + ) # We really can't test if the results are correct, so it just checks for meaningful value assert winver.major > 6 @@ -112,18 +136,14 @@ def recursive_call(n): # Test the PYTHONSAFEPATH environment variable code = "import sys; print(sys.flags.safe_path)" env = dict(os.environ) -env.pop('PYTHONSAFEPATH', None) -args = (sys.executable, '-P', '-c', code) +env.pop("PYTHONSAFEPATH", None) +args = (sys.executable, "-P", "-c", code) -proc = subprocess.run( - args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) -assert proc.stdout.rstrip() == 'True', proc +proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True, env=env) +assert proc.stdout.rstrip() == "True", proc assert proc.returncode == 0, proc -env['PYTHONSAFEPATH'] = '1' -proc = subprocess.run( - args, stdout=subprocess.PIPE, - universal_newlines=True, env=env) -assert proc.stdout.rstrip() == 'True' +env["PYTHONSAFEPATH"] = "1" +proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True, env=env) +assert proc.stdout.rstrip() == "True" assert proc.returncode == 0, proc diff --git a/extra_tests/snippets/stdlib_sys_getframe.py b/extra_tests/snippets/stdlib_sys_getframe.py index d4328286aa..50447ce882 100644 --- a/extra_tests/snippets/stdlib_sys_getframe.py +++ b/extra_tests/snippets/stdlib_sys_getframe.py @@ -2,20 +2,24 @@ value = 189 locals_dict = sys._getframe().f_locals -assert locals_dict['value'] == 189 -foo = 'bar' -assert locals_dict['foo'] == foo +assert locals_dict["value"] == 189 +foo = "bar" +assert locals_dict["foo"] == foo + def test_function(): x = 17 assert sys._getframe().f_locals is not locals_dict - assert sys._getframe().f_locals['x'] == 17 - assert sys._getframe(1).f_locals['foo'] == 'bar' + assert sys._getframe().f_locals["x"] == 17 + assert sys._getframe(1).f_locals["foo"] == "bar" + test_function() -class TestClass(): + +class TestClass: def __init__(self): - assert sys._getframe().f_locals['self'] == self + assert sys._getframe().f_locals["self"] == self + TestClass() diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index baf6755306..9a92969f5f 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -1,5 +1,3 @@ - - import time x = time.gmtime(1000) @@ -9,14 +7,13 @@ assert x.tm_sec == 40 assert x.tm_isdst == 0 -s = time.strftime('%Y-%m-%d-%H-%M-%S', x) +s = time.strftime("%Y-%m-%d-%H-%M-%S", x) # print(s) -assert s == '1970-01-01-00-16-40' +assert s == "1970-01-01-00-16-40" -x2 = time.strptime(s, '%Y-%m-%d-%H-%M-%S') +x2 = time.strptime(s, "%Y-%m-%d-%H-%M-%S") assert x2.tm_min == 16 s = time.asctime(x) # print(s) -assert s == 'Thu Jan 1 00:16:40 1970' - +assert s == "Thu Jan 1 00:16:40 1970" diff --git a/extra_tests/snippets/stdlib_traceback.py b/extra_tests/snippets/stdlib_traceback.py index 689f36e027..c2cc5773db 100644 --- a/extra_tests/snippets/stdlib_traceback.py +++ b/extra_tests/snippets/stdlib_traceback.py @@ -1,27 +1,27 @@ import traceback try: - 1/0 + 1 / 0 except ZeroDivisionError as ex: - tb = traceback.extract_tb(ex.__traceback__) - assert len(tb) == 1 + tb = traceback.extract_tb(ex.__traceback__) + assert len(tb) == 1 try: - try: - 1/0 - except ZeroDivisionError as ex: - raise KeyError().with_traceback(ex.__traceback__) + try: + 1 / 0 + except ZeroDivisionError as ex: + raise KeyError().with_traceback(ex.__traceback__) except KeyError as ex2: - tb = traceback.extract_tb(ex2.__traceback__) - assert tb[1].line == "1/0" + tb = traceback.extract_tb(ex2.__traceback__) + assert tb[1].line == "1 / 0" try: - try: - 1/0 - except ZeroDivisionError as ex: - raise ex.with_traceback(None) + try: + 1 / 0 + except ZeroDivisionError as ex: + raise ex.with_traceback(None) except ZeroDivisionError as ex2: - tb = traceback.extract_tb(ex2.__traceback__) - assert len(tb) == 1 + tb = traceback.extract_tb(ex2.__traceback__) + assert len(tb) == 1 diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index 479004b6cf..3a3872d2f4 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -2,7 +2,7 @@ from testutils import assert_raises -ns = types.SimpleNamespace(a=2, b='Rust') +ns = types.SimpleNamespace(a=2, b="Rust") assert ns.a == 2 assert ns.b == "Rust" diff --git a/extra_tests/snippets/test_threading.py b/extra_tests/snippets/test_threading.py index 41024b360e..4d7c29f509 100644 --- a/extra_tests/snippets/test_threading.py +++ b/extra_tests/snippets/test_threading.py @@ -11,7 +11,7 @@ def thread_function(name): output.append((0, 0)) -x = threading.Thread(target=thread_function, args=(1, )) +x = threading.Thread(target=thread_function, args=(1,)) output.append((0, 1)) x.start() output.append((0, 2)) diff --git a/extra_tests/snippets/testutils.py b/extra_tests/snippets/testutils.py index 437fa06ae3..aac153441e 100644 --- a/extra_tests/snippets/testutils.py +++ b/extra_tests/snippets/testutils.py @@ -1,6 +1,7 @@ import platform import sys + def assert_raises(expected, *args, _msg=None, **kw): if args: f, f_args = args[0], args[1:] @@ -22,8 +23,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: - failmsg = self.failmsg or \ - '{} was not raised'.format(self.expected.__name__) + failmsg = self.failmsg or "{} was not raised".format(self.expected.__name__) assert False, failmsg if not issubclass(exc_type, self.expected): return False @@ -36,6 +36,7 @@ class TestFailingBool: def __bool__(self): raise RuntimeError + class TestFailingIter: def __iter__(self): raise RuntimeError @@ -48,47 +49,64 @@ def _assert_print(f, args): raised = False finally: if raised: - print('Assertion Failure:', *args) + print("Assertion Failure:", *args) + def _typed(obj): - return '{}({})'.format(type(obj), obj) + return "{}({})".format(type(obj), obj) def assert_equal(a, b): - _assert_print(lambda: a == b, [_typed(a), '==', _typed(b)]) + _assert_print(lambda: a == b, [_typed(a), "==", _typed(b)]) def assert_true(e): - _assert_print(lambda: e is True, [_typed(e), 'is True']) + _assert_print(lambda: e is True, [_typed(e), "is True"]) def assert_false(e): - _assert_print(lambda: e is False, [_typed(e), 'is False']) + _assert_print(lambda: e is False, [_typed(e), "is False"]) + def assert_isinstance(obj, klass): - _assert_print(lambda: isinstance(obj, klass), ['isisntance(', _typed(obj), ',', klass, ')']) + _assert_print( + lambda: isinstance(obj, klass), ["isisntance(", _typed(obj), ",", klass, ")"] + ) + def assert_in(a, b): - _assert_print(lambda: a in b, [a, 'in', b]) + _assert_print(lambda: a in b, [a, "in", b]) + def skip_if_unsupported(req_maj_vers, req_min_vers, test_fct): def exec(): test_fct() - if platform.python_implementation() == 'RustPython': + if platform.python_implementation() == "RustPython": exec() - elif sys.version_info.major>=req_maj_vers and sys.version_info.minor>=req_min_vers: + elif ( + sys.version_info.major >= req_maj_vers + and sys.version_info.minor >= req_min_vers + ): exec() else: - print(f'Skipping test as a higher python version is required. Using {platform.python_implementation()} {platform.python_version()}') + print( + f"Skipping test as a higher python version is required. Using {platform.python_implementation()} {platform.python_version()}" + ) + def fail_if_unsupported(req_maj_vers, req_min_vers, test_fct): def exec(): test_fct() - if platform.python_implementation() == 'RustPython': + if platform.python_implementation() == "RustPython": exec() - elif sys.version_info.major>=req_maj_vers and sys.version_info.minor>=req_min_vers: + elif ( + sys.version_info.major >= req_maj_vers + and sys.version_info.minor >= req_min_vers + ): exec() else: - assert False, f'Test cannot performed on this python version. {platform.python_implementation()} {platform.python_version()}' + assert False, ( + f"Test cannot performed on this python version. {platform.python_implementation()} {platform.python_version()}" + ) diff --git a/extra_tests/test_snippets.py b/extra_tests/test_snippets.py index c191c1e638..5ff944c772 100644 --- a/extra_tests/test_snippets.py +++ b/extra_tests/test_snippets.py @@ -42,23 +42,27 @@ def perform_test(filename, method, test_type): def run_via_cpython(filename): - """ Simply invoke python itself on the script """ + """Simply invoke python itself on the script""" env = os.environ.copy() subprocess.check_call([sys.executable, filename], env=env) -RUSTPYTHON_BINARY = os.environ.get("RUSTPYTHON") or os.path.join(ROOT_DIR, "target/release/rustpython") + +RUSTPYTHON_BINARY = os.environ.get("RUSTPYTHON") or os.path.join( + ROOT_DIR, "target/release/rustpython" +) RUSTPYTHON_BINARY = os.path.abspath(RUSTPYTHON_BINARY) + def run_via_rustpython(filename, test_type): env = os.environ.copy() - env['RUST_LOG'] = 'info,cargo=error,jobserver=error' - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" subprocess.check_call([RUSTPYTHON_BINARY, filename], env=env) def create_test_function(cls, filename, method, test_type): - """ Create a test function for a single snippet """ + """Create a test function for a single snippet""" core_test_directory, snippet_filename = os.path.split(filename) test_function_name = "test_{}_".format(method) + os.path.splitext(snippet_filename)[ 0 @@ -74,7 +78,7 @@ def test_function(self): def populate(method): def wrapper(cls): - """ Decorator function which can populate a unittest.TestCase class """ + """Decorator function which can populate a unittest.TestCase class""" for test_type, filename in get_test_files(): create_test_function(cls, filename, method, test_type) return cls @@ -83,7 +87,7 @@ def wrapper(cls): def get_test_files(): - """ Retrieve test files """ + """Retrieve test files""" for test_type, test_dir in TEST_DIRS.items(): for filepath in sorted(glob.iglob(os.path.join(test_dir, "*.py"))): filename = os.path.split(filepath)[1] @@ -122,7 +126,9 @@ class SampleTestCase(unittest.TestCase): @classmethod def setUpClass(cls): # Here add resource files - cls.slices_resource_path = Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + cls.slices_resource_path = ( + Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + ) if cls.slices_resource_path.exists(): cls.slices_resource_path.unlink() diff --git a/ruff.toml b/ruff.toml index 5e2fb8c4f5..15aaa07c8a 100644 --- a/ruff.toml +++ b/ruff.toml @@ -9,6 +9,8 @@ exclude = [ "Lib", "vm/Lib", "benches", + "syntax_*.py", # Do not format files that are specifically testing for syntax + "badsyntax_*.py", ] [lint] diff --git a/wasm/demo/snippets/asyncbrowser.py b/wasm/demo/snippets/asyncbrowser.py index 5cd2f7b0a0..d3a9dca85f 100644 --- a/wasm/demo/snippets/asyncbrowser.py +++ b/wasm/demo/snippets/asyncbrowser.py @@ -1,6 +1,7 @@ import browser import asyncweb + async def main(delay): url = f"https://httpbin.org/delay/{delay}" print(f"fetching {url}...") diff --git a/wasm/demo/snippets/fetch.py b/wasm/demo/snippets/fetch.py index f507057b22..80e1775c76 100644 --- a/wasm/demo/snippets/fetch.py +++ b/wasm/demo/snippets/fetch.py @@ -1,12 +1,12 @@ from browser import fetch + def fetch_handler(res): print(f"headers: {res['headers']}") + fetch( "https://httpbin.org/get", response_format="json", - headers={ - "X-Header-Thing": "rustpython is neat!" - }, + headers={"X-Header-Thing": "rustpython is neat!"}, ).then(fetch_handler, lambda err: print(f"error: {err}")) diff --git a/wasm/demo/snippets/import_pypi.py b/wasm/demo/snippets/import_pypi.py index a7c8a8e7fd..e3325d56f4 100644 --- a/wasm/demo/snippets/import_pypi.py +++ b/wasm/demo/snippets/import_pypi.py @@ -8,12 +8,14 @@ whlimport.setup() + @asyncweb.main async def main(): await whlimport.load_package("pygments") import pygments import pygments.lexers import pygments.formatters.html + lexer = pygments.lexers.get_lexer_by_name("python") fmter = pygments.formatters.html.HtmlFormatter(noclasses=True, style="default") print(pygments.highlight("print('hi, mom!')", lexer, fmter)) diff --git a/wasm/demo/snippets/mandelbrot.py b/wasm/demo/snippets/mandelbrot.py index b4010c7539..ea4fade56d 100644 --- a/wasm/demo/snippets/mandelbrot.py +++ b/wasm/demo/snippets/mandelbrot.py @@ -1,6 +1,7 @@ w = 50.0 h = 50.0 + def mandel(): """Print a mandelbrot fractal to the console, yielding after each character is printed""" y = 0.0 @@ -20,9 +21,9 @@ def mandel(): i += 1 if Tr + Ti <= 4: - print('*', end='') + print("*", end="") else: - print('·', end='') + print("·", end="") x += 1 yield @@ -31,14 +32,24 @@ def mandel(): y += 1 yield + # run the mandelbrot -try: from browser import request_animation_frame -except: request_animation_frame = None +try: + from browser import request_animation_frame +except: + request_animation_frame = None gen = mandel() + + def gen_cb(_time=None): - for _ in range(4): gen.__next__() + for _ in range(4): + gen.__next__() request_animation_frame(gen_cb) -if request_animation_frame: gen_cb() -else: any(gen) + + +if request_animation_frame: + gen_cb() +else: + any(gen) diff --git a/wasm/example/src/main.py b/wasm/example/src/main.py index 5447d078af..b5a1bda7c0 100644 --- a/wasm/example/src/main.py +++ b/wasm/example/src/main.py @@ -1,12 +1,14 @@ from browser import fetch, alert + def fetch_handler(repos): star_sum = 0 for repo in repos: - star_sum += repo['stars'] - alert(f'Average github trending star count: {star_sum / len(repos)}') + star_sum += repo["stars"] + alert(f"Average github trending star count: {star_sum / len(repos)}") + fetch( - 'https://github-trending-api.now.sh/repositories', - response_format='json', -).then(fetch_handler, lambda err: alert(f"Error: {err}")) \ No newline at end of file + "https://github-trending-api.now.sh/repositories", + response_format="json", +).then(fetch_handler, lambda err: alert(f"Error: {err}")) From 5c0f70c3616e40d8a1bf0405a7fbc3863210ae23 Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Sat, 10 May 2025 11:09:11 -0700 Subject: [PATCH 309/317] add instructions --- Cargo.lock | 76 +++++++++++++++++++++++------------------ jit/Cargo.toml | 6 ++-- jit/src/instructions.rs | 12 +++++++ 3 files changed, 58 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 095a3dca37..63608aefb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,9 +472,9 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e482b051275b415cf7627bb6b26e9902ce6aec058b443266c2a1e7a0de148960" +checksum = "6d07c374d4da962eca0833c1d14621d5b4e32e68c8ca185b046a3b6b924ad334" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -483,39 +483,42 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4b56ebe316895d3fa37775d0a87b0c889cc933f5c8b253dbcc7c7bcb7fe7e4" +checksum = "263cc79b8a23c29720eb596d251698f604546b48c34d0d84f8fd2761e5bf8888" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cabbc01dfbd7dcd6c329ca44f0212910309c221797ac736a67a5bc8857fe1b" +checksum = "5b4a113455f8c0e13e3b3222a9c38d6940b958ff22573108be083495c72820e1" +dependencies = [ + "cranelift-srcgen", +] [[package]] name = "cranelift-bforest" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ffe46df300a45f1dc6f609dc808ce963f0e3a2e971682c479a2d13e3b9b8ef" +checksum = "58f96dca41c5acf5d4312c1d04b3391e21a312f8d64ce31a2723a3bb8edd5d4d" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b265bed7c51e1921fdae6419791d31af77d33662ee56d7b0fa0704dc8d231cab" +checksum = "7d821ed698dd83d9c012447eb63a5406c1e9c23732a2f674fb5b5015afd42202" [[package]] name = "cranelift-codegen" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e606230a7e3a6897d603761baee0d19f88d077f17b996bb5089488a29ae96e41" +checksum = "06c52fdec4322cb8d5545a648047819aaeaa04e630f88d3a609c0d3c1a00e9a0" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -538,43 +541,44 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a63bffafc23bc60969ad528e138788495999d935f0adcfd6543cb151ca8637d" +checksum = "af2c215e0c9afa8069aafb71d22aa0e0dde1048d9a5c3c72a83cacf9b61fcf4a" dependencies = [ - "cranelift-assembler-x64", + "cranelift-assembler-x64-meta", "cranelift-codegen-shared", + "cranelift-srcgen", ] [[package]] name = "cranelift-codegen-shared" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af50281b67324b58e843170a6a5943cf6d387c06f7eeacc9f5696e4ab7ae7d7e" +checksum = "97524b2446fc26a78142132d813679dda19f620048ebc9a9fbb0ac9f2d320dcb" [[package]] name = "cranelift-control" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c20c1b38d1abfbcebb0032e497e71156c0e3b8dcb3f0a92b9863b7bcaec290c" +checksum = "8e32e900aee81f9e3cc493405ef667a7812cb5c79b5fc6b669e0a2795bda4b22" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2c67d95507c51b4a1ff3f3555fe4bfec36b9e13c1b684ccc602736f5d5f4a2" +checksum = "d16a2e28e0fa6b9108d76879d60fe1cc95ba90e1bcf52bac96496371044484ee" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e002691cc69c38b54fc7ec93e5be5b744f627d027031d991cc845d1d512d0ce" +checksum = "328181a9083d99762d85954a16065d2560394a862b8dc10239f39668df528b95" dependencies = [ "cranelift-codegen", "log", @@ -584,15 +588,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93588ed1796cbcb0e2ad160403509e2c5d330d80dd6e0014ac6774c7ebac496" +checksum = "e916f36f183e377e9a3ed71769f2721df88b72648831e95bb9fa6b0cd9b1c709" [[package]] name = "cranelift-jit" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6682f0b193d6b7873cc8e7ed67e8776a8a26f50eeabf88534e9be618b9a03" +checksum = "d6bb584ac927f1076d552504b0075b833b9d61e2e9178ba55df6b2d966b4375d" dependencies = [ "anyhow", "cranelift-codegen", @@ -610,9 +614,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff19784c6de05116e63e6a34791012bd927b2a4eac56233039c46f1b6a4edac8" +checksum = "40c18ccb8e4861cf49cec79998af73b772a2b47212d12d3d63bf57cc4293a1e3" dependencies = [ "anyhow", "cranelift-codegen", @@ -621,15 +625,21 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.118.0" +version = "0.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b09bdd6407bf5d89661b80cf926ce731c9e8cc184bf49102267a2369a8358e" +checksum = "fc852cf04128877047dc2027aa1b85c64f681dc3a6a37ff45dcbfa26e4d52d2f" dependencies = [ "cranelift-codegen", "libc", "target-lexicon", ] +[[package]] +name = "cranelift-srcgen" +version = "0.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1a86340a16e74b4285cc86ac69458fa1c8e7aaff313da4a89d10efd3535ee" + [[package]] name = "crc32fast" version = "1.4.2" @@ -3361,9 +3371,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "31.0.0" +version = "32.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a54f6c6c7e9d7eeee32dfcc10db7f29d505ee7dd28d00593ea241d5f70698e64" +checksum = "eb399eaabd7594f695e1159d236bf40ef55babcb3af97f97c027864ed2104db6" dependencies = [ "anyhow", "cfg-if", diff --git a/jit/Cargo.toml b/jit/Cargo.toml index 0c7f39af07..bc21063192 100644 --- a/jit/Cargo.toml +++ b/jit/Cargo.toml @@ -17,9 +17,9 @@ num-traits = { workspace = true } thiserror = { workspace = true } libffi = { workspace = true, features = ["system"] } -cranelift = "0.118" -cranelift-jit = "0.118" -cranelift-module = "0.118" +cranelift = "0.119" +cranelift-jit = "0.119" +cranelift-module = "0.119" [dev-dependencies] rustpython-derive = { path = "../derive", version = "0.4.0" } diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index 830a578562..407a33226a 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -600,6 +600,18 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { _ => Err(JitCompileError::BadBytecode), } } + Instruction::Nop => Ok(()), + Instruction::Swap { index } => { + let len = self.stack.len(); + let i = len - 1; + let j = len - 1 - index.get(arg) as usize; + self.stack.swap(i, j); + Ok(()) + } + Instruction::Pop => { + self.stack.pop(); + Ok(()) + } _ => Err(JitCompileError::NotSupported), } } From d11d5c65e6aab500a90b6129599d1d2dad7048a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 14:10:25 +0000 Subject: [PATCH 310/317] Bump streetsidesoftware/cspell-action in the github-actions group Bumps the github-actions group with 1 update: [streetsidesoftware/cspell-action](https://github.com/streetsidesoftware/cspell-action). Updates `streetsidesoftware/cspell-action` from 6 to 7 - [Release notes](https://github.com/streetsidesoftware/cspell-action/releases) - [Changelog](https://github.com/streetsidesoftware/cspell-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/streetsidesoftware/cspell-action/compare/v6...v7) --- updated-dependencies: - dependency-name: streetsidesoftware/cspell-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index db8f9056b1..487cb3e0c9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -340,7 +340,7 @@ jobs: - name: install extra dictionaries run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell - name: spell checker - uses: streetsidesoftware/cspell-action@v6 + uses: streetsidesoftware/cspell-action@v7 with: files: '**/*.rs' incremental_files_only: true From b696e56c5fba3f5d5c5300288f3d5e21a5087f9e Mon Sep 17 00:00:00 2001 From: Rex Ledesma Date: Thu, 8 May 2025 00:37:54 -0400 Subject: [PATCH 311/317] chore: rely on the default inclusions for `ruff` --- crawl_sourcecode.py | 62 ++++++++++++++++++--------------- demo_closures.py | 5 ++- ruff.toml | 7 ---- scripts/cargo-llvm-cov.py | 9 +++-- scripts/fix_test.py | 34 +++++++++++++++--- vm/sre_engine/generate_tests.py | 33 ++++++++++++------ whats_left.py | 43 ++++++++++++++++++----- 7 files changed, 126 insertions(+), 67 deletions(-) diff --git a/crawl_sourcecode.py b/crawl_sourcecode.py index 2daad4f682..a96ec283fa 100644 --- a/crawl_sourcecode.py +++ b/crawl_sourcecode.py @@ -1,4 +1,4 @@ -""" This script can be used to test the equivalence in parsing between +"""This script can be used to test the equivalence in parsing between rustpython and cpython. Usage example: @@ -8,76 +8,80 @@ $ diff cpython.txt rustpython.txt """ - import ast import sys import symtable import dis filename = sys.argv[1] -print('Crawling file:', filename) +print("Crawling file:", filename) -with open(filename, 'r') as f: +with open(filename, "r") as f: source = f.read() t = ast.parse(source) print(t) shift = 3 + + def print_node(node, indent=0): - indents = ' ' * indent + indents = " " * indent if isinstance(node, ast.AST): - lineno = 'row={}'.format(node.lineno) if hasattr(node, 'lineno') else '' + lineno = "row={}".format(node.lineno) if hasattr(node, "lineno") else "" print(indents, "NODE", node.__class__.__name__, lineno) for field in node._fields: - print(indents,'-', field) + print(indents, "-", field) f = getattr(node, field) if isinstance(f, list): for f2 in f: - print_node(f2, indent=indent+shift) + print_node(f2, indent=indent + shift) else: - print_node(f, indent=indent+shift) + print_node(f, indent=indent + shift) else: - print(indents, 'OBJ', node) + print(indents, "OBJ", node) + print_node(t) # print(ast.dump(t)) flag_names = [ - 'is_referenced', - 'is_assigned', - 'is_global', - 'is_local', - 'is_parameter', - 'is_free', + "is_referenced", + "is_assigned", + "is_global", + "is_local", + "is_parameter", + "is_free", ] + def print_table(table, indent=0): - indents = ' ' * indent - print(indents, 'table:', table.get_name()) - print(indents, ' ', 'name:', table.get_name()) - print(indents, ' ', 'type:', table.get_type()) - print(indents, ' ', 'line:', table.get_lineno()) - print(indents, ' ', 'identifiers:', table.get_identifiers()) - print(indents, ' ', 'Syms:') + indents = " " * indent + print(indents, "table:", table.get_name()) + print(indents, " ", "name:", table.get_name()) + print(indents, " ", "type:", table.get_type()) + print(indents, " ", "line:", table.get_lineno()) + print(indents, " ", "identifiers:", table.get_identifiers()) + print(indents, " ", "Syms:") for sym in table.get_symbols(): flags = [] for flag_name in flag_names: func = getattr(sym, flag_name) if func(): flags.append(flag_name) - print(indents, ' sym:', sym.get_name(), 'flags:', ' '.join(flags)) + print(indents, " sym:", sym.get_name(), "flags:", " ".join(flags)) if table.has_children(): - print(indents, ' ', 'Child tables:') + print(indents, " ", "Child tables:") for child in table.get_children(): - print_table(child, indent=indent+shift) + print_table(child, indent=indent + shift) + -table = symtable.symtable(source, 'a', 'exec') +table = symtable.symtable(source, "a", "exec") print_table(table) print() -print('======== dis.dis ========') +print("======== dis.dis ========") print() -co = compile(source, filename, 'exec') +co = compile(source, filename, "exec") dis.dis(co) diff --git a/demo_closures.py b/demo_closures.py index 00242407e6..0ed673a94f 100644 --- a/demo_closures.py +++ b/demo_closures.py @@ -1,13 +1,12 @@ - - def foo(x): def bar(z): return z + x + return bar + f = foo(9) g = foo(10) print(f(2)) print(g(2)) - diff --git a/ruff.toml b/ruff.toml index 15aaa07c8a..2ed67851f0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,11 +1,4 @@ -include = [ - "examples/**/*.py", - "extra_tests/**/*.py", - "wasm/**/*.py", -] - exclude = [ - ".*", "Lib", "vm/Lib", "benches", diff --git a/scripts/cargo-llvm-cov.py b/scripts/cargo-llvm-cov.py index a77d56a87c..9a7b24dd04 100644 --- a/scripts/cargo-llvm-cov.py +++ b/scripts/cargo-llvm-cov.py @@ -3,18 +3,21 @@ TARGET = "extra_tests/snippets" + def run_llvm_cov(file_path: str): - """ Run cargo llvm-cov on a file. """ + """Run cargo llvm-cov on a file.""" if file_path.endswith(".py"): command = ["cargo", "llvm-cov", "--no-report", "run", "--", file_path] subprocess.call(command) + def iterate_files(folder: str): - """ Iterate over all files in a folder. """ + """Iterate over all files in a folder.""" for root, _, files in os.walk(folder): for file in files: file_path = os.path.join(root, file) run_llvm_cov(file_path) + if __name__ == "__main__": - iterate_files(TARGET) \ No newline at end of file + iterate_files(TARGET) diff --git a/scripts/fix_test.py b/scripts/fix_test.py index 99dfa2699a..a5663e3eee 100644 --- a/scripts/fix_test.py +++ b/scripts/fix_test.py @@ -10,21 +10,26 @@ 4. Ensure that there are no unexpected successes in the test. 5. Actually fix the test. """ + import argparse import ast import itertools import platform from pathlib import Path + def parse_args(): parser = argparse.ArgumentParser(description="Fix test.") parser.add_argument("--path", type=Path, help="Path to test file") parser.add_argument("--force", action="store_true", help="Force modification") - parser.add_argument("--platform", action="store_true", help="Platform specific failure") + parser.add_argument( + "--platform", action="store_true", help="Platform specific failure" + ) args = parser.parse_args() return args + class Test: name: str = "" path: str = "" @@ -33,6 +38,7 @@ class Test: def __str__(self): return f"Test(name={self.name}, path={self.path}, result={self.result})" + class TestResult: tests_result: str = "" tests = [] @@ -52,7 +58,11 @@ def parse_results(result): in_test_results = True elif line.startswith("-----------"): in_test_results = False - if in_test_results and not line.startswith("tests") and not line.startswith("["): + if ( + in_test_results + and not line.startswith("tests") + and not line.startswith("[") + ): line = line.split(" ") if line != [] and len(line) > 3: test = Test() @@ -67,9 +77,11 @@ def parse_results(result): test_results.tests_result = res return test_results + def path_to_test(path) -> list[str]: return path.split(".")[2:] + def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: a = ast.parse(file) lines = file.splitlines() @@ -84,6 +96,7 @@ def modify_test(file: str, test: list[str], for_platform: bool = False) -> str: break return "\n".join(lines) + def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> str: a = ast.parse(file) lines = file.splitlines() @@ -101,8 +114,13 @@ def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> st if fn.name == test[-1]: assert not for_platform indent = " " * fn.col_offset - lines.insert(fn.lineno - 1, indent + fixture) - lines.insert(fn.lineno - 1, indent + "# TODO: RUSTPYTHON") + lines.insert( + fn.lineno - 1, indent + fixture + ) + lines.insert( + fn.lineno - 1, + indent + "# TODO: RUSTPYTHON", + ) break case ast.FunctionDef(): if n.name == test[0] and len(test) == 1: @@ -115,11 +133,17 @@ def modify_test_v2(file: str, test: list[str], for_platform: bool = False) -> st exit() return "\n".join(lines) + def run_test(test_name): print(f"Running test: {test_name}") rustpython_location = "./target/release/rustpython" import subprocess - result = subprocess.run([rustpython_location, "-m", "test", "-v", test_name], capture_output=True, text=True) + + result = subprocess.run( + [rustpython_location, "-m", "test", "-v", test_name], + capture_output=True, + text=True, + ) return parse_results(result) diff --git a/vm/sre_engine/generate_tests.py b/vm/sre_engine/generate_tests.py index 6621c56813..3af4d7e6a5 100644 --- a/vm/sre_engine/generate_tests.py +++ b/vm/sre_engine/generate_tests.py @@ -10,6 +10,7 @@ assert re._constants.MAGIC == sre_engine_magic + class CompiledPattern: @classmethod def compile(cls, pattern, flags=0): @@ -21,40 +22,50 @@ def compile(cls, pattern, flags=0): self.flags = re.RegexFlag(flags | p.state.flags) return self + for k, v in re.RegexFlag.__members__.items(): setattr(CompiledPattern, k, v) class EscapeRustStr: hardcoded = { - ord('\r'): '\\r', - ord('\t'): '\\t', - ord('\r'): '\\r', - ord('\n'): '\\n', - ord('\\'): '\\\\', - ord('\''): '\\\'', - ord('\"'): '\\\"', + ord("\r"): "\\r", + ord("\t"): "\\t", + ord("\r"): "\\r", + ord("\n"): "\\n", + ord("\\"): "\\\\", + ord("'"): "\\'", + ord('"'): '\\"', } + @classmethod def __class_getitem__(cls, ch): if (rpl := cls.hardcoded.get(ch)) is not None: return rpl - if ch in range(0x20, 0x7f): + if ch in range(0x20, 0x7F): return ch return f"\\u{{{ch:x}}}" + + def rust_str(s): return '"' + s.translate(EscapeRustStr) + '"' + # matches `// pattern {varname} = re.compile(...)` -pattern_pattern = re.compile(r"^((\s*)\/\/\s*pattern\s+(\w+)\s+=\s+(.+?))$(?:.+?END GENERATED)?", re.M | re.S) +pattern_pattern = re.compile( + r"^((\s*)\/\/\s*pattern\s+(\w+)\s+=\s+(.+?))$(?:.+?END GENERATED)?", re.M | re.S +) + + def replace_compiled(m): line, indent, varname, pattern = m.groups() pattern = eval(pattern, {"re": CompiledPattern}) pattern = f"Pattern {{ pattern: {rust_str(pattern.pattern)}, code: &{json.dumps(pattern.code)} }}" - return f'''{line} + return f"""{line} {indent}// START GENERATED by generate_tests.py {indent}#[rustfmt::skip] let {varname} = {pattern}; -{indent}// END GENERATED''' +{indent}// END GENERATED""" + with os.scandir("tests") as t, os.scandir("benches") as b: for f in chain(t, b): diff --git a/whats_left.py b/whats_left.py index 90200b8ff0..9ef5b48f65 100755 --- a/whats_left.py +++ b/whats_left.py @@ -39,7 +39,10 @@ if implementation != "CPython": sys.exit(f"whats_left.py must be run under CPython, got {implementation} instead") if sys.version_info[:2] < (3, 13): - sys.exit(f"whats_left.py must be run under CPython 3.13 or newer, got {implementation} {sys.version} instead") + sys.exit( + f"whats_left.py must be run under CPython 3.13 or newer, got {implementation} {sys.version} instead" + ) + def parse_args(): parser = argparse.ArgumentParser(description="Process some integers.") @@ -73,8 +76,16 @@ def parse_args(): # CPython specific modules (mostly consisting of templates/tests) CPYTHON_SPECIFIC_MODS = { - 'xxmodule', 'xxsubtype', 'xxlimited', '_xxtestfuzz', - '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_testlimitedcapi' + "xxmodule", + "xxsubtype", + "xxlimited", + "_xxtestfuzz", + "_testbuffer", + "_testcapi", + "_testimportmultiple", + "_testinternalcapi", + "_testmultiphase", + "_testlimitedcapi", } IGNORED_MODULES = {"this", "antigravity"} | CPYTHON_SPECIFIC_MODS @@ -291,7 +302,7 @@ def gen_modules(): output += gen_methods() output += f""" cpymods = {gen_modules()!r} -libdir = {os.path.abspath("Lib/").encode('utf8')!r} +libdir = {os.path.abspath("Lib/").encode("utf8")!r} """ @@ -310,6 +321,8 @@ def gen_modules(): expected_methods = {} cpymods = {} libdir = "" + + # This function holds the source code that will be run under RustPython def compare(): import inspect @@ -376,7 +389,9 @@ def method_incompatibility_reason(typ, method_name, real_method_value): if rustpymod is None: result["not_implemented"][modname] = None elif isinstance(rustpymod, Exception): - result["failed_to_import"][modname] = rustpymod.__class__.__name__ + str(rustpymod) + result["failed_to_import"][modname] = rustpymod.__class__.__name__ + str( + rustpymod + ) else: implemented_items = sorted(set(cpymod) & set(rustpymod)) mod_missing_items = set(cpymod) - set(rustpymod) @@ -418,13 +433,23 @@ def remove_one_indent(s): compare_src = inspect.getsourcelines(compare)[0][1:] output += "".join(remove_one_indent(line) for line in compare_src) -with open(GENERATED_FILE, "w", encoding='utf-8') as f: +with open(GENERATED_FILE, "w", encoding="utf-8") as f: f.write(output + "\n") -subprocess.run(["cargo", "build", "--release", f"--features={args.features}"], check=True) +subprocess.run( + ["cargo", "build", "--release", f"--features={args.features}"], check=True +) result = subprocess.run( - ["cargo", "run", "--release", f"--features={args.features}", "-q", "--", GENERATED_FILE], + [ + "cargo", + "run", + "--release", + f"--features={args.features}", + "-q", + "--", + GENERATED_FILE, + ], env={**os.environ.copy(), "RUSTPYTHONPATH": "Lib"}, text=True, capture_output=True, @@ -475,7 +500,7 @@ def remove_one_indent(s): if args.doc: print("\n# mismatching `__doc__`s (warnings)") for modname, mismatched in result["mismatched_doc_items"].items(): - for (item, rustpy_doc, cpython_doc) in mismatched: + for item, rustpy_doc, cpython_doc in mismatched: print(f"{item} {repr(rustpy_doc)} != {repr(cpython_doc)}") From 72dc4954ad8b7ecfab2f6215bc5f012a76689b8b Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Mon, 12 May 2025 09:33:14 -0700 Subject: [PATCH 312/317] dev container update --- .devcontainer/DOCKERFILE | 6 ++++++ .devcontainer/devcontainer.json | 27 ++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/DOCKERFILE diff --git a/.devcontainer/DOCKERFILE b/.devcontainer/DOCKERFILE new file mode 100644 index 0000000000..339cdb69bb --- /dev/null +++ b/.devcontainer/DOCKERFILE @@ -0,0 +1,6 @@ +FROM mcr.microsoft.com/vscode/devcontainers/rust:1-bullseye + +# Install clang +RUN apt-get update \ + && apt-get install -y clang \ + && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d60eee2130..8838cf6a96 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,25 @@ { - "image": "mcr.microsoft.com/devcontainers/base:jammy", - "onCreateCommand": "curl https://sh.rustup.rs -sSf | sh -s -- -y" -} \ No newline at end of file + "name": "Rust", + "build": { + "dockerfile": "Dockerfile" + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + "customizations": { + "vscode": { + "settings": { + "lldb.executable": "/usr/bin/lldb", + // VS Code don't watch files under ./target + "files.watcherExclude": { + "**/target/**": true + }, + "extensions": [ + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml", + "vadimcn.vscode-lldb", + "mutantdino.resourcemonitor" + ] + } + } + }, + "remoteUser": "vscode" +} From 2c02e2776ba88dfeac9d7ce4fa57813f61889454 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 15 May 2025 13:01:00 -0500 Subject: [PATCH 313/317] Fix warnings for rust 1.87 --- jit/src/instructions.rs | 3 +-- jit/src/lib.rs | 8 +++++++- jit/tests/common.rs | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/jit/src/instructions.rs b/jit/src/instructions.rs index 407a33226a..fac947a98a 100644 --- a/jit/src/instructions.rs +++ b/jit/src/instructions.rs @@ -1165,8 +1165,7 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { // ----- Merge: Return the final result. self.builder.switch_to_block(merge_block); - let final_val = self.builder.block_params(merge_block)[0]; - final_val + self.builder.block_params(merge_block)[0] } fn compile_ipow(&mut self, a: Value, b: Value) -> Value { diff --git a/jit/src/lib.rs b/jit/src/lib.rs index 33054b1c95..91911fd8d1 100644 --- a/jit/src/lib.rs +++ b/jit/src/lib.rs @@ -15,7 +15,13 @@ pub enum JitCompileError { #[error("bad bytecode")] BadBytecode, #[error("error while compiling to machine code: {0}")] - CraneliftError(#[from] ModuleError), + CraneliftError(Box), +} + +impl From for JitCompileError { + fn from(err: ModuleError) -> Self { + Self::CraneliftError(Box::new(err)) + } } #[derive(Debug, thiserror::Error, Eq, PartialEq)] diff --git a/jit/tests/common.rs b/jit/tests/common.rs index a4ac8a7967..680090eb5b 100644 --- a/jit/tests/common.rs +++ b/jit/tests/common.rs @@ -78,7 +78,7 @@ impl StackMachine { pub fn run(&mut self, code: CodeObject) { let mut oparg_state = OpArgState::default(); - code.instructions.iter().try_for_each(|&word| { + let _ = code.instructions.iter().try_for_each(|&word| { let (instruction, arg) = oparg_state.get(word); self.process_instruction(instruction, arg, &code.constants, &code.names) }); From 1ae98ee177a47d8794a42eb4349e86cd5be3bec1 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 16 May 2025 14:17:25 -0500 Subject: [PATCH 314/317] Fix panic with high precision --- compiler/literal/src/float.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/literal/src/float.rs b/compiler/literal/src/float.rs index 49771b8184..3764323de3 100644 --- a/compiler/literal/src/float.rs +++ b/compiler/literal/src/float.rs @@ -55,6 +55,7 @@ pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form match magnitude { magnitude if magnitude.is_finite() => { let point = decimal_point_or_empty(precision, alternate_form); + let precision = std::cmp::min(precision, u16::MAX as usize); format!("{magnitude:.precision$}{point}") } magnitude if magnitude.is_nan() => format_nan(case), From 163296d306206698dfb03c9a4a4239b15b99627b Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 16 May 2025 15:20:10 -0500 Subject: [PATCH 315/317] Fix test_memoryio --- vm/src/stdlib/io.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 3e1979e3d0..e4339ae61a 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -259,6 +259,9 @@ mod _io { } fn write(&mut self, data: &[u8]) -> Option { + if data.is_empty() { + return Some(0); + } let length = data.len(); self.cursor.write_all(data).ok()?; Some(length as u64) From 18521290bf012e9f131458e8fb019f4e16763bb9 Mon Sep 17 00:00:00 2001 From: Noa Date: Thu, 8 May 2025 15:29:47 -0500 Subject: [PATCH 316/317] Update dependencies --- Cargo.lock | 263 ++++++++++++++++++++++------------------------------- 1 file changed, 107 insertions(+), 156 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63608aefb9..7dffc22e40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,15 +16,15 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.2.15", + "getrandom 0.3.2", "once_cell", "version_check", - "zerocopy 0.7.35", + "zerocopy", ] [[package]] @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "approx" @@ -178,7 +178,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.3" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "regex-automata", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" [[package]] name = "bzip2" @@ -283,9 +283,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.18" +version = "1.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" +checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0" dependencies = [ "shlex", ] @@ -313,9 +313,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", @@ -365,18 +365,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.36" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04" +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.36" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5" +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstyle", "clap_lex", @@ -850,9 +850,9 @@ dependencies = [ [[package]] name = "error-code" -version = "3.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "exitcode" @@ -963,9 +963,9 @@ dependencies = [ [[package]] name = "gethostname" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7131e57abbde63513e0e6636f76668a1ca9798dcae2df4e283cae9ee83859e" +checksum = "fc257fdb4038301ce4b9cd1b3b51704509692bb3ff716a410cbd07925d9dae55" dependencies = [ "rustix", "windows-targets 0.52.6", @@ -982,9 +982,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -1026,9 +1026,9 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "half" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -1036,9 +1036,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" dependencies = [ "foldhash", ] @@ -1057,9 +1057,9 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hermit-abi" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" [[package]] name = "hex" @@ -1124,14 +1124,12 @@ checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "insta" -version = "1.42.2" +version = "1.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50259abbaa67d11d2bcafc7ba1d094ed7a0c70e3ce893f0d0997f73558cb3084" +checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" dependencies = [ "console", - "linked-hash-map", "once_cell", - "pin-project", "similar", ] @@ -1144,7 +1142,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1153,7 +1151,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ - "hermit-abi 0.5.0", + "hermit-abi 0.5.1", "libc", "windows-sys 0.59.0", ] @@ -1199,9 +1197,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.5" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" +checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" dependencies = [ "jiff-static", "log", @@ -1212,13 +1210,13 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.5" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" +checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1300,9 +1298,9 @@ checksum = "0864a00c8d019e36216b69c2c4ce50b83b7bd966add3cf5ba554ec44f8bebcf5" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libffi" @@ -1335,9 +1333,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" @@ -1369,17 +1367,11 @@ dependencies = [ "zlib-rs", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "lock_api" @@ -1537,9 +1529,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff70ce3e48ae43fa075863cef62e8b43b71a4f2382229920e0df362592919430" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -1639,7 +1631,7 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1677,7 +1669,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1688,18 +1680,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.4.2+3.4.1" +version = "300.5.0+3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2" +checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.107" +version = "0.9.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847" dependencies = [ "cc", "libc", @@ -1742,7 +1734,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.11", + "redox_syscall 0.5.12", "smallvec", "windows-targets 0.52.6", ] @@ -1791,26 +1783,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "pkg-config" version = "0.3.32" @@ -1853,7 +1825,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -1877,7 +1849,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.24", + "zerocopy", ] [[package]] @@ -1887,14 +1859,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -1910,9 +1882,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" +checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" dependencies = [ "cfg-if", "indoc", @@ -1928,9 +1900,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" +checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" dependencies = [ "once_cell", "target-lexicon", @@ -1938,9 +1910,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" +checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" dependencies = [ "libc", "pyo3-build-config", @@ -1948,27 +1920,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" +checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] name = "pyo3-macros-backend" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" +checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2017,13 +1989,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.24", ] [[package]] @@ -2052,7 +2023,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -2092,9 +2063,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ "bitflags 2.9.0", ] @@ -2105,7 +2076,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "libredox", "thiserror 1.0.69", ] @@ -2183,7 +2154,7 @@ dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2255,9 +2226,9 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ "bitflags 2.9.0", "errno", @@ -2346,7 +2317,7 @@ dependencies = [ name = "rustpython-compiler" version = "0.4.0" dependencies = [ - "rand 0.9.0", + "rand 0.9.1", "ruff_python_ast", "ruff_python_parser", "ruff_source_file", @@ -2386,7 +2357,7 @@ dependencies = [ "proc-macro2", "rustpython-compiler", "rustpython-derive-impl", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2399,7 +2370,7 @@ dependencies = [ "quote", "rustpython-compiler-core", "rustpython-doc", - "syn 2.0.100", + "syn 2.0.101", "syn-ext", "textwrap", ] @@ -2435,7 +2406,7 @@ dependencies = [ "is-macro", "lexical-parse-float", "num-traits", - "rand 0.9.0", + "rand 0.9.1", "rustpython-wtf8", "unic-ucd-category", ] @@ -2631,7 +2602,7 @@ name = "rustpython_wasm" version = "0.4.0" dependencies = [ "console_error_panic_hook", - "getrandom 0.2.15", + "getrandom 0.2.16", "js-sys", "ruff_python_parser", "rustpython-common", @@ -2732,7 +2703,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2760,9 +2731,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -2849,7 +2820,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2871,9 +2842,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -2888,7 +2859,7 @@ checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2968,7 +2939,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -2979,7 +2950,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3320,7 +3291,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -3355,7 +3326,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3480,7 +3451,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3491,7 +3462,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] @@ -3693,9 +3664,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" +checksum = "a62ce76d9b56901b19a74f19431b0d8b3bc7ca4ad685a746dfd78ca8f4fc6bda" [[package]] name = "xz2" @@ -3708,42 +3679,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" -dependencies = [ - "zerocopy-derive 0.8.24", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.101", ] [[package]] From 81a9002ef2b222b479ec5842a1c85cce22b9628f Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 20 May 2025 11:09:36 +0900 Subject: [PATCH 317/317] Rename Dockerfile --- .devcontainer/{DOCKERFILE => Dockerfile} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .devcontainer/{DOCKERFILE => Dockerfile} (100%) diff --git a/.devcontainer/DOCKERFILE b/.devcontainer/Dockerfile similarity index 100% rename from .devcontainer/DOCKERFILE rename to .devcontainer/Dockerfile