From 23c8dd78c4cf691533d262037358263adf107ee1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 29 Jul 2018 15:51:09 +0300 Subject: [PATCH 1/2] bpo-34272: Reorganize C API tests. Move most C API tests into Lib/test/test_capi/. --- Lib/test/test_bytes.py | 108 ----- Lib/test/test_capi/__init__.py | 5 + Lib/test/test_capi/__main__.py | 3 + Lib/test/test_capi/test_bytes.py | 118 +++++ Lib/test/test_capi/test_dict.py | 36 ++ .../test_getargs.py} | 0 Lib/test/test_capi/test_marshal.py | 94 ++++ .../{test_capi.py => test_capi/test_misc.py} | 0 .../{ => test_capi}/test_structmembers.py | 0 Lib/test/test_capi/test_unicode.py | 431 ++++++++++++++++++ Lib/test/test_code.py | 103 +---- Lib/test/test_dict.py | 30 -- Lib/test/test_marshal.py | 87 ---- Lib/test/test_unicode.py | 425 ----------------- .../2018-07-29-15-59-51.bpo-34272.lVX2uR.rst | 1 + 15 files changed, 689 insertions(+), 752 deletions(-) create mode 100644 Lib/test/test_capi/__init__.py create mode 100644 Lib/test/test_capi/__main__.py create mode 100644 Lib/test/test_capi/test_bytes.py create mode 100644 Lib/test/test_capi/test_dict.py rename Lib/test/{test_getargs2.py => test_capi/test_getargs.py} (100%) create mode 100644 Lib/test/test_capi/test_marshal.py rename Lib/test/{test_capi.py => test_capi/test_misc.py} (100%) rename Lib/test/{ => test_capi}/test_structmembers.py (100%) create mode 100644 Lib/test/test_capi/test_unicode.py create mode 100644 Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 1408fb4fb84b0b..6b80b8d9598a9b 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -820,114 +820,6 @@ def __bytes__(self): self.assertEqual(BytesSubclass(A()), b'abc') self.assertIs(type(BytesSubclass(A())), BytesSubclass) - # Test PyBytes_FromFormat() - def test_from_format(self): - ctypes = test.support.import_module('ctypes') - _testcapi = test.support.import_module('_testcapi') - from ctypes import pythonapi, py_object - from ctypes import ( - c_int, c_uint, - c_long, c_ulong, - c_size_t, c_ssize_t, - c_char_p) - - PyBytes_FromFormat = pythonapi.PyBytes_FromFormat - PyBytes_FromFormat.restype = py_object - - # basic tests - self.assertEqual(PyBytes_FromFormat(b'format'), - b'format') - self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), - b'Hello world !') - - # test formatters - self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), - b'c=\0') - self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), - b'c=@') - self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), - b'c=\xff') - self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', - c_int(1), c_long(2), - c_size_t(3)), - b'd=1 ld=2 zd=3') - self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', - c_int(-1), c_long(-2), - c_size_t(-3)), - b'd=-1 ld=-2 zd=-3') - self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', - c_uint(123), c_ulong(456), - c_size_t(789)), - b'u=123 lu=456 zu=789') - self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), - b'i=123') - self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), - b'i=-123') - self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), - b'x=abc') - - sizeof_ptr = ctypes.sizeof(c_char_p) - - if os.name == 'nt': - # Windows (MSCRT) - ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) - def ptr_formatter(ptr): - return (ptr_format % ptr) - else: - # UNIX (glibc) - def ptr_formatter(ptr): - return '%#x' % ptr - - ptr = 0xabcdef - self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), - ('ptr=' + ptr_formatter(ptr)).encode('ascii')) - self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), - b's=cstr') - - # test minimum and maximum integer values - size_max = c_size_t(-1).value - for formatstr, ctypes_type, value, py_formatter in ( - (b'%d', c_int, _testcapi.INT_MIN, str), - (b'%d', c_int, _testcapi.INT_MAX, str), - (b'%ld', c_long, _testcapi.LONG_MIN, str), - (b'%ld', c_long, _testcapi.LONG_MAX, str), - (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), - (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), - (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), - (b'%zu', c_size_t, size_max, str), - (b'%p', c_char_p, size_max, ptr_formatter), - ): - self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), - py_formatter(value).encode('ascii')), - - # width and precision (width is currently ignored) - self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), - b'a') - self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), - b'abc') - - # '%%' formatter - self.assertEqual(PyBytes_FromFormat(b'%%'), - b'%') - self.assertEqual(PyBytes_FromFormat(b'[%%]'), - b'[%]') - self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), - b'%_') - self.assertEqual(PyBytes_FromFormat(b'%%s'), - b'%s') - - # Invalid formats and partial formatting - self.assertEqual(PyBytes_FromFormat(b'%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), - b'x=2 y=%') - - # Issue #19969: %c must raise OverflowError for values - # not in the range [0; 255] - self.assertRaises(OverflowError, - PyBytes_FromFormat, b'%c', c_int(-1)) - self.assertRaises(OverflowError, - PyBytes_FromFormat, b'%c', c_int(256)) - def test_bytes_blocking(self): class IterationBlocked(list): __bytes__ = None diff --git a/Lib/test/test_capi/__init__.py b/Lib/test/test_capi/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_capi/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_capi/__main__.py b/Lib/test/test_capi/__main__.py new file mode 100644 index 00000000000000..05d01775ddf43c --- /dev/null +++ b/Lib/test/test_capi/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_capi') diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py new file mode 100644 index 00000000000000..2f78f44e5217fd --- /dev/null +++ b/Lib/test/test_capi/test_bytes.py @@ -0,0 +1,118 @@ +import os +import unittest +from test import support + + +class CAPITest(unittest.TestCase): + + # Test PyBytes_FromFormat() + def test_from_format(self): + ctypes = support.import_module('ctypes') + _testcapi = support.import_module('_testcapi') + from ctypes import pythonapi, py_object + from ctypes import ( + c_int, c_uint, + c_long, c_ulong, + c_size_t, c_ssize_t, + c_char_p) + + PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.restype = py_object + + # basic tests + self.assertEqual(PyBytes_FromFormat(b'format'), + b'format') + self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), + b'Hello world !') + + # test formatters + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), + b'c=\0') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), + b'c=@') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), + b'c=\xff') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(1), c_long(2), + c_size_t(3)), + b'd=1 ld=2 zd=3') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(-1), c_long(-2), + c_size_t(-3)), + b'd=-1 ld=-2 zd=-3') + self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', + c_uint(123), c_ulong(456), + c_size_t(789)), + b'u=123 lu=456 zu=789') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), + b'i=123') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), + b'i=-123') + self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), + b'x=abc') + + sizeof_ptr = ctypes.sizeof(c_char_p) + + if os.name == 'nt': + # Windows (MSCRT) + ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + def ptr_formatter(ptr): + return (ptr_format % ptr) + else: + # UNIX (glibc) + def ptr_formatter(ptr): + return '%#x' % ptr + + ptr = 0xabcdef + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + ('ptr=' + ptr_formatter(ptr)).encode('ascii')) + self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), + b's=cstr') + + # test minimum and maximum integer values + size_max = c_size_t(-1).value + for formatstr, ctypes_type, value, py_formatter in ( + (b'%d', c_int, _testcapi.INT_MIN, str), + (b'%d', c_int, _testcapi.INT_MAX, str), + (b'%ld', c_long, _testcapi.LONG_MIN, str), + (b'%ld', c_long, _testcapi.LONG_MAX, str), + (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), + (b'%zu', c_size_t, size_max, str), + (b'%p', c_char_p, size_max, ptr_formatter), + ): + self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), + py_formatter(value).encode('ascii')), + + # width and precision (width is currently ignored) + self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), + b'a') + self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), + b'abc') + + # '%%' formatter + self.assertEqual(PyBytes_FromFormat(b'%%'), + b'%') + self.assertEqual(PyBytes_FromFormat(b'[%%]'), + b'[%]') + self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), + b'%_') + self.assertEqual(PyBytes_FromFormat(b'%%s'), + b'%s') + + # Invalid formats and partial formatting + self.assertEqual(PyBytes_FromFormat(b'%'), b'%') + self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), + b'x=2 y=%') + + # Issue #19969: %c must raise OverflowError for values + # not in the range [0; 255] + self.assertRaises(OverflowError, + PyBytes_FromFormat, b'%c', c_int(-1)) + self.assertRaises(OverflowError, + PyBytes_FromFormat, b'%c', c_int(256)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py new file mode 100644 index 00000000000000..210b42ca32f27f --- /dev/null +++ b/Lib/test/test_capi/test_dict.py @@ -0,0 +1,36 @@ +import unittest +from test import support + + +class CAPITest(unittest.TestCase): + + # Test _PyDict_GetItem_KnownHash() + @support.cpython_only + def test_getitem_knownhash(self): + from _testcapi import dict_getitem_knownhash + + d = {'x': 1, 'y': 2, 'z': 3} + self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) + self.assertEqual(dict_getitem_knownhash(d, 'y', hash('y')), 2) + self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) + + # not a dict + self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) + # key does not exist + self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) + + class Exc(Exception): pass + class BadEq: + def __eq__(self, other): + raise Exc + def __hash__(self): + return 7 + + k1, k2 = BadEq(), BadEq() + d = {k1: 1} + self.assertEqual(dict_getitem_knownhash(d, k1, hash(k1)), 1) + self.assertRaises(Exc, dict_getitem_knownhash, d, k2, hash(k2)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_capi/test_getargs.py similarity index 100% rename from Lib/test/test_getargs2.py rename to Lib/test/test_capi/test_getargs.py diff --git a/Lib/test/test_capi/test_marshal.py b/Lib/test/test_capi/test_marshal.py new file mode 100644 index 00000000000000..6011e9eac1e525 --- /dev/null +++ b/Lib/test/test_capi/test_marshal.py @@ -0,0 +1,94 @@ +from test import support +import marshal +import unittest + +try: + import _testcapi +except ImportError: + _testcapi = None + +@support.cpython_only +@unittest.skipUnless(_testcapi, 'requires _testcapi') +class CAPI_TestCase(unittest.TestCase): + + def test_write_long_to_file(self): + for v in range(marshal.version + 1): + _testcapi.pymarshal_write_long_to_file(0x12345678, support.TESTFN, v) + with open(support.TESTFN, 'rb') as f: + data = f.read() + support.unlink(support.TESTFN) + self.assertEqual(data, b'\x78\x56\x34\x12') + + def test_write_object_to_file(self): + obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000) + for v in range(marshal.version + 1): + _testcapi.pymarshal_write_object_to_file(obj, support.TESTFN, v) + with open(support.TESTFN, 'rb') as f: + data = f.read() + support.unlink(support.TESTFN) + self.assertEqual(marshal.loads(data), obj) + + def test_read_short_from_file(self): + with open(support.TESTFN, 'wb') as f: + f.write(b'\x34\x12xxxx') + r, p = _testcapi.pymarshal_read_short_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, 0x1234) + self.assertEqual(p, 2) + + with open(support.TESTFN, 'wb') as f: + f.write(b'\x12') + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_short_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + def test_read_long_from_file(self): + with open(support.TESTFN, 'wb') as f: + f.write(b'\x78\x56\x34\x12xxxx') + r, p = _testcapi.pymarshal_read_long_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, 0x12345678) + self.assertEqual(p, 4) + + with open(support.TESTFN, 'wb') as f: + f.write(b'\x56\x34\x12') + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_long_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + def test_read_last_object_from_file(self): + obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) + for v in range(marshal.version + 1): + data = marshal.dumps(obj, v) + with open(support.TESTFN, 'wb') as f: + f.write(data + b'xxxx') + r, p = _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, obj) + + with open(support.TESTFN, 'wb') as f: + f.write(data[:1]) + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + def test_read_object_from_file(self): + obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) + for v in range(marshal.version + 1): + data = marshal.dumps(obj, v) + with open(support.TESTFN, 'wb') as f: + f.write(data + b'xxxx') + r, p = _testcapi.pymarshal_read_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, obj) + self.assertEqual(p, len(data)) + + with open(support.TESTFN, 'wb') as f: + f.write(data[:1]) + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi/test_misc.py similarity index 100% rename from Lib/test/test_capi.py rename to Lib/test/test_capi/test_misc.py diff --git a/Lib/test/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py similarity index 100% rename from Lib/test/test_structmembers.py rename to Lib/test/test_capi/test_structmembers.py diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py new file mode 100644 index 00000000000000..df67a704d793c7 --- /dev/null +++ b/Lib/test/test_capi/test_unicode.py @@ -0,0 +1,431 @@ +import unittest +from test import support + +class CAPITest(unittest.TestCase): + + # Test PyUnicode_FromFormat() + def test_from_format(self): + support.import_module('ctypes') + from ctypes import ( + pythonapi, py_object, sizeof, + c_int, c_long, c_longlong, c_ssize_t, + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) + name = "PyUnicode_FromFormat" + _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.restype = py_object + + def PyUnicode_FromFormat(format, *args): + cargs = tuple( + py_object(arg) if isinstance(arg, str) else arg + for arg in args) + return _PyUnicode_FromFormat(format, *cargs) + + def check_format(expected, format, *args): + text = PyUnicode_FromFormat(format, *args) + self.assertEqual(expected, text) + + # ascii format, non-ascii argument + check_format('ascii\x7f=unicode\xe9', + b'ascii\x7f=%U', 'unicode\xe9') + + # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() + # raises an error + self.assertRaisesRegex(ValueError, + r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' + 'string, got a non-ASCII byte: 0xe9$', + PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') + + # test "%c" + check_format('\uabcd', + b'%c', c_int(0xabcd)) + check_format('\U0010ffff', + b'%c', c_int(0x10ffff)) + with self.assertRaises(OverflowError): + PyUnicode_FromFormat(b'%c', c_int(0x110000)) + # Issue #18183 + check_format('\U00010000\U00100000', + b'%c%c', c_int(0x10000), c_int(0x100000)) + + # test "%" + check_format('%', + b'%') + check_format('%', + b'%%') + check_format('%s', + b'%%s') + check_format('[%]', + b'[%%]') + check_format('%abc', + b'%%%s', b'abc') + + # truncated string + check_format('abc', + b'%.3s', b'abcdef') + check_format('abc[\ufffd', + b'%.5s', 'abc[\u20ac]'.encode('utf8')) + check_format("'\\u20acABC'", + b'%A', '\u20acABC') + check_format("'\\u20", + b'%.5A', '\u20acABCDEF') + check_format("'\u20acABC'", + b'%R', '\u20acABC') + check_format("'\u20acA", + b'%.3R', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3S', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3V', '\u20acABCDEF', None) + check_format('abc[\ufffd', + b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + + # following tests comes from #7330 + # test width modifier and precision modifier with %S + check_format("repr= abc", + b'repr=%5S', 'abc') + check_format("repr=ab", + b'repr=%.2S', 'abc') + check_format("repr= ab", + b'repr=%5.2S', 'abc') + + # test width modifier and precision modifier with %R + check_format("repr= 'abc'", + b'repr=%8R', 'abc') + check_format("repr='ab", + b'repr=%.3R', 'abc') + check_format("repr= 'ab", + b'repr=%5.3R', 'abc') + + # test width modifier and precision modifier with %A + check_format("repr= 'abc'", + b'repr=%8A', 'abc') + check_format("repr='ab", + b'repr=%.3A', 'abc') + check_format("repr= 'ab", + b'repr=%5.3A', 'abc') + + # test width modifier and precision modifier with %s + check_format("repr= abc", + b'repr=%5s', b'abc') + check_format("repr=ab", + b'repr=%.2s', b'abc') + check_format("repr= ab", + b'repr=%5.2s', b'abc') + + # test width modifier and precision modifier with %U + check_format("repr= abc", + b'repr=%5U', 'abc') + check_format("repr=ab", + b'repr=%.2U', 'abc') + check_format("repr= ab", + b'repr=%5.2U', 'abc') + + # test width modifier and precision modifier with %V + check_format("repr= abc", + b'repr=%5V', 'abc', b'123') + check_format("repr=ab", + b'repr=%.2V', 'abc', b'123') + check_format("repr= ab", + b'repr=%5.2V', 'abc', b'123') + check_format("repr= 123", + b'repr=%5V', None, b'123') + check_format("repr=12", + b'repr=%.2V', None, b'123') + check_format("repr= 12", + b'repr=%5.2V', None, b'123') + + # test integer formats (%i, %d, %u) + check_format('010', + b'%03i', c_int(10)) + check_format('0010', + b'%0.4i', c_int(10)) + check_format('-123', + b'%i', c_int(-123)) + check_format('-123', + b'%li', c_long(-123)) + check_format('-123', + b'%lli', c_longlong(-123)) + check_format('-123', + b'%zi', c_ssize_t(-123)) + + check_format('-123', + b'%d', c_int(-123)) + check_format('-123', + b'%ld', c_long(-123)) + check_format('-123', + b'%lld', c_longlong(-123)) + check_format('-123', + b'%zd', c_ssize_t(-123)) + + check_format('123', + b'%u', c_uint(123)) + check_format('123', + b'%lu', c_ulong(123)) + check_format('123', + b'%llu', c_ulonglong(123)) + check_format('123', + b'%zu', c_size_t(123)) + + # test long output + min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) + max_longlong = -min_longlong - 1 + check_format(str(min_longlong), + b'%lld', c_longlong(min_longlong)) + check_format(str(max_longlong), + b'%lld', c_longlong(max_longlong)) + max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 + check_format(str(max_ulonglong), + b'%llu', c_ulonglong(max_ulonglong)) + PyUnicode_FromFormat(b'%p', c_void_p(-1)) + + # test padding (width and/or precision) + check_format('123'.rjust(10, '0'), + b'%010i', c_int(123)) + check_format('123'.rjust(100), + b'%100i', c_int(123)) + check_format('123'.rjust(100, '0'), + b'%.100i', c_int(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80i', c_int(123)) + + check_format('123'.rjust(10, '0'), + b'%010u', c_uint(123)) + check_format('123'.rjust(100), + b'%100u', c_uint(123)) + check_format('123'.rjust(100, '0'), + b'%.100u', c_uint(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80u', c_uint(123)) + + check_format('123'.rjust(10, '0'), + b'%010x', c_int(0x123)) + check_format('123'.rjust(100), + b'%100x', c_int(0x123)) + check_format('123'.rjust(100, '0'), + b'%.100x', c_int(0x123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80x', c_int(0x123)) + + # test %A + check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", + b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') + + # test %V + check_format('repr=abc', + b'repr=%V', 'abc', b'xyz') + + # Test string decode from parameter of %s using utf-8. + # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of + # '\u4eba\u6c11' + check_format('repr=\u4eba\u6c11', + b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') + + #Test replace error handler. + check_format('repr=abc\ufffd', + b'repr=%V', None, b'abc\xff') + + # not supported: copy the raw format string. these tests are just here + # to check for crashes and should not be considered as specifications + check_format('%s', + b'%1%s', b'abc') + check_format('%1abc', + b'%1abc') + check_format('%+i', + b'%+i', c_int(10)) + check_format('%.%s', + b'%.%s', b'abc') + + # Test PyUnicode_AsWideChar() + @support.cpython_only + def test_aswidechar(self): + from _testcapi import unicode_aswidechar + support.import_module('ctypes') + from ctypes import c_wchar, sizeof + + wchar, size = unicode_aswidechar('abcdef', 2) + self.assertEqual(size, 2) + self.assertEqual(wchar, 'ab') + + wchar, size = unicode_aswidechar('abc', 3) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc') + + wchar, size = unicode_aswidechar('abc', 4) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc', 10) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc\0def', 20) + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + + nonbmp = chr(0x10ffff) + if sizeof(c_wchar) == 2: + buflen = 3 + nchar = 2 + else: # sizeof(c_wchar) == 4 + buflen = 2 + nchar = 1 + wchar, size = unicode_aswidechar(nonbmp, buflen) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + + # Test PyUnicode_AsWideCharString() + @support.cpython_only + def test_aswidecharstring(self): + from _testcapi import unicode_aswidecharstring + support.import_module('ctypes') + from ctypes import c_wchar, sizeof + + wchar, size = unicode_aswidecharstring('abc') + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidecharstring('abc\0def') + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + + nonbmp = chr(0x10ffff) + if sizeof(c_wchar) == 2: + nchar = 2 + else: # sizeof(c_wchar) == 4 + nchar = 1 + wchar, size = unicode_aswidecharstring(nonbmp) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + + # Test PyUnicode_AsUCS4() + @support.cpython_only + def test_asucs4(self): + from _testcapi import unicode_asucs4 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + l = len(s) + self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') + self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) + s = '\0'.join([s, s]) + self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') + + # Test PyUnicode_FindChar() + @support.cpython_only + def test_findchar(self): + from _testcapi import unicode_findchar + + for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": + for i, ch in enumerate(str): + self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) + self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) + + str = "!>_= end + self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) + # negative + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) + + # Test PyUnicode_CopyCharacters() + @support.cpython_only + def test_copycharacters(self): + from _testcapi import unicode_copycharacters + + strings = [ + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + + for idx, from_ in enumerate(strings): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises( + SystemError, + unicode_copycharacters, to, 0, from_, 0, 5 + ) + # same kind + for from_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, 0, from_, from_start, 5), + (from_[from_start:from_start+5].ljust(5, '\0'), + 5-from_start) + ) + for to_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, to_start, from_, to_start, 5), + (from_[to_start:to_start+5].rjust(5, '\0'), + 5-to_start) + ) + # narrow -> wide + # Tests omitted since this creates invalid strings. + + s = strings[0] + self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) + + @support.cpython_only + def test_encode_decimal(self): + from _testcapi import unicode_encodedecimal + self.assertEqual(unicode_encodedecimal('123'), + b'123') + self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'), + b'3.14') + self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"), + b' 3.14 ') + self.assertRaises(UnicodeEncodeError, + unicode_encodedecimal, "123\u20ac", "strict") + self.assertRaisesRegex( + ValueError, + "^'decimal' codec can't encode character", + unicode_encodedecimal, "123\u20ac", "replace") + + @support.cpython_only + def test_transform_decimal(self): + from _testcapi import unicode_transformdecimaltoascii as transform_decimal + self.assertEqual(transform_decimal('123'), + '123') + self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), + '3.14') + self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), + "\N{EM SPACE}3.14\N{EN SPACE}") + self.assertEqual(transform_decimal('123\u20ac'), + '123\u20ac') + + @support.cpython_only + def test_pep393_utf8_caching_bug(self): + # Issue #25709: Problem with string concatenation and utf-8 cache + from _testcapi import getargs_s_hash + for k in 0x24, 0xa4, 0x20ac, 0x1f40d: + s = '' + for i in range(5): + # Due to CPython specific optimization the 's' string can be + # resized in-place. + s += chr(k) + # Parsing with the "s#" format code calls indirectly + # PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + # Check that the second call returns the same result + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 55faf4c4279fb6..3e90ba337c2ff9 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -104,16 +104,9 @@ import inspect import sys -import threading import unittest import weakref -try: - import ctypes -except ImportError: - ctypes = None -from test.support import (run_doctest, run_unittest, cpython_only, - check_impl_detail) - +from test.support import run_doctest, run_unittest, cpython_only def consts(t): """Yield a doctest-safe sequence of object reprs.""" @@ -263,104 +256,10 @@ def callback(code): self.assertTrue(self.called) -if check_impl_detail(cpython=True) and ctypes is not None: - py = ctypes.pythonapi - freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) - - RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex - RequestCodeExtraIndex.argtypes = (freefunc,) - RequestCodeExtraIndex.restype = ctypes.c_ssize_t - - SetExtra = py._PyCode_SetExtra - SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) - SetExtra.restype = ctypes.c_int - - GetExtra = py._PyCode_GetExtra - GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, - ctypes.POINTER(ctypes.c_voidp)) - GetExtra.restype = ctypes.c_int - - LAST_FREED = None - def myfree(ptr): - global LAST_FREED - LAST_FREED = ptr - - FREE_FUNC = freefunc(myfree) - FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) - - class CoExtra(unittest.TestCase): - def get_func(self): - # Defining a function causes the containing function to have a - # reference to the code object. We need the code objects to go - # away, so we eval a lambda. - return eval('lambda:42') - - def test_get_non_code(self): - f = self.get_func() - - self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, - ctypes.c_voidp(100)) - self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, - ctypes.c_voidp(100)) - - def test_bad_index(self): - f = self.get_func() - self.assertRaises(SystemError, SetExtra, f.__code__, - FREE_INDEX+100, ctypes.c_voidp(100)) - self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, - ctypes.c_voidp(100)), 0) - - def test_free_called(self): - # Verify that the provided free function gets invoked - # when the code object is cleaned up. - f = self.get_func() - - SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) - del f - self.assertEqual(LAST_FREED, 100) - - def test_get_set(self): - # Test basic get/set round tripping. - f = self.get_func() - - extra = ctypes.c_voidp() - - SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) - # reset should free... - SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) - self.assertEqual(LAST_FREED, 200) - - extra = ctypes.c_voidp() - GetExtra(f.__code__, FREE_INDEX, extra) - self.assertEqual(extra.value, 300) - del f - - def test_free_different_thread(self): - # Freeing a code object on a different thread then - # where the co_extra was set should be safe. - f = self.get_func() - class ThreadTest(threading.Thread): - def __init__(self, f, test): - super().__init__() - self.f = f - self.test = test - def run(self): - del self.f - self.test.assertEqual(LAST_FREED, 500) - - SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) - tt = ThreadTest(f, self) - del f - tt.start() - tt.join() - self.assertEqual(LAST_FREED, 500) - def test_main(verbose=None): from test import test_code run_doctest(test_code, verbose) tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] - if check_impl_detail(cpython=True) and ctypes is not None: - tests.append(CoExtra) run_unittest(*tests) if __name__ == "__main__": diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index aa149d31eb0e22..7c54bf1f32f460 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1223,36 +1223,6 @@ def iter_and_mutate(): self.assertRaises(RuntimeError, iter_and_mutate) -class CAPITest(unittest.TestCase): - - # Test _PyDict_GetItem_KnownHash() - @support.cpython_only - def test_getitem_knownhash(self): - from _testcapi import dict_getitem_knownhash - - d = {'x': 1, 'y': 2, 'z': 3} - self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) - self.assertEqual(dict_getitem_knownhash(d, 'y', hash('y')), 2) - self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) - - # not a dict - self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) - # key does not exist - self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) - - class Exc(Exception): pass - class BadEq: - def __eq__(self, other): - raise Exc - def __hash__(self): - return 7 - - k1, k2 = BadEq(), BadEq() - d = {k1: 1} - self.assertEqual(dict_getitem_knownhash(d, k1, hash(k1)), 1) - self.assertRaises(Exc, dict_getitem_knownhash, d, k2, hash(k2)) - - from test import mapping_tests class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index a3bd350c77b95b..3dade7a68b22af 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -7,11 +7,6 @@ import os import types -try: - import _testcapi -except ImportError: - _testcapi = None - class HelperMixin: def helper(self, sample, *extra): new = marshal.loads(marshal.dumps(sample, *extra)) @@ -484,88 +479,6 @@ def testNoIntern(self): s2 = sys.intern(s) self.assertNotEqual(id(s2), id(s)) -@support.cpython_only -@unittest.skipUnless(_testcapi, 'requires _testcapi') -class CAPI_TestCase(unittest.TestCase, HelperMixin): - - def test_write_long_to_file(self): - for v in range(marshal.version + 1): - _testcapi.pymarshal_write_long_to_file(0x12345678, support.TESTFN, v) - with open(support.TESTFN, 'rb') as f: - data = f.read() - support.unlink(support.TESTFN) - self.assertEqual(data, b'\x78\x56\x34\x12') - - def test_write_object_to_file(self): - obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000) - for v in range(marshal.version + 1): - _testcapi.pymarshal_write_object_to_file(obj, support.TESTFN, v) - with open(support.TESTFN, 'rb') as f: - data = f.read() - support.unlink(support.TESTFN) - self.assertEqual(marshal.loads(data), obj) - - def test_read_short_from_file(self): - with open(support.TESTFN, 'wb') as f: - f.write(b'\x34\x12xxxx') - r, p = _testcapi.pymarshal_read_short_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, 0x1234) - self.assertEqual(p, 2) - - with open(support.TESTFN, 'wb') as f: - f.write(b'\x12') - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_short_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - def test_read_long_from_file(self): - with open(support.TESTFN, 'wb') as f: - f.write(b'\x78\x56\x34\x12xxxx') - r, p = _testcapi.pymarshal_read_long_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, 0x12345678) - self.assertEqual(p, 4) - - with open(support.TESTFN, 'wb') as f: - f.write(b'\x56\x34\x12') - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_long_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - def test_read_last_object_from_file(self): - obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) - for v in range(marshal.version + 1): - data = marshal.dumps(obj, v) - with open(support.TESTFN, 'wb') as f: - f.write(data + b'xxxx') - r, p = _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, obj) - - with open(support.TESTFN, 'wb') as f: - f.write(data[:1]) - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - def test_read_object_from_file(self): - obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) - for v in range(marshal.version + 1): - data = marshal.dumps(obj, v) - with open(support.TESTFN, 'wb') as f: - f.write(data + b'xxxx') - r, p = _testcapi.pymarshal_read_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, obj) - self.assertEqual(p, len(data)) - - with open(support.TESTFN, 'wb') as f: - f.write(data[:1]) - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 3cc018c0cc2caa..8b04a5588c0030 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2441,431 +2441,6 @@ def test_free_after_iterating(self): support.check_free_after_iterating(self, reversed, str) -class CAPITest(unittest.TestCase): - - # Test PyUnicode_FromFormat() - def test_from_format(self): - support.import_module('ctypes') - from ctypes import ( - pythonapi, py_object, sizeof, - c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) - name = "PyUnicode_FromFormat" - _PyUnicode_FromFormat = getattr(pythonapi, name) - _PyUnicode_FromFormat.restype = py_object - - def PyUnicode_FromFormat(format, *args): - cargs = tuple( - py_object(arg) if isinstance(arg, str) else arg - for arg in args) - return _PyUnicode_FromFormat(format, *cargs) - - def check_format(expected, format, *args): - text = PyUnicode_FromFormat(format, *args) - self.assertEqual(expected, text) - - # ascii format, non-ascii argument - check_format('ascii\x7f=unicode\xe9', - b'ascii\x7f=%U', 'unicode\xe9') - - # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() - # raises an error - self.assertRaisesRegex(ValueError, - r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' - 'string, got a non-ASCII byte: 0xe9$', - PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') - - # test "%c" - check_format('\uabcd', - b'%c', c_int(0xabcd)) - check_format('\U0010ffff', - b'%c', c_int(0x10ffff)) - with self.assertRaises(OverflowError): - PyUnicode_FromFormat(b'%c', c_int(0x110000)) - # Issue #18183 - check_format('\U00010000\U00100000', - b'%c%c', c_int(0x10000), c_int(0x100000)) - - # test "%" - check_format('%', - b'%') - check_format('%', - b'%%') - check_format('%s', - b'%%s') - check_format('[%]', - b'[%%]') - check_format('%abc', - b'%%%s', b'abc') - - # truncated string - check_format('abc', - b'%.3s', b'abcdef') - check_format('abc[\ufffd', - b'%.5s', 'abc[\u20ac]'.encode('utf8')) - check_format("'\\u20acABC'", - b'%A', '\u20acABC') - check_format("'\\u20", - b'%.5A', '\u20acABCDEF') - check_format("'\u20acABC'", - b'%R', '\u20acABC') - check_format("'\u20acA", - b'%.3R', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3S', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3U', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3V', '\u20acABCDEF', None) - check_format('abc[\ufffd', - b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) - - # following tests comes from #7330 - # test width modifier and precision modifier with %S - check_format("repr= abc", - b'repr=%5S', 'abc') - check_format("repr=ab", - b'repr=%.2S', 'abc') - check_format("repr= ab", - b'repr=%5.2S', 'abc') - - # test width modifier and precision modifier with %R - check_format("repr= 'abc'", - b'repr=%8R', 'abc') - check_format("repr='ab", - b'repr=%.3R', 'abc') - check_format("repr= 'ab", - b'repr=%5.3R', 'abc') - - # test width modifier and precision modifier with %A - check_format("repr= 'abc'", - b'repr=%8A', 'abc') - check_format("repr='ab", - b'repr=%.3A', 'abc') - check_format("repr= 'ab", - b'repr=%5.3A', 'abc') - - # test width modifier and precision modifier with %s - check_format("repr= abc", - b'repr=%5s', b'abc') - check_format("repr=ab", - b'repr=%.2s', b'abc') - check_format("repr= ab", - b'repr=%5.2s', b'abc') - - # test width modifier and precision modifier with %U - check_format("repr= abc", - b'repr=%5U', 'abc') - check_format("repr=ab", - b'repr=%.2U', 'abc') - check_format("repr= ab", - b'repr=%5.2U', 'abc') - - # test width modifier and precision modifier with %V - check_format("repr= abc", - b'repr=%5V', 'abc', b'123') - check_format("repr=ab", - b'repr=%.2V', 'abc', b'123') - check_format("repr= ab", - b'repr=%5.2V', 'abc', b'123') - check_format("repr= 123", - b'repr=%5V', None, b'123') - check_format("repr=12", - b'repr=%.2V', None, b'123') - check_format("repr= 12", - b'repr=%5.2V', None, b'123') - - # test integer formats (%i, %d, %u) - check_format('010', - b'%03i', c_int(10)) - check_format('0010', - b'%0.4i', c_int(10)) - check_format('-123', - b'%i', c_int(-123)) - check_format('-123', - b'%li', c_long(-123)) - check_format('-123', - b'%lli', c_longlong(-123)) - check_format('-123', - b'%zi', c_ssize_t(-123)) - - check_format('-123', - b'%d', c_int(-123)) - check_format('-123', - b'%ld', c_long(-123)) - check_format('-123', - b'%lld', c_longlong(-123)) - check_format('-123', - b'%zd', c_ssize_t(-123)) - - check_format('123', - b'%u', c_uint(123)) - check_format('123', - b'%lu', c_ulong(123)) - check_format('123', - b'%llu', c_ulonglong(123)) - check_format('123', - b'%zu', c_size_t(123)) - - # test long output - min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) - max_longlong = -min_longlong - 1 - check_format(str(min_longlong), - b'%lld', c_longlong(min_longlong)) - check_format(str(max_longlong), - b'%lld', c_longlong(max_longlong)) - max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 - check_format(str(max_ulonglong), - b'%llu', c_ulonglong(max_ulonglong)) - PyUnicode_FromFormat(b'%p', c_void_p(-1)) - - # test padding (width and/or precision) - check_format('123'.rjust(10, '0'), - b'%010i', c_int(123)) - check_format('123'.rjust(100), - b'%100i', c_int(123)) - check_format('123'.rjust(100, '0'), - b'%.100i', c_int(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80i', c_int(123)) - - check_format('123'.rjust(10, '0'), - b'%010u', c_uint(123)) - check_format('123'.rjust(100), - b'%100u', c_uint(123)) - check_format('123'.rjust(100, '0'), - b'%.100u', c_uint(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80u', c_uint(123)) - - check_format('123'.rjust(10, '0'), - b'%010x', c_int(0x123)) - check_format('123'.rjust(100), - b'%100x', c_int(0x123)) - check_format('123'.rjust(100, '0'), - b'%.100x', c_int(0x123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80x', c_int(0x123)) - - # test %A - check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", - b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') - - # test %V - check_format('repr=abc', - b'repr=%V', 'abc', b'xyz') - - # Test string decode from parameter of %s using utf-8. - # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of - # '\u4eba\u6c11' - check_format('repr=\u4eba\u6c11', - b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') - - #Test replace error handler. - check_format('repr=abc\ufffd', - b'repr=%V', None, b'abc\xff') - - # not supported: copy the raw format string. these tests are just here - # to check for crashes and should not be considered as specifications - check_format('%s', - b'%1%s', b'abc') - check_format('%1abc', - b'%1abc') - check_format('%+i', - b'%+i', c_int(10)) - check_format('%.%s', - b'%.%s', b'abc') - - # Test PyUnicode_AsWideChar() - @support.cpython_only - def test_aswidechar(self): - from _testcapi import unicode_aswidechar - support.import_module('ctypes') - from ctypes import c_wchar, sizeof - - wchar, size = unicode_aswidechar('abcdef', 2) - self.assertEqual(size, 2) - self.assertEqual(wchar, 'ab') - - wchar, size = unicode_aswidechar('abc', 3) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc') - - wchar, size = unicode_aswidechar('abc', 4) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc', 10) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc\0def', 20) - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - - nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - buflen = 3 - nchar = 2 - else: # sizeof(c_wchar) == 4 - buflen = 2 - nchar = 1 - wchar, size = unicode_aswidechar(nonbmp, buflen) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - - # Test PyUnicode_AsWideCharString() - @support.cpython_only - def test_aswidecharstring(self): - from _testcapi import unicode_aswidecharstring - support.import_module('ctypes') - from ctypes import c_wchar, sizeof - - wchar, size = unicode_aswidecharstring('abc') - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidecharstring('abc\0def') - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - - nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - nchar = 2 - else: # sizeof(c_wchar) == 4 - nchar = 1 - wchar, size = unicode_aswidecharstring(nonbmp) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - - # Test PyUnicode_AsUCS4() - @support.cpython_only - def test_asucs4(self): - from _testcapi import unicode_asucs4 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - l = len(s) - self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') - self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) - s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') - - # Test PyUnicode_FindChar() - @support.cpython_only - def test_findchar(self): - from _testcapi import unicode_findchar - - for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": - for i, ch in enumerate(str): - self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) - self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) - - str = "!>_= end - self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) - self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) - # negative - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) - - # Test PyUnicode_CopyCharacters() - @support.cpython_only - def test_copycharacters(self): - from _testcapi import unicode_copycharacters - - strings = [ - 'abcde', '\xa1\xa2\xa3\xa4\xa5', - '\u4f60\u597d\u4e16\u754c\uff01', - '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' - ] - - for idx, from_ in enumerate(strings): - # wide -> narrow: exceed maxchar limitation - for to in strings[:idx]: - self.assertRaises( - SystemError, - unicode_copycharacters, to, 0, from_, 0, 5 - ) - # same kind - for from_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, 0, from_, from_start, 5), - (from_[from_start:from_start+5].ljust(5, '\0'), - 5-from_start) - ) - for to_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, to_start, from_, to_start, 5), - (from_[to_start:to_start+5].rjust(5, '\0'), - 5-to_start) - ) - # narrow -> wide - # Tests omitted since this creates invalid strings. - - s = strings[0] - self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) - - @support.cpython_only - def test_encode_decimal(self): - from _testcapi import unicode_encodedecimal - self.assertEqual(unicode_encodedecimal('123'), - b'123') - self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'), - b'3.14') - self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"), - b' 3.14 ') - self.assertRaises(UnicodeEncodeError, - unicode_encodedecimal, "123\u20ac", "strict") - self.assertRaisesRegex( - ValueError, - "^'decimal' codec can't encode character", - unicode_encodedecimal, "123\u20ac", "replace") - - @support.cpython_only - def test_transform_decimal(self): - from _testcapi import unicode_transformdecimaltoascii as transform_decimal - self.assertEqual(transform_decimal('123'), - '123') - self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), - '3.14') - self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), - "\N{EM SPACE}3.14\N{EN SPACE}") - self.assertEqual(transform_decimal('123\u20ac'), - '123\u20ac') - - @support.cpython_only - def test_pep393_utf8_caching_bug(self): - # Issue #25709: Problem with string concatenation and utf-8 cache - from _testcapi import getargs_s_hash - for k in 0x24, 0xa4, 0x20ac, 0x1f40d: - s = '' - for i in range(5): - # Due to CPython specific optimization the 's' string can be - # resized in-place. - s += chr(k) - # Parsing with the "s#" format code calls indirectly - # PyUnicode_AsUTF8AndSize() which creates the UTF-8 - # encoded string cached in the Unicode object. - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - # Check that the second call returns the same result - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): def parse(format): diff --git a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst new file mode 100644 index 00000000000000..370ee8b66bc806 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst @@ -0,0 +1 @@ +Most C API tests were moved into the new Lib/test/test_capi/ directory. From bf90140406b99b017f50b07398f142964d9073d2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 30 Jul 2018 09:55:24 +0300 Subject: [PATCH 2/2] Move just test_capi.py, test_getargs2.py and test_structmembers.py. --- Lib/test/test_bytes.py | 108 +++++ Lib/test/test_capi/test_bytes.py | 118 ----- Lib/test/test_capi/test_dict.py | 36 -- Lib/test/test_capi/test_marshal.py | 94 ---- Lib/test/test_capi/test_unicode.py | 431 ------------------ Lib/test/test_code.py | 103 ++++- Lib/test/test_dict.py | 30 ++ Lib/test/test_marshal.py | 87 ++++ Lib/test/test_unicode.py | 425 +++++++++++++++++ .../2018-07-29-15-59-51.bpo-34272.lVX2uR.rst | 2 +- 10 files changed, 753 insertions(+), 681 deletions(-) delete mode 100644 Lib/test/test_capi/test_bytes.py delete mode 100644 Lib/test/test_capi/test_dict.py delete mode 100644 Lib/test/test_capi/test_marshal.py delete mode 100644 Lib/test/test_capi/test_unicode.py diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 6b80b8d9598a9b..1408fb4fb84b0b 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -820,6 +820,114 @@ def __bytes__(self): self.assertEqual(BytesSubclass(A()), b'abc') self.assertIs(type(BytesSubclass(A())), BytesSubclass) + # Test PyBytes_FromFormat() + def test_from_format(self): + ctypes = test.support.import_module('ctypes') + _testcapi = test.support.import_module('_testcapi') + from ctypes import pythonapi, py_object + from ctypes import ( + c_int, c_uint, + c_long, c_ulong, + c_size_t, c_ssize_t, + c_char_p) + + PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.restype = py_object + + # basic tests + self.assertEqual(PyBytes_FromFormat(b'format'), + b'format') + self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), + b'Hello world !') + + # test formatters + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), + b'c=\0') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), + b'c=@') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), + b'c=\xff') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(1), c_long(2), + c_size_t(3)), + b'd=1 ld=2 zd=3') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(-1), c_long(-2), + c_size_t(-3)), + b'd=-1 ld=-2 zd=-3') + self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', + c_uint(123), c_ulong(456), + c_size_t(789)), + b'u=123 lu=456 zu=789') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), + b'i=123') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), + b'i=-123') + self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), + b'x=abc') + + sizeof_ptr = ctypes.sizeof(c_char_p) + + if os.name == 'nt': + # Windows (MSCRT) + ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + def ptr_formatter(ptr): + return (ptr_format % ptr) + else: + # UNIX (glibc) + def ptr_formatter(ptr): + return '%#x' % ptr + + ptr = 0xabcdef + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + ('ptr=' + ptr_formatter(ptr)).encode('ascii')) + self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), + b's=cstr') + + # test minimum and maximum integer values + size_max = c_size_t(-1).value + for formatstr, ctypes_type, value, py_formatter in ( + (b'%d', c_int, _testcapi.INT_MIN, str), + (b'%d', c_int, _testcapi.INT_MAX, str), + (b'%ld', c_long, _testcapi.LONG_MIN, str), + (b'%ld', c_long, _testcapi.LONG_MAX, str), + (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), + (b'%zu', c_size_t, size_max, str), + (b'%p', c_char_p, size_max, ptr_formatter), + ): + self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), + py_formatter(value).encode('ascii')), + + # width and precision (width is currently ignored) + self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), + b'a') + self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), + b'abc') + + # '%%' formatter + self.assertEqual(PyBytes_FromFormat(b'%%'), + b'%') + self.assertEqual(PyBytes_FromFormat(b'[%%]'), + b'[%]') + self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), + b'%_') + self.assertEqual(PyBytes_FromFormat(b'%%s'), + b'%s') + + # Invalid formats and partial formatting + self.assertEqual(PyBytes_FromFormat(b'%'), b'%') + self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), + b'x=2 y=%') + + # Issue #19969: %c must raise OverflowError for values + # not in the range [0; 255] + self.assertRaises(OverflowError, + PyBytes_FromFormat, b'%c', c_int(-1)) + self.assertRaises(OverflowError, + PyBytes_FromFormat, b'%c', c_int(256)) + def test_bytes_blocking(self): class IterationBlocked(list): __bytes__ = None diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py deleted file mode 100644 index 2f78f44e5217fd..00000000000000 --- a/Lib/test/test_capi/test_bytes.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import unittest -from test import support - - -class CAPITest(unittest.TestCase): - - # Test PyBytes_FromFormat() - def test_from_format(self): - ctypes = support.import_module('ctypes') - _testcapi = support.import_module('_testcapi') - from ctypes import pythonapi, py_object - from ctypes import ( - c_int, c_uint, - c_long, c_ulong, - c_size_t, c_ssize_t, - c_char_p) - - PyBytes_FromFormat = pythonapi.PyBytes_FromFormat - PyBytes_FromFormat.restype = py_object - - # basic tests - self.assertEqual(PyBytes_FromFormat(b'format'), - b'format') - self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), - b'Hello world !') - - # test formatters - self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), - b'c=\0') - self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), - b'c=@') - self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), - b'c=\xff') - self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', - c_int(1), c_long(2), - c_size_t(3)), - b'd=1 ld=2 zd=3') - self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', - c_int(-1), c_long(-2), - c_size_t(-3)), - b'd=-1 ld=-2 zd=-3') - self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', - c_uint(123), c_ulong(456), - c_size_t(789)), - b'u=123 lu=456 zu=789') - self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), - b'i=123') - self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), - b'i=-123') - self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), - b'x=abc') - - sizeof_ptr = ctypes.sizeof(c_char_p) - - if os.name == 'nt': - # Windows (MSCRT) - ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) - def ptr_formatter(ptr): - return (ptr_format % ptr) - else: - # UNIX (glibc) - def ptr_formatter(ptr): - return '%#x' % ptr - - ptr = 0xabcdef - self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), - ('ptr=' + ptr_formatter(ptr)).encode('ascii')) - self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), - b's=cstr') - - # test minimum and maximum integer values - size_max = c_size_t(-1).value - for formatstr, ctypes_type, value, py_formatter in ( - (b'%d', c_int, _testcapi.INT_MIN, str), - (b'%d', c_int, _testcapi.INT_MAX, str), - (b'%ld', c_long, _testcapi.LONG_MIN, str), - (b'%ld', c_long, _testcapi.LONG_MAX, str), - (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), - (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), - (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), - (b'%zu', c_size_t, size_max, str), - (b'%p', c_char_p, size_max, ptr_formatter), - ): - self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), - py_formatter(value).encode('ascii')), - - # width and precision (width is currently ignored) - self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), - b'a') - self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), - b'abc') - - # '%%' formatter - self.assertEqual(PyBytes_FromFormat(b'%%'), - b'%') - self.assertEqual(PyBytes_FromFormat(b'[%%]'), - b'[%]') - self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), - b'%_') - self.assertEqual(PyBytes_FromFormat(b'%%s'), - b'%s') - - # Invalid formats and partial formatting - self.assertEqual(PyBytes_FromFormat(b'%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), - b'x=2 y=%') - - # Issue #19969: %c must raise OverflowError for values - # not in the range [0; 255] - self.assertRaises(OverflowError, - PyBytes_FromFormat, b'%c', c_int(-1)) - self.assertRaises(OverflowError, - PyBytes_FromFormat, b'%c', c_int(256)) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py deleted file mode 100644 index 210b42ca32f27f..00000000000000 --- a/Lib/test/test_capi/test_dict.py +++ /dev/null @@ -1,36 +0,0 @@ -import unittest -from test import support - - -class CAPITest(unittest.TestCase): - - # Test _PyDict_GetItem_KnownHash() - @support.cpython_only - def test_getitem_knownhash(self): - from _testcapi import dict_getitem_knownhash - - d = {'x': 1, 'y': 2, 'z': 3} - self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) - self.assertEqual(dict_getitem_knownhash(d, 'y', hash('y')), 2) - self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) - - # not a dict - self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) - # key does not exist - self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) - - class Exc(Exception): pass - class BadEq: - def __eq__(self, other): - raise Exc - def __hash__(self): - return 7 - - k1, k2 = BadEq(), BadEq() - d = {k1: 1} - self.assertEqual(dict_getitem_knownhash(d, k1, hash(k1)), 1) - self.assertRaises(Exc, dict_getitem_knownhash, d, k2, hash(k2)) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_marshal.py b/Lib/test/test_capi/test_marshal.py deleted file mode 100644 index 6011e9eac1e525..00000000000000 --- a/Lib/test/test_capi/test_marshal.py +++ /dev/null @@ -1,94 +0,0 @@ -from test import support -import marshal -import unittest - -try: - import _testcapi -except ImportError: - _testcapi = None - -@support.cpython_only -@unittest.skipUnless(_testcapi, 'requires _testcapi') -class CAPI_TestCase(unittest.TestCase): - - def test_write_long_to_file(self): - for v in range(marshal.version + 1): - _testcapi.pymarshal_write_long_to_file(0x12345678, support.TESTFN, v) - with open(support.TESTFN, 'rb') as f: - data = f.read() - support.unlink(support.TESTFN) - self.assertEqual(data, b'\x78\x56\x34\x12') - - def test_write_object_to_file(self): - obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000) - for v in range(marshal.version + 1): - _testcapi.pymarshal_write_object_to_file(obj, support.TESTFN, v) - with open(support.TESTFN, 'rb') as f: - data = f.read() - support.unlink(support.TESTFN) - self.assertEqual(marshal.loads(data), obj) - - def test_read_short_from_file(self): - with open(support.TESTFN, 'wb') as f: - f.write(b'\x34\x12xxxx') - r, p = _testcapi.pymarshal_read_short_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, 0x1234) - self.assertEqual(p, 2) - - with open(support.TESTFN, 'wb') as f: - f.write(b'\x12') - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_short_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - def test_read_long_from_file(self): - with open(support.TESTFN, 'wb') as f: - f.write(b'\x78\x56\x34\x12xxxx') - r, p = _testcapi.pymarshal_read_long_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, 0x12345678) - self.assertEqual(p, 4) - - with open(support.TESTFN, 'wb') as f: - f.write(b'\x56\x34\x12') - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_long_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - def test_read_last_object_from_file(self): - obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) - for v in range(marshal.version + 1): - data = marshal.dumps(obj, v) - with open(support.TESTFN, 'wb') as f: - f.write(data + b'xxxx') - r, p = _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, obj) - - with open(support.TESTFN, 'wb') as f: - f.write(data[:1]) - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - def test_read_object_from_file(self): - obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) - for v in range(marshal.version + 1): - data = marshal.dumps(obj, v) - with open(support.TESTFN, 'wb') as f: - f.write(data + b'xxxx') - r, p = _testcapi.pymarshal_read_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - self.assertEqual(r, obj) - self.assertEqual(p, len(data)) - - with open(support.TESTFN, 'wb') as f: - f.write(data[:1]) - with self.assertRaises(EOFError): - _testcapi.pymarshal_read_object_from_file(support.TESTFN) - support.unlink(support.TESTFN) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py deleted file mode 100644 index df67a704d793c7..00000000000000 --- a/Lib/test/test_capi/test_unicode.py +++ /dev/null @@ -1,431 +0,0 @@ -import unittest -from test import support - -class CAPITest(unittest.TestCase): - - # Test PyUnicode_FromFormat() - def test_from_format(self): - support.import_module('ctypes') - from ctypes import ( - pythonapi, py_object, sizeof, - c_int, c_long, c_longlong, c_ssize_t, - c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) - name = "PyUnicode_FromFormat" - _PyUnicode_FromFormat = getattr(pythonapi, name) - _PyUnicode_FromFormat.restype = py_object - - def PyUnicode_FromFormat(format, *args): - cargs = tuple( - py_object(arg) if isinstance(arg, str) else arg - for arg in args) - return _PyUnicode_FromFormat(format, *cargs) - - def check_format(expected, format, *args): - text = PyUnicode_FromFormat(format, *args) - self.assertEqual(expected, text) - - # ascii format, non-ascii argument - check_format('ascii\x7f=unicode\xe9', - b'ascii\x7f=%U', 'unicode\xe9') - - # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() - # raises an error - self.assertRaisesRegex(ValueError, - r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' - 'string, got a non-ASCII byte: 0xe9$', - PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') - - # test "%c" - check_format('\uabcd', - b'%c', c_int(0xabcd)) - check_format('\U0010ffff', - b'%c', c_int(0x10ffff)) - with self.assertRaises(OverflowError): - PyUnicode_FromFormat(b'%c', c_int(0x110000)) - # Issue #18183 - check_format('\U00010000\U00100000', - b'%c%c', c_int(0x10000), c_int(0x100000)) - - # test "%" - check_format('%', - b'%') - check_format('%', - b'%%') - check_format('%s', - b'%%s') - check_format('[%]', - b'[%%]') - check_format('%abc', - b'%%%s', b'abc') - - # truncated string - check_format('abc', - b'%.3s', b'abcdef') - check_format('abc[\ufffd', - b'%.5s', 'abc[\u20ac]'.encode('utf8')) - check_format("'\\u20acABC'", - b'%A', '\u20acABC') - check_format("'\\u20", - b'%.5A', '\u20acABCDEF') - check_format("'\u20acABC'", - b'%R', '\u20acABC') - check_format("'\u20acA", - b'%.3R', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3S', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3U', '\u20acABCDEF') - check_format('\u20acAB', - b'%.3V', '\u20acABCDEF', None) - check_format('abc[\ufffd', - b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) - - # following tests comes from #7330 - # test width modifier and precision modifier with %S - check_format("repr= abc", - b'repr=%5S', 'abc') - check_format("repr=ab", - b'repr=%.2S', 'abc') - check_format("repr= ab", - b'repr=%5.2S', 'abc') - - # test width modifier and precision modifier with %R - check_format("repr= 'abc'", - b'repr=%8R', 'abc') - check_format("repr='ab", - b'repr=%.3R', 'abc') - check_format("repr= 'ab", - b'repr=%5.3R', 'abc') - - # test width modifier and precision modifier with %A - check_format("repr= 'abc'", - b'repr=%8A', 'abc') - check_format("repr='ab", - b'repr=%.3A', 'abc') - check_format("repr= 'ab", - b'repr=%5.3A', 'abc') - - # test width modifier and precision modifier with %s - check_format("repr= abc", - b'repr=%5s', b'abc') - check_format("repr=ab", - b'repr=%.2s', b'abc') - check_format("repr= ab", - b'repr=%5.2s', b'abc') - - # test width modifier and precision modifier with %U - check_format("repr= abc", - b'repr=%5U', 'abc') - check_format("repr=ab", - b'repr=%.2U', 'abc') - check_format("repr= ab", - b'repr=%5.2U', 'abc') - - # test width modifier and precision modifier with %V - check_format("repr= abc", - b'repr=%5V', 'abc', b'123') - check_format("repr=ab", - b'repr=%.2V', 'abc', b'123') - check_format("repr= ab", - b'repr=%5.2V', 'abc', b'123') - check_format("repr= 123", - b'repr=%5V', None, b'123') - check_format("repr=12", - b'repr=%.2V', None, b'123') - check_format("repr= 12", - b'repr=%5.2V', None, b'123') - - # test integer formats (%i, %d, %u) - check_format('010', - b'%03i', c_int(10)) - check_format('0010', - b'%0.4i', c_int(10)) - check_format('-123', - b'%i', c_int(-123)) - check_format('-123', - b'%li', c_long(-123)) - check_format('-123', - b'%lli', c_longlong(-123)) - check_format('-123', - b'%zi', c_ssize_t(-123)) - - check_format('-123', - b'%d', c_int(-123)) - check_format('-123', - b'%ld', c_long(-123)) - check_format('-123', - b'%lld', c_longlong(-123)) - check_format('-123', - b'%zd', c_ssize_t(-123)) - - check_format('123', - b'%u', c_uint(123)) - check_format('123', - b'%lu', c_ulong(123)) - check_format('123', - b'%llu', c_ulonglong(123)) - check_format('123', - b'%zu', c_size_t(123)) - - # test long output - min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) - max_longlong = -min_longlong - 1 - check_format(str(min_longlong), - b'%lld', c_longlong(min_longlong)) - check_format(str(max_longlong), - b'%lld', c_longlong(max_longlong)) - max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 - check_format(str(max_ulonglong), - b'%llu', c_ulonglong(max_ulonglong)) - PyUnicode_FromFormat(b'%p', c_void_p(-1)) - - # test padding (width and/or precision) - check_format('123'.rjust(10, '0'), - b'%010i', c_int(123)) - check_format('123'.rjust(100), - b'%100i', c_int(123)) - check_format('123'.rjust(100, '0'), - b'%.100i', c_int(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80i', c_int(123)) - - check_format('123'.rjust(10, '0'), - b'%010u', c_uint(123)) - check_format('123'.rjust(100), - b'%100u', c_uint(123)) - check_format('123'.rjust(100, '0'), - b'%.100u', c_uint(123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80u', c_uint(123)) - - check_format('123'.rjust(10, '0'), - b'%010x', c_int(0x123)) - check_format('123'.rjust(100), - b'%100x', c_int(0x123)) - check_format('123'.rjust(100, '0'), - b'%.100x', c_int(0x123)) - check_format('123'.rjust(80, '0').rjust(100), - b'%100.80x', c_int(0x123)) - - # test %A - check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", - b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') - - # test %V - check_format('repr=abc', - b'repr=%V', 'abc', b'xyz') - - # Test string decode from parameter of %s using utf-8. - # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of - # '\u4eba\u6c11' - check_format('repr=\u4eba\u6c11', - b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') - - #Test replace error handler. - check_format('repr=abc\ufffd', - b'repr=%V', None, b'abc\xff') - - # not supported: copy the raw format string. these tests are just here - # to check for crashes and should not be considered as specifications - check_format('%s', - b'%1%s', b'abc') - check_format('%1abc', - b'%1abc') - check_format('%+i', - b'%+i', c_int(10)) - check_format('%.%s', - b'%.%s', b'abc') - - # Test PyUnicode_AsWideChar() - @support.cpython_only - def test_aswidechar(self): - from _testcapi import unicode_aswidechar - support.import_module('ctypes') - from ctypes import c_wchar, sizeof - - wchar, size = unicode_aswidechar('abcdef', 2) - self.assertEqual(size, 2) - self.assertEqual(wchar, 'ab') - - wchar, size = unicode_aswidechar('abc', 3) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc') - - wchar, size = unicode_aswidechar('abc', 4) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc', 10) - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidechar('abc\0def', 20) - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - - nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - buflen = 3 - nchar = 2 - else: # sizeof(c_wchar) == 4 - buflen = 2 - nchar = 1 - wchar, size = unicode_aswidechar(nonbmp, buflen) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - - # Test PyUnicode_AsWideCharString() - @support.cpython_only - def test_aswidecharstring(self): - from _testcapi import unicode_aswidecharstring - support.import_module('ctypes') - from ctypes import c_wchar, sizeof - - wchar, size = unicode_aswidecharstring('abc') - self.assertEqual(size, 3) - self.assertEqual(wchar, 'abc\0') - - wchar, size = unicode_aswidecharstring('abc\0def') - self.assertEqual(size, 7) - self.assertEqual(wchar, 'abc\0def\0') - - nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - nchar = 2 - else: # sizeof(c_wchar) == 4 - nchar = 1 - wchar, size = unicode_aswidecharstring(nonbmp) - self.assertEqual(size, nchar) - self.assertEqual(wchar, nonbmp + '\0') - - # Test PyUnicode_AsUCS4() - @support.cpython_only - def test_asucs4(self): - from _testcapi import unicode_asucs4 - for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', - 'a\ud800b\udfffc', '\ud834\udd1e']: - l = len(s) - self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') - self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) - s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') - - # Test PyUnicode_FindChar() - @support.cpython_only - def test_findchar(self): - from _testcapi import unicode_findchar - - for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": - for i, ch in enumerate(str): - self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) - self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) - - str = "!>_= end - self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) - self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) - # negative - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) - self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) - - # Test PyUnicode_CopyCharacters() - @support.cpython_only - def test_copycharacters(self): - from _testcapi import unicode_copycharacters - - strings = [ - 'abcde', '\xa1\xa2\xa3\xa4\xa5', - '\u4f60\u597d\u4e16\u754c\uff01', - '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' - ] - - for idx, from_ in enumerate(strings): - # wide -> narrow: exceed maxchar limitation - for to in strings[:idx]: - self.assertRaises( - SystemError, - unicode_copycharacters, to, 0, from_, 0, 5 - ) - # same kind - for from_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, 0, from_, from_start, 5), - (from_[from_start:from_start+5].ljust(5, '\0'), - 5-from_start) - ) - for to_start in range(5): - self.assertEqual( - unicode_copycharacters(from_, to_start, from_, to_start, 5), - (from_[to_start:to_start+5].rjust(5, '\0'), - 5-to_start) - ) - # narrow -> wide - # Tests omitted since this creates invalid strings. - - s = strings[0] - self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) - self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) - self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) - - @support.cpython_only - def test_encode_decimal(self): - from _testcapi import unicode_encodedecimal - self.assertEqual(unicode_encodedecimal('123'), - b'123') - self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'), - b'3.14') - self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"), - b' 3.14 ') - self.assertRaises(UnicodeEncodeError, - unicode_encodedecimal, "123\u20ac", "strict") - self.assertRaisesRegex( - ValueError, - "^'decimal' codec can't encode character", - unicode_encodedecimal, "123\u20ac", "replace") - - @support.cpython_only - def test_transform_decimal(self): - from _testcapi import unicode_transformdecimaltoascii as transform_decimal - self.assertEqual(transform_decimal('123'), - '123') - self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), - '3.14') - self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), - "\N{EM SPACE}3.14\N{EN SPACE}") - self.assertEqual(transform_decimal('123\u20ac'), - '123\u20ac') - - @support.cpython_only - def test_pep393_utf8_caching_bug(self): - # Issue #25709: Problem with string concatenation and utf-8 cache - from _testcapi import getargs_s_hash - for k in 0x24, 0xa4, 0x20ac, 0x1f40d: - s = '' - for i in range(5): - # Due to CPython specific optimization the 's' string can be - # resized in-place. - s += chr(k) - # Parsing with the "s#" format code calls indirectly - # PyUnicode_AsUTF8AndSize() which creates the UTF-8 - # encoded string cached in the Unicode object. - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - # Check that the second call returns the same result - self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 3e90ba337c2ff9..55faf4c4279fb6 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -104,9 +104,16 @@ import inspect import sys +import threading import unittest import weakref -from test.support import run_doctest, run_unittest, cpython_only +try: + import ctypes +except ImportError: + ctypes = None +from test.support import (run_doctest, run_unittest, cpython_only, + check_impl_detail) + def consts(t): """Yield a doctest-safe sequence of object reprs.""" @@ -256,10 +263,104 @@ def callback(code): self.assertTrue(self.called) +if check_impl_detail(cpython=True) and ctypes is not None: + py = ctypes.pythonapi + freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) + + RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = py._PyCode_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = py._PyCode_GetExtra + GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp)) + GetExtra.restype = ctypes.c_int + + LAST_FREED = None + def myfree(ptr): + global LAST_FREED + LAST_FREED = ptr + + FREE_FUNC = freefunc(myfree) + FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + + class CoExtra(unittest.TestCase): + def get_func(self): + # Defining a function causes the containing function to have a + # reference to the code object. We need the code objects to go + # away, so we eval a lambda. + return eval('lambda:42') + + def test_get_non_code(self): + f = self.get_func() + + self.assertRaises(SystemError, SetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + self.assertRaises(SystemError, GetExtra, 42, FREE_INDEX, + ctypes.c_voidp(100)) + + def test_bad_index(self): + f = self.get_func() + self.assertRaises(SystemError, SetExtra, f.__code__, + FREE_INDEX+100, ctypes.c_voidp(100)) + self.assertEqual(GetExtra(f.__code__, FREE_INDEX+100, + ctypes.c_voidp(100)), 0) + + def test_free_called(self): + # Verify that the provided free function gets invoked + # when the code object is cleaned up. + f = self.get_func() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(100)) + del f + self.assertEqual(LAST_FREED, 100) + + def test_get_set(self): + # Test basic get/set round tripping. + f = self.get_func() + + extra = ctypes.c_voidp() + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(200)) + # reset should free... + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(300)) + self.assertEqual(LAST_FREED, 200) + + extra = ctypes.c_voidp() + GetExtra(f.__code__, FREE_INDEX, extra) + self.assertEqual(extra.value, 300) + del f + + def test_free_different_thread(self): + # Freeing a code object on a different thread then + # where the co_extra was set should be safe. + f = self.get_func() + class ThreadTest(threading.Thread): + def __init__(self, f, test): + super().__init__() + self.f = f + self.test = test + def run(self): + del self.f + self.test.assertEqual(LAST_FREED, 500) + + SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500)) + tt = ThreadTest(f, self) + del f + tt.start() + tt.join() + self.assertEqual(LAST_FREED, 500) + def test_main(verbose=None): from test import test_code run_doctest(test_code, verbose) tests = [CodeTest, CodeConstsTest, CodeWeakRefTest] + if check_impl_detail(cpython=True) and ctypes is not None: + tests.append(CoExtra) run_unittest(*tests) if __name__ == "__main__": diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 7c54bf1f32f460..aa149d31eb0e22 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1223,6 +1223,36 @@ def iter_and_mutate(): self.assertRaises(RuntimeError, iter_and_mutate) +class CAPITest(unittest.TestCase): + + # Test _PyDict_GetItem_KnownHash() + @support.cpython_only + def test_getitem_knownhash(self): + from _testcapi import dict_getitem_knownhash + + d = {'x': 1, 'y': 2, 'z': 3} + self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) + self.assertEqual(dict_getitem_knownhash(d, 'y', hash('y')), 2) + self.assertEqual(dict_getitem_knownhash(d, 'z', hash('z')), 3) + + # not a dict + self.assertRaises(SystemError, dict_getitem_knownhash, [], 1, hash(1)) + # key does not exist + self.assertRaises(KeyError, dict_getitem_knownhash, {}, 1, hash(1)) + + class Exc(Exception): pass + class BadEq: + def __eq__(self, other): + raise Exc + def __hash__(self): + return 7 + + k1, k2 = BadEq(), BadEq() + d = {k1: 1} + self.assertEqual(dict_getitem_knownhash(d, k1, hash(k1)), 1) + self.assertRaises(Exc, dict_getitem_knownhash, d, k2, hash(k2)) + + from test import mapping_tests class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 3dade7a68b22af..a3bd350c77b95b 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -7,6 +7,11 @@ import os import types +try: + import _testcapi +except ImportError: + _testcapi = None + class HelperMixin: def helper(self, sample, *extra): new = marshal.loads(marshal.dumps(sample, *extra)) @@ -479,6 +484,88 @@ def testNoIntern(self): s2 = sys.intern(s) self.assertNotEqual(id(s2), id(s)) +@support.cpython_only +@unittest.skipUnless(_testcapi, 'requires _testcapi') +class CAPI_TestCase(unittest.TestCase, HelperMixin): + + def test_write_long_to_file(self): + for v in range(marshal.version + 1): + _testcapi.pymarshal_write_long_to_file(0x12345678, support.TESTFN, v) + with open(support.TESTFN, 'rb') as f: + data = f.read() + support.unlink(support.TESTFN) + self.assertEqual(data, b'\x78\x56\x34\x12') + + def test_write_object_to_file(self): + obj = ('\u20ac', b'abc', 123, 45.6, 7+8j, 'long line '*1000) + for v in range(marshal.version + 1): + _testcapi.pymarshal_write_object_to_file(obj, support.TESTFN, v) + with open(support.TESTFN, 'rb') as f: + data = f.read() + support.unlink(support.TESTFN) + self.assertEqual(marshal.loads(data), obj) + + def test_read_short_from_file(self): + with open(support.TESTFN, 'wb') as f: + f.write(b'\x34\x12xxxx') + r, p = _testcapi.pymarshal_read_short_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, 0x1234) + self.assertEqual(p, 2) + + with open(support.TESTFN, 'wb') as f: + f.write(b'\x12') + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_short_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + def test_read_long_from_file(self): + with open(support.TESTFN, 'wb') as f: + f.write(b'\x78\x56\x34\x12xxxx') + r, p = _testcapi.pymarshal_read_long_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, 0x12345678) + self.assertEqual(p, 4) + + with open(support.TESTFN, 'wb') as f: + f.write(b'\x56\x34\x12') + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_long_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + def test_read_last_object_from_file(self): + obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) + for v in range(marshal.version + 1): + data = marshal.dumps(obj, v) + with open(support.TESTFN, 'wb') as f: + f.write(data + b'xxxx') + r, p = _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, obj) + + with open(support.TESTFN, 'wb') as f: + f.write(data[:1]) + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_last_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + + def test_read_object_from_file(self): + obj = ('\u20ac', b'abc', 123, 45.6, 7+8j) + for v in range(marshal.version + 1): + data = marshal.dumps(obj, v) + with open(support.TESTFN, 'wb') as f: + f.write(data + b'xxxx') + r, p = _testcapi.pymarshal_read_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + self.assertEqual(r, obj) + self.assertEqual(p, len(data)) + + with open(support.TESTFN, 'wb') as f: + f.write(data[:1]) + with self.assertRaises(EOFError): + _testcapi.pymarshal_read_object_from_file(support.TESTFN) + support.unlink(support.TESTFN) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 8b04a5588c0030..3cc018c0cc2caa 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2441,6 +2441,431 @@ def test_free_after_iterating(self): support.check_free_after_iterating(self, reversed, str) +class CAPITest(unittest.TestCase): + + # Test PyUnicode_FromFormat() + def test_from_format(self): + support.import_module('ctypes') + from ctypes import ( + pythonapi, py_object, sizeof, + c_int, c_long, c_longlong, c_ssize_t, + c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) + name = "PyUnicode_FromFormat" + _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.restype = py_object + + def PyUnicode_FromFormat(format, *args): + cargs = tuple( + py_object(arg) if isinstance(arg, str) else arg + for arg in args) + return _PyUnicode_FromFormat(format, *cargs) + + def check_format(expected, format, *args): + text = PyUnicode_FromFormat(format, *args) + self.assertEqual(expected, text) + + # ascii format, non-ascii argument + check_format('ascii\x7f=unicode\xe9', + b'ascii\x7f=%U', 'unicode\xe9') + + # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() + # raises an error + self.assertRaisesRegex(ValueError, + r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' + 'string, got a non-ASCII byte: 0xe9$', + PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') + + # test "%c" + check_format('\uabcd', + b'%c', c_int(0xabcd)) + check_format('\U0010ffff', + b'%c', c_int(0x10ffff)) + with self.assertRaises(OverflowError): + PyUnicode_FromFormat(b'%c', c_int(0x110000)) + # Issue #18183 + check_format('\U00010000\U00100000', + b'%c%c', c_int(0x10000), c_int(0x100000)) + + # test "%" + check_format('%', + b'%') + check_format('%', + b'%%') + check_format('%s', + b'%%s') + check_format('[%]', + b'[%%]') + check_format('%abc', + b'%%%s', b'abc') + + # truncated string + check_format('abc', + b'%.3s', b'abcdef') + check_format('abc[\ufffd', + b'%.5s', 'abc[\u20ac]'.encode('utf8')) + check_format("'\\u20acABC'", + b'%A', '\u20acABC') + check_format("'\\u20", + b'%.5A', '\u20acABCDEF') + check_format("'\u20acABC'", + b'%R', '\u20acABC') + check_format("'\u20acA", + b'%.3R', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3S', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3U', '\u20acABCDEF') + check_format('\u20acAB', + b'%.3V', '\u20acABCDEF', None) + check_format('abc[\ufffd', + b'%.5V', None, 'abc[\u20ac]'.encode('utf8')) + + # following tests comes from #7330 + # test width modifier and precision modifier with %S + check_format("repr= abc", + b'repr=%5S', 'abc') + check_format("repr=ab", + b'repr=%.2S', 'abc') + check_format("repr= ab", + b'repr=%5.2S', 'abc') + + # test width modifier and precision modifier with %R + check_format("repr= 'abc'", + b'repr=%8R', 'abc') + check_format("repr='ab", + b'repr=%.3R', 'abc') + check_format("repr= 'ab", + b'repr=%5.3R', 'abc') + + # test width modifier and precision modifier with %A + check_format("repr= 'abc'", + b'repr=%8A', 'abc') + check_format("repr='ab", + b'repr=%.3A', 'abc') + check_format("repr= 'ab", + b'repr=%5.3A', 'abc') + + # test width modifier and precision modifier with %s + check_format("repr= abc", + b'repr=%5s', b'abc') + check_format("repr=ab", + b'repr=%.2s', b'abc') + check_format("repr= ab", + b'repr=%5.2s', b'abc') + + # test width modifier and precision modifier with %U + check_format("repr= abc", + b'repr=%5U', 'abc') + check_format("repr=ab", + b'repr=%.2U', 'abc') + check_format("repr= ab", + b'repr=%5.2U', 'abc') + + # test width modifier and precision modifier with %V + check_format("repr= abc", + b'repr=%5V', 'abc', b'123') + check_format("repr=ab", + b'repr=%.2V', 'abc', b'123') + check_format("repr= ab", + b'repr=%5.2V', 'abc', b'123') + check_format("repr= 123", + b'repr=%5V', None, b'123') + check_format("repr=12", + b'repr=%.2V', None, b'123') + check_format("repr= 12", + b'repr=%5.2V', None, b'123') + + # test integer formats (%i, %d, %u) + check_format('010', + b'%03i', c_int(10)) + check_format('0010', + b'%0.4i', c_int(10)) + check_format('-123', + b'%i', c_int(-123)) + check_format('-123', + b'%li', c_long(-123)) + check_format('-123', + b'%lli', c_longlong(-123)) + check_format('-123', + b'%zi', c_ssize_t(-123)) + + check_format('-123', + b'%d', c_int(-123)) + check_format('-123', + b'%ld', c_long(-123)) + check_format('-123', + b'%lld', c_longlong(-123)) + check_format('-123', + b'%zd', c_ssize_t(-123)) + + check_format('123', + b'%u', c_uint(123)) + check_format('123', + b'%lu', c_ulong(123)) + check_format('123', + b'%llu', c_ulonglong(123)) + check_format('123', + b'%zu', c_size_t(123)) + + # test long output + min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) + max_longlong = -min_longlong - 1 + check_format(str(min_longlong), + b'%lld', c_longlong(min_longlong)) + check_format(str(max_longlong), + b'%lld', c_longlong(max_longlong)) + max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 + check_format(str(max_ulonglong), + b'%llu', c_ulonglong(max_ulonglong)) + PyUnicode_FromFormat(b'%p', c_void_p(-1)) + + # test padding (width and/or precision) + check_format('123'.rjust(10, '0'), + b'%010i', c_int(123)) + check_format('123'.rjust(100), + b'%100i', c_int(123)) + check_format('123'.rjust(100, '0'), + b'%.100i', c_int(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80i', c_int(123)) + + check_format('123'.rjust(10, '0'), + b'%010u', c_uint(123)) + check_format('123'.rjust(100), + b'%100u', c_uint(123)) + check_format('123'.rjust(100, '0'), + b'%.100u', c_uint(123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80u', c_uint(123)) + + check_format('123'.rjust(10, '0'), + b'%010x', c_int(0x123)) + check_format('123'.rjust(100), + b'%100x', c_int(0x123)) + check_format('123'.rjust(100, '0'), + b'%.100x', c_int(0x123)) + check_format('123'.rjust(80, '0').rjust(100), + b'%100.80x', c_int(0x123)) + + # test %A + check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", + b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') + + # test %V + check_format('repr=abc', + b'repr=%V', 'abc', b'xyz') + + # Test string decode from parameter of %s using utf-8. + # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of + # '\u4eba\u6c11' + check_format('repr=\u4eba\u6c11', + b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') + + #Test replace error handler. + check_format('repr=abc\ufffd', + b'repr=%V', None, b'abc\xff') + + # not supported: copy the raw format string. these tests are just here + # to check for crashes and should not be considered as specifications + check_format('%s', + b'%1%s', b'abc') + check_format('%1abc', + b'%1abc') + check_format('%+i', + b'%+i', c_int(10)) + check_format('%.%s', + b'%.%s', b'abc') + + # Test PyUnicode_AsWideChar() + @support.cpython_only + def test_aswidechar(self): + from _testcapi import unicode_aswidechar + support.import_module('ctypes') + from ctypes import c_wchar, sizeof + + wchar, size = unicode_aswidechar('abcdef', 2) + self.assertEqual(size, 2) + self.assertEqual(wchar, 'ab') + + wchar, size = unicode_aswidechar('abc', 3) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc') + + wchar, size = unicode_aswidechar('abc', 4) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc', 10) + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidechar('abc\0def', 20) + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + + nonbmp = chr(0x10ffff) + if sizeof(c_wchar) == 2: + buflen = 3 + nchar = 2 + else: # sizeof(c_wchar) == 4 + buflen = 2 + nchar = 1 + wchar, size = unicode_aswidechar(nonbmp, buflen) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + + # Test PyUnicode_AsWideCharString() + @support.cpython_only + def test_aswidecharstring(self): + from _testcapi import unicode_aswidecharstring + support.import_module('ctypes') + from ctypes import c_wchar, sizeof + + wchar, size = unicode_aswidecharstring('abc') + self.assertEqual(size, 3) + self.assertEqual(wchar, 'abc\0') + + wchar, size = unicode_aswidecharstring('abc\0def') + self.assertEqual(size, 7) + self.assertEqual(wchar, 'abc\0def\0') + + nonbmp = chr(0x10ffff) + if sizeof(c_wchar) == 2: + nchar = 2 + else: # sizeof(c_wchar) == 4 + nchar = 1 + wchar, size = unicode_aswidecharstring(nonbmp) + self.assertEqual(size, nchar) + self.assertEqual(wchar, nonbmp + '\0') + + # Test PyUnicode_AsUCS4() + @support.cpython_only + def test_asucs4(self): + from _testcapi import unicode_asucs4 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + l = len(s) + self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') + self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) + s = '\0'.join([s, s]) + self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') + + # Test PyUnicode_FindChar() + @support.cpython_only + def test_findchar(self): + from _testcapi import unicode_findchar + + for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": + for i, ch in enumerate(str): + self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) + self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) + + str = "!>_= end + self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) + self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) + # negative + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) + self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) + + # Test PyUnicode_CopyCharacters() + @support.cpython_only + def test_copycharacters(self): + from _testcapi import unicode_copycharacters + + strings = [ + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + + for idx, from_ in enumerate(strings): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises( + SystemError, + unicode_copycharacters, to, 0, from_, 0, 5 + ) + # same kind + for from_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, 0, from_, from_start, 5), + (from_[from_start:from_start+5].ljust(5, '\0'), + 5-from_start) + ) + for to_start in range(5): + self.assertEqual( + unicode_copycharacters(from_, to_start, from_, to_start, 5), + (from_[to_start:to_start+5].rjust(5, '\0'), + 5-to_start) + ) + # narrow -> wide + # Tests omitted since this creates invalid strings. + + s = strings[0] + self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) + self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) + + @support.cpython_only + def test_encode_decimal(self): + from _testcapi import unicode_encodedecimal + self.assertEqual(unicode_encodedecimal('123'), + b'123') + self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'), + b'3.14') + self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"), + b' 3.14 ') + self.assertRaises(UnicodeEncodeError, + unicode_encodedecimal, "123\u20ac", "strict") + self.assertRaisesRegex( + ValueError, + "^'decimal' codec can't encode character", + unicode_encodedecimal, "123\u20ac", "replace") + + @support.cpython_only + def test_transform_decimal(self): + from _testcapi import unicode_transformdecimaltoascii as transform_decimal + self.assertEqual(transform_decimal('123'), + '123') + self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), + '3.14') + self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), + "\N{EM SPACE}3.14\N{EN SPACE}") + self.assertEqual(transform_decimal('123\u20ac'), + '123\u20ac') + + @support.cpython_only + def test_pep393_utf8_caching_bug(self): + # Issue #25709: Problem with string concatenation and utf-8 cache + from _testcapi import getargs_s_hash + for k in 0x24, 0xa4, 0x20ac, 0x1f40d: + s = '' + for i in range(5): + # Due to CPython specific optimization the 's' string can be + # resized in-place. + s += chr(k) + # Parsing with the "s#" format code calls indirectly + # PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + # Check that the second call returns the same result + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): def parse(format): diff --git a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst index 370ee8b66bc806..479299e545472b 100644 --- a/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst +++ b/Misc/NEWS.d/next/Tests/2018-07-29-15-59-51.bpo-34272.lVX2uR.rst @@ -1 +1 @@ -Most C API tests were moved into the new Lib/test/test_capi/ directory. +Some C API tests were moved into the new Lib/test/test_capi/ directory.