From a052bf7f4dc2bce34753f3335fa154d9c109ac06 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 31 May 2022 09:52:20 +0300 Subject: [PATCH 1/8] Port server cases to IsolatedAsyncioTestCase --- Lib/test/test_asyncio/test_streams.py | 234 +++++++----------- ...2-05-31-09-58-10.gh-issue-93357.GL6KS0.rst | 2 + 2 files changed, 88 insertions(+), 148 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 098a0da344d0fb..c7a8cb575a7937 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1,5 +1,6 @@ """Tests for streams.py.""" +import asyncio import gc import os import queue @@ -9,15 +10,10 @@ import threading import unittest from unittest import mock -from test.support import socket_helper -try: - import ssl -except ImportError: - ssl = None - -import asyncio +from test.support import import_helper, socket_helper from test.test_asyncio import utils as test_utils +ssl = import_helper.import_module('ssl') def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -566,46 +562,10 @@ def test_exception_cancel(self): test_utils.run_briefly(self.loop) self.assertIs(stream._waiter, None) - def test_start_server(self): - - class MyServer: - def __init__(self, loop): - self.server = None - self.loop = loop +class NewStreamTests(unittest.IsolatedAsyncioTestCase): - async def handle_client(self, client_reader, client_writer): - data = await client_reader.readline() - client_writer.write(data) - await client_writer.drain() - client_writer.close() - await client_writer.wait_closed() - - def start(self): - sock = socket.create_server(('127.0.0.1', 0)) - self.server = self.loop.run_until_complete( - asyncio.start_server(self.handle_client, - sock=sock)) - return sock.getsockname() - - def handle_client_callback(self, client_reader, client_writer): - self.loop.create_task(self.handle_client(client_reader, - client_writer)) - - def start_callback(self): - sock = socket.create_server(('127.0.0.1', 0)) - addr = sock.getsockname() - sock.close() - self.server = self.loop.run_until_complete( - asyncio.start_server(self.handle_client_callback, - host=addr[0], port=addr[1])) - return addr - - def stop(self): - if self.server is not None: - self.server.close() - self.loop.run_until_complete(self.server.wait_closed()) - self.server = None + async def test_start_server(self): async def client(addr): reader, writer = await asyncio.open_connection(*addr) @@ -617,61 +577,42 @@ async def client(addr): await writer.wait_closed() return msgback - messages = [] - self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + async def handle_client(client_reader, client_writer): + data = await client_reader.readline() + client_writer.write(data) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() # test the server variant with a coroutine as client handler - server = MyServer(self.loop) - addr = server.start() - msg = self.loop.run_until_complete(self.loop.create_task(client(addr))) - server.stop() + server = await asyncio.start_server( + handle_client, + host=socket_helper.HOSTv4 + ) + addr = server.sockets[0].getsockname() + msg = await client(addr) + server.close() + await server.wait_closed() self.assertEqual(msg, b"hello world!\n") # test the server variant with a callback as client handler - server = MyServer(self.loop) - addr = server.start_callback() - msg = self.loop.run_until_complete(self.loop.create_task(client(addr))) - server.stop() - self.assertEqual(msg, b"hello world!\n") + async def handle_client_with_callback(client_reader, client_writer): + asyncio.get_running_loop().create_task( + handle_client(client_reader, client_writer) + ) - self.assertEqual(messages, []) + server = await asyncio.start_server( + handle_client_with_callback, + host=socket_helper.HOSTv4 + ) + address, port = server.sockets[0].getsockname() + msg = await client(addr) + server.close() + await server.wait_closed() + self.assertEqual(msg, b"hello world!\n") @socket_helper.skip_unless_bind_unix_socket - def test_start_unix_server(self): - - class MyServer: - - def __init__(self, loop, path): - self.server = None - self.loop = loop - self.path = path - - async def handle_client(self, client_reader, client_writer): - data = await client_reader.readline() - client_writer.write(data) - await client_writer.drain() - client_writer.close() - await client_writer.wait_closed() - - def start(self): - self.server = self.loop.run_until_complete( - asyncio.start_unix_server(self.handle_client, - path=self.path)) - - def handle_client_callback(self, client_reader, client_writer): - self.loop.create_task(self.handle_client(client_reader, - client_writer)) - - def start_callback(self): - start = asyncio.start_unix_server(self.handle_client_callback, - path=self.path) - self.server = self.loop.run_until_complete(start) - - def stop(self): - if self.server is not None: - self.server.close() - self.loop.run_until_complete(self.server.wait_closed()) - self.server = None + async def test_start_unix_server(self): async def client(path): reader, writer = await asyncio.open_unix_connection(path) @@ -683,64 +624,43 @@ async def client(path): await writer.wait_closed() return msgback - messages = [] - self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + async def handle_client(client_reader, client_writer): + data = await client_reader.readline() + client_writer.write(data) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() # test the server variant with a coroutine as client handler with test_utils.unix_socket_path() as path: - server = MyServer(self.loop, path) - server.start() - msg = self.loop.run_until_complete( - self.loop.create_task(client(path))) - server.stop() + server = await asyncio.start_unix_server( + handle_client, + path=path + ) + msg = await client(path) + server.close() + await server.wait_closed() self.assertEqual(msg, b"hello world!\n") # test the server variant with a callback as client handler + async def handle_client_with_callback(client_reader, client_writer): + asyncio.get_running_loop().create_task( + handle_client(client_reader, client_writer) + ) + with test_utils.unix_socket_path() as path: - server = MyServer(self.loop, path) + server = await asyncio.start_unix_server( + handle_client, + path=path + ) server.start_callback() - msg = self.loop.run_until_complete( - self.loop.create_task(client(path))) - server.stop() + msg = await client(path) + server.close() + await server.wait_closed() self.assertEqual(msg, b"hello world!\n") - self.assertEqual(messages, []) - @unittest.skipIf(ssl is None, 'No ssl module') - def test_start_tls(self): - - class MyServer: - - def __init__(self, loop): - self.server = None - self.loop = loop - - async def handle_client(self, client_reader, client_writer): - data1 = await client_reader.readline() - client_writer.write(data1) - await client_writer.drain() - assert client_writer.get_extra_info('sslcontext') is None - await client_writer.start_tls( - test_utils.simple_server_sslcontext()) - assert client_writer.get_extra_info('sslcontext') is not None - data2 = await client_reader.readline() - client_writer.write(data2) - await client_writer.drain() - client_writer.close() - await client_writer.wait_closed() - - def start(self): - sock = socket.create_server(('127.0.0.1', 0)) - self.server = self.loop.run_until_complete( - asyncio.start_server(self.handle_client, - sock=sock)) - return sock.getsockname() - - def stop(self): - if self.server is not None: - self.server.close() - self.loop.run_until_complete(self.server.wait_closed()) - self.server = None + async def test_start_tls(self): async def client(addr): reader, writer = await asyncio.open_connection(*addr) @@ -757,18 +677,36 @@ async def client(addr): await writer.wait_closed() return msgback1, msgback2 - messages = [] - self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) - - server = MyServer(self.loop) - addr = server.start() - msg1, msg2 = self.loop.run_until_complete(client(addr)) - server.stop() - - self.assertEqual(messages, []) + async def handle_client(client_reader, client_writer): + data1 = await client_reader.readline() + client_writer.write(data1) + await client_writer.drain() + assert client_writer.get_extra_info('sslcontext') is None + await client_writer.start_tls( + test_utils.simple_server_sslcontext()) + assert client_writer.get_extra_info('sslcontext') is not None + + data2 = await client_reader.readline() + client_writer.write(data2) + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + server = await asyncio.start_server( + handle_client, + host=socket_helper.HOSTv4 + ) + addr = server.sockets[0].getsockname() + + msg1, msg2 = await client(addr) + server.close() + await server.wait_closed() self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") + +class StreamTests2(test_utils.TestCase): + @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") def test_read_all_from_pipe_reader(self): # See asyncio issue 168. This test is derived from the example diff --git a/Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst b/Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst new file mode 100644 index 00000000000000..3775a5d3ee1bc1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst @@ -0,0 +1,2 @@ +:mod:`test.test_asyncio.test_streams` temporarily introduced two new tests, +``NewStreamTests`` and ``StreamTests2``. From 1f8977b0c223167031a845c3c225a53c64433183 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 31 May 2022 11:00:38 +0300 Subject: [PATCH 2/8] Add forgotten setUp and tearDown into StreamTests2 --- Lib/test/test_asyncio/test_streams.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index c7a8cb575a7937..7ec489f306b09f 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -707,6 +707,19 @@ async def handle_client(client_reader, client_writer): class StreamTests2(test_utils.TestCase): + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def tearDown(self): + # just in case if we have transport close callbacks + test_utils.run_briefly(self.loop) + + self.loop.close() + gc.collect() + super().tearDown() + @unittest.skipIf(sys.platform == 'win32', "Don't have pipes") def test_read_all_from_pipe_reader(self): # See asyncio issue 168. This test is derived from the example From 3205970760b83a4e7ea7eb1f7ca52d75fab6055f Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 31 May 2022 11:12:24 +0300 Subject: [PATCH 3/8] Use subtests to localize errors --- Lib/test/test_asyncio/test_streams.py | 127 ++++++++++++-------------- 1 file changed, 59 insertions(+), 68 deletions(-) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 7ec489f306b09f..90abc20793d42d 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -584,32 +584,32 @@ async def handle_client(client_reader, client_writer): client_writer.close() await client_writer.wait_closed() - # test the server variant with a coroutine as client handler - server = await asyncio.start_server( - handle_client, - host=socket_helper.HOSTv4 - ) - addr = server.sockets[0].getsockname() - msg = await client(addr) - server.close() - await server.wait_closed() - self.assertEqual(msg, b"hello world!\n") - - # test the server variant with a callback as client handler - async def handle_client_with_callback(client_reader, client_writer): - asyncio.get_running_loop().create_task( - handle_client(client_reader, client_writer) + with self.subTest(msg="coroutine"): + server = await asyncio.start_server( + handle_client, + host=socket_helper.HOSTv4 ) + addr = server.sockets[0].getsockname() + msg = await client(addr) + server.close() + await server.wait_closed() + self.assertEqual(msg, b"hello world!\n") - server = await asyncio.start_server( - handle_client_with_callback, - host=socket_helper.HOSTv4 - ) - address, port = server.sockets[0].getsockname() - msg = await client(addr) - server.close() - await server.wait_closed() - self.assertEqual(msg, b"hello world!\n") + with self.subTest(msg="callback"): + async def handle_client_callback(client_reader, client_writer): + asyncio.get_running_loop().create_task( + handle_client(client_reader, client_writer) + ) + + server = await asyncio.start_server( + handle_client_callback, + host=socket_helper.HOSTv4 + ) + address, port = server.sockets[0].getsockname() + msg = await client(addr) + server.close() + await server.wait_closed() + self.assertEqual(msg, b"hello world!\n") @socket_helper.skip_unless_bind_unix_socket async def test_start_unix_server(self): @@ -631,33 +631,33 @@ async def handle_client(client_reader, client_writer): client_writer.close() await client_writer.wait_closed() - # test the server variant with a coroutine as client handler - with test_utils.unix_socket_path() as path: - server = await asyncio.start_unix_server( - handle_client, - path=path - ) - msg = await client(path) - server.close() - await server.wait_closed() - self.assertEqual(msg, b"hello world!\n") - - # test the server variant with a callback as client handler - async def handle_client_with_callback(client_reader, client_writer): - asyncio.get_running_loop().create_task( - handle_client(client_reader, client_writer) - ) - - with test_utils.unix_socket_path() as path: - server = await asyncio.start_unix_server( - handle_client, - path=path - ) - server.start_callback() - msg = await client(path) - server.close() - await server.wait_closed() - self.assertEqual(msg, b"hello world!\n") + with self.subTest(msg="coroutine"): + with test_utils.unix_socket_path() as path: + server = await asyncio.start_unix_server( + handle_client, + path=path + ) + msg = await client(path) + server.close() + await server.wait_closed() + self.assertEqual(msg, b"hello world!\n") + + with self.subTest(msg="callback"): + async def handle_client_callback(client_reader, client_writer): + asyncio.get_running_loop().create_task( + handle_client(client_reader, client_writer) + ) + + with test_utils.unix_socket_path() as path: + server = await asyncio.start_unix_server( + handle_client_callback, + path=path + ) + server.start_callback() + msg = await client(path) + server.close() + await server.wait_closed() + self.assertEqual(msg, b"hello world!\n") @unittest.skipIf(ssl is None, 'No ssl module') async def test_start_tls(self): @@ -918,22 +918,20 @@ def test_LimitOverrunError_pickleable(self): self.assertEqual(str(e), str(e2)) self.assertEqual(e.consumed, e2.consumed) - def test_wait_closed_on_close(self): - with test_utils.run_test_server() as httpd: + async def test_wait_closed_on_close(self): + async with test_utils.run_test_server() as httpd: rd, wr = self.loop.run_until_complete( asyncio.open_connection(*httpd.address)) wr.write(b'GET / HTTP/1.0\r\n\r\n') - f = rd.readline() - data = self.loop.run_until_complete(f) + await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') - f = rd.read() - data = self.loop.run_until_complete(f) + await rd.read() self.assertTrue(data.endswith(b'\r\n\r\nTest message')) self.assertFalse(wr.is_closing()) wr.close() self.assertTrue(wr.is_closing()) - self.loop.run_until_complete(wr.wait_closed()) + await wr.wait_closed() def test_wait_closed_on_close_with_unread_data(self): with test_utils.run_test_server() as httpd: @@ -989,15 +987,10 @@ async def inner(httpd): self.assertEqual(messages, []) - def test_eof_feed_when_closing_writer(self): + async def test_eof_feed_when_closing_writer(self): # See http://bugs.python.org/issue35065 - messages = [] - self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) - - with test_utils.run_test_server() as httpd: - rd, wr = self.loop.run_until_complete( - asyncio.open_connection(*httpd.address)) - + async with test_utils.run_test_server() as httpd: + rd, wr = await asyncio.open_connection(*httpd.address) wr.close() f = wr.wait_closed() self.loop.run_until_complete(f) @@ -1006,8 +999,6 @@ def test_eof_feed_when_closing_writer(self): data = self.loop.run_until_complete(f) self.assertEqual(data, b'') - self.assertEqual(messages, []) - if __name__ == '__main__': unittest.main() From 9f32482796c85955c2e074e7ea205d22505b3b4d Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 31 May 2022 11:30:19 +0300 Subject: [PATCH 4/8] Fix a wrongly used address --- Lib/test/test_asyncio/test_streams.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 90abc20793d42d..6628182b55e32e 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -605,7 +605,8 @@ async def handle_client_callback(client_reader, client_writer): handle_client_callback, host=socket_helper.HOSTv4 ) - address, port = server.sockets[0].getsockname() + addr = server.sockets[0].getsockname() + reader, writer = await asyncio.open_connection(*addr) msg = await client(addr) server.close() await server.wait_closed() From e41c7ca738e4428681fc4d08b9b11b6c28d5cdbc Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 31 May 2022 11:37:09 +0300 Subject: [PATCH 5/8] Fix another typo --- Lib/test/test_asyncio/test_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 6628182b55e32e..2847cfccdc55e9 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -654,7 +654,7 @@ async def handle_client_callback(client_reader, client_writer): handle_client_callback, path=path ) - server.start_callback() + server.start() msg = await client(path) server.close() await server.wait_closed() From 35e4f269177aea4ecd692ef79816a900baa3893f Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Tue, 31 May 2022 11:53:02 +0300 Subject: [PATCH 6/8] Remove manual server starting --- Lib/test/test_asyncio/test_streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 2847cfccdc55e9..4a53a60b45ac28 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -654,7 +654,6 @@ async def handle_client_callback(client_reader, client_writer): handle_client_callback, path=path ) - server.start() msg = await client(path) server.close() await server.wait_closed() From 92e30e52506567b141d5bbe1efc7355908a3579a Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 1 Jun 2022 09:39:10 +0300 Subject: [PATCH 7/8] Address a review of Serhiy Storchaka --- Lib/test/test_asyncio/test_streams.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 4a53a60b45ac28..3c53f960ef5db1 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1,6 +1,5 @@ """Tests for streams.py.""" -import asyncio import gc import os import queue @@ -10,10 +9,15 @@ import threading import unittest from unittest import mock -from test.support import import_helper, socket_helper +from test.support import socket_helper +try: + import ssl +except ImportError: + ssl = None + +import asyncio from test.test_asyncio import utils as test_utils -ssl = import_helper.import_module('ssl') def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -924,7 +928,7 @@ async def test_wait_closed_on_close(self): asyncio.open_connection(*httpd.address)) wr.write(b'GET / HTTP/1.0\r\n\r\n') - await rd.readline() + data = await rd.readline() self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') await rd.read() self.assertTrue(data.endswith(b'\r\n\r\nTest message')) From dcc10ddf47c13e8e2700e7ba776e5764f3b05c9f Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 2 Oct 2022 21:25:36 +0400 Subject: [PATCH 8/8] Remove a What's New entry as proposed by Nikita Sobolev --- .../next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst diff --git a/Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst b/Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst deleted file mode 100644 index 3775a5d3ee1bc1..00000000000000 --- a/Misc/NEWS.d/next/Tests/2022-05-31-09-58-10.gh-issue-93357.GL6KS0.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`test.test_asyncio.test_streams` temporarily introduced two new tests, -``NewStreamTests`` and ``StreamTests2``.