bpo-38614: Add timeout constants to test.support (GH-16964)

vstinner · web-flow · commit 24c6258269ac · 2019-10-30T12:41:43.000+01:00
Add timeout constants to test.support:

* LOOPBACK_TIMEOUT
* INTERNET_TIMEOUT
* SHORT_TIMEOUT
* LONG_TIMEOUT
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
@@ -287,6 +287,67 @@ The :mod:`test.support` module defines the following constants:
    Set to a filename containing the :data:`FS_NONASCII` character.
 
 
+.. data:: LOOPBACK_TIMEOUT
+
+   Timeout in seconds for tests using a network server listening on the network
+   local loopback interface like ``127.0.0.1``.
+
+   The timeout is long enough to prevent test failure: it takes into account
+   that the client and the server can run in different threads or even
+   different processes.
+
+   The timeout should be long enough for :meth:`~socket.socket.connect`,
+   :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` methods of
+   :class:`socket.socket`.
+
+   Its default value is 5 seconds.
+
+   See also :data:`INTERNET_TIMEOUT`.
+
+
+.. data:: INTERNET_TIMEOUT
+
+   Timeout in seconds for network requests going to the Internet.
+
+   The timeout is short enough to prevent a test to wait for too long if the
+   Internet request is blocked for whatever reason.
+
+   Usually, a timeout using :data:`INTERNET_TIMEOUT` should not mark a test as
+   failed, but skip the test instead: see
+   :func:`~test.support.transient_internet`.
+
+   Its default value is 1 minute.
+
+   See also :data:`LOOPBACK_TIMEOUT`.
+
+
+.. data:: SHORT_TIMEOUT
+
+   Timeout in seconds to mark a test as failed if the test takes "too long".
+
+   The timeout value depends on the regrtest ``--timeout`` command line option.
+
+   If a test using :data:`SHORT_TIMEOUT` starts to fail randomly on slow
+   buildbots, use :data:`LONG_TIMEOUT` instead.
+
+   Its default value is 30 seconds.
+
+
+.. data:: LONG_TIMEOUT
+
+   Timeout in seconds to detect when a test hangs.
+
+   It is long enough to reduce the risk of test failure on the slowest Python
+   buildbots. It should not be used to mark a test as failed if the test takes
+   "too long".  The timeout value depends on the regrtest ``--timeout`` command
+   line option.
+
+   Its default value is 5 minutes.
+
+   See also :data:`LOOPBACK_TIMEOUT`, :data:`INTERNET_TIMEOUT` and
+   :data:`SHORT_TIMEOUT`.
+
+
 .. data:: IPV6_ENABLED
 
     Set to ``True`` if IPV6 is enabled on this host, ``False`` otherwise.
diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py
@@ -66,12 +66,6 @@
 except ImportError:
     msvcrt = None
 
-#
-#
-#
-
-# Timeout to wait until a process completes
-TIMEOUT = 60.0 # seconds
 
 def latin(s):
     return s.encode('latin')
@@ -86,7 +80,7 @@ def close_queue(queue):
 def join_process(process):
     # Since multiprocessing.Process has the same API than threading.Thread
     # (join() and is_alive(), the support function can be reused
-    support.join_thread(process, timeout=TIMEOUT)
+    support.join_thread(process)
 
 
 if os.name == "posix":
@@ -1128,7 +1122,7 @@ def __reduce__(self):
             q = self.Queue()
             q.put(NotSerializable())
             q.put(True)
-            self.assertTrue(q.get(timeout=TIMEOUT))
+            self.assertTrue(q.get(timeout=support.LONG_TIMEOUT))
             close_queue(q)
 
         with test.support.captured_stderr():
@@ -1531,7 +1525,7 @@ def test_waitfor_timeout(self):
                          args=(cond, state, success, sem))
         p.daemon = True
         p.start()
-        self.assertTrue(sem.acquire(timeout=TIMEOUT))
+        self.assertTrue(sem.acquire(timeout=support.LONG_TIMEOUT))
 
         # Only increment 3 times, so state == 4 is never reached.
         for i in range(3):
@@ -3388,7 +3382,7 @@ class _TestPicklingConnections(BaseTestCase):
     @classmethod
     def tearDownClass(cls):
         from multiprocessing import resource_sharer
-        resource_sharer.stop(timeout=TIMEOUT)
+        resource_sharer.stop(timeout=support.LONG_TIMEOUT)
 
     @classmethod
     def _listener(cls, conn, families):
@@ -4033,7 +4027,7 @@ def test_shared_memory_cleaned_after_process_termination(self):
             p.terminate()
             p.wait()
 
-            deadline = time.monotonic() + 60
+            deadline = time.monotonic() + support.LONG_TIMEOUT
             t = 0.1
             while time.monotonic() < deadline:
                 time.sleep(t)
@@ -5040,7 +5034,7 @@ def create_and_register_resource(rtype):
                 p.terminate()
                 p.wait()
 
-                deadline = time.monotonic() + 60
+                deadline = time.monotonic() + support.LONG_TIMEOUT
                 while time.monotonic() < deadline:
                     time.sleep(.5)
                     try:
diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py
@@ -81,6 +81,17 @@ def _test_audit_hook(name, args):
 
     setup_unraisable_hook()
 
+    if ns.timeout is not None:
+        # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT
+        support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, ns.timeout / 40)
+        support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, ns.timeout / 4)
+
+        # If --timeout is short: reduce timeouts
+        support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, ns.timeout)
+        support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, ns.timeout)
+        support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, ns.timeout)
+        support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, ns.timeout)
+
 
 def suppress_msvcrt_asserts(verbose):
     try:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -119,9 +119,53 @@
     "run_with_locale", "swap_item",
     "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict",
     "run_with_tz", "PGO", "missing_compiler_executable", "fd_count",
-    "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST"
+    "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
+    "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
     ]
 
+
+# Timeout in seconds for tests using a network server listening on the network
+# local loopback interface like 127.0.0.1.
+#
+# The timeout is long enough to prevent test failure: it takes into account
+# that the client and the server can run in different threads or even different
+# processes.
+#
+# The timeout should be long enough for connect(), recv() and send() methods
+# of socket.socket.
+LOOPBACK_TIMEOUT = 5.0
+if sys.platform == 'win32' and platform.machine() == 'ARM':
+    # bpo-37553: test_socket.SendfileUsingSendTest is taking longer than 2
+    # seconds on Windows ARM32 buildbot
+    LOOPBACK_TIMEOUT = 10
+
+# Timeout in seconds for network requests going to the Internet. The timeout is
+# short enough to prevent a test to wait for too long if the Internet request
+# is blocked for whatever reason.
+#
+# Usually, a timeout using INTERNET_TIMEOUT should not mark a test as failed,
+# but skip the test instead: see transient_internet().
+INTERNET_TIMEOUT = 60.0
+
+# Timeout in seconds to mark a test as failed if the test takes "too long".
+#
+# The timeout value depends on the regrtest --timeout command line option.
+#
+# If a test using SHORT_TIMEOUT starts to fail randomly on slow buildbots, use
+# LONG_TIMEOUT instead.
+SHORT_TIMEOUT = 30.0
+
+# Timeout in seconds to detect when a test hangs.
+#
+# It is long enough to reduce the risk of test failure on the slowest Python
+# buildbots. It should not be used to mark a test as failed if the test takes
+# "too long". The timeout value depends on the regrtest --timeout command line
+# option.
+LONG_TIMEOUT = 5 * 60.0
+
+_NOT_SET = object()
+
+
 class Error(Exception):
     """Base class for regression test exceptions."""
 
@@ -1231,7 +1275,7 @@ def check_valid_file(fn):
     opener = urllib.request.build_opener()
     if gzip:
         opener.addheaders.append(('Accept-Encoding', 'gzip'))
-    f = opener.open(url, timeout=15)
+    f = opener.open(url, timeout=INTERNET_TIMEOUT)
     if gzip and f.headers.get('Content-Encoding') == 'gzip':
         f = gzip.GzipFile(fileobj=f)
     try:
@@ -1542,9 +1586,12 @@ def get_socket_conn_refused_errs():
 
 
 @contextlib.contextmanager
-def transient_internet(resource_name, *, timeout=30.0, errnos=()):
+def transient_internet(resource_name, *, timeout=_NOT_SET, errnos=()):
     """Return a context manager that raises ResourceDenied when various issues
     with the Internet connection manifest themselves as exceptions."""
+    if timeout is _NOT_SET:
+        timeout = INTERNET_TIMEOUT
+
     default_errnos = [
         ('ECONNREFUSED', 111),
         ('ECONNRESET', 104),
@@ -2264,7 +2311,7 @@ def decorator(*args):
 
 
 @contextlib.contextmanager
-def wait_threads_exit(timeout=60.0):
+def wait_threads_exit(timeout=None):
     """
     bpo-31234: Context manager to wait until all threads created in the with
     statement exit.
@@ -2278,6 +2325,8 @@ def wait_threads_exit(timeout=60.0):
     which doesn't allow to wait for thread exit, whereas thread.Thread has a
     join() method.
     """
+    if timeout is None:
+        timeout = SHORT_TIMEOUT
     old_count = _thread._count()
     try:
         yield
@@ -2298,10 +2347,12 @@ def wait_threads_exit(timeout=60.0):
             gc_collect()
 
 
-def join_thread(thread, timeout=30.0):
+def join_thread(thread, timeout=None):
     """Join a thread. Raise an AssertionError if the thread is still alive
     after timeout seconds.
     """
+    if timeout is None:
+        timeout = SHORT_TIMEOUT
     thread.join(timeout)
     if thread.is_alive():
         msg = f"failed to join the thread in {timeout:.1f} seconds"
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
@@ -37,7 +37,6 @@
 HOST = support.HOST
 # test unicode string and carriage return
 MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8')
-MAIN_TIMEOUT = 60.0
 
 VSOCKPORT = 1234
 AIX = platform.system() == "AIX"
@@ -2527,7 +2526,7 @@ class SendrecvmsgBase(ThreadSafeCleanupTestCase):
 
     # Time in seconds to wait before considering a test failed, or
     # None for no timeout.  Not all tests actually set a timeout.
-    fail_timeout = 3.0
+    fail_timeout = support.LOOPBACK_TIMEOUT
 
     def setUp(self):
         self.misc_event = threading.Event()
@@ -4320,7 +4319,7 @@ def setUp(self):
         self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler)
 
     # Timeout for socket operations
-    timeout = 4.0
+    timeout = support.LOOPBACK_TIMEOUT
 
     # Provide setAlarm() method to schedule delivery of SIGALRM after
     # given number of seconds, or cancel it if zero, and an
@@ -4610,7 +4609,7 @@ def testAccept(self):
 
         self.event.set()
 
-        read, write, err = select.select([self.serv], [], [], MAIN_TIMEOUT)
+        read, write, err = select.select([self.serv], [], [], support.LONG_TIMEOUT)
         if self.serv not in read:
             self.fail("Error trying to do accept after select.")
 
@@ -4638,7 +4637,7 @@ def testRecv(self):
 
         self.event.set()
 
-        read, write, err = select.select([conn], [], [], MAIN_TIMEOUT)
+        read, write, err = select.select([conn], [], [], support.LONG_TIMEOUT)
         if conn not in read:
             self.fail("Error during select call to non-blocking socket.")
 
@@ -5838,8 +5837,7 @@ class SendfileUsingSendTest(ThreadedTCPSocketTest):
     FILESIZE = (10 * 1024 * 1024)  # 10 MiB
     BUFSIZE = 8192
     FILEDATA = b""
-    # bpo-37553: This is taking longer than 2 seconds on Windows ARM32 buildbot
-    TIMEOUT = 10 if sys.platform == 'win32' and platform.machine() == 'ARM' else 2
+    TIMEOUT = support.LOOPBACK_TIMEOUT
 
     @classmethod
     def setUpClass(cls):
@@ -5865,7 +5863,7 @@ def tearDownClass(cls):
         support.unlink(support.TESTFN)
 
     def accept_conn(self):
-        self.serv.settimeout(MAIN_TIMEOUT)
+        self.serv.settimeout(support.LONG_TIMEOUT)
         conn, addr = self.serv.accept()
         conn.settimeout(self.TIMEOUT)
         self.addCleanup(conn.close)
@@ -6369,7 +6367,7 @@ def test_dualstack_ipv6_family(self):
 
 
 class CreateServerFunctionalTest(unittest.TestCase):
-    timeout = 3
+    timeout = support.LOOPBACK_TIMEOUT
 
     def setUp(self):
         self.thread = None
diff --git a/Misc/NEWS.d/next/Tests/2019-10-28-15-56-02.bpo-38614.aDdDYE.rst b/Misc/NEWS.d/next/Tests/2019-10-28-15-56-02.bpo-38614.aDdDYE.rst
@@ -0,0 +1,4 @@
+Add timeout constants to :mod:`test.support`:
+:data:`~test.support.LOOPBACK_TIMEOUT`,
+:data:`~test.support.INTERNET_TIMEOUT`, :data:`~test.support.SHORT_TIMEOUT`
+and :data:`~test.support.LONG_TIMEOUT`.
