gh-110171: libregrtest always sets random.seed (#110172)

sobolevn · web-flow · commit 1465386720cd · 2023-10-04T06:42:12.000Z
diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
@@ -27,8 +27,10 @@
 Additional option details:
 
 -r randomizes test execution order. You can use --randseed=int to provide an
-int seed value for the randomizer; this is useful for reproducing troublesome
-test orders.
+int seed value for the randomizer. The randseed value will be used
+to set seeds for all random usages in tests
+(including randomizing the tests order if -r is set).
+By default we always set random seed, but do not randomize test order.
 
 -s On the first invocation of regrtest using -s, the first test file found
 or the first test file given on the command line is run, and the name of
@@ -229,6 +231,9 @@ def _create_parser():
                             more_details)
     group.add_argument('-p', '--python', metavar='PYTHON',
                        help='Command to run Python test subprocesses with.')
+    group.add_argument('--randseed', metavar='SEED',
+                       dest='random_seed', type=int,
+                       help='pass a global random seed')
 
     group = parser.add_argument_group('Verbosity')
     group.add_argument('-v', '--verbose', action='count',
@@ -249,10 +254,6 @@ def _create_parser():
     group = parser.add_argument_group('Selecting tests')
     group.add_argument('-r', '--randomize', action='store_true',
                        help='randomize test execution order.' + more_details)
-    group.add_argument('--randseed', metavar='SEED',
-                       dest='random_seed', type=int,
-                       help='pass a random seed to reproduce a previous '
-                            'random run')
     group.add_argument('-f', '--fromfile', metavar='FILE',
                        help='read names of tests to run from a file.' +
                             more_details)
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
@@ -129,7 +129,11 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
 
         # Randomize
         self.randomize: bool = ns.randomize
-        self.random_seed: int | None = ns.random_seed
+        self.random_seed: int | None =  (
+            ns.random_seed
+            if ns.random_seed is not None
+            else random.getrandbits(32)
+        )
         if 'SOURCE_DATE_EPOCH' in os.environ:
             self.randomize = False
             self.random_seed = None
@@ -214,10 +218,8 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList
                 print(f"Cannot find starting test: {self.starting_test}")
                 sys.exit(1)
 
+        random.seed(self.random_seed)
         if self.randomize:
-            if self.random_seed is None:
-                self.random_seed = random.randrange(100_000_000)
-            random.seed(self.random_seed)
             random.shuffle(selected)
 
         return (tuple(selected), tests)
@@ -439,8 +441,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
                    or tests or self.cmdline_args)):
             display_header(self.use_resources, self.python_cmd)
 
-        if self.randomize:
-            print("Using random seed", self.random_seed)
+        print("Using random seed", self.random_seed)
 
         runtests = self.create_run_tests(selected)
         self.first_runtests = runtests
diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py
@@ -126,5 +126,4 @@ def setup_tests(runtests: RunTests):
     if runtests.gc_threshold is not None:
         gc.set_threshold(runtests.gc_threshold)
 
-    if runtests.randomize:
-        random.seed(runtests.random_seed)
+    random.seed(runtests.random_seed)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
@@ -392,7 +392,7 @@ def check_ci_mode(self, args, use_resources, rerun=True):
         self.assertEqual(regrtest.num_workers, -1)
         self.assertEqual(regrtest.want_rerun, rerun)
         self.assertTrue(regrtest.randomize)
-        self.assertIsNone(regrtest.random_seed)
+        self.assertIsInstance(regrtest.random_seed, int)
         self.assertTrue(regrtest.fail_env_changed)
         self.assertTrue(regrtest.fail_rerun)
         self.assertTrue(regrtest.print_slowest)
@@ -663,7 +663,7 @@ def list_regex(line_format, tests):
     def parse_random_seed(self, output):
         match = self.regex_search(r'Using random seed ([0-9]+)', output)
         randseed = int(match.group(1))
-        self.assertTrue(0 <= randseed <= 100_000_000, randseed)
+        self.assertTrue(0 <= randseed, randseed)
         return randseed
 
     def run_command(self, args, input=None, exitcode=0, **kw):
@@ -950,6 +950,10 @@ def test_random(self):
         test_random2 = int(match.group(1))
         self.assertEqual(test_random2, test_random)
 
+        # check that random.seed is used by default
+        output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN)
+        self.assertIsInstance(self.parse_random_seed(output), int)
+
     def test_fromfile(self):
         # test --fromfile
         tests = [self.create_test() for index in range(5)]
diff --git a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst
@@ -0,0 +1,3 @@
+``libregrtest`` now always sets and shows ``random.seed``,
+so tests are more reproducible. Use ``--randseed`` flag
+to pass the explicit random seed for tests.

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	+``libregrtest`` now always sets and shows ``random.seed``,
	`2`	+so tests are more reproducible. Use ``--randseed`` flag
	`3`	`+to pass the explicit random seed for tests.`