From c5ff3f564046f7ad61282946c82ce2c1427e05b8 Mon Sep 17 00:00:00 2001 From: ankit Date: Sat, 15 Apr 2023 21:04:24 +0530 Subject: [PATCH 01/63] add support for os fork --- Lib/test/test_os.py | 29 +++++++++++++++++++++++++++++ vm/src/stdlib/os.rs | 5 +++++ 2 files changed, 34 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d801bd8ec0..0883b1be52 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4596,6 +4596,7 @@ def test_times(self): self.assertEqual(times.elapsed, 0) + @requires_os_func('fork') class ForkTests(unittest.TestCase): def test_fork(self): @@ -4611,6 +4612,34 @@ def test_fork(self): assert_python_ok("-c", code) assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + @unittest.skipUnless(sys.platform in ("linux", "darwin"), + "Only Linux and macOS detect this today.") + def test_fork_warns_when_non_python_thread_exists(self): + code = """if 1: + import os, threading, warnings + from _testcapi import _spawn_pthread_waiter, _end_spawned_pthread + _spawn_pthread_waiter() + try: + with warnings.catch_warnings(record=True) as ws: + warnings.filterwarnings( + "always", category=DeprecationWarning) + if os.fork() == 0: + assert not ws, f"unexpected warnings in child: {ws}" + os._exit(0) # child + else: + assert ws[0].category == DeprecationWarning, ws[0] + assert 'fork' in str(ws[0].message), ws[0] + # Waiting allows an error in the child to hit stderr. + exitcode = os.wait()[1] + assert exitcode == 0, f"child exited {exitcode}" + assert threading.active_count() == 1, threading.enumerate() + finally: + _end_spawned_pthread() + """ + _, out, err = assert_python_ok("-c", code, PYTHONOPTIMIZE='0') + self.assertEqual(err.decode("utf-8"), "") + self.assertEqual(out.decode("utf-8"), "") + # Only test if the C version is provided, otherwise TestPEP519 already tested # the pure Python implementation. diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 6c822822f4..642b1ef199 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1127,6 +1127,11 @@ pub(super) mod _os { OutputMode::String.process_path(curdir_inner(vm)?, vm) } + #[pyfunction] + fn fork(vm: &VirtualMachine) -> PyObjectRef { + unsafe { vm.ctx.new_int(libc::fork()).into() } + } + #[pyfunction] fn getcwdb(vm: &VirtualMachine) -> PyResult { OutputMode::Bytes.process_path(curdir_inner(vm)?, vm) From c2dd77d34b1d3c08a35a48b3686ba5864a3eb421 Mon Sep 17 00:00:00 2001 From: ankit Date: Sun, 16 Apr 2023 15:10:18 +0530 Subject: [PATCH 02/63] WIP: add stub functions for registring callbacks --- vm/src/stdlib/os.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 642b1ef199..1017416f8b 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1127,9 +1127,41 @@ pub(super) mod _os { OutputMode::String.process_path(curdir_inner(vm)?, vm) } + #[pyfunction] + fn register_at_fork( + func: PyObjectRef, + before: OptionalArg, + after_in_parent: OptionalArg, + after_in_child: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<()> { + Ok(()) + } + fn run_at_forkers() {} + fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } + fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } + + fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { + Ok(()) + } + #[pyfunction] fn fork(vm: &VirtualMachine) -> PyObjectRef { - unsafe { vm.ctx.new_int(libc::fork()).into() } + let mut pid: i32 = 0; + //PyOS_BeforeFork(); + unsafe { + pid = libc::fork(); + } + if (pid == 0) { + // PyOS_AfterFork_Child(); + } else { + // PyOS_AfterFork_Parent(); + } + vm.ctx.new_int(pid).into() } #[pyfunction] From bd25a548fcea6210abc0f6ea0ebaab1de032bf99 Mon Sep 17 00:00:00 2001 From: ankit Date: Sun, 16 Apr 2023 19:52:14 +0530 Subject: [PATCH 03/63] fill stubs --- vm/src/stdlib/os.rs | 25 ++++++++++++++++++++++++- vm/src/vm/mod.rs | 6 ++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 1017416f8b..c9595c1e23 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1129,23 +1129,46 @@ pub(super) mod _os { #[pyfunction] fn register_at_fork( - func: PyObjectRef, before: OptionalArg, after_in_parent: OptionalArg, after_in_child: OptionalArg, vm: &VirtualMachine, ) -> PyResult<()> { + match before { + OptionalArg::Present(before) => vm.state.before_forkers.lock().push(before), + _ => {} + } + match after_in_parent { + OptionalArg::Present(after_in_parent) => { + vm.state.after_forkers_parent.lock().push(after_in_parent) + } + _ => {} + } + + match after_in_child { + OptionalArg::Present(after_in_child) => { + vm.state.after_forkers_child.lock().push(after_in_child) + } + _ => {} + } + Ok(()) } fn run_at_forkers() {} fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { + let mut before_forkers: Vec = + std::mem::take(&mut *vm.state.before_forkers.lock()); Ok(()) } fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { + let mut after_forkers_child: Vec = + std::mem::take(&mut *vm.state.after_forkers_child.lock()); Ok(()) } fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { + let mut after_forkers_parent: Vec = + std::mem::take(&mut *vm.state.after_forkers_parent.lock()); Ok(()) } diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 11991fa6c8..3c0fc000d9 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -98,6 +98,9 @@ pub struct PyGlobalState { pub finalizing: AtomicBool, pub warnings: WarningsState, pub override_frozen_modules: AtomicCell, + pub before_forkers: PyMutex>, + pub after_forkers_child: PyMutex>, + pub after_forkers_parent: PyMutex>, } pub fn process_hash_secret_seed() -> u32 { @@ -175,6 +178,9 @@ impl VirtualMachine { finalizing: AtomicBool::new(false), warnings, override_frozen_modules: AtomicCell::new(0), + before_forkers: PyMutex::default(), + after_forkers_child: PyMutex::default(), + after_forkers_parent: PyMutex::default(), }), initialized: false, recursion_depth: Cell::new(0), From 2efe9bd0c5a0c7f11284df620110a3feb1b3cec8 Mon Sep 17 00:00:00 2001 From: ankit Date: Sun, 16 Apr 2023 20:52:16 +0530 Subject: [PATCH 04/63] make things functional --- vm/src/stdlib/os.rs | 48 +++++++++++++++++++++++---------------------- vm/src/vm/mod.rs | 8 ++++---- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index c9595c1e23..623580223d 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1129,60 +1129,62 @@ pub(super) mod _os { #[pyfunction] fn register_at_fork( - before: OptionalArg, - after_in_parent: OptionalArg, - after_in_child: OptionalArg, + before: PyObjectRef, + after_in_parent: PyObjectRef, + after_in_child: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { - match before { - OptionalArg::Present(before) => vm.state.before_forkers.lock().push(before), - _ => {} - } - match after_in_parent { - OptionalArg::Present(after_in_parent) => { - vm.state.after_forkers_parent.lock().push(after_in_parent) - } - _ => {} - } + vm.state.before_forkers.lock().push(before); - match after_in_child { - OptionalArg::Present(after_in_child) => { - vm.state.after_forkers_child.lock().push(after_in_child) - } - _ => {} - } + vm.state.after_forkers_parent.lock().push(after_in_parent); + + vm.state.after_forkers_child.lock().push(after_in_child); Ok(()) } - fn run_at_forkers() {} + fn run_at_forkers(funcs: Vec, vm: &VirtualMachine) { + for (func) in funcs.into_iter().rev() { + if let Err(e) = func.call((), vm) { + let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); + vm.run_unraisable(e, Some("Error in atexit._run_exitfuncs".to_owned()), func); + if exit { + break; + } + } + } + } fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { let mut before_forkers: Vec = std::mem::take(&mut *vm.state.before_forkers.lock()); + run_at_forkers(before_forkers, vm); + Ok(()) } fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { let mut after_forkers_child: Vec = std::mem::take(&mut *vm.state.after_forkers_child.lock()); + run_at_forkers(after_forkers_child, vm); Ok(()) } fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { let mut after_forkers_parent: Vec = std::mem::take(&mut *vm.state.after_forkers_parent.lock()); + run_at_forkers(after_forkers_parent, vm); Ok(()) } #[pyfunction] fn fork(vm: &VirtualMachine) -> PyObjectRef { let mut pid: i32 = 0; - //PyOS_BeforeFork(); + py_os_before_fork(vm); unsafe { pid = libc::fork(); } if (pid == 0) { - // PyOS_AfterFork_Child(); + py_os_after_fork_child(vm); } else { - // PyOS_AfterFork_Parent(); + py_os_after_fork_parent(vm); } vm.ctx.new_int(pid).into() } diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 3c0fc000d9..2fac1f0150 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -28,7 +28,7 @@ use crate::{ convert::ToPyObject, frame::{ExecutionResult, Frame, FrameRef}, frozen, - function::{ArgMapping, FuncArgs, PySetterValue}, + function::{ArgCallable, ArgMapping, FuncArgs, PySetterValue}, import, protocol::PyIterIter, scope::Scope, @@ -98,9 +98,9 @@ pub struct PyGlobalState { pub finalizing: AtomicBool, pub warnings: WarningsState, pub override_frozen_modules: AtomicCell, - pub before_forkers: PyMutex>, - pub after_forkers_child: PyMutex>, - pub after_forkers_parent: PyMutex>, + pub before_forkers: PyMutex>, + pub after_forkers_child: PyMutex>, + pub after_forkers_parent: PyMutex>, } pub fn process_hash_secret_seed() -> u32 { From 3ee4da8130260388f57403f6548141d8c3772025 Mon Sep 17 00:00:00 2001 From: ankit Date: Sun, 16 Apr 2023 21:15:30 +0530 Subject: [PATCH 05/63] make args as kwargs for register at fork --- vm/src/stdlib/os.rs | 62 ++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 623580223d..7db470ed54 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1128,65 +1128,75 @@ pub(super) mod _os { } #[pyfunction] - fn register_at_fork( - before: PyObjectRef, - after_in_parent: PyObjectRef, - after_in_child: PyObjectRef, - vm: &VirtualMachine, - ) -> PyResult<()> { - vm.state.before_forkers.lock().push(before); - - vm.state.after_forkers_parent.lock().push(after_in_parent); + fn register_at_fork(kwargs: crate::function::KwArgs, vm: &VirtualMachine) -> PyResult<()> { + let mut match_found = false; // better way to handle this? + for (key, value) in kwargs.into_iter() { + if key == "before" { + match_found = true; + vm.state.before_forkers.lock().push(value.clone()); + } + if key == "after_in_parent" { + match_found = true; + vm.state.after_forkers_parent.lock().push(value.clone()); + } + if key == "after_in_child" { + match_found = true; + vm.state.after_forkers_child.lock().push(value.clone()); + } + } - vm.state.after_forkers_child.lock().push(after_in_child); + if !match_found { + return Err(vm.new_value_error("At least one argument is required.".to_owned())); + } Ok(()) } fn run_at_forkers(funcs: Vec, vm: &VirtualMachine) { - for (func) in funcs.into_iter().rev() { - if let Err(e) = func.call((), vm) { - let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); - vm.run_unraisable(e, Some("Error in atexit._run_exitfuncs".to_owned()), func); - if exit { - break; + if funcs.len() > 0 { + for func in funcs.into_iter().rev() { + if let Err(e) = func.call((), vm) { + let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); + vm.run_unraisable(e, Some("Error in atexit._run_exitfuncs".to_owned()), func); + if exit { + // Do nothing! + } } } } } fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { - let mut before_forkers: Vec = - std::mem::take(&mut *vm.state.before_forkers.lock()); + let before_forkers: Vec = std::mem::take(&mut *vm.state.before_forkers.lock()); run_at_forkers(before_forkers, vm); Ok(()) } fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { - let mut after_forkers_child: Vec = + let after_forkers_child: Vec = std::mem::take(&mut *vm.state.after_forkers_child.lock()); run_at_forkers(after_forkers_child, vm); Ok(()) } fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { - let mut after_forkers_parent: Vec = + let after_forkers_parent: Vec = std::mem::take(&mut *vm.state.after_forkers_parent.lock()); run_at_forkers(after_forkers_parent, vm); Ok(()) } #[pyfunction] - fn fork(vm: &VirtualMachine) -> PyObjectRef { + fn fork(vm: &VirtualMachine) -> PyResult { let mut pid: i32 = 0; - py_os_before_fork(vm); + py_os_before_fork(vm)?; unsafe { pid = libc::fork(); } - if (pid == 0) { - py_os_after_fork_child(vm); + if pid == 0 { + py_os_after_fork_child(vm)?; } else { - py_os_after_fork_parent(vm); + py_os_after_fork_parent(vm)?; } - vm.ctx.new_int(pid).into() + Ok(vm.ctx.new_int(pid).into()) } #[pyfunction] From c6da3798b61815ecfcaf669982cad67336a67c4a Mon Sep 17 00:00:00 2001 From: ankit Date: Sun, 16 Apr 2023 21:44:27 +0530 Subject: [PATCH 06/63] change error text as per as cpython --- vm/src/stdlib/os.rs | 6 +++++- vm/src/vm/mod.rs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 7db470ed54..cb1f476c8c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1156,7 +1156,11 @@ pub(super) mod _os { for func in funcs.into_iter().rev() { if let Err(e) = func.call((), vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); - vm.run_unraisable(e, Some("Error in atexit._run_exitfuncs".to_owned()), func); + vm.run_unraisable( + e, + Some("Exception ignored in".to_owned()), + func, + ); if exit { // Do nothing! } diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 2fac1f0150..f03bba3ceb 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -28,7 +28,7 @@ use crate::{ convert::ToPyObject, frame::{ExecutionResult, Frame, FrameRef}, frozen, - function::{ArgCallable, ArgMapping, FuncArgs, PySetterValue}, + function::{ArgMapping, FuncArgs, PySetterValue}, import, protocol::PyIterIter, scope::Scope, From d51daaf7e05cae7d842c5894597dbe410f99f896 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 18:50:04 +0530 Subject: [PATCH 07/63] add skip test for os.fork on windows --- Lib/test/test_os.py | 2 +- Lib/test/test_random.py | 1 + Lib/test/test_socketserver.py | 2 +- Lib/test/test_support.py | 1 + Lib/test/test_tempfile.py | 1 + Lib/test/test_thread.py | 1 + Lib/test/test_threading.py | 12 ++++++++++-- Lib/test/test_uuid.py | 1 + vm/src/stdlib/os.rs | 8 +++----- 9 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 0883b1be52..f740aab903 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4596,7 +4596,7 @@ def test_times(self): self.assertEqual(times.elapsed, 0) - +@unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @requires_os_func('fork') class ForkTests(unittest.TestCase): def test_fork(self): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 4bf00d5dbb..9bb807148a 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1338,6 +1338,7 @@ def test__all__(self): # tests validity but not completeness of the __all__ list self.assertTrue(set(random.__all__) <= set(dir(random))) + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @test.support.requires_fork() def test_after_fork(self): # Test the global Random instance gets reseeded in child diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 4e01c5bf58..945062fca3 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -55,7 +55,7 @@ class ForkingUnixDatagramServer(socketserver.ForkingMixIn, socketserver.UnixDatagramServer): pass - +@unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @contextlib.contextmanager def simple_subprocess(testcase): """Tests that a custom child process is not waited on (Issue 1540386)""" diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index d93eda773d..31897032fe 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -201,6 +201,7 @@ def test_temp_dir__existing_dir__quiet_true(self): f'temporary directory {path!r}: '), warn) + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @support.requires_fork() def test_temp_dir__forked_child(self): """Test that a forked child process does not remove the directory.""" diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index a2d06d323a..67c38fd98f 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -198,6 +198,7 @@ def supports_iter(self): if i == 20: break + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "os.fork is required for this test") def test_process_awareness(self): diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index f55cf3656e..26f43e9ca6 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -225,6 +225,7 @@ class TestForkInThread(unittest.TestCase): def setUp(self): self.read_fd, self.write_fd = os.pipe() + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') @threading_helper.reap_threads def test_forkinthread(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0f6eceda73..b80b66b65d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -509,6 +509,7 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()') def test_fork_at_exit(self): # bpo-42350: Calling os.fork() after threading._shutdown() must @@ -564,6 +565,7 @@ def background_thread(evt): self.assertEqual(out, b'') self.assertEqual(err, b'') + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on @@ -598,6 +600,7 @@ def f(): th.start() th.join() + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): @@ -619,6 +622,7 @@ def test_main_thread_after_fork(self): self.assertEqual(err, b"") self.assertEqual(data, "MainThread\nTrue\nTrue\n") + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") @@ -1002,7 +1006,8 @@ def test_1_join_on_shutdown(self): print('end of main') """ self._run_and_join(script) - + + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_2_join_in_forked_process(self): @@ -1024,6 +1029,7 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_3_join_in_forked_from_thread(self): @@ -1094,6 +1100,7 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_reinit_tls_after_fork(self): @@ -1117,7 +1124,8 @@ def do_fork_and_wait(): for t in threads: t.join() - + + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index eb13657a21..3d29bfe8ad 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -640,6 +640,7 @@ def test_uuid5(self): equal(str(u), v) # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.expectedFailure @unittest.skipUnless(os.name == 'posix', 'requires Posix') def testIssue8621(self): diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index cb1f476c8c..870efcc3f6 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1127,6 +1127,7 @@ pub(super) mod _os { OutputMode::String.process_path(curdir_inner(vm)?, vm) } + #[cfg(unix)] #[pyfunction] fn register_at_fork(kwargs: crate::function::KwArgs, vm: &VirtualMachine) -> PyResult<()> { let mut match_found = false; // better way to handle this? @@ -1156,11 +1157,7 @@ pub(super) mod _os { for func in funcs.into_iter().rev() { if let Err(e) = func.call((), vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); - vm.run_unraisable( - e, - Some("Exception ignored in".to_owned()), - func, - ); + vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func); if exit { // Do nothing! } @@ -1188,6 +1185,7 @@ pub(super) mod _os { Ok(()) } + #[cfg(unix)] #[pyfunction] fn fork(vm: &VirtualMachine) -> PyResult { let mut pid: i32 = 0; From 9fc89520beb13a3695c480d16545c47a5a8327b2 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 19:08:06 +0530 Subject: [PATCH 08/63] chore! fix clippy errors --- vm/src/stdlib/os.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 870efcc3f6..5fc7cc6b01 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1153,7 +1153,7 @@ pub(super) mod _os { Ok(()) } fn run_at_forkers(funcs: Vec, vm: &VirtualMachine) { - if funcs.len() > 0 { + if !funcs.is_empty() { for func in funcs.into_iter().rev() { if let Err(e) = func.call((), vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); @@ -1188,7 +1188,7 @@ pub(super) mod _os { #[cfg(unix)] #[pyfunction] fn fork(vm: &VirtualMachine) -> PyResult { - let mut pid: i32 = 0; + let pid: i32; py_os_before_fork(vm)?; unsafe { pid = libc::fork(); From 137fd293fbca380132fb10701bd95f428c4cf795 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 19:17:55 +0530 Subject: [PATCH 09/63] skip fork functions in windows --- vm/src/stdlib/os.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 5fc7cc6b01..0c213adb9a 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1152,6 +1152,8 @@ pub(super) mod _os { Ok(()) } + + #[cfg(unix)] fn run_at_forkers(funcs: Vec, vm: &VirtualMachine) { if !funcs.is_empty() { for func in funcs.into_iter().rev() { @@ -1165,12 +1167,16 @@ pub(super) mod _os { } } } + + #[cfg(unix)] fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { let before_forkers: Vec = std::mem::take(&mut *vm.state.before_forkers.lock()); run_at_forkers(before_forkers, vm); Ok(()) } + + #[cfg(unix)] fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { let after_forkers_child: Vec = std::mem::take(&mut *vm.state.after_forkers_child.lock()); @@ -1178,6 +1184,7 @@ pub(super) mod _os { Ok(()) } + #[cfg(unix)] fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { let after_forkers_parent: Vec = std::mem::take(&mut *vm.state.after_forkers_parent.lock()); From a58e4296366e9a2637b7d7a5e61956a2c3921e76 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 19:59:29 +0530 Subject: [PATCH 10/63] skip unsupported function calls --- Lib/concurrent/futures/thread.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 51c942f51a..493861d314 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -37,7 +37,8 @@ def _python_exit(): threading._register_atexit(_python_exit) # At fork, reinitialize the `_global_shutdown_lock` lock in the child process -if hasattr(os, 'register_at_fork'): +# TODO RUSTPYTHON - _at_fork_reinit is not implemented yet +if hasattr(os, 'register_at_fork') and hasattr(_global_shutdown_lock, '_at_fork_reinit'): os.register_at_fork(before=_global_shutdown_lock.acquire, after_in_child=_global_shutdown_lock._at_fork_reinit, after_in_parent=_global_shutdown_lock.release) From cb5166f789674bb1f7ae85eae6c102942c4b9710 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 20:27:03 +0530 Subject: [PATCH 11/63] remove un-needed skip decorators --- Lib/test/test_os.py | 1 - Lib/test/test_random.py | 1 - Lib/test/test_socketserver.py | 2 +- Lib/test/test_support.py | 1 - Lib/test/test_tempfile.py | 1 - Lib/test/test_thread.py | 1 - Lib/test/test_threading.py | 8 -------- Lib/test/test_uuid.py | 2 +- 8 files changed, 2 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f740aab903..2d3526fb83 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4596,7 +4596,6 @@ def test_times(self): self.assertEqual(times.elapsed, 0) -@unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @requires_os_func('fork') class ForkTests(unittest.TestCase): def test_fork(self): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 9bb807148a..4bf00d5dbb 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1338,7 +1338,6 @@ def test__all__(self): # tests validity but not completeness of the __all__ list self.assertTrue(set(random.__all__) <= set(dir(random))) - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @test.support.requires_fork() def test_after_fork(self): # Test the global Random instance gets reseeded in child diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 945062fca3..a3ecd974c9 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -55,7 +55,7 @@ class ForkingUnixDatagramServer(socketserver.ForkingMixIn, socketserver.UnixDatagramServer): pass -@unittest.expectedFailureIfWindows("Fork is for Unix based systems only") +@test.support.requires_fork() @contextlib.contextmanager def simple_subprocess(testcase): """Tests that a custom child process is not waited on (Issue 1540386)""" diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 31897032fe..d93eda773d 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -201,7 +201,6 @@ def test_temp_dir__existing_dir__quiet_true(self): f'temporary directory {path!r}: '), warn) - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @support.requires_fork() def test_temp_dir__forked_child(self): """Test that a forked child process does not remove the directory.""" diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 67c38fd98f..a2d06d323a 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -198,7 +198,6 @@ def supports_iter(self): if i == 20: break - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "os.fork is required for this test") def test_process_awareness(self): diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index 26f43e9ca6..f55cf3656e 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -225,7 +225,6 @@ class TestForkInThread(unittest.TestCase): def setUp(self): self.read_fd, self.write_fd = os.pipe() - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork') @threading_helper.reap_threads def test_forkinthread(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index b80b66b65d..c18ce27ab4 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -509,7 +509,6 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()') def test_fork_at_exit(self): # bpo-42350: Calling os.fork() after threading._shutdown() must @@ -565,7 +564,6 @@ def background_thread(evt): self.assertEqual(out, b'') self.assertEqual(err, b'') - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on @@ -600,7 +598,6 @@ def f(): th.start() th.join() - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): @@ -622,7 +619,6 @@ def test_main_thread_after_fork(self): self.assertEqual(err, b"") self.assertEqual(data, "MainThread\nTrue\nTrue\n") - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") @@ -1007,7 +1003,6 @@ def test_1_join_on_shutdown(self): """ self._run_and_join(script) - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_2_join_in_forked_process(self): @@ -1029,7 +1024,6 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_3_join_in_forked_from_thread(self): @@ -1100,7 +1094,6 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_reinit_tls_after_fork(self): @@ -1125,7 +1118,6 @@ def do_fork_and_wait(): for t in threads: t.join() - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 3d29bfe8ad..13669d3620 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -640,8 +640,8 @@ def test_uuid5(self): equal(str(u), v) # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("Fork is for Unix based systems only") @unittest.expectedFailure + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(os.name == 'posix', 'requires Posix') def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates From ddcae2331ce901d410471753c4472f7af592e5ba Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Mon, 17 Apr 2023 20:52:00 +0530 Subject: [PATCH 12/63] Update Lib/test/test_socketserver.py Co-authored-by: fanninpm --- Lib/test/test_socketserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index a3ecd974c9..113f959ff2 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -55,7 +55,7 @@ class ForkingUnixDatagramServer(socketserver.ForkingMixIn, socketserver.UnixDatagramServer): pass -@test.support.requires_fork() +@test.support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems @contextlib.contextmanager def simple_subprocess(testcase): """Tests that a custom child process is not waited on (Issue 1540386)""" From 63a881c209b1d4e486c661d93e58d2765e0803fc Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 20:53:47 +0530 Subject: [PATCH 13/63] remove un-needed decorator --- Lib/test/test_uuid.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 13669d3620..47714dd678 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -641,8 +641,7 @@ def test_uuid5(self): # TODO: RUSTPYTHON @unittest.expectedFailure - @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") - @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates # the same sequence of UUIDs in the parent and any From abce797cba07037bfa879dd1f19047ece7d75b19 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 20:59:09 +0530 Subject: [PATCH 14/63] fix lint --- Lib/test/test_threading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index c18ce27ab4..0f6eceda73 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1002,7 +1002,7 @@ def test_1_join_on_shutdown(self): print('end of main') """ self._run_and_join(script) - + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_2_join_in_forked_process(self): @@ -1117,7 +1117,7 @@ def do_fork_and_wait(): for t in threads: t.join() - + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() From 7aa1311fa5b70d6465ce10f818f0e56f437697c8 Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 21:01:09 +0530 Subject: [PATCH 15/63] add todo items --- Lib/test/test_uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 47714dd678..d0ee47b808 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -641,7 +641,7 @@ def test_uuid5(self): # TODO: RUSTPYTHON @unittest.expectedFailure - @support.requires_fork() + @support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates # the same sequence of UUIDs in the parent and any From 5779c893ccec00800dd1c3935c1e4ceff376258a Mon Sep 17 00:00:00 2001 From: ankit Date: Mon, 17 Apr 2023 21:07:11 +0530 Subject: [PATCH 16/63] remove un-needed comment --- Lib/test/test_uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index d0ee47b808..47714dd678 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -641,7 +641,7 @@ def test_uuid5(self): # TODO: RUSTPYTHON @unittest.expectedFailure - @support.requires_fork() # TODO: RUSTPYTHON, os.fork is currently only supported on Unix-based systems + @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates # the same sequence of UUIDs in the parent and any From 37b73f37f783158152763d9d32cf4b60835943c5 Mon Sep 17 00:00:00 2001 From: ankit Date: Tue, 18 Apr 2023 22:35:51 +0530 Subject: [PATCH 17/63] fix test case getting stuck --- Lib/test/test_httpservers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index d31582c0db..8e9ee6311b 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -607,7 +607,8 @@ def test_html_escape_filename(self): print("") """ - +@unittest.skipIf(not hasattr(os, '_exit'), + "run_cgi in http/server.py gets stuck as if os._exit(127) is not defined to kill forked process") @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") class CGIHTTPServerTestCase(BaseTestCase): From eaa3e634dd73c83ec6371e588b4d3633b40ee99d Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Tue, 18 Apr 2023 22:41:13 +0530 Subject: [PATCH 18/63] Update Lib/test/test_httpservers.py Co-authored-by: fanninpm --- Lib/test/test_httpservers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 8e9ee6311b..bdb25b9ef3 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -608,7 +608,7 @@ def test_html_escape_filename(self): """ @unittest.skipIf(not hasattr(os, '_exit'), - "run_cgi in http/server.py gets stuck as if os._exit(127) is not defined to kill forked process") + "TODO: RUSTPYTHON, run_cgi in http/server.py gets stuck as os._exit(127) doesn't currently kill forked processes") @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") class CGIHTTPServerTestCase(BaseTestCase): From e041cbdef4a05cc3dde888c86dfc50627b05ef67 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Tue, 18 Apr 2023 22:35:51 +0530 Subject: [PATCH 19/63] fix test case getting stuck --- Lib/test/test_httpservers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index d31582c0db..8e9ee6311b 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -607,7 +607,8 @@ def test_html_escape_filename(self): print("") """ - +@unittest.skipIf(not hasattr(os, '_exit'), + "run_cgi in http/server.py gets stuck as if os._exit(127) is not defined to kill forked process") @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") class CGIHTTPServerTestCase(BaseTestCase): From ae634d20dc189ec25820783ed9b08b4fd6a8e3ec Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Thu, 20 Apr 2023 00:09:07 +0530 Subject: [PATCH 20/63] skip failures due to missing module --- Lib/test/test_threading.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0f6eceda73..a2ce443502 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -509,6 +509,7 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'exit_handler needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()') def test_fork_at_exit(self): # bpo-42350: Calling os.fork() after threading._shutdown() must @@ -537,6 +538,7 @@ def exit_handler(): self.assertEqual(out, b'') self.assertEqual(err.rstrip(), b'child process ok') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up @@ -564,6 +566,7 @@ def background_thread(evt): self.assertEqual(out, b'') self.assertEqual(err, b'') + @unittest.skipUnless(hasattr(sys, 'getswitchinterval'), "needs sys.getswitchinterval()") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on @@ -598,6 +601,7 @@ def f(): th.start() th.join() + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): @@ -1003,6 +1007,7 @@ def test_1_join_on_shutdown(self): """ self._run_and_join(script) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_2_join_in_forked_process(self): @@ -1024,6 +1029,7 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_3_join_in_forked_from_thread(self): @@ -1094,6 +1100,7 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_reinit_tls_after_fork(self): @@ -1118,6 +1125,7 @@ def do_fork_and_wait(): for t in threads: t.join() + @unittest.skipUnless(hasattr(sys, '_current_frames'), "needs sys._current_frames()") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() From f54f715275ed149351d2216ba75ed5d92e7d5fb5 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:53:37 +0530 Subject: [PATCH 21/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index a2ce443502..e2d2cc448f 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1125,7 +1125,7 @@ def do_fork_and_wait(): for t in threads: t.join() - @unittest.skipUnless(hasattr(sys, '_current_frames'), "needs sys._current_frames()") + @unittest.skipUnless(hasattr(sys, '_current_frames'), "TODO: RUSTPYTHON, needs sys._current_frames()") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_clear_threads_states_after_fork(self): # Issue #17094: check that threads states are cleared after fork() From e27e9e9628a11c08468e7d02a84b42f1b8e35b9c Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:53:47 +0530 Subject: [PATCH 22/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index e2d2cc448f..c1f490c8df 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1100,7 +1100,7 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_reinit_tls_after_fork(self): From 6199f765ccb0549e62c0e22e1e7f5d0549ce96e2 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:53:57 +0530 Subject: [PATCH 23/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index c1f490c8df..06e1fe9a36 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1029,7 +1029,7 @@ def test_2_join_in_forked_process(self): """ self._run_and_join(script) - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_3_join_in_forked_from_thread(self): From 9ebdf0b6e41ab696e2f6b635ef3bad99c394e8a1 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:54:04 +0530 Subject: [PATCH 24/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 06e1fe9a36..888668d2cc 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1007,7 +1007,7 @@ def test_1_join_on_shutdown(self): """ self._run_and_join(script) - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") def test_2_join_in_forked_process(self): From 9c525af2a5eddc7ad9eb7626e9e39165db7509dd Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:54:11 +0530 Subject: [PATCH 25/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 888668d2cc..facf5b617d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -601,7 +601,7 @@ def f(): th.start() th.join() - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork(self): From 84fda437d54b17603374181820457ad2916181ea Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:54:17 +0530 Subject: [PATCH 26/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index facf5b617d..a92e5de309 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -566,7 +566,7 @@ def background_thread(evt): self.assertEqual(out, b'') self.assertEqual(err, b'') - @unittest.skipUnless(hasattr(sys, 'getswitchinterval'), "needs sys.getswitchinterval()") + @unittest.skipUnless(hasattr(sys, 'getswitchinterval'), "TODO: RUSTPYTHON, needs sys.getswitchinterval()") @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") def test_is_alive_after_fork(self): # Try hard to trigger #18418: is_alive() could sometimes be True on From 891861748212425afa509c96791b4e00c0005787 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:54:24 +0530 Subject: [PATCH 27/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index a92e5de309..e11d545ba5 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -538,7 +538,7 @@ def exit_handler(): self.assertEqual(out, b'') self.assertEqual(err.rstrip(), b'child process ok') - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'test needs lock._at_fork_reinit') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') def test_dummy_thread_after_fork(self): # Issue #14308: a dummy thread in the active list doesn't mess up From 123bc2dfa527c7b8a3944b91fd3aacee07e524b6 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Thu, 20 Apr 2023 18:54:30 +0530 Subject: [PATCH 28/63] Update Lib/test/test_threading.py Co-authored-by: fanninpm --- Lib/test/test_threading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index e11d545ba5..420fbc2cb3 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -509,7 +509,7 @@ def test_daemon_param(self): t = threading.Thread(daemon=True) self.assertTrue(t.daemon) - @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'exit_handler needs lock._at_fork_reinit') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, exit_handler needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), 'needs os.fork()') def test_fork_at_exit(self): # bpo-42350: Calling os.fork() after threading._shutdown() must From 20f5abf2e91b05698bbd3e54c3f1168a7e0a2a6e Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 14:08:51 +0530 Subject: [PATCH 29/63] skip cases which needs _at_fork_reinit support --- Lib/test/test_os.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 8ef24151a1..d1f43beea9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -69,6 +69,10 @@ except ImportError: INT_MAX = PY_SSIZE_T_MAX = sys.maxsize +try: + import _testcapi +except ImportError: + _testcapi = None from test.support.script_helper import assert_python_ok from test.support import unix_shell @@ -3067,11 +3071,13 @@ def check_waitpid(self, code, exitcode, callback=None): self.assertEqual(pid2, pid) # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.expectedFailure def test_waitpid(self): self.check_waitpid(code='pass', exitcode=0) # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.expectedFailure def test_waitstatus_to_exitcode(self): exitcode = 23 @@ -3103,7 +3109,8 @@ def test_waitstatus_to_exitcode_windows(self): os.waitstatus_to_exitcode((max_exitcode + 1) << 8) with self.assertRaises(OverflowError): os.waitstatus_to_exitcode(-1) - + + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv') @unittest.expectedFailure # Skip the test on Windows @@ -3146,31 +3153,36 @@ def create_args(self, *, with_env=False, use_bytes=False): for k, v in self.env.items()} return args - + + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnl') def test_spawnl(self): args = self.create_args() exitcode = os.spawnl(os.P_WAIT, args[0], *args) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnle') def test_spawnle(self): args = self.create_args(with_env=True) exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnlp') def test_spawnlp(self): args = self.create_args() exitcode = os.spawnlp(os.P_WAIT, args[0], *args) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnlpe') def test_spawnlpe(self): args = self.create_args(with_env=True) exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnv') def test_spawnv(self): args = self.create_args() @@ -3181,30 +3193,35 @@ def test_spawnv(self): exitcode = os.spawnv(os.P_WAIT, FakePath(args[0]), args) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnve') def test_spawnve(self): args = self.create_args(with_env=True) exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnvp') def test_spawnvp(self): args = self.create_args() exitcode = os.spawnvp(os.P_WAIT, args[0], args) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnvpe') def test_spawnvpe(self): args = self.create_args(with_env=True) exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env) self.assertEqual(exitcode, self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnv') def test_nowait(self): args = self.create_args() pid = os.spawnv(os.P_NOWAIT, args[0], args) support.wait_process(pid, exitcode=self.exitcode) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -3286,10 +3303,12 @@ def _test_invalid_env(self, spawn): exitcode = spawn(os.P_WAIT, args[0], args, newenv) self.assertEqual(exitcode, 0) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnve') def test_spawnve_invalid_env(self): self._test_invalid_env(os.spawnve) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @requires_os_func('spawnvpe') def test_spawnvpe_invalid_env(self): self._test_invalid_env(os.spawnvpe) @@ -4660,6 +4679,7 @@ def test_fork(self): assert_python_ok("-c", code) assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + @unittest.skipIf(_testcapi is None, 'needs _testcapi') @unittest.skipUnless(sys.platform in ("linux", "darwin"), "Only Linux and macOS detect this today.") def test_fork_warns_when_non_python_thread_exists(self): From 3bfe8c875b770fcf013a721bab5bade57a7b7d71 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 14:18:34 +0530 Subject: [PATCH 30/63] add type error if arg is not callable --- Lib/test/test_fcntl.py | 5 +++-- vm/src/stdlib/os.rs | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 06a84118e1..c3efaed0ea 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -4,6 +4,7 @@ import os import struct import sys +import threading import unittest from multiprocessing import Process from test.support import verbose, cpython_only @@ -155,8 +156,8 @@ def test_flock(self): self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH) self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) - # TODO: RUSTPYTHON, AttributeError: module 'os' has no attribute 'fork' @unittest.expectedFailure + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") def test_lockf_exclusive(self): self.f = open(TESTFN, 'wb+') @@ -168,8 +169,8 @@ def test_lockf_exclusive(self): fcntl.lockf(self.f, fcntl.LOCK_UN) self.assertEqual(p.exitcode, 0) - # TODO: RUSTPYTHON, AttributeError: module 'os' has no attribute 'fork' @unittest.expectedFailure + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") def test_lockf_share(self): self.f = open(TESTFN, 'wb+') diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 323610f02a..cc31a71f1f 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1132,6 +1132,11 @@ pub(super) mod _os { fn register_at_fork(kwargs: crate::function::KwArgs, vm: &VirtualMachine) -> PyResult<()> { let mut match_found = false; // better way to handle this? for (key, value) in kwargs.into_iter() { + if !value.is_callable() { + return Err(vm.new_type_error( + "Args must be callable".to_owned() + )); + } if key == "before" { match_found = true; vm.state.before_forkers.lock().push(value.clone()); From e2e6844022d59aaa2c876a5d2462dafb6ee9b652 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 15:30:38 +0530 Subject: [PATCH 31/63] reverse function calls for before forkers as per as cpython implementation --- vm/src/stdlib/os.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index cc31a71f1f..a886e73622 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1159,9 +1159,14 @@ pub(super) mod _os { } #[cfg(unix)] - fn run_at_forkers(funcs: Vec, vm: &VirtualMachine) { + fn run_at_forkers(mut funcs: Vec, reversed: bool, vm: &VirtualMachine) { + if !funcs.is_empty() { - for func in funcs.into_iter().rev() { + + if reversed { + funcs.reverse(); + } + for func in funcs.into_iter() { if let Err(e) = func.call((), vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func); @@ -1176,8 +1181,10 @@ pub(super) mod _os { #[cfg(unix)] fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { let before_forkers: Vec = std::mem::take(&mut *vm.state.before_forkers.lock()); - run_at_forkers(before_forkers, vm); + // functions must be executed in reversed order as they are registered + // only for before_forkers, refer: test_register_at_fork in test_posix + run_at_forkers(before_forkers, true, vm); Ok(()) } @@ -1185,7 +1192,7 @@ pub(super) mod _os { fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { let after_forkers_child: Vec = std::mem::take(&mut *vm.state.after_forkers_child.lock()); - run_at_forkers(after_forkers_child, vm); + run_at_forkers(after_forkers_child, false, vm); Ok(()) } @@ -1193,7 +1200,7 @@ pub(super) mod _os { fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { let after_forkers_parent: Vec = std::mem::take(&mut *vm.state.after_forkers_parent.lock()); - run_at_forkers(after_forkers_parent, vm); + run_at_forkers(after_forkers_parent, false, vm); Ok(()) } From e5d9bb527d0d318e7e2ecec5e97feae38283e117 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 15:31:07 +0530 Subject: [PATCH 32/63] handle non callable args --- vm/src/stdlib/os.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index a886e73622..4b28c907bc 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1133,9 +1133,7 @@ pub(super) mod _os { let mut match_found = false; // better way to handle this? for (key, value) in kwargs.into_iter() { if !value.is_callable() { - return Err(vm.new_type_error( - "Args must be callable".to_owned() - )); + return Err(vm.new_type_error("Args must be callable".to_owned())); } if key == "before" { match_found = true; @@ -1160,9 +1158,7 @@ pub(super) mod _os { #[cfg(unix)] fn run_at_forkers(mut funcs: Vec, reversed: bool, vm: &VirtualMachine) { - if !funcs.is_empty() { - if reversed { funcs.reverse(); } From 0fd1a9f37cbd615165965a2d4e52bdf6199c7a59 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 15:56:19 +0530 Subject: [PATCH 33/63] skip test cases which has dependencies on Lock._at_fork_reinit --- Lib/test/test_httpservers.py | 10 ++++++++++ Lib/test/test_pty.py | 3 +++ Lib/test/test_support.py | 2 ++ Lib/test/test_tempfile.py | 3 +++ Lib/test/test_uuid.py | 2 ++ 5 files changed, 20 insertions(+) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index bdb25b9ef3..bcac6bc940 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -766,6 +766,7 @@ def test_url_collapse_path(self): msg='path = %r\nGot: %r\nWanted: %r' % (path, actual, expected)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_headers_and_content(self): res = self.request('/cgi-bin/file1.py') self.assertEqual( @@ -776,6 +777,7 @@ def test_issue19435(self): res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') self.assertEqual(res.status, HTTPStatus.NOT_FOUND) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_post(self): params = urllib.parse.urlencode( {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) @@ -797,6 +799,7 @@ def test_authorization(self): (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_no_leading_slash(self): # http://bugs.python.org/issue2254 res = self.request('cgi-bin/file1.py') @@ -804,6 +807,7 @@ def test_no_leading_slash(self): (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_os_environ_is_not_altered(self): signature = "Test CGI Server" os.environ['SERVER_SOFTWARE'] = signature @@ -813,24 +817,28 @@ def test_os_environ_is_not_altered(self): (res.read(), res.getheader('Content-type'), res.status)) self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_urlquote_decoding_in_cgi_check(self): res = self.request('/cgi-bin%2ffile1.py') self.assertEqual( (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_nested_cgi_path_issue21323(self): res = self.request('/cgi-bin/child-dir/file3.py') self.assertEqual( (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_query_with_multiple_question_mark(self): res = self.request('/cgi-bin/file4.py?a=b?c=d') self.assertEqual( (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_query_with_continuous_slashes(self): res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') self.assertEqual( @@ -838,6 +846,7 @@ def test_query_with_continuous_slashes(self): 'text/html', HTTPStatus.OK), (res.read(), res.getheader('Content-type'), res.status)) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_cgi_path_in_sub_directories(self): try: CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin') @@ -848,6 +857,7 @@ def test_cgi_path_in_sub_directories(self): finally: CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin') + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_accept(self): browser_accept = \ 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 7c38b64f78..184e2205a8 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,3 +1,4 @@ +import threading from test.support import verbose, reap_children from test.support.import_helper import import_module @@ -211,6 +212,7 @@ def test_openpty(self): self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) # TODO: RUSTPYTHON + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.expectedFailure def test_fork(self): debug("calling pty.fork()") @@ -314,6 +316,7 @@ def test_master_read(self): self.assertEqual(data, b"") # TODO: RUSTPYTHON; no os.fork + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.expectedFailure def test_spawn_doesnt_hang(self): pty.spawn([sys.executable, '-c', 'print("hi there")']) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index d93eda773d..efc86131a5 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -9,6 +9,7 @@ import sys import tempfile import textwrap +import threading import time import unittest import warnings @@ -453,6 +454,7 @@ def test_check__all__(self): # TODO: RUSTPYTHON @unittest.expectedFailure + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG'), 'need os.waitpid() and os.WNOHANG') @support.requires_fork() diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index a2d06d323a..50e7ffc2eb 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -6,6 +6,7 @@ import pathlib import sys import re +import threading import warnings import contextlib import stat @@ -198,6 +199,7 @@ def supports_iter(self): if i == 20: break + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(hasattr(os, 'fork'), "os.fork is required for this test") def test_process_awareness(self): @@ -465,6 +467,7 @@ def test_file_mode(self): expected = user * (1 + 8 + 64) self.assertEqual(mode, expected) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipUnless(has_spawnl, 'os.spawnl not available') def test_noinherit(self): # _mkstemp_inner file handles are not inherited by child processes diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 47714dd678..0fdd78aba4 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,3 +1,4 @@ +import threading import unittest from test import support from test.support import import_helper @@ -641,6 +642,7 @@ def test_uuid5(self): # TODO: RUSTPYTHON @unittest.expectedFailure + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates From 47f13b269b75eef790d89c8239f395241cfa496c Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 15:57:24 +0530 Subject: [PATCH 34/63] skip test case --- Lib/test/test_httpservers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index bcac6bc940..7480a27add 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -791,6 +791,7 @@ def test_invaliduri(self): res.read() self.assertEqual(res.status, HTTPStatus.NOT_FOUND) + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_authorization(self): headers = {b'Authorization' : b'Basic ' + base64.b64encode(b'username:pass')} From 35f23be34e411236642412e05c3282fd40740be5 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 16:51:54 +0530 Subject: [PATCH 35/63] remove un-needed expected failure decorator as test is passing --- Lib/test/test_platform.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 0d2a04c795..c6a1cc3417 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -346,8 +346,6 @@ def test_mac_ver(self): else: self.assertEqual(res[2], 'PowerPC') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.platform == 'darwin', "OSX only test") def test_mac_ver_with_fork(self): # Issue7895: platform.mac_ver() crashes when using fork without exec From e19c8fdaa65018c4ee84bab33619747ab0f16fcb Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 18:21:26 +0530 Subject: [PATCH 36/63] debug test failure --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c901ada030..1d33a33ab3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -138,7 +138,7 @@ jobs: - name: run rust tests run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} - if: runner.os != 'macOS' + if: runner.os != 'macOS' # temp skip ssl linking for Mac to avoid CI failure - name: run rust tests (MacOS no ssl) run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit ${{ env.NON_WASM_PACKAGES }} @@ -285,7 +285,7 @@ jobs: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} - if: runner.os != 'Windows' name: run cpython platform-dependent tests - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v test_builtin - if: runner.os == 'Windows' name: run cpython platform-dependent tests (windows partial - fixme) run: From c79e2fd349c976a2969b5910c1383176b7df3144 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 18:38:27 +0530 Subject: [PATCH 37/63] increase verbosity --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d33a33ab3..46b8751394 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -285,7 +285,7 @@ jobs: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} - if: runner.os != 'Windows' name: run cpython platform-dependent tests - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v test_builtin + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -vv test_builtin - if: runner.os == 'Windows' name: run cpython platform-dependent tests (windows partial - fixme) run: From f237e5e96265df95d1226e95feb33811724a9676 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 18:50:36 +0530 Subject: [PATCH 38/63] print more lines for debug --- Lib/test/test_builtin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 3d89f85a5a..54d265b571 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2199,7 +2199,7 @@ def _run_child(self, child, terminal_input): child_output = bytearray() while True: try: - chunk = os.read(fd, 3000) + chunk = os.read(fd, 30000) except OSError: # Assume EIO break if not chunk: From 870ef53f6c887525703fbcbb7cb5186c95559c9e Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 19:14:35 +0530 Subject: [PATCH 39/63] disable assertion failure to check error --- Lib/test/test_builtin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 54d265b571..09367bb3e8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2207,8 +2207,8 @@ def _run_child(self, child, terminal_input): child_output.extend(chunk) os.close(fd) child_output = child_output.decode("ascii", "ignore") - self.fail("got %d lines in pipe but expected 2, child output was:\n%s" - % (len(lines), child_output)) + # self.fail("got %d lines in pipe but expected 2, child output was:\n%s" + # % (len(lines), child_output)) # bpo-40155: Close the PTY before waiting for the child process # completion, otherwise the child process hangs on AIX. From fdaefd030e89828df65eae46949e7cdf7e0b7ab9 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 19:36:48 +0530 Subject: [PATCH 40/63] more debugging --- Lib/test/test_builtin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 09367bb3e8..b782166f0a 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2176,7 +2176,8 @@ def _run_child(self, child, terminal_input): traceback.print_exc() finally: # We don't want to return to unittest... - os._exit(0) + #os._exit(0) + ... # Parent os.close(w) @@ -2207,8 +2208,8 @@ def _run_child(self, child, terminal_input): child_output.extend(chunk) os.close(fd) child_output = child_output.decode("ascii", "ignore") - # self.fail("got %d lines in pipe but expected 2, child output was:\n%s" - # % (len(lines), child_output)) + self.fail("got %d lines in pipe but expected 2, child output was:\n%s" + % (len(lines), child_output)) # bpo-40155: Close the PTY before waiting for the child process # completion, otherwise the child process hangs on AIX. From eb011607cbdefe6d258f9f45caced274df5e1f2c Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 19:46:38 +0530 Subject: [PATCH 41/63] minor change --- Lib/test/test_builtin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index b782166f0a..239a25a5db 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2173,11 +2173,11 @@ def _run_child(self, child, terminal_input): with open(w, "w") as wpipe: child(wpipe) except: + print(traceback.format_exc()) traceback.print_exc() finally: # We don't want to return to unittest... - #os._exit(0) - ... + os._exit(0) # Parent os.close(w) From 669ff5912df65d3020420def73c7ca6b3bd4d182 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 20:09:37 +0530 Subject: [PATCH 42/63] add logs --- Lib/test/test_builtin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 239a25a5db..a677c97187 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2278,6 +2278,7 @@ def child(wpipe): sys.stdout = io.StringIO() # Does not support fileno() input("prompt") print("captured:", ascii(sys.stdout.getvalue()), file=wpipe) + child(b"quux\r") lines = self.run_child(child, b"quux\r") expected = ( "stdin.isatty(): True", From 76b89cec1efdf04ea4ba7cd08e537e4aa6649609 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 20:25:25 +0530 Subject: [PATCH 43/63] more experimentations to get error --- Lib/test/test_builtin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index a677c97187..3977d19a4e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2173,8 +2173,8 @@ def _run_child(self, child, terminal_input): with open(w, "w") as wpipe: child(wpipe) except: - print(traceback.format_exc()) traceback.print_exc() + self.fail(traceback.format_exc()) finally: # We don't want to return to unittest... os._exit(0) @@ -2278,7 +2278,6 @@ def child(wpipe): sys.stdout = io.StringIO() # Does not support fileno() input("prompt") print("captured:", ascii(sys.stdout.getvalue()), file=wpipe) - child(b"quux\r") lines = self.run_child(child, b"quux\r") expected = ( "stdin.isatty(): True", From 68a78e4a20bb222420c70e4dc6d66666904cfbbe Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 20:41:00 +0530 Subject: [PATCH 44/63] last shot to find issue --- .github/workflows/ci.yaml | 4 ++-- Lib/test/test_builtin.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 46b8751394..c901ada030 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -138,7 +138,7 @@ jobs: - name: run rust tests run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} - if: runner.os != 'macOS' + if: runner.os != 'macOS' # temp skip ssl linking for Mac to avoid CI failure - name: run rust tests (MacOS no ssl) run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit ${{ env.NON_WASM_PACKAGES }} @@ -285,7 +285,7 @@ jobs: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} - if: runner.os != 'Windows' name: run cpython platform-dependent tests - run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -vv test_builtin + run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} - if: runner.os == 'Windows' name: run cpython platform-dependent tests (windows partial - fixme) run: diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 3977d19a4e..d9241e9a93 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2174,9 +2174,10 @@ def _run_child(self, child, terminal_input): child(wpipe) except: traceback.print_exc() - self.fail(traceback.format_exc()) finally: # We don't want to return to unittest... + from time import sleep + sleep(60) os._exit(0) # Parent @@ -2200,7 +2201,7 @@ def _run_child(self, child, terminal_input): child_output = bytearray() while True: try: - chunk = os.read(fd, 30000) + chunk = os.read(fd, 3000) except OSError: # Assume EIO break if not chunk: From a854bb63c9cf1ef806b532da5f2e91f11866dbed Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 21:10:40 +0530 Subject: [PATCH 45/63] remove debug logs --- Lib/test/test_builtin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index d9241e9a93..3d89f85a5a 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2176,8 +2176,6 @@ def _run_child(self, child, terminal_input): traceback.print_exc() finally: # We don't want to return to unittest... - from time import sleep - sleep(60) os._exit(0) # Parent From 110fc33802d28f4255919f996388b352d2ee62fd Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Mon, 24 Apr 2023 01:24:21 +0900 Subject: [PATCH 46/63] skip test_input_no_stdout_fileno for macos --- Lib/test/test_builtin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 3d89f85a5a..bfa2802fb8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -37,6 +37,8 @@ except ImportError: pty = signal = None +import threading # XXX: RUSTPYTHON; to skip _at_fork_reinit + class Squares: @@ -2269,6 +2271,7 @@ def test_input_tty_non_ascii_unicode_errors(self): # Check stdin/stdout error handler is used when invoking PyOS_Readline() self.check_input_tty("prompté", b"quux\xe9", "ascii") + @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') def test_input_no_stdout_fileno(self): # Issue #24402: If stdin is the original terminal but stdout.fileno() # fails, do not use the original stdout file descriptor From 0a2c6fd38d68ff7eeebaf0e1fa047a65cedfa74c Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:07:04 +0530 Subject: [PATCH 47/63] Add todo item Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- Lib/test/test_support.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index efc86131a5..0a855571fb 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -9,7 +9,7 @@ import sys import tempfile import textwrap -import threading +import threading # XXX: RUSTPYTHON import time import unittest import warnings From af2831148b4325f491c576d0e3fbf2b8a5ab1f9f Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:17:50 +0530 Subject: [PATCH 48/63] Add todo item Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- Lib/test/test_tempfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 50e7ffc2eb..fee1969503 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -6,7 +6,7 @@ import pathlib import sys import re -import threading +import threading # XXX: RUSTPYTHON; to check `_at_fork_reinit` import warnings import contextlib import stat From dc087c3264aeece52a7ce4e44edbcffc1128d5f4 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:18:21 +0530 Subject: [PATCH 49/63] not to remove it when update to next CPython version Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- Lib/test/test_uuid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 0fdd78aba4..66b8720ffc 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,4 +1,4 @@ -import threading +import threading # XXX: RUSTPYTHON; to check `_at_fork_reinit` import unittest from test import support from test.support import import_helper From 511a448c6e353c419309a8cad3e06c95a0c05ca9 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Sun, 23 Apr 2023 22:19:20 +0530 Subject: [PATCH 50/63] add todo item --- Lib/test/test_fcntl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index c3efaed0ea..08b26d2ee3 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -156,6 +156,7 @@ def test_flock(self): self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH) self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) + # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") @@ -168,7 +169,7 @@ def test_lockf_exclusive(self): p.join() fcntl.lockf(self.f, fcntl.LOCK_UN) self.assertEqual(p.exitcode, 0) - + # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipUnless(hasattr(threading.Lock(), '_at_fork_reinit'), 'TODO: RUSTPYTHON, test needs lock._at_fork_reinit') @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") From e4aacf950a7e06c83b9c61de8bf24401a62e6a3c Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 00:09:35 +0530 Subject: [PATCH 51/63] handle optional args for register at fork --- vm/src/stdlib/os.rs | 67 ++++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 4b28c907bc..b312ed2dc2 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1127,30 +1127,59 @@ pub(super) mod _os { OutputMode::String.process_path(curdir_inner(vm)?, vm) } + #[derive(FromArgs)] + struct RegisterAtForkArgs { + #[pyarg(named, optional)] + before: OptionalArg, + #[pyarg(named, optional)] + after_in_parent: OptionalArg, + #[pyarg(named, optional)] + after_in_child: OptionalArg, + } + #[cfg(unix)] #[pyfunction] - fn register_at_fork(kwargs: crate::function::KwArgs, vm: &VirtualMachine) -> PyResult<()> { - let mut match_found = false; // better way to handle this? - for (key, value) in kwargs.into_iter() { - if !value.is_callable() { - return Err(vm.new_type_error("Args must be callable".to_owned())); - } - if key == "before" { - match_found = true; - vm.state.before_forkers.lock().push(value.clone()); - } - if key == "after_in_parent" { - match_found = true; - vm.state.after_forkers_parent.lock().push(value.clone()); + fn register_at_fork(args: RegisterAtForkArgs, vm: &VirtualMachine) -> PyResult<()> { + let mut found_arg: bool = false; // hack to find atleas one arg + match args.before { + OptionalArg::Present(before) => { + if !before.is_callable() { + return Err(vm.new_type_error("Args must be callable".to_owned())); + } + vm.state.before_forkers.lock().push(before.clone()); + found_arg = true } - if key == "after_in_child" { - match_found = true; - vm.state.after_forkers_child.lock().push(value.clone()); + OptionalArg::Missing => (), + }; + match args.after_in_parent { + OptionalArg::Present(after_in_parent) => { + if !after_in_parent.is_callable() { + return Err(vm.new_type_error("Args must be callable".to_owned())); + } + vm.state + .after_forkers_parent + .lock() + .push(after_in_parent.clone()); + found_arg = true } - } - if !match_found { - return Err(vm.new_value_error("At least one argument is required.".to_owned())); + OptionalArg::Missing => (), + }; + match args.after_in_child { + OptionalArg::Present(after_in_child) => { + if !after_in_child.is_callable() { + return Err(vm.new_type_error("Args must be callable".to_owned())); + } + vm.state + .after_forkers_child + .lock() + .push(after_in_child.clone()); + found_arg = true + } + OptionalArg::Missing => (), + }; + if !found_arg { + return Err(vm.new_type_error("At least one argument is required.".to_owned())); } Ok(()) From 72b53a98a7c07c58d7effadb4ae2179969faef27 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 00:24:55 +0530 Subject: [PATCH 52/63] add ignore kwargs --- vm/src/stdlib/os.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b312ed2dc2..1a14be619e 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1139,14 +1139,18 @@ pub(super) mod _os { #[cfg(unix)] #[pyfunction] - fn register_at_fork(args: RegisterAtForkArgs, vm: &VirtualMachine) -> PyResult<()> { + fn register_at_fork( + args: RegisterAtForkArgs, + _ignored: Kwargs, + vm: &VirtualMachine, + ) -> PyResult<()> { let mut found_arg: bool = false; // hack to find atleas one arg match args.before { OptionalArg::Present(before) => { if !before.is_callable() { return Err(vm.new_type_error("Args must be callable".to_owned())); } - vm.state.before_forkers.lock().push(before.clone()); + vm.state.before_forkers.lock().push(before); found_arg = true } OptionalArg::Missing => (), @@ -1159,7 +1163,7 @@ pub(super) mod _os { vm.state .after_forkers_parent .lock() - .push(after_in_parent.clone()); + .push(after_in_parent); found_arg = true } @@ -1173,7 +1177,7 @@ pub(super) mod _os { vm.state .after_forkers_child .lock() - .push(after_in_child.clone()); + .push(after_in_child); found_arg = true } OptionalArg::Missing => (), From f8b966d00316dd0c9ae2a7ff7f745de4f8269a72 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 00:25:26 +0530 Subject: [PATCH 53/63] remove clone --- vm/src/stdlib/os.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 1a14be619e..cd9f0eff93 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1160,10 +1160,7 @@ pub(super) mod _os { if !after_in_parent.is_callable() { return Err(vm.new_type_error("Args must be callable".to_owned())); } - vm.state - .after_forkers_parent - .lock() - .push(after_in_parent); + vm.state.after_forkers_parent.lock().push(after_in_parent); found_arg = true } @@ -1174,10 +1171,7 @@ pub(super) mod _os { if !after_in_child.is_callable() { return Err(vm.new_type_error("Args must be callable".to_owned())); } - vm.state - .after_forkers_child - .lock() - .push(after_in_child); + vm.state.after_forkers_child.lock().push(after_in_child); found_arg = true } OptionalArg::Missing => (), From 9963280ecbfd84ba3a59205901239ae3dd17d949 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 00:31:37 +0530 Subject: [PATCH 54/63] chore! add ignorable kwargs --- vm/src/stdlib/os.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index cd9f0eff93..ed871e45cf 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -310,7 +310,7 @@ pub(super) mod _os { common::lock::{OnceCell, PyRwLock}, common::suppress_iph, convert::{IntoPyException, ToPyObject}, - function::{ArgBytesLike, Either, FsPath, FuncArgs, OptionalArg}, + function::{ArgBytesLike, Either, FsPath, FuncArgs, KwArgs, OptionalArg}, protocol::PyIterReturn, recursion::ReprGuard, types::{IterNext, IterNextIterable, PyStructSequence, Representable}, @@ -1141,7 +1141,7 @@ pub(super) mod _os { #[pyfunction] fn register_at_fork( args: RegisterAtForkArgs, - _ignored: Kwargs, + _ignored: KwArgs, vm: &VirtualMachine, ) -> PyResult<()> { let mut found_arg: bool = false; // hack to find atleas one arg From 6cd534be57a9efc62c9e58c8e3527f71d6f7c462 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 01:18:00 +0530 Subject: [PATCH 55/63] add cfg to unix only structs --- vm/src/stdlib/os.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index ed871e45cf..b3df02945c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -310,7 +310,7 @@ pub(super) mod _os { common::lock::{OnceCell, PyRwLock}, common::suppress_iph, convert::{IntoPyException, ToPyObject}, - function::{ArgBytesLike, Either, FsPath, FuncArgs, KwArgs, OptionalArg}, + function::{ArgBytesLike, Either, FsPath, FuncArgs, OptionalArg}, protocol::PyIterReturn, recursion::ReprGuard, types::{IterNext, IterNextIterable, PyStructSequence, Representable}, @@ -327,6 +327,9 @@ pub(super) mod _os { time::{Duration, SystemTime}, }; + #[cfg(unix)] + use crate::function::KwArgs; + const OPEN_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); pub(crate) const MKDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); const STAT_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); @@ -1127,6 +1130,7 @@ pub(super) mod _os { OutputMode::String.process_path(curdir_inner(vm)?, vm) } + #[cfg(unix)] #[derive(FromArgs)] struct RegisterAtForkArgs { #[pyarg(named, optional)] From 0b8f237130c78f3b52df4097660a24a6665724fe Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Mon, 24 Apr 2023 18:39:13 +0530 Subject: [PATCH 56/63] Update Lib/test/test_pty.py Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- Lib/test/test_pty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 184e2205a8..7bd1d0877b 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,4 +1,4 @@ -import threading +import threading # XXX: RUSTPYTHON from test.support import verbose, reap_children from test.support.import_helper import import_module From a85ac15a4e9eb554a960c4c31d8fc41df31d6100 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Mon, 24 Apr 2023 18:44:58 +0530 Subject: [PATCH 57/63] Update vm/src/stdlib/os.rs Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- vm/src/stdlib/os.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index b3df02945c..e7b002be77 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1140,6 +1140,29 @@ pub(super) mod _os { #[pyarg(named, optional)] after_in_child: OptionalArg, } + + impl RegisterAtForkArgs { + fn into_validated(self, vm: &VirtualMachine) -> PyResult<(Option, Option, Option)> { + fn into_option(arg: OptionalArg, vm: &VirtualMachine) -> PyResult> { + match arg { + OptionalArg::Present(obj) => { + if !obj.is_callable() { + return Err(vm.new_type_error("Args must be callable".to_owned())); + } + Ok(Some(obj)) + } + OptionalArg::Missing => Ok(None), + } + } + let before = into_option(self.before, vm)?; + let after_in_parent = into_option(self.after_in_parent, vm)?; + let after_in_child = into_option(self.after_in_child, vm)?; + if before.is_none() && after_in_parent.is_none() && after_in_child.is_none() { + return Err(vm.new_type_error("At least one arg must be present".to_owned())); + } + Ok((before, after_in_parent, after_in_child)) + } + } #[cfg(unix)] #[pyfunction] From 68fc959bb2a65cefa2b435b6a1f67eb9ee248971 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 19:19:42 +0530 Subject: [PATCH 58/63] use cleaner way of param validation and remove take on state --- vm/src/stdlib/os.rs | 67 +++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index e7b002be77..8e612b6e39 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1140,10 +1140,20 @@ pub(super) mod _os { #[pyarg(named, optional)] after_in_child: OptionalArg, } - + impl RegisterAtForkArgs { - fn into_validated(self, vm: &VirtualMachine) -> PyResult<(Option, Option, Option)> { - fn into_option(arg: OptionalArg, vm: &VirtualMachine) -> PyResult> { + fn into_validated( + self, + vm: &VirtualMachine, + ) -> PyResult<( + Option, + Option, + Option, + )> { + fn into_option( + arg: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { match arg { OptionalArg::Present(obj) => { if !obj.is_callable() { @@ -1171,42 +1181,17 @@ pub(super) mod _os { _ignored: KwArgs, vm: &VirtualMachine, ) -> PyResult<()> { - let mut found_arg: bool = false; // hack to find atleas one arg - match args.before { - OptionalArg::Present(before) => { - if !before.is_callable() { - return Err(vm.new_type_error("Args must be callable".to_owned())); - } - vm.state.before_forkers.lock().push(before); - found_arg = true - } - OptionalArg::Missing => (), - }; - match args.after_in_parent { - OptionalArg::Present(after_in_parent) => { - if !after_in_parent.is_callable() { - return Err(vm.new_type_error("Args must be callable".to_owned())); - } - vm.state.after_forkers_parent.lock().push(after_in_parent); - found_arg = true - } + let (before, after_in_parent, after_in_child) = args.into_validated(vm)?; - OptionalArg::Missing => (), - }; - match args.after_in_child { - OptionalArg::Present(after_in_child) => { - if !after_in_child.is_callable() { - return Err(vm.new_type_error("Args must be callable".to_owned())); - } - vm.state.after_forkers_child.lock().push(after_in_child); - found_arg = true - } - OptionalArg::Missing => (), - }; - if !found_arg { - return Err(vm.new_type_error("At least one argument is required.".to_owned())); + if let Some(before) = before { + vm.state.before_forkers.lock().push(before); + } + if let Some(after_in_parent) = after_in_parent { + vm.state.after_forkers_parent.lock().push(after_in_parent); + } + if let Some(after_in_child) = after_in_child { + vm.state.after_forkers_child.lock().push(after_in_child); } - Ok(()) } @@ -1230,7 +1215,7 @@ pub(super) mod _os { #[cfg(unix)] fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { - let before_forkers: Vec = std::mem::take(&mut *vm.state.before_forkers.lock()); + let before_forkers: Vec = vm.state.before_forkers.lock().clone(); // functions must be executed in reversed order as they are registered // only for before_forkers, refer: test_register_at_fork in test_posix @@ -1240,16 +1225,14 @@ pub(super) mod _os { #[cfg(unix)] fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { - let after_forkers_child: Vec = - std::mem::take(&mut *vm.state.after_forkers_child.lock()); + let after_forkers_child: Vec = vm.state.after_forkers_child.lock().clone(); run_at_forkers(after_forkers_child, false, vm); Ok(()) } #[cfg(unix)] fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { - let after_forkers_parent: Vec = - std::mem::take(&mut *vm.state.after_forkers_parent.lock()); + let after_forkers_parent: Vec = vm.state.after_forkers_parent.lock().clone(); run_at_forkers(after_forkers_parent, false, vm); Ok(()) } From 90b472b78f6b531600520bfda4c1d989eda2225d Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey Date: Mon, 24 Apr 2023 19:47:49 +0530 Subject: [PATCH 59/63] add skip cfg --- vm/src/stdlib/os.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 8e612b6e39..12dec3e397 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1141,6 +1141,7 @@ pub(super) mod _os { after_in_child: OptionalArg, } + #[cfg(unix)] impl RegisterAtForkArgs { fn into_validated( self, From 7b52282448b614a668218d5157f4a21dc3178abf Mon Sep 17 00:00:00 2001 From: Ankit Kumar Pandey <93041495+itsankitkp@users.noreply.github.com> Date: Mon, 24 Apr 2023 22:02:36 +0530 Subject: [PATCH 60/63] Update Lib/test/test_os.py Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- Lib/test/test_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d1f43beea9..d7776a0255 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4679,7 +4679,7 @@ def test_fork(self): assert_python_ok("-c", code) assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - @unittest.skipIf(_testcapi is None, 'needs _testcapi') + @unittest.skipIf(_testcapi is None, 'TODO: RUSTPYTHON; needs _testcapi') @unittest.skipUnless(sys.platform in ("linux", "darwin"), "Only Linux and macOS detect this today.") def test_fork_warns_when_non_python_thread_exists(self): From 898e0846738d9825f48152272bf5950b07270e9e Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 25 Apr 2023 01:35:37 +0900 Subject: [PATCH 61/63] Update Lib/test/test_fcntl.py --- Lib/test/test_fcntl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 08b26d2ee3..84bdf83e93 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -4,7 +4,7 @@ import os import struct import sys -import threading +import threading # XXX: RUSTPYTHON import unittest from multiprocessing import Process from test.support import verbose, cpython_only From 5ab1706bf4e24ce96856996b902d8f50104bf2b9 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 25 Apr 2023 01:40:31 +0900 Subject: [PATCH 62/63] relocate fork to posix --- vm/src/stdlib/os.rs | 127 ----------------------------------------- vm/src/stdlib/posix.rs | 118 +++++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 128 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 12dec3e397..d03e40261e 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -327,9 +327,6 @@ pub(super) mod _os { time::{Duration, SystemTime}, }; - #[cfg(unix)] - use crate::function::KwArgs; - const OPEN_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); pub(crate) const MKDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); const STAT_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox"))); @@ -1130,130 +1127,6 @@ pub(super) mod _os { OutputMode::String.process_path(curdir_inner(vm)?, vm) } - #[cfg(unix)] - #[derive(FromArgs)] - struct RegisterAtForkArgs { - #[pyarg(named, optional)] - before: OptionalArg, - #[pyarg(named, optional)] - after_in_parent: OptionalArg, - #[pyarg(named, optional)] - after_in_child: OptionalArg, - } - - #[cfg(unix)] - impl RegisterAtForkArgs { - fn into_validated( - self, - vm: &VirtualMachine, - ) -> PyResult<( - Option, - Option, - Option, - )> { - fn into_option( - arg: OptionalArg, - vm: &VirtualMachine, - ) -> PyResult> { - match arg { - OptionalArg::Present(obj) => { - if !obj.is_callable() { - return Err(vm.new_type_error("Args must be callable".to_owned())); - } - Ok(Some(obj)) - } - OptionalArg::Missing => Ok(None), - } - } - let before = into_option(self.before, vm)?; - let after_in_parent = into_option(self.after_in_parent, vm)?; - let after_in_child = into_option(self.after_in_child, vm)?; - if before.is_none() && after_in_parent.is_none() && after_in_child.is_none() { - return Err(vm.new_type_error("At least one arg must be present".to_owned())); - } - Ok((before, after_in_parent, after_in_child)) - } - } - - #[cfg(unix)] - #[pyfunction] - fn register_at_fork( - args: RegisterAtForkArgs, - _ignored: KwArgs, - vm: &VirtualMachine, - ) -> PyResult<()> { - let (before, after_in_parent, after_in_child) = args.into_validated(vm)?; - - if let Some(before) = before { - vm.state.before_forkers.lock().push(before); - } - if let Some(after_in_parent) = after_in_parent { - vm.state.after_forkers_parent.lock().push(after_in_parent); - } - if let Some(after_in_child) = after_in_child { - vm.state.after_forkers_child.lock().push(after_in_child); - } - Ok(()) - } - - #[cfg(unix)] - fn run_at_forkers(mut funcs: Vec, reversed: bool, vm: &VirtualMachine) { - if !funcs.is_empty() { - if reversed { - funcs.reverse(); - } - for func in funcs.into_iter() { - if let Err(e) = func.call((), vm) { - let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); - vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func); - if exit { - // Do nothing! - } - } - } - } - } - - #[cfg(unix)] - fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { - let before_forkers: Vec = vm.state.before_forkers.lock().clone(); - // functions must be executed in reversed order as they are registered - // only for before_forkers, refer: test_register_at_fork in test_posix - - run_at_forkers(before_forkers, true, vm); - Ok(()) - } - - #[cfg(unix)] - fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { - let after_forkers_child: Vec = vm.state.after_forkers_child.lock().clone(); - run_at_forkers(after_forkers_child, false, vm); - Ok(()) - } - - #[cfg(unix)] - fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { - let after_forkers_parent: Vec = vm.state.after_forkers_parent.lock().clone(); - run_at_forkers(after_forkers_parent, false, vm); - Ok(()) - } - - #[cfg(unix)] - #[pyfunction] - fn fork(vm: &VirtualMachine) -> PyResult { - let pid: i32; - py_os_before_fork(vm)?; - unsafe { - pid = libc::fork(); - } - if pid == 0 { - py_os_after_fork_child(vm)?; - } else { - py_os_after_fork_parent(vm)?; - } - Ok(vm.ctx.new_int(pid).into()) - } - #[pyfunction] fn getcwdb(vm: &VirtualMachine) -> PyResult { OutputMode::Bytes.process_path(curdir_inner(vm)?, vm) diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 2366cb622a..89975c0715 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -23,7 +23,7 @@ pub mod module { use crate::{ builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, convert::{IntoPyException, ToPyObject, TryFromObject}, - function::{Either, OptionalArg}, + function::{Either, OptionalArg, KwArgs}, stdlib::os::{ errno_err, DirFd, FollowSymlinks, OsPath, OsPathOrFd, SupportFunc, TargetIsDirectory, _os, fs_metadata, IOErrorBuilder, @@ -421,6 +421,122 @@ pub mod module { ) } + #[derive(FromArgs)] + struct RegisterAtForkArgs { + #[pyarg(named, optional)] + before: OptionalArg, + #[pyarg(named, optional)] + after_in_parent: OptionalArg, + #[pyarg(named, optional)] + after_in_child: OptionalArg, + } + + impl RegisterAtForkArgs { + fn into_validated( + self, + vm: &VirtualMachine, + ) -> PyResult<( + Option, + Option, + Option, + )> { + fn into_option( + arg: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult> { + match arg { + OptionalArg::Present(obj) => { + if !obj.is_callable() { + return Err(vm.new_type_error("Args must be callable".to_owned())); + } + Ok(Some(obj)) + } + OptionalArg::Missing => Ok(None), + } + } + let before = into_option(self.before, vm)?; + let after_in_parent = into_option(self.after_in_parent, vm)?; + let after_in_child = into_option(self.after_in_child, vm)?; + if before.is_none() && after_in_parent.is_none() && after_in_child.is_none() { + return Err(vm.new_type_error("At least one arg must be present".to_owned())); + } + Ok((before, after_in_parent, after_in_child)) + } + } + + #[pyfunction] + fn register_at_fork( + args: RegisterAtForkArgs, + _ignored: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + let (before, after_in_parent, after_in_child) = args.into_validated(vm)?; + + if let Some(before) = before { + vm.state.before_forkers.lock().push(before); + } + if let Some(after_in_parent) = after_in_parent { + vm.state.after_forkers_parent.lock().push(after_in_parent); + } + if let Some(after_in_child) = after_in_child { + vm.state.after_forkers_child.lock().push(after_in_child); + } + Ok(()) + } + + fn run_at_forkers(mut funcs: Vec, reversed: bool, vm: &VirtualMachine) { + if !funcs.is_empty() { + if reversed { + funcs.reverse(); + } + for func in funcs.into_iter() { + if let Err(e) = func.call((), vm) { + let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); + vm.run_unraisable(e, Some("Exception ignored in".to_owned()), func); + if exit { + // Do nothing! + } + } + } + } + } + + fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { + let before_forkers: Vec = vm.state.before_forkers.lock().clone(); + // functions must be executed in reversed order as they are registered + // only for before_forkers, refer: test_register_at_fork in test_posix + + run_at_forkers(before_forkers, true, vm); + Ok(()) + } + + fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { + let after_forkers_child: Vec = vm.state.after_forkers_child.lock().clone(); + run_at_forkers(after_forkers_child, false, vm); + Ok(()) + } + + fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { + let after_forkers_parent: Vec = vm.state.after_forkers_parent.lock().clone(); + run_at_forkers(after_forkers_parent, false, vm); + Ok(()) + } + + #[pyfunction] + fn fork(vm: &VirtualMachine) -> PyResult { + let pid: i32; + py_os_before_fork(vm)?; + unsafe { + pid = libc::fork(); + } + if pid == 0 { + py_os_after_fork_child(vm)?; + } else { + py_os_after_fork_parent(vm)?; + } + Ok(vm.ctx.new_int(pid).into()) + } + #[cfg(not(target_os = "redox"))] const MKNOD_DIR_FD: bool = cfg!(not(target_vendor = "apple")); From 0b61077eb3959a5cd9a89fbc95fd83cf561fe29e Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 25 Apr 2023 01:44:11 +0900 Subject: [PATCH 63/63] simplify fork functoins --- vm/src/stdlib/posix.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 89975c0715..68a77019e2 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -23,7 +23,7 @@ pub mod module { use crate::{ builtins::{PyDictRef, PyInt, PyListRef, PyStrRef, PyTupleRef, PyTypeRef}, convert::{IntoPyException, ToPyObject, TryFromObject}, - function::{Either, OptionalArg, KwArgs}, + function::{Either, KwArgs, OptionalArg}, stdlib::os::{ errno_err, DirFd, FollowSymlinks, OsPath, OsPathOrFd, SupportFunc, TargetIsDirectory, _os, fs_metadata, IOErrorBuilder, @@ -501,40 +501,37 @@ pub mod module { } } - fn py_os_before_fork(vm: &VirtualMachine) -> PyResult<()> { + fn py_os_before_fork(vm: &VirtualMachine) { let before_forkers: Vec = vm.state.before_forkers.lock().clone(); // functions must be executed in reversed order as they are registered // only for before_forkers, refer: test_register_at_fork in test_posix run_at_forkers(before_forkers, true, vm); - Ok(()) } - fn py_os_after_fork_child(vm: &VirtualMachine) -> PyResult<()> { + fn py_os_after_fork_child(vm: &VirtualMachine) { let after_forkers_child: Vec = vm.state.after_forkers_child.lock().clone(); run_at_forkers(after_forkers_child, false, vm); - Ok(()) } - fn py_os_after_fork_parent(vm: &VirtualMachine) -> PyResult<()> { + fn py_os_after_fork_parent(vm: &VirtualMachine) { let after_forkers_parent: Vec = vm.state.after_forkers_parent.lock().clone(); run_at_forkers(after_forkers_parent, false, vm); - Ok(()) } #[pyfunction] - fn fork(vm: &VirtualMachine) -> PyResult { + fn fork(vm: &VirtualMachine) -> i32 { let pid: i32; - py_os_before_fork(vm)?; + py_os_before_fork(vm); unsafe { pid = libc::fork(); } if pid == 0 { - py_os_after_fork_child(vm)?; + py_os_after_fork_child(vm); } else { - py_os_after_fork_parent(vm)?; + py_os_after_fork_parent(vm); } - Ok(vm.ctx.new_int(pid).into()) + pid } #[cfg(not(target_os = "redox"))]