From b3e39a26165c46c70fd651eca4896bec2bd4aa63 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 09:46:07 +0700 Subject: [PATCH 01/29] Add localeconv function to locale module --- stdlib/src/lib.rs | 2 + stdlib/src/locale.rs | 103 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 stdlib/src/locale.rs diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index a4bbabf5ed..36ccbaf2b4 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -15,6 +15,7 @@ mod dis; mod gc; mod hashlib; mod json; +mod locale; mod math; #[cfg(unix)] mod mmap; @@ -102,6 +103,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "math" => math::make_module, "pyexpat" => pyexpat::make_module, "_random" => random::make_module, + "_locale" => locale::make_module, "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, "unicodedata" => unicodedata::make_module, diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs new file mode 100644 index 0000000000..50ddb45522 --- /dev/null +++ b/stdlib/src/locale.rs @@ -0,0 +1,103 @@ +pub(crate) use _locale::make_module; + +#[pymodule] +mod _locale { + extern crate libc; + use rustpython_vm::{ + builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef}, + function::OptionalArg, + PyObjectRef, PyResult, VirtualMachine, + }; + + #[pyattr] + use libc::{ + ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_10, ABMON_11, + ABMON_12, ABMON_2, ABMON_3, ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, + ALT_DIGITS, AM_STR, CODESET, CRNCYSTR, DAY_1, DAY_2, DAY_3, DAY_4, DAY_5, DAY_6, DAY_7, + D_FMT, D_T_FMT, ERA, ERA_D_FMT, ERA_D_T_FMT, ERA_T_FMT, LC_ALL, LC_COLLATE, LC_CTYPE, + LC_MESSAGES, LC_MONETARY, LC_NUMERIC, LC_TIME, MON_1, MON_10, MON_11, MON_12, MON_2, MON_3, + MON_4, MON_5, MON_6, MON_7, MON_8, MON_9, NOEXPR, PM_STR, RADIXCHAR, THOUSEP, T_FMT, + T_FMT_AMPM, YESEXPR, + }; + + use libc::{c_char, c_int}; + use std::ffi::CStr; + use std::ptr; + + #[pyattr(name = "CHAR_MAX")] + fn char_max(vm: &VirtualMachine) -> PyIntRef { + vm.ctx.new_int(libc::c_char::MAX) + } + + unsafe fn _get_grouping(group: *mut c_char, vm: &VirtualMachine) -> PyListRef { + let mut group_vec: Vec = Vec::new(); + let ptr = group; + + while *ptr != (u8::MIN as i8) && *ptr != i8::MAX { + let val = vm.ctx.new_int(*ptr); + group_vec.push(val.into()); + } + vm.ctx.new_list(group_vec) + } + + unsafe fn _parse_ptr_to_str(vm: &VirtualMachine, raw_ptr: *mut c_char) -> PyResult { + let slice = unsafe { CStr::from_ptr(raw_ptr) }; + let cstr = slice + .to_str() + .map(|s| s.to_owned()) + .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode: {e}")))?; + + Ok(vm.new_pyobj(cstr)) + } + + #[pyfunction] + fn localeconv(vm: &VirtualMachine) -> PyResult { + let result = vm.ctx.new_dict(); + unsafe { + let lc = libc::localeconv(); + + let mon_grouping = (*lc).mon_grouping; + let int_frac_digits = vm.ctx.new_int((*lc).int_frac_digits).into(); + let frac_digits = vm.ctx.new_int((*lc).frac_digits).into(); + let p_cs_precedes = vm.ctx.new_int((*lc).p_cs_precedes).into(); + let p_sep_by_space = vm.ctx.new_int((*lc).p_sep_by_space).into(); + let n_cs_precedes = vm.ctx.new_int((*lc).n_cs_precedes).into(); + let p_sign_posn = vm.ctx.new_int((*lc).p_sign_posn).into(); + let n_sign_posn = vm.ctx.new_int((*lc).n_sign_posn).into(); + let grouping = (*lc).grouping; + let decimal_point = _parse_ptr_to_str(vm, (*lc).decimal_point)?; + let thousands_sep = _parse_ptr_to_str(vm, (*lc).thousands_sep)?; + let int_curr_symbol = _parse_ptr_to_str(vm, (*lc).int_curr_symbol)?; + let currency_symbol = _parse_ptr_to_str(vm, (*lc).currency_symbol)?; + let mon_decimal_point = _parse_ptr_to_str(vm, (*lc).mon_decimal_point)?; + let mon_thousands_sep = _parse_ptr_to_str(vm, (*lc).mon_thousands_sep)?; + let n_sep_by_space = vm.ctx.new_int((*lc).n_sep_by_space).into(); + let positive_sign = _parse_ptr_to_str(vm, (*lc).positive_sign)?; + let negative_sign = _parse_ptr_to_str(vm, (*lc).negative_sign)?; + + result.set_item( + stringify!(mon_grouping), + _get_grouping(mon_grouping, vm).into(), + vm, + )?; + result.set_item(stringify!(int_frac_digits), int_frac_digits, vm)?; + result.set_item(stringify!(frac_digits), frac_digits, vm)?; + result.set_item(stringify!(p_cs_precedes), p_cs_precedes, vm)?; + result.set_item(stringify!(p_sep_by_space), p_sep_by_space, vm)?; + result.set_item(stringify!(n_cs_precedes), n_cs_precedes, vm)?; + result.set_item(stringify!(p_sign_posn), p_sign_posn, vm)?; + result.set_item(stringify!(n_sign_posn), n_sign_posn, vm)?; + result.set_item(stringify!(grouping), _get_grouping(grouping, vm).into(), vm)?; + result.set_item(stringify!(decimal_point), decimal_point, vm)?; + result.set_item(stringify!(thousands_sep), thousands_sep, vm)?; + result.set_item(stringify!(int_curr_symbol), int_curr_symbol, vm)?; + result.set_item(stringify!(currency_symbol), currency_symbol, vm)?; + result.set_item(stringify!(mon_decimal_point), mon_decimal_point, vm)?; + result.set_item(stringify!(mon_thousands_sep), mon_thousands_sep, vm)?; + result.set_item(stringify!(n_sep_by_space), n_sep_by_space, vm)?; + result.set_item(stringify!(positive_sign), positive_sign, vm)?; + result.set_item(stringify!(negative_sign), negative_sign, vm)?; + } + Ok(result) + } +} From 1ed1b295fb289f9eb71a253bbfa625175a556df2 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 09:56:45 +0700 Subject: [PATCH 02/29] Fix potential infinite loop --- stdlib/src/locale.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 50ddb45522..b7f2181953 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -4,8 +4,7 @@ pub(crate) use _locale::make_module; mod _locale { extern crate libc; use rustpython_vm::{ - builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef}, - function::OptionalArg, + builtins::{PyDictRef, PyIntRef, PyListRef}, PyObjectRef, PyResult, VirtualMachine, }; @@ -20,9 +19,8 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use libc::{c_char, c_int}; + use libc::c_char; use std::ffi::CStr; - use std::ptr; #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { @@ -31,11 +29,12 @@ mod _locale { unsafe fn _get_grouping(group: *mut c_char, vm: &VirtualMachine) -> PyListRef { let mut group_vec: Vec = Vec::new(); - let ptr = group; + let mut ptr = group; while *ptr != (u8::MIN as i8) && *ptr != i8::MAX { let val = vm.ctx.new_int(*ptr); group_vec.push(val.into()); + ptr = ptr.offset(1); } vm.ctx.new_list(group_vec) } From 7e9390c3a9e527ca14afcabce2777de66679eeff Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 10:06:45 +0700 Subject: [PATCH 03/29] Skip locale test --- Lib/test/test_locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 196dc6e3a1..afd75d33e4 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -6,7 +6,7 @@ import sys import codecs - +@unittest.SkipTest(reason="Library not completed") class BaseLocalizedTest(unittest.TestCase): # # Base class for tests using a real locale From b2c1e5f44cda6b2615419027051253e2a50f8594 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 16:05:44 +0700 Subject: [PATCH 04/29] Refactor and add platform def for localeconv --- stdlib/src/locale.rs | 91 ++++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index b7f2181953..db4fdc8767 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -49,53 +49,68 @@ mod _locale { Ok(vm.new_pyobj(cstr)) } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[pyfunction] fn localeconv(vm: &VirtualMachine) -> PyResult { let result = vm.ctx.new_dict(); + macro_rules! set_string_field { + ($field:expr) => {{ + result.set_item(stringify!($field), _parse_ptr_to_str(vm, $field)?, vm)? + }}; + } + + macro_rules! set_int_field { + ($field:expr) => {{ + result.set_item(stringify!($field), vm.new_pyobj($field), vm)? + }}; + } + + macro_rules! set_group_field { + ($field:expr) => {{ + result.set_item(stringify!($field), _get_grouping($field, vm).into(), vm)? + }}; + } + unsafe { let lc = libc::localeconv(); let mon_grouping = (*lc).mon_grouping; - let int_frac_digits = vm.ctx.new_int((*lc).int_frac_digits).into(); - let frac_digits = vm.ctx.new_int((*lc).frac_digits).into(); - let p_cs_precedes = vm.ctx.new_int((*lc).p_cs_precedes).into(); - let p_sep_by_space = vm.ctx.new_int((*lc).p_sep_by_space).into(); - let n_cs_precedes = vm.ctx.new_int((*lc).n_cs_precedes).into(); - let p_sign_posn = vm.ctx.new_int((*lc).p_sign_posn).into(); - let n_sign_posn = vm.ctx.new_int((*lc).n_sign_posn).into(); + let int_frac_digits = (*lc).int_frac_digits; + let frac_digits = (*lc).frac_digits; + let p_cs_precedes = (*lc).p_cs_precedes; + let p_sep_by_space = (*lc).p_sep_by_space; + let n_cs_precedes = (*lc).n_cs_precedes; + let p_sign_posn = (*lc).p_sign_posn; + let n_sign_posn = (*lc).n_sign_posn; let grouping = (*lc).grouping; - let decimal_point = _parse_ptr_to_str(vm, (*lc).decimal_point)?; - let thousands_sep = _parse_ptr_to_str(vm, (*lc).thousands_sep)?; - let int_curr_symbol = _parse_ptr_to_str(vm, (*lc).int_curr_symbol)?; - let currency_symbol = _parse_ptr_to_str(vm, (*lc).currency_symbol)?; - let mon_decimal_point = _parse_ptr_to_str(vm, (*lc).mon_decimal_point)?; - let mon_thousands_sep = _parse_ptr_to_str(vm, (*lc).mon_thousands_sep)?; - let n_sep_by_space = vm.ctx.new_int((*lc).n_sep_by_space).into(); - let positive_sign = _parse_ptr_to_str(vm, (*lc).positive_sign)?; - let negative_sign = _parse_ptr_to_str(vm, (*lc).negative_sign)?; + let decimal_point = (*lc).decimal_point; + let thousands_sep = (*lc).thousands_sep; + let int_curr_symbol = (*lc).int_curr_symbol; + let currency_symbol = (*lc).currency_symbol; + let mon_decimal_point = (*lc).mon_decimal_point; + let mon_thousands_sep = (*lc).mon_thousands_sep; + let n_sep_by_space = (*lc).n_sep_by_space; + let positive_sign = (*lc).positive_sign; + let negative_sign = (*lc).negative_sign; - result.set_item( - stringify!(mon_grouping), - _get_grouping(mon_grouping, vm).into(), - vm, - )?; - result.set_item(stringify!(int_frac_digits), int_frac_digits, vm)?; - result.set_item(stringify!(frac_digits), frac_digits, vm)?; - result.set_item(stringify!(p_cs_precedes), p_cs_precedes, vm)?; - result.set_item(stringify!(p_sep_by_space), p_sep_by_space, vm)?; - result.set_item(stringify!(n_cs_precedes), n_cs_precedes, vm)?; - result.set_item(stringify!(p_sign_posn), p_sign_posn, vm)?; - result.set_item(stringify!(n_sign_posn), n_sign_posn, vm)?; - result.set_item(stringify!(grouping), _get_grouping(grouping, vm).into(), vm)?; - result.set_item(stringify!(decimal_point), decimal_point, vm)?; - result.set_item(stringify!(thousands_sep), thousands_sep, vm)?; - result.set_item(stringify!(int_curr_symbol), int_curr_symbol, vm)?; - result.set_item(stringify!(currency_symbol), currency_symbol, vm)?; - result.set_item(stringify!(mon_decimal_point), mon_decimal_point, vm)?; - result.set_item(stringify!(mon_thousands_sep), mon_thousands_sep, vm)?; - result.set_item(stringify!(n_sep_by_space), n_sep_by_space, vm)?; - result.set_item(stringify!(positive_sign), positive_sign, vm)?; - result.set_item(stringify!(negative_sign), negative_sign, vm)?; + set_group_field!(mon_grouping); + set_group_field!(grouping); + set_int_field!(int_frac_digits); + set_int_field!(frac_digits); + set_int_field!(p_cs_precedes); + set_int_field!(p_sep_by_space); + set_int_field!(n_cs_precedes); + set_int_field!(p_sign_posn); + set_int_field!(n_sign_posn); + set_string_field!(decimal_point); + set_string_field!(thousands_sep); + set_string_field!(int_curr_symbol); + set_string_field!(currency_symbol); + set_string_field!(mon_decimal_point); + set_string_field!(mon_thousands_sep); + set_int_field!(n_sep_by_space); + set_string_field!(positive_sign); + set_string_field!(negative_sign); } Ok(result) } From 0dd95e02281eff1f45ca4e95ae913fd3abc31532 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 16:13:11 +0700 Subject: [PATCH 05/29] Add missing platform def on constant --- stdlib/src/locale.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index db4fdc8767..6f45ca401b 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -7,7 +7,8 @@ mod _locale { builtins::{PyDictRef, PyIntRef, PyListRef}, PyObjectRef, PyResult, VirtualMachine, }; - + + #[cfg(any(target_os = "macos", target_os = "linux"))] #[pyattr] use libc::{ ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_10, ABMON_11, From 65f5036119ca159bd015d8b5b408d727d3cea58f Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 16:15:43 +0700 Subject: [PATCH 06/29] Fix wrong format --- stdlib/src/locale.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 6f45ca401b..4b172fe0e7 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -7,7 +7,7 @@ mod _locale { builtins::{PyDictRef, PyIntRef, PyListRef}, PyObjectRef, PyResult, VirtualMachine, }; - + #[cfg(any(target_os = "macos", target_os = "linux"))] #[pyattr] use libc::{ From 359c6966468a1673df0143dbc24d069320e24184 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 16:41:00 +0700 Subject: [PATCH 07/29] Fix platform dependent error build --- stdlib/src/lib.rs | 5 ++++- stdlib/src/locale.rs | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 36ccbaf2b4..30afcaa436 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -103,7 +103,6 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "math" => math::make_module, "pyexpat" => pyexpat::make_module, "_random" => random::make_module, - "_locale" => locale::make_module, "_statistics" => statistics::make_module, "_struct" => pystruct::make_module, "unicodedata" => unicodedata::make_module, @@ -161,5 +160,9 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit { "_uuid" => uuid::make_module, } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + "_locale" => locale::make_module, + } } } diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 4b172fe0e7..0fec3d895d 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -8,7 +8,6 @@ mod _locale { PyObjectRef, PyResult, VirtualMachine, }; - #[cfg(any(target_os = "macos", target_os = "linux"))] #[pyattr] use libc::{ ABDAY_1, ABDAY_2, ABDAY_3, ABDAY_4, ABDAY_5, ABDAY_6, ABDAY_7, ABMON_1, ABMON_10, ABMON_11, @@ -20,8 +19,7 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use libc::c_char; - use std::ffi::CStr; + use std::ffi::{c_char, CStr}; #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { @@ -50,7 +48,6 @@ mod _locale { Ok(vm.new_pyobj(cstr)) } - #[cfg(any(target_os = "macos", target_os = "linux"))] #[pyfunction] fn localeconv(vm: &VirtualMachine) -> PyResult { let result = vm.ctx.new_dict(); From 1533f7b1edafd42e64c402739c93942c4bf3f771 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 16:51:30 +0700 Subject: [PATCH 08/29] Add platform def at the top of locale module --- stdlib/src/locale.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 0fec3d895d..b52bf75874 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -1,5 +1,7 @@ +#[cfg(any(target_os = "linux", target_os = "macos"))] pub(crate) use _locale::make_module; +#[cfg(any(target_os = "linux", target_os = "macos"))] #[pymodule] mod _locale { extern crate libc; From 09c750ce8bee86bf0a162af7d3d1303be8e22b6a Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 21:42:02 +0700 Subject: [PATCH 09/29] Refactor code --- stdlib/src/lib.rs | 5 +--- stdlib/src/locale.rs | 68 +++++++++++++++++--------------------------- 2 files changed, 27 insertions(+), 46 deletions(-) diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 30afcaa436..b735eb0581 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -139,6 +139,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_posixsubprocess" => posixsubprocess::make_module, "syslog" => syslog::make_module, "mmap" => mmap::make_module, + "_locale" => locale::make_module, } #[cfg(all(unix, not(target_os = "redox")))] { @@ -160,9 +161,5 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit { "_uuid" => uuid::make_module, } - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - "_locale" => locale::make_module, - } } } diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index b52bf75874..175704d150 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -1,10 +1,9 @@ -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(unix)] pub(crate) use _locale::make_module; -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(unix)] #[pymodule] mod _locale { - extern crate libc; use rustpython_vm::{ builtins::{PyDictRef, PyIntRef, PyListRef}, PyObjectRef, PyResult, VirtualMachine, @@ -28,11 +27,11 @@ mod _locale { vm.ctx.new_int(libc::c_char::MAX) } - unsafe fn _get_grouping(group: *mut c_char, vm: &VirtualMachine) -> PyListRef { + unsafe fn copy_grouping(group: *mut c_char, vm: &VirtualMachine) -> PyListRef { let mut group_vec: Vec = Vec::new(); let mut ptr = group; - while *ptr != (u8::MIN as i8) && *ptr != i8::MAX { + while ![0 as i8, c_char::MAX].contains(&*ptr) { let val = vm.ctx.new_int(*ptr); group_vec.push(val.into()); ptr = ptr.offset(1); @@ -44,8 +43,7 @@ mod _locale { let slice = unsafe { CStr::from_ptr(raw_ptr) }; let cstr = slice .to_str() - .map(|s| s.to_owned()) - .map_err(|e| vm.new_unicode_decode_error(format!("unable to decode: {e}")))?; + .expect("localeconv always return decodable string"); Ok(vm.new_pyobj(cstr)) } @@ -53,45 +51,31 @@ mod _locale { #[pyfunction] fn localeconv(vm: &VirtualMachine) -> PyResult { let result = vm.ctx.new_dict(); - macro_rules! set_string_field { - ($field:expr) => {{ - result.set_item(stringify!($field), _parse_ptr_to_str(vm, $field)?, vm)? - }}; - } - - macro_rules! set_int_field { - ($field:expr) => {{ - result.set_item(stringify!($field), vm.new_pyobj($field), vm)? - }}; - } - - macro_rules! set_group_field { - ($field:expr) => {{ - result.set_item(stringify!($field), _get_grouping($field, vm).into(), vm)? - }}; - } unsafe { let lc = libc::localeconv(); - let mon_grouping = (*lc).mon_grouping; - let int_frac_digits = (*lc).int_frac_digits; - let frac_digits = (*lc).frac_digits; - let p_cs_precedes = (*lc).p_cs_precedes; - let p_sep_by_space = (*lc).p_sep_by_space; - let n_cs_precedes = (*lc).n_cs_precedes; - let p_sign_posn = (*lc).p_sign_posn; - let n_sign_posn = (*lc).n_sign_posn; - let grouping = (*lc).grouping; - let decimal_point = (*lc).decimal_point; - let thousands_sep = (*lc).thousands_sep; - let int_curr_symbol = (*lc).int_curr_symbol; - let currency_symbol = (*lc).currency_symbol; - let mon_decimal_point = (*lc).mon_decimal_point; - let mon_thousands_sep = (*lc).mon_thousands_sep; - let n_sep_by_space = (*lc).n_sep_by_space; - let positive_sign = (*lc).positive_sign; - let negative_sign = (*lc).negative_sign; + macro_rules! set_string_field { + ($field:ident) => {{ + result.set_item(stringify!($field), _parse_ptr_to_str(vm, (*lc).$field)?, vm)? + }}; + } + + macro_rules! set_int_field { + ($field:ident) => {{ + result.set_item(stringify!($field), vm.new_pyobj((*lc).$field), vm)? + }}; + } + + macro_rules! set_group_field { + ($field:ident) => {{ + result.set_item( + stringify!($field), + copy_grouping((*lc).$field, vm).into(), + vm, + )? + }}; + } set_group_field!(mon_grouping); set_group_field!(grouping); From 844a30af6987ff47dc2e2751192b5b29990cb127 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 22:20:20 +0700 Subject: [PATCH 10/29] Fix mismatch typing --- stdlib/src/locale.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 175704d150..dc2f5ce00d 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -31,7 +31,7 @@ mod _locale { let mut group_vec: Vec = Vec::new(); let mut ptr = group; - while ![0 as i8, c_char::MAX].contains(&*ptr) { + while ![0_i8, c_char::MAX as i8].contains(&*ptr) { let val = vm.ctx.new_int(*ptr); group_vec.push(val.into()); ptr = ptr.offset(1); From 26f103ac3e93185ca4365ff8194d05061264cee0 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 23:54:18 +0700 Subject: [PATCH 11/29] Use libc::c_char instead --- stdlib/src/locale.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index dc2f5ce00d..79583c5779 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -20,18 +20,18 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use std::ffi::{c_char, CStr}; + use std::ffi::CStr; #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { vm.ctx.new_int(libc::c_char::MAX) } - unsafe fn copy_grouping(group: *mut c_char, vm: &VirtualMachine) -> PyListRef { + unsafe fn copy_grouping(group: *mut libc::c_char, vm: &VirtualMachine) -> PyListRef { let mut group_vec: Vec = Vec::new(); let mut ptr = group; - while ![0_i8, c_char::MAX as i8].contains(&*ptr) { + while ![0_i8, libc::c_char::MAX].contains(&*ptr) { let val = vm.ctx.new_int(*ptr); group_vec.push(val.into()); ptr = ptr.offset(1); @@ -39,7 +39,7 @@ mod _locale { vm.ctx.new_list(group_vec) } - unsafe fn _parse_ptr_to_str(vm: &VirtualMachine, raw_ptr: *mut c_char) -> PyResult { + unsafe fn _parse_ptr_to_str(vm: &VirtualMachine, raw_ptr: *mut libc::c_char) -> PyResult { let slice = unsafe { CStr::from_ptr(raw_ptr) }; let cstr = slice .to_str() From 937400533a3908f51a6575b403467697ee447e56 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sat, 25 Feb 2023 01:17:45 +0700 Subject: [PATCH 12/29] Fix skip test reason for locale --- Lib/test/test_locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index afd75d33e4..897f47196f 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -6,7 +6,7 @@ import sys import codecs -@unittest.SkipTest(reason="Library not completed") +@unittest.SkipTest(reason="TODO: RUSTPYTHON, Library not completed") class BaseLocalizedTest(unittest.TestCase): # # Base class for tests using a real locale From 757545fe37c347c75a5734e0fa9ef9d13a3bcce8 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Fri, 24 Feb 2023 23:38:10 +0700 Subject: [PATCH 13/29] Add setlocale function --- stdlib/src/locale.rs | 54 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index 79583c5779..d69b4720b9 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -5,8 +5,8 @@ pub(crate) use _locale::make_module; #[pymodule] mod _locale { use rustpython_vm::{ - builtins::{PyDictRef, PyIntRef, PyListRef}, - PyObjectRef, PyResult, VirtualMachine, + builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, + PyObjectRef, PyResult, VirtualMachine, function::OptionalArg, }; #[pyattr] @@ -20,7 +20,7 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use std::ffi::CStr; + use std::{ptr, ffi::CStr}; #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { @@ -47,6 +47,15 @@ mod _locale { Ok(vm.new_pyobj(cstr)) } + + #[pyattr(name="Error")] + fn error(vm: &VirtualMachine) -> PyTypeRef { + vm.ctx.new_exception_type( + "locale", + "locale.Error", + Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), + ) + } #[pyfunction] fn localeconv(vm: &VirtualMachine) -> PyResult { @@ -98,4 +107,43 @@ mod _locale { } Ok(result) } + + #[derive(FromArgs)] + struct LocaleArgs { + #[pyarg(any)] + category: i32, + #[pyarg(any, optional)] + locale: OptionalArg> + } + + #[pyfunction] + fn setlocale(args: LocaleArgs, vm: &VirtualMachine) -> PyResult { + unsafe { + let result = match args.locale { + OptionalArg::Missing => { + let null_ptr: *const i8 = ptr::null(); + libc::setlocale(args.category, null_ptr) + }, + OptionalArg::Present(locale) => match locale { + None => { + let null_ptr: *const i8 = ptr::null(); + libc::setlocale(args.category, null_ptr) + }, + Some(l) => { + let mut l_str = l.to_string(); + if l_str.is_empty() { + l_str.push_str("\0"); + } + let l_ptr = l_str.as_ptr() as *const i8; + libc::setlocale(args.category, l_ptr) + }, + } + }; + if result.is_null() { + let error = error(vm); + return Err(vm.new_exception_msg(error, String::from("unsupported locale setting"))) + } + _parse_ptr_to_str(vm, result) + } + } } From ecc21e862f6c213c9bc1f7e7328646c0848757ea Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sat, 25 Feb 2023 14:25:49 +0700 Subject: [PATCH 14/29] Fix clippy requirements --- stdlib/src/locale.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index d69b4720b9..a5534f50db 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -132,7 +132,7 @@ mod _locale { Some(l) => { let mut l_str = l.to_string(); if l_str.is_empty() { - l_str.push_str("\0"); + l_str.push('\0'); } let l_ptr = l_str.as_ptr() as *const i8; libc::setlocale(args.category, l_ptr) From 973d2b2858742d37af06ac955d71ca38318619f2 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sat, 25 Feb 2023 14:28:16 +0700 Subject: [PATCH 15/29] Fix rustfmt issues --- stdlib/src/locale.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index a5534f50db..d24daa785d 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -6,7 +6,8 @@ pub(crate) use _locale::make_module; mod _locale { use rustpython_vm::{ builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, - PyObjectRef, PyResult, VirtualMachine, function::OptionalArg, + function::OptionalArg, + PyObjectRef, PyResult, VirtualMachine, }; #[pyattr] @@ -20,7 +21,7 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use std::{ptr, ffi::CStr}; + use std::{ffi::CStr, ptr}; #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { @@ -47,8 +48,8 @@ mod _locale { Ok(vm.new_pyobj(cstr)) } - - #[pyattr(name="Error")] + + #[pyattr(name = "Error")] fn error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.new_exception_type( "locale", @@ -113,7 +114,7 @@ mod _locale { #[pyarg(any)] category: i32, #[pyarg(any, optional)] - locale: OptionalArg> + locale: OptionalArg>, } #[pyfunction] @@ -122,13 +123,13 @@ mod _locale { let result = match args.locale { OptionalArg::Missing => { let null_ptr: *const i8 = ptr::null(); - libc::setlocale(args.category, null_ptr) - }, + libc::setlocale(args.category, null_ptr) + } OptionalArg::Present(locale) => match locale { None => { let null_ptr: *const i8 = ptr::null(); libc::setlocale(args.category, null_ptr) - }, + } Some(l) => { let mut l_str = l.to_string(); if l_str.is_empty() { @@ -136,12 +137,12 @@ mod _locale { } let l_ptr = l_str.as_ptr() as *const i8; libc::setlocale(args.category, l_ptr) - }, - } + } + }, }; if result.is_null() { let error = error(vm); - return Err(vm.new_exception_msg(error, String::from("unsupported locale setting"))) + return Err(vm.new_exception_msg(error, String::from("unsupported locale setting"))); } _parse_ptr_to_str(vm, result) } From 2d379337b7ee4b37deded926f25741dd2b3cea3e Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sat, 25 Feb 2023 15:16:11 +0700 Subject: [PATCH 16/29] Remove test_locale skip testcase --- Lib/test/test_locale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 897f47196f..196dc6e3a1 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -6,7 +6,7 @@ import sys import codecs -@unittest.SkipTest(reason="TODO: RUSTPYTHON, Library not completed") + class BaseLocalizedTest(unittest.TestCase): # # Base class for tests using a real locale From 082d1efd51d3d0ce0808dee2b8688b9cf20c67e0 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sat, 25 Feb 2023 18:22:59 +0700 Subject: [PATCH 17/29] Fix unittest --- Lib/test/test_locale.py | 2 -- stdlib/src/lib.rs | 5 ++++- stdlib/src/locale.rs | 12 ++++++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 196dc6e3a1..e4aa430212 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -506,8 +506,6 @@ def test_japanese(self): class TestMiscellaneous(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is # valid. Furthermore LC_CTYPE=UTF is used by the UTF-8 locale coercing diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index b735eb0581..f3a35ace64 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -139,7 +139,6 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_posixsubprocess" => posixsubprocess::make_module, "syslog" => syslog::make_module, "mmap" => mmap::make_module, - "_locale" => locale::make_module, } #[cfg(all(unix, not(target_os = "redox")))] { @@ -161,5 +160,9 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit { "_uuid" => uuid::make_module, } + #[cfg(all(unix, not(any(target_os = "ios", target_os = "android"))))] + { + "_locale" => locale::make_module, + } } } diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index d24daa785d..c013c6cb32 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -1,7 +1,7 @@ -#[cfg(unix)] +#[cfg(all(unix, not(any(target_os = "ios", target_os = "android"))))] pub(crate) use _locale::make_module; -#[cfg(unix)] +#[cfg(all(unix, not(any(target_os = "ios", target_os = "android"))))] #[pymodule] mod _locale { use rustpython_vm::{ @@ -37,6 +37,10 @@ mod _locale { group_vec.push(val.into()); ptr = ptr.offset(1); } + // https://github.com/python/cpython/blob/677320348728ce058fa3579017e985af74a236d4/Modules/_localemodule.c#L80 + if group_vec.len() > 0 { + group_vec.push(vm.ctx.new_int(0i32).into()); + } vm.ctx.new_list(group_vec) } @@ -49,11 +53,11 @@ mod _locale { Ok(vm.new_pyobj(cstr)) } - #[pyattr(name = "Error")] + #[pyattr(name = "Error", once)] fn error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.new_exception_type( "locale", - "locale.Error", + "Error", Some(vec![vm.ctx.exceptions.exception_type.to_owned()]), ) } From f5730c421940b1bb2317565bcd906b588f418f12 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sat, 25 Feb 2023 18:49:11 +0700 Subject: [PATCH 18/29] Fix clippy issues --- stdlib/src/locale.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index c013c6cb32..e157de840b 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -38,7 +38,7 @@ mod _locale { ptr = ptr.offset(1); } // https://github.com/python/cpython/blob/677320348728ce058fa3579017e985af74a236d4/Modules/_localemodule.c#L80 - if group_vec.len() > 0 { + if !group_vec.is_empty() { group_vec.push(vm.ctx.new_int(0i32).into()); } vm.ctx.new_list(group_vec) From 31941a46b7336a43253d5182748f6a167f30ca19 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sun, 26 Feb 2023 00:06:45 +0700 Subject: [PATCH 19/29] Clean up locale module code --- stdlib/src/locale.rs | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index e157de840b..c0aef545f3 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -21,7 +21,10 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use std::{ffi::CStr, ptr}; + use std::{ + ffi::{CStr, CString}, + ptr, + }; #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { @@ -44,7 +47,7 @@ mod _locale { vm.ctx.new_list(group_vec) } - unsafe fn _parse_ptr_to_str(vm: &VirtualMachine, raw_ptr: *mut libc::c_char) -> PyResult { + unsafe fn _parse_ptr_to_str(vm: &VirtualMachine, raw_ptr: *const libc::c_char) -> PyResult { let slice = unsafe { CStr::from_ptr(raw_ptr) }; let cstr = slice .to_str() @@ -124,25 +127,13 @@ mod _locale { #[pyfunction] fn setlocale(args: LocaleArgs, vm: &VirtualMachine) -> PyResult { unsafe { - let result = match args.locale { - OptionalArg::Missing => { - let null_ptr: *const i8 = ptr::null(); - libc::setlocale(args.category, null_ptr) + let result = match args.locale.flatten() { + None => libc::setlocale(args.category, ptr::null()), + Some(l) => { + let l_str = CString::new(l.to_string()).expect("expect to be always converted"); + let l_ptr = CStr::as_ptr(&l_str); + libc::setlocale(args.category, l_ptr) } - OptionalArg::Present(locale) => match locale { - None => { - let null_ptr: *const i8 = ptr::null(); - libc::setlocale(args.category, null_ptr) - } - Some(l) => { - let mut l_str = l.to_string(); - if l_str.is_empty() { - l_str.push('\0'); - } - let l_ptr = l_str.as_ptr() as *const i8; - libc::setlocale(args.category, l_ptr) - } - }, }; if result.is_null() { let error = error(vm); From 0c5ecb5d39ccfcd17ffaf6b00a008e8cb1e35887 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Sun, 26 Feb 2023 00:39:26 +0700 Subject: [PATCH 20/29] Remove setlocale test expected failure --- Lib/test/test_locale.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index e4aa430212..9d47824e11 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -563,8 +563,6 @@ def test_strcoll_3303(self): self.assertRaises(TypeError, locale.strcoll, "a", None) self.assertRaises(TypeError, locale.strcoll, b"a", None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setlocale_category(self): locale.setlocale(locale.LC_ALL) locale.setlocale(locale.LC_TIME) From 26f04e235a9fff9700b978ca6a650adf87afd175 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Tue, 28 Feb 2023 12:53:20 +0700 Subject: [PATCH 21/29] Skip format and strcol test case --- Lib/test/test_locale.py | 26 ++++++++++++++------------ Lib/test/test_types.py | 1 + 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 9d47824e11..7e8b32a5a6 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -375,21 +375,23 @@ def setUp(self): raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) - @unittest.skipIf(sys.platform.startswith('aix'), - 'bpo-29972: broken test on AIX') - @unittest.skipIf( - is_emscripten or is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) + # @unittest.skipIf(sys.platform.startswith('aix'), + # 'bpo-29972: broken test on AIX') + # @unittest.skipIf( + # is_emscripten or is_wasi, + # "musl libc issue on Emscripten/WASI, bpo-46390" + # ) + @unittest.SkipTest("TODO: strcoll hasn't been implemented") def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) - @unittest.skipIf(sys.platform.startswith('aix'), - 'bpo-29972: broken test on AIX') - @unittest.skipIf( - is_emscripten or is_wasi, - "musl libc issue on Emscripten/WASI, bpo-46390" - ) + # @unittest.skipIf(sys.platform.startswith('aix'), + # 'bpo-29972: broken test on AIX') + # @unittest.skipIf( + # is_emscripten or is_wasi, + # "musl libc issue on Emscripten/WASI, bpo-46390" + # ) + @unittest.SkipTest("TODO: strcoll hasn't been implemented") def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 339c370088..64ad4ea6d7 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -405,6 +405,7 @@ def test_float__format__locale(self): @run_with_locale('LC_NUMERIC', 'en_US.UTF8') def test_int__format__locale(self): + self.skipTest("TODO: format with type 'n' is not integrated with locale") # test locale support for __format__ code 'n' for integers x = 123456789012345678901234567890 From 0268af5fcc6f417939e283ff4ac532e6f9aebfc5 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Tue, 28 Feb 2023 13:19:49 +0700 Subject: [PATCH 22/29] Fix skpTest bug --- Lib/test/test_locale.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 7e8b32a5a6..01a0bd1df9 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -381,8 +381,8 @@ def setUp(self): # is_emscripten or is_wasi, # "musl libc issue on Emscripten/WASI, bpo-46390" # ) - @unittest.SkipTest("TODO: strcoll hasn't been implemented") def test_strcoll_with_diacritic(self): + self.skipTest("TODO: strcoll hasn't been implemented") self.assertLess(locale.strcoll('à', 'b'), 0) # @unittest.skipIf(sys.platform.startswith('aix'), @@ -391,8 +391,8 @@ def test_strcoll_with_diacritic(self): # is_emscripten or is_wasi, # "musl libc issue on Emscripten/WASI, bpo-46390" # ) - @unittest.SkipTest("TODO: strcoll hasn't been implemented") def test_strxfrm_with_diacritic(self): + self.skipTest("TODO: strxfrm hasn't been implemented") self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) From 44a36afca9cee61fecb893354c748de0cc150745 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Tue, 28 Feb 2023 13:58:41 +0700 Subject: [PATCH 23/29] Skip locale test for window --- Lib/test/test_locale.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 01a0bd1df9..b23528c0a4 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -512,6 +512,9 @@ def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is # valid. Furthermore LC_CTYPE=UTF is used by the UTF-8 locale coercing # during interpreter startup (on macOS). + if hasattr(sys, 'getwindowsversion'): + self.skipTest("TODO: _locale has not been implemented for window") + import _locale import os @@ -566,6 +569,9 @@ def test_strcoll_3303(self): self.assertRaises(TypeError, locale.strcoll, b"a", None) def test_setlocale_category(self): + if hasattr(sys, "getwindowsversion"): + self.skipTest("TODO: locale has not been implemented for window") + locale.setlocale(locale.LC_ALL) locale.setlocale(locale.LC_TIME) locale.setlocale(locale.LC_CTYPE) From 98014090fc8f7ca82f0b1eaee15fe456730bbf0e Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Tue, 28 Feb 2023 17:53:10 +0700 Subject: [PATCH 24/29] Fix test locale skip convention --- Lib/test/test_locale.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index b23528c0a4..a89cc16efa 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -375,24 +375,12 @@ def setUp(self): raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) - # @unittest.skipIf(sys.platform.startswith('aix'), - # 'bpo-29972: broken test on AIX') - # @unittest.skipIf( - # is_emscripten or is_wasi, - # "musl libc issue on Emscripten/WASI, bpo-46390" - # ) + @unittest.skip("TODO: RUSTPYTHON") def test_strcoll_with_diacritic(self): - self.skipTest("TODO: strcoll hasn't been implemented") self.assertLess(locale.strcoll('à', 'b'), 0) - # @unittest.skipIf(sys.platform.startswith('aix'), - # 'bpo-29972: broken test on AIX') - # @unittest.skipIf( - # is_emscripten or is_wasi, - # "musl libc issue on Emscripten/WASI, bpo-46390" - # ) + @unittest.skip("TODO: RUSTPYTHON") def test_strxfrm_with_diacritic(self): - self.skipTest("TODO: strxfrm hasn't been implemented") self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) @@ -547,6 +535,10 @@ def test_defaults_UTF8(self): if orig_getlocale is not None: _locale._getdefaultlocale = orig_getlocale + + # TODO: RUSTPYTHON + if sys.platform == "win32": + test_defaults_UTF8 = unittest.expectedFailure(test_defaults_UTF8) def test_getencoding(self): # Invoke getencoding to make sure it does not cause exceptions. @@ -569,9 +561,6 @@ def test_strcoll_3303(self): self.assertRaises(TypeError, locale.strcoll, b"a", None) def test_setlocale_category(self): - if hasattr(sys, "getwindowsversion"): - self.skipTest("TODO: locale has not been implemented for window") - locale.setlocale(locale.LC_ALL) locale.setlocale(locale.LC_TIME) locale.setlocale(locale.LC_CTYPE) @@ -581,6 +570,10 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) + + # TODO: RUSTPYTHON + if sys.platform == "win32": + test_setlocale_category = unittest.expectedFailure(test_setlocale_category) def test_getsetlocale_issue1813(self): # Issue #1813: setting and getting the locale under a Turkish locale From fcfa35ab35d1d196683e031998379858b43f5d55 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Wed, 1 Mar 2023 00:59:46 +0700 Subject: [PATCH 25/29] Revert original skip decorator and polish code --- Lib/test/test_locale.py | 15 +++++++++++++-- Lib/test/test_types.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index a89cc16efa..3796ad144c 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -375,10 +375,22 @@ def setUp(self): raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) + @unittest.skipIf(sys.platform.startswith('aix'), + 'bpo-29972: broken test on AIX') + @unittest.skipIf( + is_emscripten or is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" + ) @unittest.skip("TODO: RUSTPYTHON") def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) + @unittest.skipIf(sys.platform.startswith('aix'), + 'bpo-29972: broken test on AIX') + @unittest.skipIf( + is_emscripten or is_wasi, + "musl libc issue on Emscripten/WASI, bpo-46390" + ) @unittest.skip("TODO: RUSTPYTHON") def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) @@ -500,8 +512,6 @@ def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is # valid. Furthermore LC_CTYPE=UTF is used by the UTF-8 locale coercing # during interpreter startup (on macOS). - if hasattr(sys, 'getwindowsversion'): - self.skipTest("TODO: _locale has not been implemented for window") import _locale import os @@ -555,6 +565,7 @@ def test_getpreferredencoding(self): # If encoding non-empty, make sure it is valid codecs.lookup(enc) + @unittest.skip("TODO: RustPython strcoll has not been implemented") def test_strcoll_3303(self): # test crasher from bug #3303 self.assertRaises(TypeError, locale.strcoll, "a", None) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 64ad4ea6d7..2212d2323d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -403,9 +403,9 @@ def test_float__format__locale(self): self.assertEqual(locale.format_string('%g', x, grouping=True), format(x, 'n')) self.assertEqual(locale.format_string('%.10g', x, grouping=True), format(x, '.10n')) + @unittest.skip("TODO: RustPython format code n is not integrated with locale") @run_with_locale('LC_NUMERIC', 'en_US.UTF8') def test_int__format__locale(self): - self.skipTest("TODO: format with type 'n' is not integrated with locale") # test locale support for __format__ code 'n' for integers x = 123456789012345678901234567890 From a44a59327ae3346a571d4a36e40de43fe2a9c8b7 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Wed, 1 Mar 2023 01:09:01 +0700 Subject: [PATCH 26/29] Allow test_strcoll_3303 --- Lib/test/test_locale.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 3796ad144c..2f9290d700 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -565,7 +565,6 @@ def test_getpreferredencoding(self): # If encoding non-empty, make sure it is valid codecs.lookup(enc) - @unittest.skip("TODO: RustPython strcoll has not been implemented") def test_strcoll_3303(self): # test crasher from bug #3303 self.assertRaises(TypeError, locale.strcoll, "a", None) From 9e5be11c43806d2a4faa42a9348b47cbaa2d03c6 Mon Sep 17 00:00:00 2001 From: "minh.pham2000" Date: Wed, 1 Mar 2023 01:19:07 +0700 Subject: [PATCH 27/29] Expect failure instead of skip --- Lib/test/test_locale.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 2f9290d700..fb75a98a24 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -381,7 +381,8 @@ def setUp(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) - @unittest.skip("TODO: RUSTPYTHON") + # TODO: RUSTPYTHON", strcoll has not been implemented + @unittest.expectedFailure def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) @@ -391,7 +392,8 @@ def test_strcoll_with_diacritic(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) - @unittest.skip("TODO: RUSTPYTHON") + # TODO: RUSTPYTHON", strxfrm has not been implemented + @unittest.expectedFailure def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) From 6291ca95b95afc20c0ff4e56b73cf7a9c921ad90 Mon Sep 17 00:00:00 2001 From: Jeong YunWon <69878+youknowone@users.noreply.github.com> Date: Wed, 1 Mar 2023 03:22:02 +0900 Subject: [PATCH 28/29] Update Lib/test/test_locale.py --- Lib/test/test_locale.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index fb75a98a24..fff605d9b6 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -514,7 +514,6 @@ def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is # valid. Furthermore LC_CTYPE=UTF is used by the UTF-8 locale coercing # during interpreter startup (on macOS). - import _locale import os From 38d22537080f71c6f4014cbaf21b1952731bc889 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 1 Mar 2023 04:37:48 +0900 Subject: [PATCH 29/29] clean up and additional error handling for NulError --- stdlib/src/locale.rs | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/stdlib/src/locale.rs b/stdlib/src/locale.rs index c0aef545f3..631fda1e08 100644 --- a/stdlib/src/locale.rs +++ b/stdlib/src/locale.rs @@ -6,9 +6,14 @@ pub(crate) use _locale::make_module; mod _locale { use rustpython_vm::{ builtins::{PyDictRef, PyIntRef, PyListRef, PyStrRef, PyTypeRef}, + convert::ToPyException, function::OptionalArg, PyObjectRef, PyResult, VirtualMachine, }; + use std::{ + ffi::{CStr, CString}, + ptr, + }; #[pyattr] use libc::{ @@ -21,39 +26,36 @@ mod _locale { T_FMT_AMPM, YESEXPR, }; - use std::{ - ffi::{CStr, CString}, - ptr, - }; - #[pyattr(name = "CHAR_MAX")] fn char_max(vm: &VirtualMachine) -> PyIntRef { vm.ctx.new_int(libc::c_char::MAX) } - unsafe fn copy_grouping(group: *mut libc::c_char, vm: &VirtualMachine) -> PyListRef { + unsafe fn copy_grouping(group: *const libc::c_char, vm: &VirtualMachine) -> PyListRef { let mut group_vec: Vec = Vec::new(); - let mut ptr = group; + if group.is_null() { + return vm.ctx.new_list(group_vec); + } + let mut ptr = group; while ![0_i8, libc::c_char::MAX].contains(&*ptr) { let val = vm.ctx.new_int(*ptr); group_vec.push(val.into()); - ptr = ptr.offset(1); + ptr = ptr.add(1); } // https://github.com/python/cpython/blob/677320348728ce058fa3579017e985af74a236d4/Modules/_localemodule.c#L80 if !group_vec.is_empty() { - group_vec.push(vm.ctx.new_int(0i32).into()); + group_vec.push(vm.ctx.new_int(0).into()); } vm.ctx.new_list(group_vec) } - unsafe fn _parse_ptr_to_str(vm: &VirtualMachine, raw_ptr: *const libc::c_char) -> PyResult { + unsafe fn pystr_from_raw_cstr(vm: &VirtualMachine, raw_ptr: *const libc::c_char) -> PyResult { let slice = unsafe { CStr::from_ptr(raw_ptr) }; - let cstr = slice + let string = slice .to_str() .expect("localeconv always return decodable string"); - - Ok(vm.new_pyobj(cstr)) + Ok(vm.new_pyobj(string)) } #[pyattr(name = "Error", once)] @@ -74,7 +76,11 @@ mod _locale { macro_rules! set_string_field { ($field:ident) => {{ - result.set_item(stringify!($field), _parse_ptr_to_str(vm, (*lc).$field)?, vm)? + result.set_item( + stringify!($field), + pystr_from_raw_cstr(vm, (*lc).$field)?, + vm, + )? }}; } @@ -129,17 +135,17 @@ mod _locale { unsafe { let result = match args.locale.flatten() { None => libc::setlocale(args.category, ptr::null()), - Some(l) => { - let l_str = CString::new(l.to_string()).expect("expect to be always converted"); - let l_ptr = CStr::as_ptr(&l_str); - libc::setlocale(args.category, l_ptr) + Some(locale) => { + let c_locale: CString = + CString::new(locale.as_str()).map_err(|e| e.to_pyexception(vm))?; + libc::setlocale(args.category, c_locale.as_ptr()) } }; if result.is_null() { let error = error(vm); return Err(vm.new_exception_msg(error, String::from("unsupported locale setting"))); } - _parse_ptr_to_str(vm, result) + pystr_from_raw_cstr(vm, result) } } }