Skip to content

Commit adc05e6

Browse files
coolreader18coolreader18
authored
Merge pull request RustPython#5443 from coolreader18/incremental-newline-decoder
Implement IncrementalNewlineDecoder in rust
2 parents 0bc236a + fbd0c7a commit adc05e6

File tree

16 files changed

+324
-122
lines changed

16 files changed

+324
-122
lines changed

‎Lib/io.py

+1-10
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,7 @@
5757
from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
5858
open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
5959
BufferedWriter, BufferedRWPair, BufferedRandom,
60-
# XXX RUSTPYTHON TODO: IncrementalNewlineDecoder
61-
# IncrementalNewlineDecoder,
62-
text_encoding, TextIOWrapper)
60+
IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
6361

6462
# Pretend this exception was created here.
6563
UnsupportedOperation.__module__ = "io"
@@ -100,10 +98,3 @@ class TextIOBase(_io._TextIOBase, IOBase):
10098
pass
10199
else:
102100
RawIOBase.register(_WindowsConsoleIO)
103-
104-
105-
# XXX: RUSTPYTHON; borrow IncrementalNewlineDecoder from _pyio
106-
try:
107-
from _pyio import IncrementalNewlineDecoder
108-
except ImportError:
109-
pass

‎Lib/test/test_difflib.py

-1
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,6 @@ def test_mdiff_catch_stop_iteration(self):
186186
the end"""
187187

188188
class TestSFpatches(unittest.TestCase):
189-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
190189
def test_html_diff(self):
191190
# Check SF patch 914575 for generating HTML differences
192191
f1a = ((patch914575_from1 + '123\n'*10)*3)

‎Lib/test/test_fileinput.py

-2
Original file line numberDiff line numberDiff line change
@@ -280,8 +280,6 @@ def __call__(self, *args, **kargs):
280280
fi.readline()
281281
self.assertTrue(custom_open_hook.invoked, "openhook not invoked")
282282

283-
# TODO: RUSTPYTHON
284-
@unittest.expectedFailure
285283
def test_readline(self):
286284
with open(TESTFN, 'wb') as f:
287285
f.write(b'A\nB\r\nC\r')

‎Lib/test/test_imp.py

-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ def test_find_module_encoding(self):
8484
with self.assertRaises(SyntaxError):
8585
imp.find_module('badsyntax_pep3120', path)
8686

87-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
8887
def test_issue1267(self):
8988
for mod, encoding, _ in self.test_strings:
9089
fp, filename, info = imp.find_module('module_' + mod,

‎Lib/test/test_importlib/test_abc.py

-10
Original file line numberDiff line numberDiff line change
@@ -701,8 +701,6 @@ def verify_code(self, code_object):
701701
class SourceOnlyLoaderTests(SourceLoaderTestHarness):
702702
"""Test importlib.abc.SourceLoader for source-only loading."""
703703

704-
# TODO: RUSTPYTHON
705-
@unittest.expectedFailure
706704
def test_get_source(self):
707705
# Verify the source code is returned as a string.
708706
# If an OSError is raised by get_data then raise ImportError.
@@ -761,8 +759,6 @@ def test_package_settings(self):
761759
self.verify_module(module)
762760
self.assertFalse(hasattr(module, '__path__'))
763761

764-
# TODO: RUSTPYTHON
765-
@unittest.expectedFailure
766762
def test_get_source_encoding(self):
767763
# Source is considered encoded in UTF-8 by default unless otherwise
768764
# specified by an encoding line.
@@ -882,8 +878,6 @@ class SourceLoaderGetSourceTests:
882878

883879
"""Tests for importlib.abc.SourceLoader.get_source()."""
884880

885-
# TODO: RUSTPYTHON
886-
@unittest.expectedFailure
887881
def test_default_encoding(self):
888882
# Should have no problems with UTF-8 text.
889883
name = 'mod'
@@ -893,8 +887,6 @@ def test_default_encoding(self):
893887
returned_source = mock.get_source(name)
894888
self.assertEqual(returned_source, source)
895889

896-
# TODO: RUSTPYTHON
897-
@unittest.expectedFailure
898890
def test_decoded_source(self):
899891
# Decoding should work.
900892
name = 'mod'
@@ -905,8 +897,6 @@ def test_decoded_source(self):
905897
returned_source = mock.get_source(name)
906898
self.assertEqual(returned_source, source)
907899

908-
# TODO: RUSTPYTHON
909-
@unittest.expectedFailure
910900
def test_universal_newlines(self):
911901
# PEP 302 says universal newlines should be used.
912902
name = 'mod'

‎Lib/test/test_importlib/test_util.py

-6
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,16 @@ class DecodeSourceBytesTests:
3636

3737
source = "string ='ü'"
3838

39-
# TODO: RUSTPYTHON
40-
@unittest.expectedFailure
4139
def test_ut8_default(self):
4240
source_bytes = self.source.encode('utf-8')
4341
self.assertEqual(self.util.decode_source(source_bytes), self.source)
4442

45-
# TODO: RUSTPYTHON
46-
@unittest.expectedFailure
4743
def test_specified_encoding(self):
4844
source = '# coding=latin-1\n' + self.source
4945
source_bytes = source.encode('latin-1')
5046
assert source_bytes != source.encode('utf-8')
5147
self.assertEqual(self.util.decode_source(source_bytes), source)
5248

53-
# TODO: RUSTPYTHON
54-
@unittest.expectedFailure
5549
def test_universal_newlines(self):
5650
source = '\r\n'.join([self.source, self.source])
5751
source_bytes = source.encode('utf-8')

‎Lib/test/test_io.py

-40
Original file line numberDiff line numberDiff line change
@@ -4121,41 +4121,6 @@ def test_newlines(self):
41214121
def test_newlines_input(self):
41224122
super().test_newlines_input()
41234123

4124-
# TODO: RUSTPYTHON
4125-
@unittest.expectedFailure
4126-
def test_read_one_by_one(self):
4127-
super().test_read_one_by_one()
4128-
4129-
# TODO: RUSTPYTHON
4130-
@unittest.expectedFailure
4131-
def test_read_by_chunk(self):
4132-
super().test_read_by_chunk()
4133-
4134-
# TODO: RUSTPYTHON
4135-
@unittest.expectedFailure
4136-
def test_issue1395_1(self):
4137-
super().test_issue1395_1()
4138-
4139-
# TODO: RUSTPYTHON
4140-
@unittest.expectedFailure
4141-
def test_issue1395_2(self):
4142-
super().test_issue1395_2()
4143-
4144-
# TODO: RUSTPYTHON
4145-
@unittest.expectedFailure
4146-
def test_issue1395_3(self):
4147-
super().test_issue1395_3()
4148-
4149-
# TODO: RUSTPYTHON
4150-
@unittest.expectedFailure
4151-
def test_issue1395_4(self):
4152-
super().test_issue1395_4()
4153-
4154-
# TODO: RUSTPYTHON
4155-
@unittest.expectedFailure
4156-
def test_issue1395_5(self):
4157-
super().test_issue1395_5()
4158-
41594124
# TODO: RUSTPYTHON
41604125
@unittest.expectedFailure
41614126
def test_reconfigure_write_through(self):
@@ -4812,11 +4777,6 @@ class CMiscIOTest(MiscIOTest):
48124777
name_of_module = "io", "_io"
48134778
extra_exported = "BlockingIOError",
48144779

4815-
# TODO: RUSTPYTHON
4816-
@unittest.expectedFailure
4817-
def test___all__(self):
4818-
super().test___all__()
4819-
48204780
def test_readinto_buffer_overflow(self):
48214781
# Issue #18025
48224782
class BadReader(self.io.BufferedIOBase):

‎Lib/test/test_linecache.py

-6
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,6 @@ def test_lazycache_no_globals(self):
187187
self.assertEqual(False, linecache.lazycache(FILENAME, None))
188188
self.assertEqual(lines, linecache.getlines(FILENAME))
189189

190-
# TODO: RUSTPYTHON
191-
@unittest.expectedFailure
192190
def test_lazycache_smoke(self):
193191
lines = linecache.getlines(NONEXISTENT_FILENAME, globals())
194192
linecache.clearcache()
@@ -199,8 +197,6 @@ def test_lazycache_smoke(self):
199197
# globals: this would error if the lazy value wasn't resolved.
200198
self.assertEqual(lines, linecache.getlines(NONEXISTENT_FILENAME))
201199

202-
# TODO: RUSTPYTHON
203-
@unittest.expectedFailure
204200
def test_lazycache_provide_after_failed_lookup(self):
205201
linecache.clearcache()
206202
lines = linecache.getlines(NONEXISTENT_FILENAME, globals())
@@ -219,8 +215,6 @@ def test_lazycache_bad_filename(self):
219215
self.assertEqual(False, linecache.lazycache('', globals()))
220216
self.assertEqual(False, linecache.lazycache('<foo>', globals()))
221217

222-
# TODO: RUSTPYTHON
223-
@unittest.expectedFailure
224218
def test_lazycache_already_cached(self):
225219
linecache.clearcache()
226220
lines = linecache.getlines(NONEXISTENT_FILENAME, globals())

‎Lib/test/test_popen.py

-2
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,10 @@ def test_return_code(self):
5454
else:
5555
self.assertEqual(os.waitstatus_to_exitcode(status), 42)
5656

57-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
5857
def test_contextmanager(self):
5958
with os.popen("echo hello") as f:
6059
self.assertEqual(f.read(), "hello\n")
6160

62-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
6361
def test_iterating(self):
6462
with os.popen("echo hello") as f:
6563
self.assertEqual(list(f), ["hello\n"])

‎Lib/test/test_subprocess.py

-5
Original file line numberDiff line numberDiff line change
@@ -1056,8 +1056,6 @@ def test_writes_before_communicate(self):
10561056
self.assertEqual(stdout, b"bananasplit")
10571057
self.assertEqual(stderr, b"")
10581058

1059-
# TODO: RUSTPYTHON
1060-
@unittest.expectedFailure
10611059
def test_universal_newlines_and_text(self):
10621060
args = [
10631061
sys.executable, "-c",
@@ -1097,7 +1095,6 @@ def test_universal_newlines_and_text(self):
10971095
self.assertEqual(p.stdout.read(),
10981096
"line4\nline5\nline6\nline7\nline8")
10991097

1100-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
11011098
def test_universal_newlines_communicate(self):
11021099
# universal newlines through communicate()
11031100
p = subprocess.Popen([sys.executable, "-c",
@@ -1149,7 +1146,6 @@ def test_universal_newlines_communicate_input_none(self):
11491146
p.communicate()
11501147
self.assertEqual(p.returncode, 0)
11511148

1152-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
11531149
def test_universal_newlines_communicate_stdin_stdout_stderr(self):
11541150
# universal newlines through communicate(), with stdin, stdout, stderr
11551151
p = subprocess.Popen([sys.executable, "-c",
@@ -3805,7 +3801,6 @@ def popen_via_context_manager(*args, **kwargs):
38053801
raise KeyboardInterrupt # Test how __exit__ handles ^C.
38063802
self._test_keyboardinterrupt_no_kill(popen_via_context_manager)
38073803

3808-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
38093804
def test_getoutput(self):
38103805
self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy')
38113806
self.assertEqual(subprocess.getstatusoutput('echo xyzzy'),

‎Lib/test/test_traceback.py

-18
Original file line numberDiff line numberDiff line change
@@ -1040,8 +1040,6 @@ def extract():
10401040

10411041
class TestFrame(unittest.TestCase):
10421042

1043-
# TODO: RUSTPYTHON
1044-
@unittest.expectedFailure
10451043
def test_basics(self):
10461044
linecache.clearcache()
10471045
linecache.lazycache("f", globals())
@@ -1059,8 +1057,6 @@ def test_basics(self):
10591057
self.assertNotEqual(f, object())
10601058
self.assertEqual(f, ALWAYS_EQ)
10611059

1062-
# TODO: RUSTPYTHON
1063-
@unittest.expectedFailure
10641060
def test_lazy_lines(self):
10651061
linecache.clearcache()
10661062
f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False)
@@ -1109,8 +1105,6 @@ def test_extract_stack_limit(self):
11091105
s = traceback.StackSummary.extract(traceback.walk_stack(None), limit=5)
11101106
self.assertEqual(len(s), 5)
11111107

1112-
# TODO: RUSTPYTHON
1113-
@unittest.expectedFailure
11141108
def test_extract_stack_lookup_lines(self):
11151109
linecache.clearcache()
11161110
linecache.updatecache('/foo.py', globals())
@@ -1120,8 +1114,6 @@ def test_extract_stack_lookup_lines(self):
11201114
linecache.clearcache()
11211115
self.assertEqual(s[0].line, "import sys")
11221116

1123-
# TODO: RUSTPYTHON
1124-
@unittest.expectedFailure
11251117
def test_extract_stackup_deferred_lookup_lines(self):
11261118
linecache.clearcache()
11271119
c = test_code('/foo.py', 'method')
@@ -1153,17 +1145,13 @@ def test_format_smoke(self):
11531145
[' File "foo.py", line 1, in fred\n line\n'],
11541146
s.format())
11551147

1156-
# TODO: RUSTPYTHON
1157-
@unittest.expectedFailure
11581148
def test_locals(self):
11591149
linecache.updatecache('/foo.py', globals())
11601150
c = test_code('/foo.py', 'method')
11611151
f = test_frame(c, globals(), {'something': 1})
11621152
s = traceback.StackSummary.extract(iter([(f, 6)]), capture_locals=True)
11631153
self.assertEqual(s[0].locals, {'something': '1'})
11641154

1165-
# TODO: RUSTPYTHON
1166-
@unittest.expectedFailure
11671155
def test_no_locals(self):
11681156
linecache.updatecache('/foo.py', globals())
11691157
c = test_code('/foo.py', 'method')
@@ -1444,8 +1432,6 @@ def recurse(n):
14441432
traceback.walk_tb(exc_info[2]), limit=5)
14451433
self.assertEqual(expected_stack, exc.stack)
14461434

1447-
# TODO: RUSTPYTHON
1448-
@unittest.expectedFailure
14491435
def test_lookup_lines(self):
14501436
linecache.clearcache()
14511437
e = Exception("uh oh")
@@ -1457,8 +1443,6 @@ def test_lookup_lines(self):
14571443
linecache.updatecache('/foo.py', globals())
14581444
self.assertEqual(exc.stack[0].line, "import sys")
14591445

1460-
# TODO: RUSTPYTHON
1461-
@unittest.expectedFailure
14621446
def test_locals(self):
14631447
linecache.updatecache('/foo.py', globals())
14641448
e = Exception("uh oh")
@@ -1470,8 +1454,6 @@ def test_locals(self):
14701454
self.assertEqual(
14711455
exc.stack[0].locals, {'something': '1', 'other': "'string'"})
14721456

1473-
# TODO: RUSTPYTHON
1474-
@unittest.expectedFailure
14751457
def test_no_locals(self):
14761458
linecache.updatecache('/foo.py', globals())
14771459
e = Exception("uh oh")

‎common/src/lock/thread_mutex.rs

+5
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,11 @@ impl<R: RawMutex, G: GetThreadId, T: Default> Default for ThreadMutex<R, G, T> {
9292
Self::new(T::default())
9393
}
9494
}
95+
impl<R: RawMutex, G: GetThreadId, T> From<T> for ThreadMutex<R, G, T> {
96+
fn from(val: T) -> Self {
97+
Self::new(val)
98+
}
99+
}
95100
impl<R: RawMutex, G: GetThreadId, T: ?Sized> ThreadMutex<R, G, T> {
96101
pub fn lock(&self) -> Option<ThreadMutexGuard<R, G, T>> {
97102
if self.raw.lock() {

‎derive-impl/src/from_args.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use proc_macro2::TokenStream;
22
use quote::{quote, ToTokens};
3+
use syn::ext::IdentExt;
34
use syn::{
45
parse_quote, Attribute, Data, DeriveInput, Expr, Field, Ident, Lit, Meta, NestedMeta, Result,
56
};
@@ -138,7 +139,7 @@ fn generate_field((i, field): (usize, &Field)) -> Result<TokenStream> {
138139
};
139140

140141
let name = field.ident.as_ref();
141-
let name_string = name.map(Ident::to_string);
142+
let name_string = name.map(|ident| ident.unraw().to_string());
142143
if matches!(&name_string, Some(s) if s.starts_with("_phantom")) {
143144
return Ok(quote! {
144145
#name: ::std::marker::PhantomData,

0 commit comments

Comments
 (0)