Skip to content

Commit 0799052

Browse files
windelbouwmanwindelbouwman
committed
Add time module and improve lexer handling of numeric constants
1 parent d72abeb commit 0799052

File tree

9 files changed

+141
-52
lines changed

9 files changed

+141
-52
lines changed

‎parser/src/ast.rs

+6
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,12 @@ pub enum Statement {
7171
Expression {
7272
expression: Expression,
7373
},
74+
Global {
75+
names: Vec<String>,
76+
},
77+
Nonlocal {
78+
names: Vec<String>,
79+
},
7480
If {
7581
test: Expression,
7682
body: Vec<LocatedStatement>,

‎parser/src/lexer.rs

+37-23
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
44
pub use super::token::Tok;
55
use std::collections::HashMap;
6+
use std::str::FromStr;
67

78
pub struct Lexer<T: Iterator<Item = char>> {
89
chars: T,
@@ -287,18 +288,37 @@ where
287288
}
288289

289290
// If float:
290-
if let Some('.') = self.chr0 {
291-
value_text.push(self.next_char().unwrap());
292-
while self.is_number() {
291+
if self.chr0 == Some('.') || self.chr0 == Some('e') {
292+
// Take '.':
293+
if self.chr0 == Some('.') {
293294
value_text.push(self.next_char().unwrap());
295+
while self.is_number() {
296+
value_text.push(self.next_char().unwrap());
297+
}
294298
}
295-
}
296299

297-
let end_pos = self.get_pos();
300+
// 1e6 for example:
301+
if self.chr0 == Some('e') {
302+
value_text.push(self.next_char().unwrap());
303+
304+
// Optional +/-
305+
if self.chr0 == Some('-') || self.chr0 == Some('+') {
306+
value_text.push(self.next_char().unwrap());
307+
}
298308

299-
let value = value_text;
309+
while self.is_number() {
310+
value_text.push(self.next_char().unwrap());
311+
}
312+
}
300313

301-
return Ok((start_pos, Tok::Number { value: value }, end_pos));
314+
let end_pos = self.get_pos();
315+
let value = f64::from_str(&value_text).unwrap();
316+
Ok((start_pos, Tok::Float { value: value }, end_pos))
317+
} else {
318+
let end_pos = self.get_pos();
319+
let value = i32::from_str(&value_text).unwrap();
320+
Ok((start_pos, Tok::Int { value: value }, end_pos))
321+
}
302322
}
303323

304324
fn lex_comment(&mut self) {
@@ -946,7 +966,7 @@ mod tests {
946966
fn $name() {
947967
let source = String::from(format!(r"99232 # {}", $eol));
948968
let tokens = lex_source(&source);
949-
assert_eq!(tokens, vec![Tok::Number { value: "99232".to_string() }]);
969+
assert_eq!(tokens, vec![Tok::Int { value: 99232 }]);
950970
}
951971
)*
952972
}
@@ -969,9 +989,9 @@ mod tests {
969989
assert_eq!(
970990
tokens,
971991
vec![
972-
Tok::Number { value: "123".to_string() },
992+
Tok::Int { value: 123 },
973993
Tok::Newline,
974-
Tok::Number { value: "456".to_string() },
994+
Tok::Int { value: 456 },
975995
]
976996
)
977997
}
@@ -996,17 +1016,11 @@ mod tests {
9961016
name: String::from("avariable"),
9971017
},
9981018
Tok::Equal,
999-
Tok::Number {
1000-
value: "99".to_string()
1001-
},
1019+
Tok::Int { value: 99 },
10021020
Tok::Plus,
1003-
Tok::Number {
1004-
value: "2".to_string()
1005-
},
1021+
Tok::Int { value: 2 },
10061022
Tok::Minus,
1007-
Tok::Number {
1008-
value: "0".to_string()
1009-
},
1023+
Tok::Int { value: 0 },
10101024
]
10111025
);
10121026
}
@@ -1031,7 +1045,7 @@ mod tests {
10311045
Tok::Newline,
10321046
Tok::Indent,
10331047
Tok::Return,
1034-
Tok::Number { value: "99".to_string() },
1048+
Tok::Int { value: 99 },
10351049
Tok::Newline,
10361050
Tok::Dedent,
10371051
]
@@ -1074,7 +1088,7 @@ mod tests {
10741088
Tok::Newline,
10751089
Tok::Indent,
10761090
Tok::Return,
1077-
Tok::Number { value: "99".to_string() },
1091+
Tok::Int { value: 99 },
10781092
Tok::Newline,
10791093
Tok::Dedent,
10801094
Tok::Dedent,
@@ -1106,9 +1120,9 @@ mod tests {
11061120
},
11071121
Tok::Equal,
11081122
Tok::Lsqb,
1109-
Tok::Number { value: "1".to_string() },
1123+
Tok::Int { value: 1 },
11101124
Tok::Comma,
1111-
Tok::Number { value: "2".to_string() },
1125+
Tok::Int { value: 2 },
11121126
Tok::Rsqb,
11131127
Tok::Newline,
11141128
]

‎parser/src/python.lalrpop

+46-26
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
use super::ast;
88
use super::lexer;
99
use std::iter::FromIterator;
10-
use std::str::FromStr;
1110

1211
grammar;
1312

@@ -53,6 +52,8 @@ SmallStatement: ast::LocatedStatement = {
5352
DelStatement,
5453
FlowStatement,
5554
ImportStatement,
55+
GlobalStatement,
56+
NonlocalStatement,
5657
AssertStatement,
5758
};
5859

@@ -282,6 +283,24 @@ DottedName: String = {
282283
},
283284
};
284285

286+
GlobalStatement: ast::LocatedStatement = {
287+
<loc:@L> "global" <names:OneOrMore<Identifier>> => {
288+
ast::LocatedStatement {
289+
location: loc,
290+
node: ast::Statement::Global { names }
291+
}
292+
},
293+
};
294+
295+
NonlocalStatement: ast::LocatedStatement = {
296+
<loc:@L> "nonlocal" <names:OneOrMore<Identifier>> => {
297+
ast::LocatedStatement {
298+
location: loc,
299+
node: ast::Statement::Nonlocal { names }
300+
}
301+
},
302+
};
303+
285304
AssertStatement: ast::LocatedStatement = {
286305
<loc:@L> "assert" <t:Test> <m: ("," Test)?> => {
287306
ast::LocatedStatement {
@@ -400,11 +419,7 @@ ExceptClause: ast::ExceptHandler = {
400419
};
401420

402421
WithStatement: ast::LocatedStatement = {
403-
<loc:@L> "with" <i1:WithItem> <i2:("," WithItem)*> ":" <s:Suite> => {
404-
let mut items = vec![i1];
405-
for item in i2 {
406-
items.push(item.1);
407-
}
422+
<loc:@L> "with" <items:OneOrMore<WithItem>> ":" <s:Suite> => {
408423
ast::LocatedStatement {
409424
location: loc,
410425
node: ast::Statement::With { items: items, body: s },
@@ -808,10 +823,8 @@ Atom: ast::Expression = {
808823
};
809824

810825
TestListComp: Vec<ast::Expression> = {
811-
<e:TestOrStarExpr> <e2:("," TestOrStarExpr)*> <_trailing_comma:","?> => {
812-
let mut res = vec![e];
813-
res.extend(e2.into_iter().map(|x| x.1));
814-
res
826+
<e:OneOrMore<TestOrStarExpr>> <_trailing_comma:","?> => {
827+
e
815828
},
816829
};
817830

@@ -825,10 +838,8 @@ TestListComp2: ast::Expression = {
825838
};
826839

827840
TestDict: Vec<(ast::Expression, ast::Expression)> = {
828-
<e1:DictEntry> <e2:("," DictEntry)*> <_trailing_comma:","?> => {
829-
let mut d = vec![e1];
830-
d.extend(e2.into_iter().map(|x| x.1));
831-
d
841+
<e1:OneOrMore<DictEntry>> <_trailing_comma:","?> => {
842+
e1
832843
}
833844
};
834845

@@ -846,10 +857,8 @@ DictEntry: (ast::Expression, ast::Expression) = {
846857
};
847858

848859
TestSet: Vec<ast::Expression> = {
849-
<e1:Test> <e2:("," Test)*> ","? => {
850-
let mut e = vec![e1];
851-
e.extend(e2.into_iter().map(|x| x.1));
852-
e
860+
<e1:OneOrMore<Test>> ","? => {
861+
e1
853862
}
854863
};
855864

@@ -962,14 +971,22 @@ Comma<T>: Vec<T> = {
962971
}
963972
};
964973

965-
Number: ast::Number = {
966-
<s:number> => {
967-
if s.contains(".") {
968-
ast::Number::Float { value: f64::from_str(&s).unwrap() }
969-
} else {
970-
ast::Number::Integer { value: i32::from_str(&s).unwrap() }
974+
#[inline]
975+
OneOrMore<T>: Vec<T> = {
976+
<i1: T> <i2:("," T)*> => {
977+
let mut items = vec![i1];
978+
items.extend(i2.into_iter().map(|e| e.1));
979+
items
971980
}
972-
}
981+
};
982+
983+
Number: ast::Number = {
984+
<s:int> => {
985+
ast::Number::Integer { value: s }
986+
},
987+
<s:float> => {
988+
ast::Number::Float { value: s }
989+
},
973990
};
974991

975992
StringConstant: ast::Expression = {
@@ -1052,12 +1069,14 @@ extern {
10521069
"finally" => lexer::Tok::Finally,
10531070
"for" => lexer::Tok::For,
10541071
"from" => lexer::Tok::From,
1072+
"global" => lexer::Tok::Global,
10551073
"if" => lexer::Tok::If,
10561074
"in" => lexer::Tok::In,
10571075
"is" => lexer::Tok::Is,
10581076
"import" => lexer::Tok::Import,
10591077
"from" => lexer::Tok::From,
10601078
"lambda" => lexer::Tok::Lambda,
1079+
"nonlocal" => lexer::Tok::Nonlocal,
10611080
"not" => lexer::Tok::Not,
10621081
"or" => lexer::Tok::Or,
10631082
"pass" => lexer::Tok::Pass,
@@ -1070,7 +1089,8 @@ extern {
10701089
"True" => lexer::Tok::True,
10711090
"False" => lexer::Tok::False,
10721091
"None" => lexer::Tok::None,
1073-
number => lexer::Tok::Number { value: <String> },
1092+
int => lexer::Tok::Int { value: <i32> },
1093+
float => lexer::Tok::Float { value: <f64> },
10741094
string => lexer::Tok::String { value: <String> },
10751095
bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
10761096
name => lexer::Tok::Name { name: <String> },

‎parser/src/token.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
#[derive(Debug, PartialEq)]
33
pub enum Tok {
44
Name { name: String },
5-
Number { value: String },
5+
Int { value: i32 },
6+
Float { value: f64 },
67
Complex { real: f64, imag: f64 },
78
String { value: String },
89
Bytes { value: Vec<u8> },

‎src/main.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ fn main() {
7070

7171
fn _run_string(vm: &mut VirtualMachine, source: &str, source_path: Option<String>) -> PyResult {
7272
let code_obj = compile::compile(vm, &source.to_string(), compile::Mode::Exec, source_path)?;
73-
debug!("Code object: {:?}", code_obj.borrow());
73+
// trace!("Code object: {:?}", code_obj.borrow());
7474
let builtins = vm.get_builtin_scope();
7575
let vars = vm.context().new_scope(Some(builtins)); // Keep track of local variables
7676
vm.run_code_obj(code_obj, vars)

‎vm/src/compile.rs

+6
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,12 @@ impl Compiler {
177177
// Pop result of stack, since we not use it:
178178
self.emit(Instruction::Pop);
179179
}
180+
ast::Statement::Global { names } => {
181+
unimplemented!("global {:?}", names);
182+
}
183+
ast::Statement::Nonlocal { names } => {
184+
unimplemented!("nonlocal {:?}", names);
185+
}
180186
ast::Statement::If { test, body, orelse } => {
181187
let end_label = self.new_label();
182188
match orelse {

‎vm/src/import.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ fn import_uncached_module(
3737
compile::Mode::Exec,
3838
Some(filepath.to_str().unwrap().to_string()),
3939
)?;
40-
debug!("Code object: {:?}", code_obj);
40+
// trace!("Code object: {:?}", code_obj);
4141

4242
let builtins = vm.get_builtin_scope();
4343
let scope = vm.ctx.new_scope(Some(builtins));

‎vm/src/stdlib/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ mod keyword;
55
mod math;
66
mod pystruct;
77
mod re;
8+
mod time_module;
89
mod tokenize;
910
mod types;
1011
use std::collections::HashMap;
@@ -22,6 +23,7 @@ pub fn get_module_inits() -> HashMap<String, StdlibInitFunc> {
2223
modules.insert("math".to_string(), math::mk_module as StdlibInitFunc);
2324
modules.insert("re".to_string(), re::mk_module as StdlibInitFunc);
2425
modules.insert("struct".to_string(), pystruct::mk_module as StdlibInitFunc);
26+
modules.insert("time".to_string(), time_module::mk_module as StdlibInitFunc);
2527
modules.insert(
2628
"tokenize".to_string(),
2729
tokenize::mk_module as StdlibInitFunc,

‎vm/src/stdlib/time_module.rs

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
//! The python `time` module.
2+
3+
use super::super::obj::{objfloat, objtype};
4+
use super::super::pyobject::{
5+
DictProtocol, PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol,
6+
};
7+
use super::super::VirtualMachine;
8+
use std::thread;
9+
use std::time::{Duration, SystemTime, UNIX_EPOCH};
10+
11+
fn time_sleep(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
12+
arg_check!(vm, args, required = [(seconds, Some(vm.ctx.float_type()))]);
13+
let seconds = objfloat::get_value(seconds);
14+
let secs: u64 = seconds.trunc() as u64;
15+
let nanos: u32 = (seconds.fract() * 1e9) as u32;
16+
let duration = Duration::new(secs, nanos);
17+
thread::sleep(duration);
18+
Ok(vm.get_none())
19+
}
20+
21+
fn duration_to_f64(d: Duration) -> f64 {
22+
(d.as_secs() as f64) + ((d.subsec_nanos() as f64) / 1e9)
23+
}
24+
25+
fn time_time(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
26+
arg_check!(vm, args);
27+
let x = match SystemTime::now().duration_since(UNIX_EPOCH) {
28+
Ok(v) => duration_to_f64(v),
29+
Err(err) => panic!("Error: {:?}", err),
30+
};
31+
let value = vm.ctx.new_float(x);
32+
Ok(value)
33+
}
34+
35+
pub fn mk_module(ctx: &PyContext) -> PyObjectRef {
36+
let py_mod = ctx.new_module(&"time".to_string(), ctx.new_scope(None));
37+
py_mod.set_item("sleep", ctx.new_rustfunc(time_sleep));
38+
py_mod.set_item("time", ctx.new_rustfunc(time_time));
39+
py_mod
40+
}

0 commit comments

Comments
 (0)