From 6293117886089b79f5985ef945c38b1518dc4fa5 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Tue, 4 Sep 2018 21:07:57 +0200 Subject: [PATCH 1/5] Initial attempt at ast builtin module --- tests/snippets/ast_snippet.py | 9 +++++ vm/src/stdlib/ast.rs | 70 +++++++++++++++++++++++++++++++++++ vm/src/stdlib/mod.rs | 2 + 3 files changed, 81 insertions(+) create mode 100644 tests/snippets/ast_snippet.py create mode 100644 vm/src/stdlib/ast.rs diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py new file mode 100644 index 0000000000..d3715f6951 --- /dev/null +++ b/tests/snippets/ast_snippet.py @@ -0,0 +1,9 @@ + +import ast + +source = """ +def foo(): + print('bar') +""" +n = ast.parse(source) +print(n) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs new file mode 100644 index 0000000000..baf88f6c78 --- /dev/null +++ b/vm/src/stdlib/ast.rs @@ -0,0 +1,70 @@ +/* + * Ast standard module + * + * This module makes use of the parser logic, and translates all ast nodes + * into python ast.AST objects. + */ + +extern crate rustpython_parser; + +use self::rustpython_parser::{ast, parser}; +use super::super::obj::{objdict, objfloat, objint, objlist, objstr, objtuple, objtype}; +use super::super::objbool; +use super::super::pyobject::{ + DictProtocol, PyContext, PyFuncArgs, PyObjectKind, PyObjectRef, PyResult, TypeProtocol, +}; +use super::super::VirtualMachine; + +fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { + let mut body = vec![]; + for statement in &program.statements { + body.push(statement_to_ast(ctx, statement)); + } + // TODO: create Module node and set attributes: + let ast_node = ctx.new_list(body); + ast_node +} + +fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObjectRef { + match &statement.node { + ast::Statement::FunctionDef { name, args, body } => { + // TODO: create ast.FunctionDef object and set attributes + // let node = ctx.new_object(); + let mut py_body = vec![]; + py_body.push(ctx.new_str(format!("{:?}", name))); + // node.set_attr("name", new_str(name)); + for statement in body { + py_body.push(statement_to_ast(ctx, statement)); + } + + ctx.new_list(py_body) + }, + _ => { + ctx.new_str(format!("{:?}", statement)) + } + } + + // TODO: set lineno on object +} + +fn ast_parse(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]); + + let source_string = objstr::get_value(source); + let internal_ast = match parser::parse_program(&source_string) { + Ok(ast) => ast, + Err(msg) => { + return Err(vm.new_value_error(msg)); + } + }; + + // source.clone(); + let ast_node = program_to_ast(&vm.ctx, &internal_ast); + Ok(ast_node) +} + +pub fn mk_module(ctx: &PyContext) -> PyObjectRef { + let ast_mod = ctx.new_module(&"ast".to_string(), ctx.new_scope(None)); + ast_mod.set_item("parse", ctx.new_rustfunc(ast_parse)); + ast_mod +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index dbccc03c28..6a88463be5 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,3 +1,4 @@ +mod ast; mod json; use std::collections::HashMap; @@ -8,5 +9,6 @@ pub type StdlibInitFunc = fn(&PyContext) -> PyObjectRef; pub fn get_module_inits() -> HashMap { let mut modules = HashMap::new(); modules.insert("json".to_string(), json::mk_module as StdlibInitFunc); + modules.insert("ast".to_string(), ast::mk_module as StdlibInitFunc); modules } From f914b4848f6ae4a6c7531387db3e9a7576cf6a03 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 5 Sep 2018 12:36:28 +0200 Subject: [PATCH 2/5] Add two asserts to the ast snippets to check attributes of ast nodes. --- tests/snippets/ast_snippet.py | 7 +++ vm/src/pyobject.rs | 9 +++ vm/src/stdlib/ast.rs | 105 ++++++++++++++++++++++++++++------ 3 files changed, 105 insertions(+), 16 deletions(-) diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py index d3715f6951..1a96451cea 100644 --- a/tests/snippets/ast_snippet.py +++ b/tests/snippets/ast_snippet.py @@ -7,3 +7,10 @@ def foo(): """ n = ast.parse(source) print(n) +print(n.body) +print(n.body[0].name) +assert n.body[0].name == 'foo' +print(n.body[0].body) +print(n.body[0].body[0]) +print(n.body[0].body[0].value.func.id) +assert n.body[0].body[0].value.func.id == 'print' diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 5dbcaebfb8..41860b64ad 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -213,6 +213,15 @@ impl PyContext { self.object.clone() } + pub fn new_object(&self) -> PyObjectRef { + PyObject::new( + PyObjectKind::Instance { + dict: self.new_dict(), + }, + self.object(), + ) + } + pub fn new_int(&self, i: i32) -> PyObjectRef { PyObject::new(PyObjectKind::Integer { value: i }, self.int_type()) } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index baf88f6c78..c8cef5956e 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -8,10 +8,9 @@ extern crate rustpython_parser; use self::rustpython_parser::{ast, parser}; -use super::super::obj::{objdict, objfloat, objint, objlist, objstr, objtuple, objtype}; -use super::super::objbool; +use super::super::obj::{objstr, objtype}; use super::super::pyobject::{ - DictProtocol, PyContext, PyFuncArgs, PyObjectKind, PyObjectRef, PyResult, TypeProtocol, + AttributeProtocol, DictProtocol, PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol, }; use super::super::VirtualMachine; @@ -20,31 +19,92 @@ fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { for statement in &program.statements { body.push(statement_to_ast(ctx, statement)); } - // TODO: create Module node and set attributes: - let ast_node = ctx.new_list(body); + // TODO: create Module node: + // let ast_node = ctx.new_instance(this.Module); + let ast_node = ctx.new_object(); + let py_body = ctx.new_list(body); + ast_node.set_attr("body", py_body); ast_node } fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObjectRef { - match &statement.node { - ast::Statement::FunctionDef { name, args, body } => { - // TODO: create ast.FunctionDef object and set attributes - // let node = ctx.new_object(); + let node = match &statement.node { + ast::Statement::FunctionDef { + name, + args: _, + body, + } => { + // TODO: create ast.FunctionDef object: + let node = ctx.new_object(); + + // Set name: + node.set_attr("name", ctx.new_str(name.to_string())); + + // Set body: let mut py_body = vec![]; - py_body.push(ctx.new_str(format!("{:?}", name))); - // node.set_attr("name", new_str(name)); for statement in body { py_body.push(statement_to_ast(ctx, statement)); } - ctx.new_list(py_body) - }, - _ => { - ctx.new_str(format!("{:?}", statement)) + node.set_attr("body", ctx.new_list(py_body)); + node } - } + ast::Statement::Expression { expression } => { + let value = expression_to_ast(ctx, expression); + // TODO: create proper class: + let node = ctx.new_object(); + node.set_attr("value", value); + node + } + x => { + unimplemented!("{:?}", x); + } + }; + + // set lineno on node: + let lineno = ctx.new_int(statement.location.get_row() as i32); + node.set_attr("lineno", lineno); + + node +} + +fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectRef { + let node = match &expression { + ast::Expression::Call { function, args } => { + // TODO: create ast.Call instance + let node = ctx.new_object(); + + let py_func_ast = expression_to_ast(ctx, function); + node.set_attr("func", py_func_ast); + + let mut py_args = vec![]; + for arg in args { + py_args.push(expression_to_ast(ctx, &arg.1)); + } + let py_args = ctx.new_list(py_args); + node.set_attr("args", py_args); + + node + } + ast::Expression::Identifier { name } => { + // TODO: create ast.Identifier instance + let node = ctx.new_object(); + let py_name = ctx.new_str(name.clone()); + node.set_attr("id", py_name); + node + } + ast::Expression::String { value } => { + let node = ctx.new_object(); + node.set_attr("s", ctx.new_str(value.clone())); + node + } + n => { + unimplemented!("{:?}", n); + } + }; // TODO: set lineno on object + node } fn ast_parse(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -66,5 +126,18 @@ fn ast_parse(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { pub fn mk_module(ctx: &PyContext) -> PyObjectRef { let ast_mod = ctx.new_module(&"ast".to_string(), ctx.new_scope(None)); ast_mod.set_item("parse", ctx.new_rustfunc(ast_parse)); + ast_mod.set_item( + "Module", + ctx.new_class(&"_ast.Module".to_string(), ctx.object()), + ); + // TODO: maybe we can use some clever macro to generate this? + ast_mod.set_item( + "FunctionDef", + ctx.new_class(&"_ast.FunctionDef".to_string(), ctx.object()), + ); + ast_mod.set_item( + "Call", + ctx.new_class(&"_ast.Call".to_string(), ctx.object()), + ); ast_mod } From 51885f29fccf626ea3995ef4dae8936bc044b0e4 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 5 Sep 2018 14:06:16 +0200 Subject: [PATCH 3/5] Added more ast to pyast transformations --- vm/src/stdlib/ast.rs | 212 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 197 insertions(+), 15 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index c8cef5956e..d26b65b747 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -14,6 +14,22 @@ use super::super::pyobject::{ }; use super::super::VirtualMachine; +/* + * Idea: maybe we can create a sort of struct with some helper functions? +struct AstToPyAst { + ctx: &PyContext, +} + +impl AstToPyAst { + fn new(ctx: &PyContext) -> Self { + AstToPyAst { + ctx: ctx, + } + } + +} +*/ + fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { let mut body = vec![]; for statement in &program.statements { @@ -27,33 +43,166 @@ fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { ast_node } +// Create a node class instance +fn create_node(ctx: &PyContext, name: &str) -> PyObjectRef { + // TODO: instantiate a class of type given by name + // TODO: lookup in the current module? + let node = ctx.new_object(); + node +} + +fn statements_to_ast(ctx: &PyContext, statements: &Vec) -> PyObjectRef { + let mut py_statements = vec![]; + for statement in statements { + py_statements.push(statement_to_ast(ctx, statement)); + } + ctx.new_list(py_statements) +} + fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObjectRef { let node = match &statement.node { + ast::Statement::ClassDef { + name, + body, + args: _, + } => { + let node = create_node(ctx, "ClassDef"); + + // Set name: + node.set_attr("name", ctx.new_str(name.to_string())); + + // Set body: + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + node + } ast::Statement::FunctionDef { name, args: _, body, } => { - // TODO: create ast.FunctionDef object: - let node = ctx.new_object(); + let node = create_node(ctx, "FunctionDef"); // Set name: node.set_attr("name", ctx.new_str(name.to_string())); // Set body: - let mut py_body = vec![]; - for statement in body { - py_body.push(statement_to_ast(ctx, statement)); - } + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + node + } + ast::Statement::Continue => { + let node = create_node(ctx, "Continue"); + node + } + ast::Statement::Break => { + let node = create_node(ctx, "Break"); + node + } + ast::Statement::Pass => { + let node = create_node(ctx, "Pass"); + node + } + ast::Statement::Delete { targets } => { + let node = create_node(ctx, "Delete"); + + let py_targets = ctx.new_tuple( + targets + .into_iter() + .map(|v| expression_to_ast(ctx, v)) + .collect(), + ); + node.set_attr("targets", py_targets); + + node + } + ast::Statement::Return { value } => { + let node = create_node(ctx, "Return"); + + let py_value = if let Some(value) = value { + ctx.new_tuple( + value + .into_iter() + .map(|v| expression_to_ast(ctx, v)) + .collect(), + ) + } else { + ctx.none() + }; + node.set_attr("value", py_value); + + node + } + ast::Statement::If { test, body, orelse } => { + let node = create_node(ctx, "If"); + + let py_test = expression_to_ast(ctx, test); + node.set_attr("test", py_test); + + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = if let Some(orelse) = orelse { + statements_to_ast(ctx, orelse) + } else { + ctx.none() + }; + node.set_attr("orelse", py_orelse); + + node + } + ast::Statement::For { + target, + iter, + body, + orelse, + } => { + let node = create_node(ctx, "For"); + + /* + let py_target = expression_to_ast(ctx, target); + node.set_attr("target", py_target); + + let py_iter = expression_to_ast(ctx, iter); + node.set_attr("iter", py_iter); + */ + + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = if let Some(orelse) = orelse { + statements_to_ast(ctx, orelse) + } else { + ctx.none() + }; + node.set_attr("orelse", py_orelse); + + node + } + ast::Statement::While { test, body, orelse } => { + let node = create_node(ctx, "While"); + + let py_test = expression_to_ast(ctx, test); + node.set_attr("test", py_test); + + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = if let Some(orelse) = orelse { + statements_to_ast(ctx, orelse) + } else { + ctx.none() + }; + node.set_attr("orelse", py_orelse); - node.set_attr("body", ctx.new_list(py_body)); node } ast::Statement::Expression { expression } => { + let node = create_node(ctx, "Expr"); + let value = expression_to_ast(ctx, expression); - // TODO: create proper class: - let node = ctx.new_object(); node.set_attr("value", value); + node } x => { @@ -71,8 +220,7 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectRef { let node = match &expression { ast::Expression::Call { function, args } => { - // TODO: create ast.Call instance - let node = ctx.new_object(); + let node = create_node(ctx, "Call"); let py_func_ast = expression_to_ast(ctx, function); node.set_attr("func", py_func_ast); @@ -86,15 +234,45 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR node } + ast::Expression::Binop { a, op, b } => { + let node = create_node(ctx, "BinOp"); + + let py_a = expression_to_ast(ctx, a); + node.set_attr("left", py_a); + + // Operator: + let str_op = match op { + ast::Operator::Add => "Add", + ast::Operator::Sub => "Sub", + ast::Operator::Mult => "Mult", + ast::Operator::MatMult => "MatMult", + ast::Operator::Div => "Div", + ast::Operator::Mod => "Mod", + ast::Operator::Pow => "Pow", + ast::Operator::LShift => "LShift", + ast::Operator::RShift => "RShift", + ast::Operator::BitOr => "BitOr", + ast::Operator::BitXor => "BitXor", + ast::Operator::BitAnd => "BitAnd", + ast::Operator::FloorDiv => "FloorDiv", + }; + let py_op = ctx.new_str(str_op.to_string()); + node.set_attr("op", py_op); + + let py_b = expression_to_ast(ctx, b); + node.set_attr("right", py_b); + node + } ast::Expression::Identifier { name } => { - // TODO: create ast.Identifier instance - let node = ctx.new_object(); + let node = create_node(ctx, "Identifier"); + + // Id: let py_name = ctx.new_str(name.clone()); node.set_attr("id", py_name); node } ast::Expression::String { value } => { - let node = ctx.new_object(); + let node = create_node(ctx, "Str"); node.set_attr("s", ctx.new_str(value.clone())); node } @@ -103,7 +281,10 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR } }; - // TODO: set lineno on object + // TODO: retrieve correct lineno: + let lineno = ctx.new_int(1); + node.set_attr("lineno", lineno); + node } @@ -130,6 +311,7 @@ pub fn mk_module(ctx: &PyContext) -> PyObjectRef { "Module", ctx.new_class(&"_ast.Module".to_string(), ctx.object()), ); + // TODO: maybe we can use some clever macro to generate this? ast_mod.set_item( "FunctionDef", From ac3c18cf4a624e18a89d99f7a9803fc2be646b42 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 5 Sep 2018 14:16:36 +0200 Subject: [PATCH 4/5] Add module type --- vm/src/pyobject.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 41860b64ad..cff8cca82a 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -62,6 +62,7 @@ pub struct PyContext { pub tuple_type: PyObjectRef, pub str_type: PyObjectRef, pub function_type: PyObjectRef, + pub module_type: PyObjectRef, pub bound_method_type: PyObjectRef, pub member_descriptor_type: PyObjectRef, pub object: PyObjectRef, @@ -111,6 +112,7 @@ impl PyContext { objobject::create_object(type_type.clone(), object_type.clone(), dict_type.clone()); objdict::create_type(type_type.clone(), object_type.clone(), dict_type.clone()); + let module_type = create_type("module", &type_type, &object_type, &dict_type); let function_type = create_type("function", &type_type, &object_type, &dict_type); let bound_method_type = create_type("method", &type_type, &object_type, &dict_type); let member_descriptor_type = @@ -145,6 +147,7 @@ impl PyContext { str_type: str_type, object: object_type, function_type: function_type, + module_type: module_type, bound_method_type: bound_method_type, member_descriptor_type: member_descriptor_type, type_type: type_type, @@ -284,7 +287,7 @@ impl PyContext { name: name.clone(), dict: scope.clone(), }, - self.type_type(), + self.module_type.clone(), ) } From da7745cf27e4cf23b5f3137d5e75a4a6e2da1065 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Wed, 5 Sep 2018 14:31:06 +0200 Subject: [PATCH 5/5] Fix module type --- tests/snippets/ast_snippet.py | 1 + vm/src/import.rs | 21 ++++++++------------- vm/src/stdlib/ast.rs | 6 +++--- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py index 1a96451cea..033d596966 100644 --- a/tests/snippets/ast_snippet.py +++ b/tests/snippets/ast_snippet.py @@ -1,5 +1,6 @@ import ast +print(ast) source = """ def foo(): diff --git a/vm/src/import.rs b/vm/src/import.rs index e9aae8f68b..67824f0e32 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; use self::rustpython_parser::parser; use super::compile; -use super::pyobject::{DictProtocol, PyObject, PyObjectKind, PyResult}; +use super::pyobject::{DictProtocol, PyObjectKind, PyResult}; use super::vm::VirtualMachine; fn import_module(vm: &mut VirtualMachine, module: &String) -> PyResult { @@ -50,28 +50,23 @@ fn import_module(vm: &mut VirtualMachine, module: &String) -> PyResult { }; let builtins = vm.get_builtin_scope(); - let scope = vm.context().new_scope(Some(builtins)); + let scope = vm.ctx.new_scope(Some(builtins)); match vm.run_code_obj(code_obj, scope.clone()) { Ok(_) => {} Err(value) => return Err(value), } - Ok(scope) + let py_module = vm.ctx.new_module(module, scope); + Ok(py_module) } -pub fn import(vm: &mut VirtualMachine, module: &String, symbol: &Option) -> PyResult { - let scope = import_module(vm, module)?; +pub fn import(vm: &mut VirtualMachine, module_name: &String, symbol: &Option) -> PyResult { + let module = import_module(vm, module_name)?; // If we're importing a symbol, look it up and use it, otherwise construct a module and return // that let obj = match symbol { - Some(symbol) => scope.get_item(symbol).unwrap(), - None => PyObject::new( - PyObjectKind::Module { - name: module.clone(), - dict: scope.clone(), - }, - vm.get_type(), - ), + Some(symbol) => module.get_item(symbol).unwrap(), + None => module, }; Ok(obj) } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index d26b65b747..46a4317c8a 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -44,7 +44,7 @@ fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { } // Create a node class instance -fn create_node(ctx: &PyContext, name: &str) -> PyObjectRef { +fn create_node(ctx: &PyContext, _name: &str) -> PyObjectRef { // TODO: instantiate a class of type given by name // TODO: lookup in the current module? let node = ctx.new_object(); @@ -152,8 +152,8 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj node } ast::Statement::For { - target, - iter, + target: _, + iter: _, body, orelse, } => {