diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py new file mode 100644 index 0000000000..033d596966 --- /dev/null +++ b/tests/snippets/ast_snippet.py @@ -0,0 +1,17 @@ + +import ast +print(ast) + +source = """ +def foo(): + print('bar') +""" +n = ast.parse(source) +print(n) +print(n.body) +print(n.body[0].name) +assert n.body[0].name == 'foo' +print(n.body[0].body) +print(n.body[0].body[0]) +print(n.body[0].body[0].value.func.id) +assert n.body[0].body[0].value.func.id == 'print' diff --git a/vm/src/import.rs b/vm/src/import.rs index e9aae8f68b..67824f0e32 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; use self::rustpython_parser::parser; use super::compile; -use super::pyobject::{DictProtocol, PyObject, PyObjectKind, PyResult}; +use super::pyobject::{DictProtocol, PyObjectKind, PyResult}; use super::vm::VirtualMachine; fn import_module(vm: &mut VirtualMachine, module: &String) -> PyResult { @@ -50,28 +50,23 @@ fn import_module(vm: &mut VirtualMachine, module: &String) -> PyResult { }; let builtins = vm.get_builtin_scope(); - let scope = vm.context().new_scope(Some(builtins)); + let scope = vm.ctx.new_scope(Some(builtins)); match vm.run_code_obj(code_obj, scope.clone()) { Ok(_) => {} Err(value) => return Err(value), } - Ok(scope) + let py_module = vm.ctx.new_module(module, scope); + Ok(py_module) } -pub fn import(vm: &mut VirtualMachine, module: &String, symbol: &Option) -> PyResult { - let scope = import_module(vm, module)?; +pub fn import(vm: &mut VirtualMachine, module_name: &String, symbol: &Option) -> PyResult { + let module = import_module(vm, module_name)?; // If we're importing a symbol, look it up and use it, otherwise construct a module and return // that let obj = match symbol { - Some(symbol) => scope.get_item(symbol).unwrap(), - None => PyObject::new( - PyObjectKind::Module { - name: module.clone(), - dict: scope.clone(), - }, - vm.get_type(), - ), + Some(symbol) => module.get_item(symbol).unwrap(), + None => module, }; Ok(obj) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 5dbcaebfb8..cff8cca82a 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -62,6 +62,7 @@ pub struct PyContext { pub tuple_type: PyObjectRef, pub str_type: PyObjectRef, pub function_type: PyObjectRef, + pub module_type: PyObjectRef, pub bound_method_type: PyObjectRef, pub member_descriptor_type: PyObjectRef, pub object: PyObjectRef, @@ -111,6 +112,7 @@ impl PyContext { objobject::create_object(type_type.clone(), object_type.clone(), dict_type.clone()); objdict::create_type(type_type.clone(), object_type.clone(), dict_type.clone()); + let module_type = create_type("module", &type_type, &object_type, &dict_type); let function_type = create_type("function", &type_type, &object_type, &dict_type); let bound_method_type = create_type("method", &type_type, &object_type, &dict_type); let member_descriptor_type = @@ -145,6 +147,7 @@ impl PyContext { str_type: str_type, object: object_type, function_type: function_type, + module_type: module_type, bound_method_type: bound_method_type, member_descriptor_type: member_descriptor_type, type_type: type_type, @@ -213,6 +216,15 @@ impl PyContext { self.object.clone() } + pub fn new_object(&self) -> PyObjectRef { + PyObject::new( + PyObjectKind::Instance { + dict: self.new_dict(), + }, + self.object(), + ) + } + pub fn new_int(&self, i: i32) -> PyObjectRef { PyObject::new(PyObjectKind::Integer { value: i }, self.int_type()) } @@ -275,7 +287,7 @@ impl PyContext { name: name.clone(), dict: scope.clone(), }, - self.type_type(), + self.module_type.clone(), ) } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs new file mode 100644 index 0000000000..46a4317c8a --- /dev/null +++ b/vm/src/stdlib/ast.rs @@ -0,0 +1,325 @@ +/* + * Ast standard module + * + * This module makes use of the parser logic, and translates all ast nodes + * into python ast.AST objects. + */ + +extern crate rustpython_parser; + +use self::rustpython_parser::{ast, parser}; +use super::super::obj::{objstr, objtype}; +use super::super::pyobject::{ + AttributeProtocol, DictProtocol, PyContext, PyFuncArgs, PyObjectRef, PyResult, TypeProtocol, +}; +use super::super::VirtualMachine; + +/* + * Idea: maybe we can create a sort of struct with some helper functions? +struct AstToPyAst { + ctx: &PyContext, +} + +impl AstToPyAst { + fn new(ctx: &PyContext) -> Self { + AstToPyAst { + ctx: ctx, + } + } + +} +*/ + +fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { + let mut body = vec![]; + for statement in &program.statements { + body.push(statement_to_ast(ctx, statement)); + } + // TODO: create Module node: + // let ast_node = ctx.new_instance(this.Module); + let ast_node = ctx.new_object(); + let py_body = ctx.new_list(body); + ast_node.set_attr("body", py_body); + ast_node +} + +// Create a node class instance +fn create_node(ctx: &PyContext, _name: &str) -> PyObjectRef { + // TODO: instantiate a class of type given by name + // TODO: lookup in the current module? + let node = ctx.new_object(); + node +} + +fn statements_to_ast(ctx: &PyContext, statements: &Vec) -> PyObjectRef { + let mut py_statements = vec![]; + for statement in statements { + py_statements.push(statement_to_ast(ctx, statement)); + } + ctx.new_list(py_statements) +} + +fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObjectRef { + let node = match &statement.node { + ast::Statement::ClassDef { + name, + body, + args: _, + } => { + let node = create_node(ctx, "ClassDef"); + + // Set name: + node.set_attr("name", ctx.new_str(name.to_string())); + + // Set body: + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + node + } + ast::Statement::FunctionDef { + name, + args: _, + body, + } => { + let node = create_node(ctx, "FunctionDef"); + + // Set name: + node.set_attr("name", ctx.new_str(name.to_string())); + + // Set body: + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + node + } + ast::Statement::Continue => { + let node = create_node(ctx, "Continue"); + node + } + ast::Statement::Break => { + let node = create_node(ctx, "Break"); + node + } + ast::Statement::Pass => { + let node = create_node(ctx, "Pass"); + node + } + ast::Statement::Delete { targets } => { + let node = create_node(ctx, "Delete"); + + let py_targets = ctx.new_tuple( + targets + .into_iter() + .map(|v| expression_to_ast(ctx, v)) + .collect(), + ); + node.set_attr("targets", py_targets); + + node + } + ast::Statement::Return { value } => { + let node = create_node(ctx, "Return"); + + let py_value = if let Some(value) = value { + ctx.new_tuple( + value + .into_iter() + .map(|v| expression_to_ast(ctx, v)) + .collect(), + ) + } else { + ctx.none() + }; + node.set_attr("value", py_value); + + node + } + ast::Statement::If { test, body, orelse } => { + let node = create_node(ctx, "If"); + + let py_test = expression_to_ast(ctx, test); + node.set_attr("test", py_test); + + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = if let Some(orelse) = orelse { + statements_to_ast(ctx, orelse) + } else { + ctx.none() + }; + node.set_attr("orelse", py_orelse); + + node + } + ast::Statement::For { + target: _, + iter: _, + body, + orelse, + } => { + let node = create_node(ctx, "For"); + + /* + let py_target = expression_to_ast(ctx, target); + node.set_attr("target", py_target); + + let py_iter = expression_to_ast(ctx, iter); + node.set_attr("iter", py_iter); + */ + + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = if let Some(orelse) = orelse { + statements_to_ast(ctx, orelse) + } else { + ctx.none() + }; + node.set_attr("orelse", py_orelse); + + node + } + ast::Statement::While { test, body, orelse } => { + let node = create_node(ctx, "While"); + + let py_test = expression_to_ast(ctx, test); + node.set_attr("test", py_test); + + let py_body = statements_to_ast(ctx, body); + node.set_attr("body", py_body); + + let py_orelse = if let Some(orelse) = orelse { + statements_to_ast(ctx, orelse) + } else { + ctx.none() + }; + node.set_attr("orelse", py_orelse); + + node + } + ast::Statement::Expression { expression } => { + let node = create_node(ctx, "Expr"); + + let value = expression_to_ast(ctx, expression); + node.set_attr("value", value); + + node + } + x => { + unimplemented!("{:?}", x); + } + }; + + // set lineno on node: + let lineno = ctx.new_int(statement.location.get_row() as i32); + node.set_attr("lineno", lineno); + + node +} + +fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectRef { + let node = match &expression { + ast::Expression::Call { function, args } => { + let node = create_node(ctx, "Call"); + + let py_func_ast = expression_to_ast(ctx, function); + node.set_attr("func", py_func_ast); + + let mut py_args = vec![]; + for arg in args { + py_args.push(expression_to_ast(ctx, &arg.1)); + } + let py_args = ctx.new_list(py_args); + node.set_attr("args", py_args); + + node + } + ast::Expression::Binop { a, op, b } => { + let node = create_node(ctx, "BinOp"); + + let py_a = expression_to_ast(ctx, a); + node.set_attr("left", py_a); + + // Operator: + let str_op = match op { + ast::Operator::Add => "Add", + ast::Operator::Sub => "Sub", + ast::Operator::Mult => "Mult", + ast::Operator::MatMult => "MatMult", + ast::Operator::Div => "Div", + ast::Operator::Mod => "Mod", + ast::Operator::Pow => "Pow", + ast::Operator::LShift => "LShift", + ast::Operator::RShift => "RShift", + ast::Operator::BitOr => "BitOr", + ast::Operator::BitXor => "BitXor", + ast::Operator::BitAnd => "BitAnd", + ast::Operator::FloorDiv => "FloorDiv", + }; + let py_op = ctx.new_str(str_op.to_string()); + node.set_attr("op", py_op); + + let py_b = expression_to_ast(ctx, b); + node.set_attr("right", py_b); + node + } + ast::Expression::Identifier { name } => { + let node = create_node(ctx, "Identifier"); + + // Id: + let py_name = ctx.new_str(name.clone()); + node.set_attr("id", py_name); + node + } + ast::Expression::String { value } => { + let node = create_node(ctx, "Str"); + node.set_attr("s", ctx.new_str(value.clone())); + node + } + n => { + unimplemented!("{:?}", n); + } + }; + + // TODO: retrieve correct lineno: + let lineno = ctx.new_int(1); + node.set_attr("lineno", lineno); + + node +} + +fn ast_parse(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]); + + let source_string = objstr::get_value(source); + let internal_ast = match parser::parse_program(&source_string) { + Ok(ast) => ast, + Err(msg) => { + return Err(vm.new_value_error(msg)); + } + }; + + // source.clone(); + let ast_node = program_to_ast(&vm.ctx, &internal_ast); + Ok(ast_node) +} + +pub fn mk_module(ctx: &PyContext) -> PyObjectRef { + let ast_mod = ctx.new_module(&"ast".to_string(), ctx.new_scope(None)); + ast_mod.set_item("parse", ctx.new_rustfunc(ast_parse)); + ast_mod.set_item( + "Module", + ctx.new_class(&"_ast.Module".to_string(), ctx.object()), + ); + + // TODO: maybe we can use some clever macro to generate this? + ast_mod.set_item( + "FunctionDef", + ctx.new_class(&"_ast.FunctionDef".to_string(), ctx.object()), + ); + ast_mod.set_item( + "Call", + ctx.new_class(&"_ast.Call".to_string(), ctx.object()), + ); + ast_mod +} diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index dbccc03c28..6a88463be5 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,3 +1,4 @@ +mod ast; mod json; use std::collections::HashMap; @@ -8,5 +9,6 @@ pub type StdlibInitFunc = fn(&PyContext) -> PyObjectRef; pub fn get_module_inits() -> HashMap { let mut modules = HashMap::new(); modules.insert("json".to_string(), json::mk_module as StdlibInitFunc); + modules.insert("ast".to_string(), ast::mk_module as StdlibInitFunc); modules }