RustPython · windelbouwman · Jul 26, 2019 · Jul 24, 2019 · Jul 25, 2019 · coolreader18
diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs
@@ -385,35 +385,47 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            With { items, body } => {
-                let end_label = self.new_label();
-                for item in items {
-                    self.compile_expression(&item.context_expr)?;
-                    self.emit(Instruction::SetupWith { end: end_label });
-                    match &item.optional_vars {
-                        Some(var) => {
-                            self.compile_store(var)?;
-                        }
-                        None => {
-                            self.emit(Instruction::Pop);
+            With {
+                is_async,
+                items,
+                body,
+            } => {
+                if *is_async {
+                    unimplemented!("async with");
+                } else {
+                    let end_label = self.new_label();
+                    for item in items {
+                        self.compile_expression(&item.context_expr)?;
+                        self.emit(Instruction::SetupWith { end: end_label });
+                        match &item.optional_vars {
+                            Some(var) => {
+                                self.compile_store(var)?;
+                            }
+                            None => {
+                                self.emit(Instruction::Pop);
+                            }
                         }
                     }
-                }
 
-                self.compile_statements(body)?;
-                for _ in 0..items.len() {
-                    self.emit(Instruction::CleanupWith { end: end_label });
+                    self.compile_statements(body)?;
+                    for _ in 0..items.len() {
+                        self.emit(Instruction::CleanupWith { end: end_label });
+                    }
+                    self.set_label(end_label);
                 }
-                self.set_label(end_label);
             }
             For {
+                is_async,
                 target,
                 iter,
                 body,
                 orelse,
-            } => self.compile_for(target, iter, body, orelse)?,
-            AsyncFor { .. } => {
-                unimplemented!("async for");
+            } => {
+                if *is_async {
+                    unimplemented!("async for");
+                } else {
+                    self.compile_for(target, iter, body, orelse)?
+                }
             }
             Raise { exception, cause } => match exception {
                 Some(value) => {
@@ -439,14 +451,18 @@ impl Compiler {
                 finalbody,
             } => self.compile_try_statement(body, handlers, orelse, finalbody)?,
             FunctionDef {
+                is_async,
                 name,
                 args,
                 body,
                 decorator_list,
                 returns,
-            } => self.compile_function_def(name, args, body, decorator_list, returns)?,
-            AsyncFunctionDef { .. } => {
-                unimplemented!("async def");
+            } => {
+                if *is_async {
+                    unimplemented!("async def");
+                } else {
+                    self.compile_function_def(name, args, body, decorator_list, returns)?
+                }
             }
             ClassDef {
                 name,

diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs
@@ -240,13 +240,7 @@ impl SymbolTableBuilder {
                 args,
                 decorator_list,
                 returns,
-            }
-            | AsyncFunctionDef {
-                name,
-                body,
-                args,
-                decorator_list,
-                returns,
+                ..
             } => {
                 self.scan_expressions(decorator_list)?;
                 self.register_name(name, SymbolRole::Assigned)?;
@@ -289,12 +283,7 @@ impl SymbolTableBuilder {
                 iter,
                 body,
                 orelse,
-            }
-            | AsyncFor {
-                target,
-                iter,
-                body,
-                orelse,
+                ..
             } => {
                 self.scan_expression(target)?;
                 self.scan_expression(iter)?;
@@ -346,7 +335,7 @@ impl SymbolTableBuilder {
                 self.scan_expression(target)?;
                 self.scan_expression(value)?;
             }
-            With { items, body } => {
+            With { items, body, .. } => {
                 for item in items {
                     self.scan_expression(&item.context_expr)?;
                     if let Some(expression) = &item.optional_vars {

diff --git a/examples/parse_folder.rs b/examples/parse_folder.rs
@@ -0,0 +1,70 @@
+/// This an example usage of the rustpython_parser crate.
+/// This program crawls over a directory of python files and
+/// tries to parse them into an abstract syntax tree (AST)
+///
+/// example usage:
+/// $ RUST_LOG=info cargo run --release parse_folder /usr/lib/python3.7
+
+#[macro_use]
+extern crate clap;
+extern crate env_logger;
+#[macro_use]
+extern crate log;
+
+use clap::{App, Arg};
+
+use rustpython_parser::{ast, parser};
+use std::path::Path;
+
+fn main() {
+    env_logger::init();
+    let app = App::new("parse_folders")
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about("Walks over all .py files in a folder, and parses them.")
+        .arg(
+            Arg::with_name("folder")
+                .help("Folder to scan")
+                .required(true),
+        );
+    let matches = app.get_matches();
+
+    let folder = Path::new(matches.value_of("folder").unwrap());
+    if folder.exists() && folder.is_dir() {
+        println!("Parsing folder of python code: {:?}", folder);
+        let res = parse_folder(&folder).unwrap();
+        println!("Processed {:?} files", res.len());
+    } else {
+        println!("{:?} is not a folder.", folder);
+    }
+}
+
+fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
+    let mut res = vec![];
+    info!("Parsing folder of python code: {:?}", path);
+    for entry in path.read_dir()? {
+        debug!("Entry: {:?}", entry);
+        let entry = entry?;
+        let metadata = entry.metadata()?;
+
+        let path = entry.path();
+        if metadata.is_dir() {
+            let x = parse_folder(&path)?;
+            res.extend(x);
+        }
+
+        if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {
+            match parse_python_file(&path) {
+                Ok(x) => res.push(x),
+                Err(y) => error!("Erreur in file {:?} {:?}", path, y),
+            }
+        }
+    }
+    Ok(res)
+}
+
+fn parse_python_file(filename: &Path) -> Result<ast::Program, String> {
+    info!("Parsing file {:?}", filename);
+    let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?;
+    parser::parse_program(&source).map_err(|e| e.to_string())
+}
diff --git a/parser/src/ast.rs b/parser/src/ast.rs
@@ -94,16 +94,12 @@ pub enum StatementType {
         orelse: Option<Vec<Statement>>,
     },
     With {
+        is_async: bool,
         items: Vec<WithItem>,
         body: Vec<Statement>,
     },
     For {
-        target: Expression,
-        iter: Expression,
-        body: Vec<Statement>,
-        orelse: Option<Vec<Statement>>,
-    },
-    AsyncFor {
+        is_async: bool,
         target: Expression,
         iter: Expression,
         body: Vec<Statement>,
@@ -127,13 +123,7 @@ pub enum StatementType {
         decorator_list: Vec<Expression>,
     },
     FunctionDef {
-        name: String,
-        args: Parameters,
-        body: Vec<Statement>,
-        decorator_list: Vec<Expression>,
-        returns: Option<Expression>,
-    },
-    AsyncFunctionDef {
+        is_async: bool,
         name: String,
         args: Parameters,
         body: Vec<Statement>,

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
@@ -694,6 +694,13 @@ where
                     spaces = 0;
                     tabs = 0;
                 }
+                Some('\x0C') => {
+                    // Form feed character!
+                    // Reset indentation for the Emacs user.
+                    self.next_char();
+                    spaces = 0;
+                    tabs = 0;
+                }
                 Some('\n') => {
                     // Empty line!
                     self.next_char();
@@ -1157,9 +1164,13 @@ where
                     self.emit((tok_start, Tok::Newline, tok_end));
                 }
             }
-            ' ' => {
+            ' ' | '\t' | '\x0C' => {
                 // Skip whitespaces
                 self.next_char();
+                while self.chr0 == Some(' ') || self.chr0 == Some('\t') || self.chr0 == Some('\x0C')
+                {
+                    self.next_char();
+                }
             }
             _ => {
                 let c = self.next_char();

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
@@ -306,6 +306,7 @@ mod tests {
                         ast::Statement {
                             location: ast::Location::new(2, 2),
                             node: ast::StatementType::FunctionDef {
+                                is_async: false,
                                 name: String::from("__init__"),
                                 args: ast::Parameters {
                                     args: vec![ast::Parameter {
@@ -329,6 +330,7 @@ mod tests {
                         ast::Statement {
                             location: ast::Location::new(4, 2),
                             node: ast::StatementType::FunctionDef {
+                                is_async: false,
                                 name: String::from("method_with_default"),
                                 args: ast::Parameters {
                                     args: vec![

diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop
@@ -334,14 +334,11 @@ WhileStatement: ast::Statement = {
 
 ForStatement: ast::Statement = {
     <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let is_async = is_async.is_some();
         let orelse = s2.map(|s| s.2);
         ast::Statement {
             location,
-            node: if is_async.is_some() {
-                ast::StatementType::AsyncFor { target, iter, body, orelse }
-            } else {
-                ast::StatementType::For { target, iter, body, orelse }
-            },
+            node: ast::StatementType::For { is_async, target, iter, body, orelse },
         }
     },
 };
@@ -380,10 +377,11 @@ ExceptClause: ast::ExceptHandler = {
 };
 
 WithStatement: ast::Statement = {
-    <location:@L> "with" <items:OneOrMore<WithItem>> ":" <s:Suite> => {
+    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
+        let is_async = is_async.is_some();
         ast::Statement {
             location,
-            node: ast::StatementType::With { items: items, body: s },
+            node: ast::StatementType::With { is_async, items, body },
         }
     },
 };
@@ -396,26 +394,18 @@ WithItem: ast::WithItem = {
 };
 
 FuncDef: ast::Statement = {
-    <d:Decorator*> <location:@L> <is_async:"async"?> "def" <i:Identifier> <a:Parameters> <r:("->" Test)?> ":" <s:Suite> => {
+    <d:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <a:Parameters> <r:("->" Test)?> ":" <body:Suite> => {
+        let is_async = is_async.is_some();
         ast::Statement {
             location,
-            node: if is_async.is_some() {
-                    ast::StatementType::AsyncFunctionDef {
-                        name: i,
-                        args: a,
-                        body: s,
-                        decorator_list: d,
-                        returns: r.map(|x| x.1),
-                    }
-                } else {
-                    ast::StatementType::FunctionDef {
-                        name: i,
-                        args: a,
-                        body: s,
-                        decorator_list: d,
-                        returns: r.map(|x| x.1),
-                    }
-                }
+            node: ast::StatementType::FunctionDef {
+                is_async,
+                name,
+                args: a,
+                body,
+                decorator_list: d,
+                returns: r.map(|x| x.1),
+            }
         }
     },
 };