RustPython · youknowone · Jun 2, 2023 · May 18, 2023
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -29,11 +29,11 @@ rustpython-pylib = { path = "pylib" }
 rustpython-stdlib = { path = "stdlib" }
 rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" }
 
-rustpython-literal = { git = "https://github.com/youknowone/RustPython-Parser.git", rev = "fc301ab1b0f5483f2b2d3e28d8d92a2845b99fec" }
-rustpython-parser-core = { git = "https://github.com/youknowone/RustPython-Parser.git", rev = "fc301ab1b0f5483f2b2d3e28d8d92a2845b99fec" }
-rustpython-parser = { git = "https://github.com/youknowone/RustPython-Parser.git", rev = "fc301ab1b0f5483f2b2d3e28d8d92a2845b99fec" }
-rustpython-ast = { git = "https://github.com/youknowone/RustPython-Parser.git", rev = "fc301ab1b0f5483f2b2d3e28d8d92a2845b99fec" }
-rustpython-format = { git = "https://github.com/youknowone/RustPython-Parser.git", rev = "fc301ab1b0f5483f2b2d3e28d8d92a2845b99fec" }
+rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "fdec727f8068c882f99a47f0439572a71fd98b28" }
+rustpython-parser-core = { git = "https://github.com/RustPython/Parser.git", rev = "fdec727f8068c882f99a47f0439572a71fd98b28" }
+rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "fdec727f8068c882f99a47f0439572a71fd98b28" }
+rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "fdec727f8068c882f99a47f0439572a71fd98b28" }
+rustpython-format = { git = "https://github.com/RustPython/Parser.git", rev = "fdec727f8068c882f99a47f0439572a71fd98b28" }
 # rustpython-literal = { path = "../RustPython-parser/literal" }
 # rustpython-parser-core = { path = "../RustPython-parser/core" }
 # rustpython-parser = { path = "../RustPython-parser/parser" }

diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs
@@ -927,31 +927,30 @@ impl Compiler {
         name: &str,
         args: &located_ast::Arguments,
     ) -> CompileResult<bytecode::MakeFunctionFlags> {
-        let have_defaults = !args.defaults.is_empty();
+        let defaults: Vec<_> = args.defaults().collect();
+        let have_defaults = !defaults.is_empty();
         if have_defaults {
             // Construct a tuple:
-            let size = args.defaults.len().to_u32();
-            for element in &args.defaults {
+            let size = defaults.len().to_u32();
+            for element in &defaults {
                 self.compile_expression(element)?;
             }
             emit!(self, Instruction::BuildTuple { size });
         }
 
-        if !args.kw_defaults.is_empty() {
-            let required_kw_count = args.kwonlyargs.len().saturating_sub(args.kw_defaults.len());
-            for (kw, default) in args.kwonlyargs[required_kw_count..]
-                .iter()
-                .zip(&args.kw_defaults)
-            {
+        let (kw_without_defaults, kw_with_defaults) = args.split_kwonlyargs();
+        if !kw_with_defaults.is_empty() {
+            let default_kw_count = kw_with_defaults.len();
+            for (arg, default) in kw_with_defaults.iter() {
                 self.emit_constant(ConstantData::Str {
-                    value: kw.arg.to_string(),
+                    value: arg.arg.to_string(),
                 });
                 self.compile_expression(default)?;
             }
             emit!(
                 self,
                 Instruction::BuildMap {
-                    size: args.kw_defaults.len().to_u32(),
+                    size: default_kw_count.to_u32(),
                 }
             );
         }
@@ -960,7 +959,7 @@ impl Compiler {
         if have_defaults {
             func_flags |= bytecode::MakeFunctionFlags::DEFAULTS;
         }
-        if !args.kw_defaults.is_empty() {
+        if !kw_with_defaults.is_empty() {
             func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS;
         }
 
@@ -975,7 +974,9 @@ impl Compiler {
         let args_iter = std::iter::empty()
             .chain(&args.posonlyargs)
             .chain(&args.args)
-            .chain(&args.kwonlyargs);
+            .map(|arg| arg.as_arg())
+            .chain(kw_without_defaults.into_iter())
+            .chain(kw_with_defaults.into_iter().map(|(arg, _)| arg));
         for name in args_iter {
             self.varname(name.arg.as_str())?;
         }
@@ -1228,6 +1229,7 @@ impl Compiler {
             .chain(&args.posonlyargs)
             .chain(&args.args)
             .chain(&args.kwonlyargs)
+            .map(|arg| arg.as_arg())
             .chain(args.vararg.as_deref())
             .chain(args.kwarg.as_deref());
         for arg in args_iter {

diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs
@@ -599,9 +599,17 @@ impl SymbolTableBuilder {
         Ok(())
     }
 
-    fn scan_parameters(&mut self, parameters: &[ast::located::Arg]) -> SymbolTableResult {
+    fn scan_parameters(
+        &mut self,
+        parameters: &[ast::located::ArgWithDefault],
+    ) -> SymbolTableResult {
         for parameter in parameters {
-            self.scan_parameter(parameter)?;
+            let usage = if parameter.def.annotation.is_some() {
+                SymbolUsage::AnnotationParameter
+            } else {
+                SymbolUsage::Parameter
+            };
+            self.register_name(parameter.def.arg.as_str(), usage, parameter.def.location())?;
         }
         Ok(())
     }
@@ -615,23 +623,6 @@ impl SymbolTableBuilder {
         self.register_name(parameter.arg.as_str(), usage, parameter.location())
     }
 
-    fn scan_parameters_annotations(
-        &mut self,
-        parameters: &[ast::located::Arg],
-    ) -> SymbolTableResult {
-        for parameter in parameters {
-            self.scan_parameter_annotation(parameter)?;
-        }
-        Ok(())
-    }
-
-    fn scan_parameter_annotation(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult {
-        if let Some(annotation) = &parameter.annotation {
-            self.scan_annotation(annotation)?;
-        }
-        Ok(())
-    }
-
     fn scan_annotation(&mut self, annotation: &ast::located::Expr) -> SymbolTableResult {
         if self.future_annotations {
             Ok(())
@@ -1201,20 +1192,31 @@ impl SymbolTableBuilder {
         line_number: LineNumber,
     ) -> SymbolTableResult {
         // Evaluate eventual default parameters:
-        self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
-        for expression in args.kw_defaults.iter() {
-            self.scan_expression(expression, ExpressionContext::Load)?;
+        for default in args
+            .posonlyargs
+            .iter()
+            .chain(args.args.iter())
+            .chain(args.kwonlyargs.iter())
+            .filter_map(|arg| arg.default.as_ref())
+        {
+            self.scan_expression(default, ExpressionContext::Load)?; // not ExprContext?
         }
 
         // Annotations are scanned in outer scope:
-        self.scan_parameters_annotations(&args.posonlyargs)?;
-        self.scan_parameters_annotations(&args.args)?;
-        self.scan_parameters_annotations(&args.kwonlyargs)?;
-        if let Some(name) = &args.vararg {
-            self.scan_parameter_annotation(name)?;
+        for annotation in args
+            .posonlyargs
+            .iter()
+            .chain(args.args.iter())
+            .chain(args.kwonlyargs.iter())
+            .filter_map(|arg| arg.def.annotation.as_ref())
+        {
+            self.scan_annotation(annotation)?;
         }
-        if let Some(name) = &args.kwarg {
-            self.scan_parameter_annotation(name)?;
+        if let Some(annotation) = args.vararg.as_ref().and_then(|arg| arg.annotation.as_ref()) {
+            self.scan_annotation(annotation)?;
+        }
+        if let Some(annotation) = args.kwarg.as_ref().and_then(|arg| arg.annotation.as_ref()) {
+            self.scan_annotation(annotation)?;
         }
 
         self.enter_scope(name, SymbolTableType::Function, line_number.get());

diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs
@@ -1,9 +1,9 @@
 use rustpython_codegen::{compile, symboltable};
-use rustpython_parser::ast::{fold::Fold, ConstantOptimizer};
+use rustpython_parser::ast::{self as ast, fold::Fold, ConstantOptimizer};
 
 pub use rustpython_codegen::compile::CompileOpts;
 pub use rustpython_compiler_core::{bytecode::CodeObject, Mode};
-pub use rustpython_parser::source_code::SourceLocator;
+pub use rustpython_parser::{source_code::SourceLocator, Parse};
 
 // these modules are out of repository. re-exporting them here for convenience.
 pub use rustpython_codegen as codegen;
@@ -75,13 +75,13 @@ pub fn compile_symtable(
     let res = match mode {
         Mode::Exec | Mode::Single | Mode::BlockExpr => {
             let ast =
-                parser::parse_program(source, source_path).map_err(|e| locator.locate_error(e))?;
+                ast::Suite::parse(source, source_path).map_err(|e| locator.locate_error(e))?;
             let ast = locator.fold(ast).unwrap();
             symboltable::SymbolTable::scan_program(&ast)
         }
         Mode::Eval => {
-            let expr = parser::parse_expression(source, source_path)
-                .map_err(|e| locator.locate_error(e))?;
+            let expr =
+                ast::Expr::parse(source, source_path).map_err(|e| locator.locate_error(e))?;
             let expr = locator.fold(expr).unwrap();
             symboltable::SymbolTable::scan_expr(&expr)
         }

diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs
@@ -99,10 +99,6 @@ trait Node: Sized {
     fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self>;
 }
 
-trait NamedNode: Node {
-    const NAME: &'static str;
-}
-
 impl<T: Node> Node for Vec<T> {
     fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
         vm.ctx
@@ -317,6 +313,16 @@ impl Node for ast::ConversionFlag {
     }
 }
 
+impl Node for ast::located::Arguments {
+    fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
+        self.into_python_arguments().ast_to_object(vm)
+    }
+    fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
+        ast::located::PythonArguments::ast_from_object(vm, object)
+            .map(ast::located::PythonArguments::into_arguments)
+    }
+}
+
 #[cfg(feature = "rustpython-parser")]
 pub(crate) fn parse(
     vm: &VirtualMachine,