Skip to content

Commit 85cef1e

Browse files
progvalprogval
committed
doc.
1 parent 46ba4b7 commit 85cef1e

File tree

8 files changed

+106
-48
lines changed

8 files changed

+106
-48
lines changed

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ path = "src/main.rs"
1515
[dependencies]
1616
nom = "^4.0"
1717
#nom_locate = { path = "../nom_locate/" }
18-
nom_locate = { git = "https://github.com/fflorent/nom_locate" }
18+
nom_locate = "^0.3.0"
1919
unicode-xid = "^0.1"
2020
unicode_names2 = "^0.2.1"
2121
num-traits = { version="^0.2.4", optional=true }

‎src/ast.rs

Lines changed: 27 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
//! enums and structures that store the syntax tree outputed by the parser.
2+
13
use std::fmt;
24

35
#[cfg(feature="bigint")]
@@ -21,46 +23,9 @@ pub type PyStringContent = String;
2123
#[cfg(not(feature="wtf8"))]
2224
pub type PyStringCodePoint = char;
2325

24-
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25-
pub enum ArgumentError {
26-
KeywordExpression,
27-
PositionalAfterKeyword,
28-
StarargsAfterKeyword,
29-
}
30-
31-
impl ArgumentError {
32-
pub fn to_string(self) -> &'static str {
33-
match self {
34-
ArgumentError::KeywordExpression => "Keyword cannot be an expression.",
35-
ArgumentError::PositionalAfterKeyword => "Positional argument after keyword argument or **kwargs.",
36-
ArgumentError::StarargsAfterKeyword => "*args after keyword argument or **kwargs.",
37-
}
38-
}
39-
}
40-
41-
impl From<u32> for ArgumentError {
42-
fn from(i: u32) -> ArgumentError {
43-
match i {
44-
1 => ArgumentError::KeywordExpression,
45-
2 => ArgumentError::PositionalAfterKeyword,
46-
3 => ArgumentError::StarargsAfterKeyword,
47-
_ => panic!("Invalid error code.")
48-
}
49-
}
50-
}
51-
52-
impl From<ArgumentError> for u32 {
53-
fn from(e: ArgumentError) -> u32 {
54-
match e {
55-
ArgumentError::KeywordExpression => 1,
56-
ArgumentError::PositionalAfterKeyword => 2,
57-
ArgumentError::StarargsAfterKeyword => 3,
58-
}
59-
}
60-
}
61-
6226
pub type Name = String;
6327

28+
/// Represents whether a function signature has `*`, `*args`, or none of these.
6429
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
6530
pub enum StarParams<T> {
6631
/// No single star
@@ -77,6 +42,7 @@ impl<T> Default for StarParams<T> {
7742
}
7843
}
7944

45+
/// The list of parameters of a function definition.
8046
#[derive(Clone, Debug, PartialEq, Default,)]
8147
pub struct TypedArgsList {
8248
pub positional_args: Vec<(Name, Option<Expression>, Option<Expression>)>,
@@ -85,6 +51,7 @@ pub struct TypedArgsList {
8551
pub star_kwargs: Option<(Name, Option<Expression>)>,
8652
}
8753

54+
/// The list of parameters of a lambda definition.
8855
#[derive(Clone, Debug, PartialEq, Default)]
8956
pub struct UntypedArgsList {
9057
pub positional_args: Vec<(Name, Option<Expression>)>,
@@ -93,12 +60,14 @@ pub struct UntypedArgsList {
9360
pub star_kwargs: Option<Name>,
9461
}
9562

63+
/// A function or class decorator.
9664
#[derive(Clone, Debug, PartialEq)]
9765
pub struct Decorator {
9866
pub name: Vec<Name>,
9967
pub args: Option<Vec<Argument>>,
10068
}
10169

70+
/// An argument to a function call
10271
#[derive(Clone, Debug, PartialEq)]
10372
pub enum Argument {
10473
Positional(Expression),
@@ -107,13 +76,18 @@ pub enum Argument {
10776
Kwargs(Expression),
10877
}
10978

79+
/// The `foo[bar]` syntax.
11080
#[derive(Clone, Debug, PartialEq)]
11181
pub enum Subscript {
82+
/// `foo[i]`
11283
Simple(Expression),
84+
/// `foo[start:end]`, `foo[start:]`, etc.
11385
Double(Option<Expression>, Option<Expression>),
86+
/// `foo[start:end:step]`, `foo[start::]`, etc.
11487
Triple(Option<Expression>, Option<Expression>, Option<Expression>),
11588
}
11689

90+
/// Unary operators.
11791
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
11892
pub enum Uop {
11993
Plus,
@@ -134,6 +108,7 @@ impl fmt::Display for Uop {
134108
}
135109
}
136110

111+
/// Binary operators.
137112
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
138113
pub enum Bop {
139114
Add,
@@ -199,30 +174,37 @@ impl fmt::Display for Bop {
199174
}
200175
}
201176

177+
/// One of the `if` or `for` clause(s) of a comprehension list/dict/set or
178+
/// generator expression.
202179
#[derive(Clone, Debug, PartialEq)]
203180
pub enum ComprehensionChunk {
204181
If { cond: Expression },
205182
For { async: bool, item: Vec<Expression>, iterator: Expression },
206183
}
207184

185+
/// `**foo` or `foo:bar`, as in a dict comprehension.
208186
#[derive(Clone, Debug, PartialEq)]
209187
pub enum DictItem {
210188
Star(Expression),
211189
Unique(Expression, Expression),
212190
}
213191

192+
/// `*foo` or `foo`, as in a list/set comprehension or a generator expression.
214193
#[derive(Clone, Debug, PartialEq)]
215194
pub enum SetItem {
216195
Star(Expression),
217196
Unique(Expression),
218197
}
219198

199+
/// A Python string. See the doc of the crate for the boring speech about
200+
/// encoding stuff.
220201
#[derive(Clone, Debug, PartialEq, Eq)]
221202
pub struct PyString {
222203
pub prefix: String,
223204
pub content: PyStringContent,
224205
}
225206

207+
/// The big thing: a Python expression.
226208
#[derive(Clone, Debug, PartialEq)]
227209
pub enum Expression {
228210
Ellipsis,
@@ -265,6 +247,7 @@ pub enum Expression {
265247
Lambdef(UntypedArgsList, Box<Expression>),
266248
}
267249

250+
/// An import statement.
268251
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
269252
pub enum Import {
270253
/// `from x import y`
@@ -287,6 +270,7 @@ pub enum Import {
287270
Import { names: Vec<(Vec<Name>, Option<Name>)> },
288271
}
289272

273+
/// `+=` and its friends.
290274
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
291275
pub enum AugAssignOp {
292276
Add,
@@ -324,6 +308,7 @@ impl fmt::Display for AugAssignOp {
324308
}
325309
}
326310

311+
/// A Python statement.
327312
#[derive(Clone, Debug, PartialEq)]
328313
pub enum Statement {
329314
Pass,
@@ -349,6 +334,7 @@ pub enum Statement {
349334
Compound(Box<CompoundStatement>),
350335
}
351336

337+
/// A function definition, including its decorators.
352338
#[derive(Clone, Debug, PartialEq)]
353339
pub struct Funcdef {
354340
pub async: bool,
@@ -359,6 +345,7 @@ pub struct Funcdef {
359345
pub code: Vec<Statement>,
360346
}
361347

348+
/// A class definition, including its decorators.
362349
#[derive(Clone, Debug, PartialEq)]
363350
pub struct Classdef {
364351
pub decorators: Vec<Decorator>,
@@ -367,6 +354,7 @@ pub struct Classdef {
367354
pub code: Vec<Statement>,
368355
}
369356

357+
/// A try block.
370358
#[derive(Clone, Debug, PartialEq)]
371359
pub struct Try {
372360
pub try_block: Vec<Statement>,
@@ -380,6 +368,7 @@ pub struct Try {
380368
pub finally_block: Vec<Statement>,
381369
}
382370

371+
/// Statements with blocks.
383372
#[derive(Clone, Debug, PartialEq)]
384373
pub enum CompoundStatement {
385374
If(Vec<(Expression, Vec<Statement>)>, Option<Vec<Statement>>),

‎src/expressions.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
use std::marker::PhantomData;
22

3-
//use unicode_names;
4-
53
use helpers::*;
64
use functions::varargslist;
75
use bytes::bytes;

‎src/helpers.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ pub(crate) type StrSpan<'a> = LocatedSpan<CompleteStr<'a>>;
1010

1111

1212
/// Like `ws!()`, but does not allow newlines.
13-
#[macro_export]
1413
macro_rules! ws2 (
1514
($i:expr, $($args:tt)*) => (
1615
{
@@ -31,7 +30,6 @@ macro_rules! ws2 (
3130
);
3231

3332
/// Like `ws!()`, but ignores comments as well
34-
#[macro_export]
3533
macro_rules! ws4 (
3634
($i:expr, $($args:tt)*) => (
3735
{
@@ -178,6 +176,8 @@ named!(pub semicolon<StrSpan, ()>,
178176
map!(ws2!(char!(';')), |_| ())
179177
);
180178

179+
/// Helper to make an instance of `StrSpan`, that can be used as the argument
180+
/// to other parsers.
181181
pub fn make_strspan(s: &str) -> StrSpan {
182182
StrSpan::new(CompleteStr(s))
183183
}

‎src/lib.rs

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,63 @@
1+
//! A Python parser based on nom, plus some utilities.
2+
//!
3+
//! # Panics
4+
//!
5+
//! Never (except stack overflows).
6+
//!
7+
//! # Numbers
8+
//!
9+
//! Python's integer literals may be arbitrary large. This is supported
10+
//! thanks to the `num_bigint` crate.
11+
//! Disable the `bigint` feature to fall back to `u64`.
12+
//!
13+
//! # String encoding
14+
//!
15+
//! `ast::PyString`s are WTF8-encoded if the `wtf8` feature is enabled
16+
//! (the default) allowing full support for Python's string litteral.
17+
//!
18+
//! If that feature is disabled, they default to regular Rust string
19+
//! Note that without the `wtf8` feature, some valid string
20+
//! literals will be badly parsed (missing characters).
21+
//!
22+
//! # Example
23+
//!
24+
//! ```
25+
//! extern crate python_parser;
26+
//! use python_parser::ast::*;
27+
//! let code = "print(2 + 3, fd=sys.stderr)";
28+
//! let ast = python_parser::file_input(python_parser::make_strspan(code))
29+
//! .unwrap()
30+
//! .1;
31+
//! assert_eq!(ast,
32+
//! vec![
33+
//! Statement::Assignment(
34+
//! vec![
35+
//! Expression::Call(
36+
//! Box::new(Expression::Name("print".to_string())),
37+
//! vec![
38+
//! Argument::Positional(
39+
//! Expression::Bop(
40+
//! Bop::Add,
41+
//! Box::new(Expression::Int(2u32.into())),
42+
//! Box::new(Expression::Int(3u32.into())),
43+
//! )
44+
//! ),
45+
//! Argument::Keyword(
46+
//! "fd".to_string(),
47+
//! Expression::Attribute(
48+
//! Box::new(Expression::Name("sys".to_string())),
49+
//! "stderr".to_string(),
50+
//! )
51+
//! ),
52+
//! ]
53+
//! ),
54+
//! ],
55+
//! vec![],
56+
//! )
57+
//! ]
58+
//! );
59+
//! ```
60+
161
#![recursion_limit="128"]
262

363
#[macro_use]
@@ -40,15 +100,17 @@ use ast::*;
40100
pub use helpers::make_strspan;
41101

42102
// single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
43-
named!(pub parse_single_input <StrSpan, Vec<Statement>>,
103+
named_attr!(#[doc = "Parses a single interactive statement, like in the REPL."],
104+
pub parse_single_input <StrSpan, Vec<Statement>>,
44105
alt!(
45106
newline => { |_| Vec::new() }
46107
| call!(statement, 0) => { |stmts| stmts }
47108
)
48109
);
49110

50111
// file_input: (NEWLINE | stmt)* ENDMARKER
51-
named!(pub file_input <StrSpan, Vec<Statement>>,
112+
named_attr!(#[doc = "Parses a module or sequence of commands."],
113+
pub file_input <StrSpan, Vec<Statement>>,
52114
fold_many0!(
53115
alt!(
54116
call!(statement, 0) => { |s| Some(s) }
@@ -60,7 +122,8 @@ named!(pub file_input <StrSpan, Vec<Statement>>,
60122
);
61123

62124
// eval_input: testlist NEWLINE* ENDMARKER
63-
named!(pub eval_input <StrSpan, Vec<Expression>>,
125+
named_attr!(#[doc = "Parses the input of eval()."],
126+
pub eval_input <StrSpan, Vec<Expression>>,
64127
terminated!(ws2!(call!(ExpressionParser::<NewlinesAreNotSpaces>::testlist)), many0!(newline))
65128
);
66129

‎src/visitors/mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1+
//! Utilities that work on the AST.
2+
13
pub mod printer;

‎src/visitors/printer.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
//! Prints the AST as Python code.
2+
13
use super::super::ast::*;
24

35
fn comma_join<'a, T2: ToString, T: IntoIterator<Item=T2>>(i: T) -> String {

‎test_on_corpus.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import os
33
import sys
44
import ast
5+
import time
56
import difflib
67
import tempfile
78
import unittest
@@ -13,14 +14,16 @@
1314
def test_file(path):
1415
with open(path) as f:
1516
expected_ast = astpretty.pformat(ast.parse(f.read()), show_offsets=False)
16-
printer_output = subprocess.check_output(['cargo', '--quiet', 'run', path])
17+
before = time.time()
18+
printer_output = subprocess.check_output(['./target/release/prettyprint', path])
19+
total_time = time.time() - before
1720
try:
1821
received_ast = astpretty.pformat(ast.parse(printer_output), show_offsets=False)
1922
except:
2023
print('Error while parsing the output from {}:'.format(path))
2124
raise
2225
if expected_ast == received_ast:
23-
print('{}: ok'.format(path))
26+
print('({:03}ms) {}: ok'.format(int(total_time*1000), path))
2427
return
2528
print('========================')
2629
print(path)
@@ -72,6 +75,7 @@ def main():
7275
with_threads = True
7376
else:
7477
with_threads = False
78+
subprocess.check_output(['cargo', 'build', '--release'])
7579
for path in sys.argv[1:]:
7680
if os.path.isfile(path):
7781
test_file(path)

0 commit comments

Comments
 (0)