Skip to content

Commit c7ad41e

Browse files
youknowoneyouknowone
authored
Merge pull request #4119 from youknowone/compiler
Reorganize compiler crates
2 parents a9cacc7 + a52c5e5 commit c7ad41e

File tree

97 files changed

+385
-265
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

97 files changed

+385
-265
lines changed

‎.gitattributes

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
Lib/** linguist-vendored
22
Cargo.lock linguist-generated -merge
3+
*.snap linguist-generated -merge
34
ast/src/ast_gen.rs linguist-generated -merge
45
vm/src/stdlib/ast/gen.rs linguist-generated -merge
5-
parser/python.lalrpop text eol=LF
6+
compiler/parser/python.lalrpop text eol=LF
67
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
78
**/*.rs text working-tree-encoding=UTF-8 eol=LF

‎.github/workflows/ci.yaml

+19-19
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ jobs:
112112
- name: Cache generated parser
113113
uses: actions/cache@v2
114114
with:
115-
path: parser/python.rs
116-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
115+
path: compiler/parser/python.rs
116+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
117117
- uses: dtolnay/rust-toolchain@stable
118118
- name: Set up the Windows environment
119119
shell: bash
@@ -156,8 +156,8 @@ jobs:
156156
- name: Cache generated parser
157157
uses: actions/cache@v2
158158
with:
159-
path: parser/python.rs
160-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
159+
path: compiler/parser/python.rs
160+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
161161

162162
- uses: dtolnay/rust-toolchain@stable
163163
with:
@@ -226,8 +226,8 @@ jobs:
226226
- name: Cache generated parser
227227
uses: actions/cache@v2
228228
with:
229-
path: parser/python.rs
230-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
229+
path: compiler/parser/python.rs
230+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
231231

232232
- uses: dtolnay/rust-toolchain@stable
233233
- uses: actions/setup-python@v2
@@ -288,17 +288,17 @@ jobs:
288288
- name: Cache generated parser
289289
uses: actions/cache@v2
290290
with:
291-
path: parser/python.rs
292-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
291+
path: compiler/parser/python.rs
292+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
293293
- name: Check if cached generated parser exists
294294
id: generated_parser
295295
uses: andstor/file-existence-action@v1
296296
with:
297-
files: "parser/python.rs"
297+
files: "compiler/parser/python.rs"
298298
- if: runner.os == 'Windows'
299299
name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes
300300
run: |
301-
set file parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
301+
set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
302302
- name: Install lalrpop
303303
if: steps.generated_parser.outputs.files_exists == 'false'
304304
uses: baptiste0928/cargo-install@v1
@@ -307,7 +307,7 @@ jobs:
307307
version: "0.19.8"
308308
- name: Run lalrpop
309309
if: steps.generated_parser.outputs.files_exists == 'false'
310-
run: lalrpop parser/python.lalrpop
310+
run: lalrpop compiler/parser/python.lalrpop
311311

312312
lint:
313313
name: Check Rust code with rustfmt and clippy
@@ -318,8 +318,8 @@ jobs:
318318
- name: Cache generated parser
319319
uses: actions/cache@v2
320320
with:
321-
path: parser/python.rs
322-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
321+
path: compiler/parser/python.rs
322+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
323323
- uses: dtolnay/rust-toolchain@stable
324324
with:
325325
components: rustfmt, clippy
@@ -351,8 +351,8 @@ jobs:
351351
- name: Cache generated parser
352352
uses: actions/cache@v2
353353
with:
354-
path: parser/python.rs
355-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
354+
path: compiler/parser/python.rs
355+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
356356
- uses: dtolnay/rust-toolchain@master
357357
with:
358358
toolchain: nightly
@@ -372,8 +372,8 @@ jobs:
372372
- name: Cache generated parser
373373
uses: actions/cache@v2
374374
with:
375-
path: parser/python.rs
376-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
375+
path: compiler/parser/python.rs
376+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
377377
- uses: dtolnay/rust-toolchain@stable
378378
- uses: Swatinem/rust-cache@v1
379379
- name: install wasm-pack
@@ -420,8 +420,8 @@ jobs:
420420
- name: Cache generated parser
421421
uses: actions/cache@v2
422422
with:
423-
path: parser/python.rs
424-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
423+
path: compiler/parser/python.rs
424+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
425425
- uses: dtolnay/rust-toolchain@stable
426426
with:
427427
target: wasm32-wasi

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,4 @@ flamescope.json
2020
extra_tests/snippets/resources
2121
extra_tests/not_impl.py
2222

23-
parser/python.rs
23+
compiler/parser/python.rs

‎Cargo.lock

+18-19
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+6-6
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
1414
[workspace]
1515
resolver = "2"
1616
members = [
17-
".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
18-
"derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
17+
"compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
18+
".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
1919
]
2020

2121
[features]
@@ -36,10 +36,10 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
3636
log = "0.4.16"
3737
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
3838
clap = "2.34"
39-
rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
40-
rustpython-parser = { path = "parser", version = "0.1.1" }
41-
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
42-
rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]}
39+
rustpython-compiler = { path = "compiler", version = "0.1.1" }
40+
rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
41+
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
42+
rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
4343
dirs = { package = "dirs-next", version = "2.0.0" }
4444
num-traits = "0.2.14"
4545
cfg-if = "1.0.0"

‎DEVELOPMENT.md

+7-7
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,10 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
115115
Understanding a new codebase takes time. Here's a brief view of the
116116
repository's structure:
117117

118-
- `bytecode/src`: python bytecode representation in rust structures
119118
- `compiler/src`: python compilation to bytecode
119+
- `bytecode/src`: python bytecode representation in rust structures
120+
- `parser/src`: python lexing, parsing and ast
120121
- `derive/src`: Rust language extensions and macros specific to rustpython
121-
- `parser/src`: python lexing, parsing and ast
122122
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
123123
the python side of the standard library.
124124
- `test`: CPython test suite
@@ -137,7 +137,7 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`).
137137

138138
The top-level `rustpython` binary depends on several lower-level crates including:
139139

140-
- `rustpython-parser` (implementation in `parser/src`)
140+
- `rustpython-parser` (implementation in `compiler/parser/src`)
141141
- `rustpython-compiler` (implementation in `compiler/src`)
142142
- `rustpython-vm` (implementation in `vm/src`)
143143

@@ -155,15 +155,15 @@ enable a line of code to go through a series of steps:
155155
This crate contains the lexer and parser to convert a line of code to
156156
an Abstract Syntax Tree (AST):
157157

158-
- Lexer: `parser/src/lexer.rs` converts Python source code into tokens
159-
- Parser: `parser/src/parser.rs` takes the tokens generated by the lexer and parses
158+
- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
159+
- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
160160
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
161161
tree are Rust structs and enums.
162162
- The Parser relies on `LALRPOP`, a Rust parser generator framework. The
163-
LALRPOP definition of Python's grammar is in `parser/src/python.lalrpop`.
163+
LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
164164
- More information on parsers and a tutorial can be found in the
165165
[LALRPOP book](https://lalrpop.github.io/lalrpop/).
166-
- AST: `ast/` implements in Rust the Python types and expressions
166+
- AST: `compiler/ast/` implements in Rust the Python types and expressions
167167
represented by the AST nodes.
168168

169169
### rustpython-compiler

‎compiler/Cargo.toml

+6-16
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,12 @@
11
[package]
2-
name = "rustpython-compiler-core"
2+
name = "rustpython-compiler"
33
version = "0.1.2"
4-
description = "Compiler for python code into bytecode for the rustpython VM."
4+
description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
55
authors = ["RustPython Team"]
6-
repository = "https://github.com/RustPython/RustPython"
7-
license = "MIT"
86
edition = "2021"
97

108
[dependencies]
11-
indexmap = "1.8.1"
12-
itertools = "0.10.3"
13-
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
14-
rustpython-ast = { path = "../ast", features = ["unparse"] }
15-
num-complex = { version = "0.4.0", features = ["serde"] }
16-
num-traits = "0.2.14"
17-
log = "0.4.16"
18-
ahash = "0.7.6"
19-
20-
[dev-dependencies]
21-
rustpython-parser = { path = "../parser" }
22-
insta = "1.14.0"
9+
thiserror = "1.0"
10+
rustpython-codegen = { path = "codegen" }
11+
rustpython-parser = { path = "parser" }
12+
rustpython-bytecode = { path = "bytecode" }

‎ast/Cargo.toml renamed to ‎compiler/ast/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
1212

1313
[dependencies]
1414
num-bigint = "0.4.3"
15-
rustpython-common = { path = "../common", optional = true }
15+
rustpython-common = { path = "../../common", optional = true }
1616
rustpython-bytecode = { path = "../bytecode"}

‎ast/Python.asdl renamed to ‎compiler/ast/Python.asdl

File renamed without changes.

‎ast/asdl.py renamed to ‎compiler/ast/asdl.py

File renamed without changes.

‎ast/asdl_rs.py renamed to ‎compiler/ast/asdl_rs.py

File renamed without changes.

‎ast/src/ast_gen.rs renamed to ‎compiler/ast/src/ast_gen.rs

File renamed without changes.

‎ast/src/constant.rs renamed to ‎compiler/ast/src/constant.rs

File renamed without changes.

‎ast/src/fold_helpers.rs renamed to ‎compiler/ast/src/fold_helpers.rs

File renamed without changes.

‎ast/src/impls.rs renamed to ‎compiler/ast/src/impls.rs

File renamed without changes.

‎ast/src/lib.rs renamed to ‎compiler/ast/src/lib.rs

File renamed without changes.

‎ast/src/location.rs renamed to ‎compiler/ast/src/location.rs

File renamed without changes.

‎ast/src/unparse.rs renamed to ‎compiler/ast/src/unparse.rs

File renamed without changes.

‎bytecode/Cargo.toml renamed to ‎compiler/bytecode/Cargo.toml

File renamed without changes.

‎bytecode/src/lib.rs renamed to ‎compiler/bytecode/src/lib.rs

File renamed without changes.

‎compiler/codegen/Cargo.toml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[package]
2+
name = "rustpython-codegen"
3+
version = "0.1.2"
4+
description = "Compiler for python code into bytecode for the rustpython VM."
5+
authors = ["RustPython Team"]
6+
repository = "https://github.com/RustPython/RustPython"
7+
license = "MIT"
8+
edition = "2021"
9+
10+
[dependencies]
11+
indexmap = "1.8.1"
12+
itertools = "0.10.3"
13+
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
14+
rustpython-ast = { path = "../ast", features = ["unparse"] }
15+
num-complex = { version = "0.4.0", features = ["serde"] }
16+
num-traits = "0.2.14"
17+
log = "0.4.16"
18+
ahash = "0.7.6"
19+
20+
[dev-dependencies]
21+
rustpython-parser = { path = "../parser" }
22+
insta = "1.14.0"

‎compiler/src/compile.rs renamed to ‎compiler/codegen/src/compile.rs

+10-10
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,11 @@
55
//! <https://github.com/python/cpython/blob/main/Python/compile.c>
66
//! <https://github.com/micropython/micropython/blob/master/py/compile.c>
77
8-
use crate::ir::{self, CodeInfo};
9-
pub use crate::mode::Mode;
10-
use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable};
11-
use crate::IndexSet;
128
use crate::{
139
error::{CompileError, CompileErrorType},
14-
symboltable,
10+
ir,
11+
symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable},
12+
IndexSet,
1513
};
1614
use itertools::Itertools;
1715
use num_complex::Complex64;
@@ -20,6 +18,8 @@ use rustpython_ast as ast;
2018
use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
2119
use std::borrow::Cow;
2220

21+
pub use crate::mode::Mode;
22+
2323
type CompileResult<T> = Result<T, CompileError>;
2424

2525
#[derive(PartialEq, Eq, Clone, Copy)]
@@ -61,7 +61,7 @@ fn is_forbidden_name(name: &str) -> bool {
6161

6262
/// Main structure holding the state of compilation.
6363
struct Compiler {
64-
code_stack: Vec<CodeInfo>,
64+
code_stack: Vec<ir::CodeInfo>,
6565
symbol_table_stack: Vec<SymbolTable>,
6666
source_path: String,
6767
current_source_location: ast::Location,
@@ -199,7 +199,7 @@ pub fn compile_expression(
199199

200200
impl Compiler {
201201
fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
202-
let module_code = CodeInfo {
202+
let module_code = ir::CodeInfo {
203203
flags: bytecode::CodeFlags::NEW_LOCALS,
204204
posonlyarg_count: 0,
205205
arg_count: 0,
@@ -278,7 +278,7 @@ impl Compiler {
278278

279279
self.symbol_table_stack.push(table);
280280

281-
let info = CodeInfo {
281+
let info = ir::CodeInfo {
282282
flags,
283283
posonlyarg_count,
284284
arg_count,
@@ -318,7 +318,7 @@ impl Compiler {
318318
fn _name_inner(
319319
&mut self,
320320
name: &str,
321-
cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
321+
cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>,
322322
) -> bytecode::NameIdx {
323323
let name = self.mangle(name);
324324
let cache = cache(self.current_codeinfo());
@@ -2583,7 +2583,7 @@ impl Compiler {
25832583
self.emit(Instruction::LoadConst { idx })
25842584
}
25852585

2586-
fn current_codeinfo(&mut self) -> &mut CodeInfo {
2586+
fn current_codeinfo(&mut self) -> &mut ir::CodeInfo {
25872587
self.code_stack.last_mut().expect("no code on stack")
25882588
}
25892589

‎compiler/src/error.rs renamed to ‎compiler/codegen/src/error.rs

File renamed without changes.

‎compiler/src/ir.rs renamed to ‎compiler/codegen/src/ir.rs

File renamed without changes.

0 commit comments

Comments
 (0)