Skip to content

Commit fb5ac9e

Browse files
youknowoneyouknowone
committed
reorganize compiler crates
1 parent 1100df7 commit fb5ac9e

File tree

96 files changed

+364
-234
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+364
-234
lines changed

‎.gitattributes

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@ Lib/** linguist-vendored
22
Cargo.lock linguist-generated -merge
33
ast/src/ast_gen.rs linguist-generated -merge
44
vm/src/stdlib/ast/gen.rs linguist-generated -merge
5-
parser/python.lalrpop text eol=LF
5+
compiler/parser/python.lalrpop text eol=LF
66
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
77
**/*.rs text working-tree-encoding=UTF-8 eol=LF

‎.github/workflows/ci.yaml

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,8 @@ jobs:
112112
- name: Cache generated parser
113113
uses: actions/cache@v2
114114
with:
115-
path: parser/python.rs
116-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
115+
path: compiler/parser/python.rs
116+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
117117
- uses: dtolnay/rust-toolchain@stable
118118
- name: Set up the Windows environment
119119
shell: bash
@@ -156,8 +156,8 @@ jobs:
156156
- name: Cache generated parser
157157
uses: actions/cache@v2
158158
with:
159-
path: parser/python.rs
160-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
159+
path: compiler/parser/python.rs
160+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
161161

162162
- uses: dtolnay/rust-toolchain@stable
163163
with:
@@ -226,8 +226,8 @@ jobs:
226226
- name: Cache generated parser
227227
uses: actions/cache@v2
228228
with:
229-
path: parser/python.rs
230-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
229+
path: compiler/parser/python.rs
230+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
231231

232232
- uses: dtolnay/rust-toolchain@stable
233233
- uses: actions/setup-python@v2
@@ -288,17 +288,17 @@ jobs:
288288
- name: Cache generated parser
289289
uses: actions/cache@v2
290290
with:
291-
path: parser/python.rs
292-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
291+
path: compiler/parser/python.rs
292+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
293293
- name: Check if cached generated parser exists
294294
id: generated_parser
295295
uses: andstor/file-existence-action@v1
296296
with:
297-
files: "parser/python.rs"
297+
files: "compiler/parser/python.rs"
298298
- if: runner.os == 'Windows'
299299
name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes
300300
run: |
301-
set file parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
301+
set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
302302
- name: Install lalrpop
303303
if: steps.generated_parser.outputs.files_exists == 'false'
304304
uses: baptiste0928/cargo-install@v1
@@ -307,7 +307,7 @@ jobs:
307307
version: "0.19.8"
308308
- name: Run lalrpop
309309
if: steps.generated_parser.outputs.files_exists == 'false'
310-
run: lalrpop parser/python.lalrpop
310+
run: lalrpop compiler/parser/python.lalrpop
311311

312312
lint:
313313
name: Check Rust code with rustfmt and clippy
@@ -318,8 +318,8 @@ jobs:
318318
- name: Cache generated parser
319319
uses: actions/cache@v2
320320
with:
321-
path: parser/python.rs
322-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
321+
path: compiler/parser/python.rs
322+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
323323
- uses: dtolnay/rust-toolchain@stable
324324
with:
325325
components: rustfmt, clippy
@@ -351,8 +351,8 @@ jobs:
351351
- name: Cache generated parser
352352
uses: actions/cache@v2
353353
with:
354-
path: parser/python.rs
355-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
354+
path: compiler/parser/python.rs
355+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
356356
- uses: dtolnay/rust-toolchain@master
357357
with:
358358
toolchain: nightly
@@ -372,8 +372,8 @@ jobs:
372372
- name: Cache generated parser
373373
uses: actions/cache@v2
374374
with:
375-
path: parser/python.rs
376-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
375+
path: compiler/parser/python.rs
376+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
377377
- uses: dtolnay/rust-toolchain@stable
378378
- uses: Swatinem/rust-cache@v1
379379
- name: install wasm-pack
@@ -420,8 +420,8 @@ jobs:
420420
- name: Cache generated parser
421421
uses: actions/cache@v2
422422
with:
423-
path: parser/python.rs
424-
key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
423+
path: compiler/parser/python.rs
424+
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
425425
- uses: dtolnay/rust-toolchain@stable
426426
with:
427427
target: wasm32-wasi

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,4 @@ flamescope.json
2020
extra_tests/snippets/resources
2121
extra_tests/not_impl.py
2222

23-
parser/python.rs
23+
compiler/parser/python.rs

‎Cargo.lock

Lines changed: 18 additions & 18 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
1414
[workspace]
1515
resolver = "2"
1616
members = [
17-
".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
18-
"derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
17+
"compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
18+
".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
1919
]
2020

2121
[features]
@@ -36,8 +36,8 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
3636
log = "0.4.16"
3737
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
3838
clap = "2.34"
39-
rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
40-
rustpython-parser = { path = "parser", version = "0.1.1" }
39+
rustpython-compiler = { path = "compiler", version = "0.1.1" }
40+
rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
4141
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
4242
rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]}
4343
dirs = { package = "dirs-next", version = "2.0.0" }

‎DEVELOPMENT.md

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,10 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
115115
Understanding a new codebase takes time. Here's a brief view of the
116116
repository's structure:
117117

118-
- `bytecode/src`: python bytecode representation in rust structures
119118
- `compiler/src`: python compilation to bytecode
119+
- `bytecode/src`: python bytecode representation in rust structures
120+
- `parser/src`: python lexing, parsing and ast
120121
- `derive/src`: Rust language extensions and macros specific to rustpython
121-
- `parser/src`: python lexing, parsing and ast
122122
- `Lib`: Carefully selected / copied files from CPython sourcecode. This is
123123
the python side of the standard library.
124124
- `test`: CPython test suite
@@ -137,7 +137,7 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`).
137137

138138
The top-level `rustpython` binary depends on several lower-level crates including:
139139

140-
- `rustpython-parser` (implementation in `parser/src`)
140+
- `rustpython-parser` (implementation in `compiler/parser/src`)
141141
- `rustpython-compiler` (implementation in `compiler/src`)
142142
- `rustpython-vm` (implementation in `vm/src`)
143143

@@ -155,15 +155,15 @@ enable a line of code to go through a series of steps:
155155
This crate contains the lexer and parser to convert a line of code to
156156
an Abstract Syntax Tree (AST):
157157

158-
- Lexer: `parser/src/lexer.rs` converts Python source code into tokens
159-
- Parser: `parser/src/parser.rs` takes the tokens generated by the lexer and parses
158+
- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
159+
- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
160160
the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
161161
tree are Rust structs and enums.
162162
- The Parser relies on `LALRPOP`, a Rust parser generator framework. The
163-
LALRPOP definition of Python's grammar is in `parser/src/python.lalrpop`.
163+
LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
164164
- More information on parsers and a tutorial can be found in the
165165
[LALRPOP book](https://lalrpop.github.io/lalrpop/).
166-
- AST: `ast/` implements in Rust the Python types and expressions
166+
- AST: `compiler/ast/` implements in Rust the Python types and expressions
167167
represented by the AST nodes.
168168

169169
### rustpython-compiler

‎compiler/Cargo.toml

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,12 @@
11
[package]
2-
name = "rustpython-compiler-core"
2+
name = "rustpython-compiler"
33
version = "0.1.2"
4-
description = "Compiler for python code into bytecode for the rustpython VM."
4+
description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
55
authors = ["RustPython Team"]
6-
repository = "https://github.com/RustPython/RustPython"
7-
license = "MIT"
86
edition = "2021"
97

108
[dependencies]
11-
indexmap = "1.8.1"
12-
itertools = "0.10.3"
13-
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
14-
rustpython-ast = { path = "../ast", features = ["unparse"] }
15-
num-complex = { version = "0.4.0", features = ["serde"] }
16-
num-traits = "0.2.14"
17-
log = "0.4.16"
18-
ahash = "0.7.6"
19-
20-
[dev-dependencies]
21-
rustpython-parser = { path = "../parser" }
22-
insta = "1.14.0"
9+
thiserror = "1.0"
10+
rustpython-codegen = { path = "codegen" }
11+
rustpython-parser = { path = "parser" }
12+
rustpython-bytecode = { path = "bytecode" }

‎ast/Cargo.toml renamed to ‎compiler/ast/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
1212

1313
[dependencies]
1414
num-bigint = "0.4.3"
15-
rustpython-common = { path = "../common", optional = true }
15+
rustpython-common = { path = "../../common", optional = true }
1616
rustpython-bytecode = { path = "../bytecode"}

‎ast/Python.asdl renamed to ‎compiler/ast/Python.asdl

File renamed without changes.

‎ast/asdl.py renamed to ‎compiler/ast/asdl.py

File renamed without changes.

‎ast/asdl_rs.py renamed to ‎compiler/ast/asdl_rs.py

File renamed without changes.

‎ast/src/ast_gen.rs renamed to ‎compiler/ast/src/ast_gen.rs

File renamed without changes.

‎ast/src/constant.rs renamed to ‎compiler/ast/src/constant.rs

File renamed without changes.

‎ast/src/fold_helpers.rs renamed to ‎compiler/ast/src/fold_helpers.rs

File renamed without changes.

‎ast/src/impls.rs renamed to ‎compiler/ast/src/impls.rs

File renamed without changes.

‎ast/src/lib.rs renamed to ‎compiler/ast/src/lib.rs

File renamed without changes.

‎ast/src/location.rs renamed to ‎compiler/ast/src/location.rs

File renamed without changes.

‎ast/src/unparse.rs renamed to ‎compiler/ast/src/unparse.rs

File renamed without changes.

‎bytecode/Cargo.toml renamed to ‎compiler/bytecode/Cargo.toml

File renamed without changes.

‎bytecode/src/lib.rs renamed to ‎compiler/bytecode/src/lib.rs

File renamed without changes.

‎compiler/codegen/Cargo.toml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[package]
2+
name = "rustpython-codegen"
3+
version = "0.1.2"
4+
description = "Compiler for python code into bytecode for the rustpython VM."
5+
authors = ["RustPython Team"]
6+
repository = "https://github.com/RustPython/RustPython"
7+
license = "MIT"
8+
edition = "2021"
9+
10+
[dependencies]
11+
indexmap = "1.8.1"
12+
itertools = "0.10.3"
13+
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
14+
rustpython-ast = { path = "../ast", features = ["unparse"] }
15+
num-complex = { version = "0.4.0", features = ["serde"] }
16+
num-traits = "0.2.14"
17+
log = "0.4.16"
18+
ahash = "0.7.6"
19+
20+
[dev-dependencies]
21+
rustpython-parser = { path = "../parser" }
22+
insta = "1.14.0"

‎compiler/src/compile.rs renamed to ‎compiler/codegen/src/compile.rs

File renamed without changes.

‎compiler/src/error.rs renamed to ‎compiler/codegen/src/error.rs

File renamed without changes.

‎compiler/src/ir.rs renamed to ‎compiler/codegen/src/ir.rs

File renamed without changes.

‎compiler/codegen/src/lib.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
//! Compile a Python AST or source code into bytecode consumable by RustPython.
2+
#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
3+
#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
4+
5+
#[macro_use]
6+
extern crate log;
7+
8+
type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
9+
type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
10+
11+
pub mod compile;
12+
pub mod error;
13+
pub mod ir;
14+
pub mod mode;
15+
pub mod symboltable;

‎compiler/src/mode.rs renamed to ‎compiler/codegen/src/mode.rs

File renamed without changes.

‎compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
source: compiler/codegen/src/compile.rs
3+
expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")"
4+
---
5+
1 0 LoadConst (True)
6+
1 JumpIfFalse (6)
7+
2 LoadConst (False)
8+
3 JumpIfFalse (6)
9+
4 LoadConst (False)
10+
5 JumpIfFalse (6)
11+
12+
2 >> 6 LoadConst (None)
13+
7 ReturnValue
14+

‎compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
source: compiler/codegen/src/compile.rs
3+
expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")"
4+
---
5+
1 0 LoadConst (True)
6+
1 JumpIfFalse (4)
7+
2 LoadConst (False)
8+
3 JumpIfTrue (8)
9+
>> 4 LoadConst (False)
10+
5 JumpIfFalse (8)
11+
6 LoadConst (True)
12+
7 JumpIfFalse (8)
13+
14+
2 >> 8 LoadConst (None)
15+
9 ReturnValue
16+

‎compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
source: compiler/codegen/src/compile.rs
3+
expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")"
4+
---
5+
1 0 LoadConst (True)
6+
1 JumpIfTrue (6)
7+
2 LoadConst (False)
8+
3 JumpIfTrue (6)
9+
4 LoadConst (False)
10+
5 JumpIfFalse (6)
11+
12+
2 >> 6 LoadConst (None)
13+
7 ReturnValue
14+

0 commit comments

Comments
 (0)