Skip to content

Commit 0d19e0c

Browse files
youknowoneyouknowone
authored
Merge pull request #4121 from youknowone/compiler
more parser/codegen separation
2 parents c7ad41e + 60a6714 commit 0d19e0c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+670
-660
lines changed

‎.github/workflows/ci.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ concurrency:
1515
env:
1616
CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
1717
NON_WASM_PACKAGES: >-
18-
-p rustpython-bytecode
1918
-p rustpython-common
19+
-p rustpython-compiler-core
2020
-p rustpython-compiler
21+
-p rustpython-codegen
2122
-p rustpython-parser
2223
-p rustpython-vm
2324
-p rustpython-stdlib

‎Cargo.lock

+27-22
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+10-10
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
1414
[workspace]
1515
resolver = "2"
1616
members = [
17-
"compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
17+
"compiler", "compiler/ast", "compiler/core", "compiler/codegen", "compiler/parser",
1818
".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
1919
]
2020

@@ -33,28 +33,28 @@ ssl = ["rustpython-stdlib/ssl"]
3333
ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
3434

3535
[dependencies]
36-
log = "0.4.16"
37-
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
38-
clap = "2.34"
3936
rustpython-compiler = { path = "compiler", version = "0.1.1" }
4037
rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
41-
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
4238
rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
43-
dirs = { package = "dirs-next", version = "2.0.0" }
44-
num-traits = "0.2.14"
45-
cfg-if = "1.0.0"
46-
libc = "0.2.126"
39+
rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
4740

41+
cfg-if = "1.0.0"
42+
clap = "2.34"
43+
dirs = { package = "dirs-next", version = "2.0.0" }
44+
env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
4845
flame = { version = "0.2.2", optional = true }
4946
flamescope = { version = "0.1.2", optional = true }
47+
libc = "0.2.126"
48+
log = "0.4.16"
49+
num-traits = "0.2.14"
5050

5151
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
5252
rustyline = "10.0.0"
5353

5454
[dev-dependencies]
5555
cpython = "0.7.0"
56-
python3-sys = "0.7.0"
5756
criterion = "0.3.5"
57+
python3-sys = "0.7.0"
5858

5959
[[bench]]
6060
name = "execution"

‎Lib/test/test_ast.py

+2-6
Original file line numberDiff line numberDiff line change
@@ -760,8 +760,6 @@ def test_parse(self):
760760
b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST)
761761
self.assertEqual(ast.dump(a), ast.dump(b))
762762

763-
# TODO: RUSTPYTHON
764-
@unittest.expectedFailure
765763
def test_parse_in_error(self):
766764
try:
767765
1/0
@@ -1101,8 +1099,6 @@ def test_literal_eval_malformed_dict_nodes(self):
11011099
malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)])
11021100
self.assertRaises(ValueError, ast.literal_eval, malformed)
11031101

1104-
# TODO: RUSTPYTHON
1105-
@unittest.expectedFailure
11061102
def test_literal_eval_trailing_ws(self):
11071103
self.assertEqual(ast.literal_eval(" -1"), -1)
11081104
self.assertEqual(ast.literal_eval("\t\t-1"), -1)
@@ -1121,8 +1117,6 @@ def test_literal_eval_malformed_lineno(self):
11211117
with self.assertRaisesRegex(ValueError, msg):
11221118
ast.literal_eval(node)
11231119

1124-
# TODO: RUSTPYTHON
1125-
@unittest.expectedFailure
11261120
def test_literal_eval_syntax_errors(self):
11271121
with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
11281122
ast.literal_eval(r'''
@@ -1767,6 +1761,8 @@ def test_stdlib_validates(self):
17671761
ast.MatchMapping([], [], rest="_"),
17681762
]
17691763

1764+
# TODO: RUSTPYTHON
1765+
@unittest.expectedFailure
17701766
def test_match_validation_pattern(self):
17711767
name_x = ast.Name('x', ast.Load())
17721768
for pattern in self._MATCH_PATTERNS:

‎benches/execution.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ pub fn benchmark_file_execution(
5151
pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) {
5252
group.throughput(Throughput::Bytes(contents.len() as u64));
5353
group.bench_function(BenchmarkId::new("rustpython", name), |b| {
54-
b.iter(|| parse_program(contents).unwrap())
54+
b.iter(|| parse_program(contents, name).unwrap())
5555
});
5656
group.bench_function(BenchmarkId::new("cpython", name), |b| {
5757
let gil = cpython::Python::acquire_gil();

‎common/Cargo.toml

+11-11
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@ edition = "2021"
88
threading = ["parking_lot"]
99

1010
[dependencies]
11+
ascii = "1.0"
12+
cfg-if = "1.0"
13+
hexf-parse = "0.2.1"
14+
lexical-parse-float = { version = "0.8.0", features = ["format"] }
15+
libc = "0.2.126"
1116
lock_api = "0.4"
12-
parking_lot = { version = "0.12.0", optional = true }
13-
num-traits = "0.2"
14-
num-complex = "0.4.0"
1517
num-bigint = "0.4.2"
16-
lexical-parse-float = { version = "0.8.0", features = ["format"] }
17-
hexf-parse = "0.2.1"
18-
cfg-if = "1.0"
18+
num-complex = "0.4.0"
19+
num-traits = "0.2"
1920
once_cell = "1.4.1"
20-
siphasher = "0.3"
21-
rand = "0.8"
22-
volatile = "0.3"
21+
parking_lot = { version = "0.12.0", optional = true }
2322
radium = "0.7"
24-
libc = "0.2.126"
25-
ascii = "1.0"
23+
rand = "0.8"
24+
siphasher = "0.3"
2625
unic-ucd-category = "0.9"
26+
volatile = "0.3"
2727

2828
[target.'cfg(windows)'.dependencies]
2929
widestring = "0.5.1"

‎compiler/Cargo.toml

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ authors = ["RustPython Team"]
66
edition = "2021"
77

88
[dependencies]
9-
thiserror = "1.0"
9+
rustpython-compiler-core = { path = "core" }
1010
rustpython-codegen = { path = "codegen" }
1111
rustpython-parser = { path = "parser" }
12-
rustpython-bytecode = { path = "bytecode" }
12+
13+
thiserror = "1.0"

‎compiler/ast/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
1212

1313
[dependencies]
1414
num-bigint = "0.4.3"
15+
rustpython-compiler-core = { path = "../core" }
1516
rustpython-common = { path = "../../common", optional = true }
16-
rustpython-bytecode = { path = "../bytecode"}

‎compiler/ast/asdl_rs.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -645,7 +645,7 @@ def write_ast_def(mod, typeinfo, f):
645645
#![allow(clippy::derive_partial_eq_without_eq)]
646646
647647
pub use crate::constant::*;
648-
pub use crate::location::Location;
648+
pub use crate::Location;
649649
650650
type Ident = String;
651651
\n

‎compiler/ast/src/ast_gen.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#![allow(clippy::derive_partial_eq_without_eq)]
44

55
pub use crate::constant::*;
6-
pub use crate::location::Location;
6+
pub use crate::Location;
77

88
type Ident = String;
99

‎compiler/ast/src/constant.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use num_bigint::BigInt;
2-
pub use rustpython_bytecode::ConversionFlag;
2+
pub use rustpython_compiler_core::ConversionFlag;
33

44
#[derive(Debug, PartialEq)]
55
pub enum Constant {

‎compiler/ast/src/fold_helpers.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
use crate::constant;
2-
use crate::fold::Fold;
1+
use crate::{constant, fold::Fold};
32

43
pub(crate) trait Foldable<T, U> {
54
type Mapped;

‎compiler/ast/src/lib.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,10 @@ mod constant;
33
#[cfg(feature = "fold")]
44
mod fold_helpers;
55
mod impls;
6-
mod location;
76
#[cfg(feature = "unparse")]
87
mod unparse;
98

109
pub use ast_gen::*;
11-
pub use location::Location;
10+
pub use rustpython_compiler_core::Location;
1211

1312
pub type Suite<U = ()> = Vec<Stmt<U>>;

‎compiler/ast/src/location.rs

-79
This file was deleted.

0 commit comments

Comments
 (0)