diff --git a/.github/ISSUE_TEMPLATE/report-incompatibility.md b/.github/ISSUE_TEMPLATE/report-incompatibility.md new file mode 100644 index 0000000000..e917e94326 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/report-incompatibility.md @@ -0,0 +1,16 @@ +--- +name: Report incompatibility +about: Report an incompatibility between RustPython and CPython +title: '' +labels: feat +assignees: '' + +--- + +## Feature + + + +## Python Documentation + + diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md new file mode 100644 index 0000000000..84a7b19ce7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/rfc.md @@ -0,0 +1,24 @@ +--- +name: RFC +about: Make a suggestion in a Request for Comments format to RustPython +title: "[RFC] " +labels: RFC +assignees: '' + +--- + +## Summary + + + +## Detailed Explanation + + + +## Drawbacks, Rationale, and Alternatives + + + +## Unresolved Questions + + diff --git a/.gitignore b/.gitignore index 967a51a834..e9dd6d220a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .vscode wasm-pack.log .idea/ +tests/snippets/resources \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index aff28cd6a6..fc9afe7d8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,22 @@ language: rust - -rust: - - stable - - beta - - nightly - -script: - - cargo build --verbose --all - - cargo test --verbose --all - -env: - # This is used to only capture the regular nightly test in allow_failures - - REGULAR_TEST=true - +rust: stable cache: cargo matrix: fast_finish: true include: + - name: rust unittests and doctests + language: rust + rust: stable + cache: cargo + script: + - cargo build --verbose --all + - cargo test --verbose --all + # To test the snippets, we use Travis' Python environment (because # installing rust ourselves is a lot easier than installing Python) - - language: python + - name: python test snippets + language: python python: 3.6 cache: pip: true @@ -31,21 +27,6 @@ matrix: - target env: - TRAVIS_RUST_VERSION=stable - - REGULAR_TEST=false - - CODE_COVERAGE=false - script: tests/.travis-runner.sh - - language: python - python: 3.6 - cache: - pip: true - # Because we're using the Python Travis environment, we can't use - # the built-in cargo cacher - directories: - - /home/travis/.cargo - - target - env: - - TRAVIS_RUST_VERSION=beta - - REGULAR_TEST=false - CODE_COVERAGE=false script: tests/.travis-runner.sh - name: rustfmt @@ -61,8 +42,6 @@ matrix: # creates.) - echo > parser/src/python.rs - cargo fmt --all -- --check - env: - - REGULAR_TEST=false - name: publish documentation language: rust rust: stable @@ -71,11 +50,10 @@ matrix: - cargo doc --no-deps --all if: branch = release env: - - REGULAR_TEST=false - DEPLOY_DOC=true - name: WASM online demo language: rust - rust: nightly + rust: stable cache: cargo install: - nvm install node @@ -87,7 +65,6 @@ matrix: - npm run dist if: branch = release env: - - REGULAR_TEST=false - DEPLOY_DEMO=true - name: cargo-clippy language: rust @@ -97,8 +74,6 @@ matrix: - rustup component add clippy script: - cargo clippy - env: - - REGULAR_TEST=true - name: Code Coverage language: python python: 3.6 @@ -111,9 +86,10 @@ matrix: - target script: - tests/.travis-runner.sh + # Only do code coverage on master via a cron job. + if: branch = master AND type = cron env: - TRAVIS_RUST_VERSION=nightly - - REGULAR_TEST=false - CODE_COVERAGE=true - name: test WASM language: python @@ -133,11 +109,8 @@ matrix: script: - wasm/tests/.travis-runner.sh env: - - REGULAR_TEST=true - TRAVIS_RUST_VERSION=stable allow_failures: - - rust: nightly - env: REGULAR_TEST=true - name: cargo-clippy deploy: diff --git a/Cargo.lock b/Cargo.lock index 818d0af89e..c01dc468ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,10 +2,18 @@ # It is not intended for manual editing. [[package]] name = "aho-corasick" -version = "0.6.4" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -13,7 +21,7 @@ name = "ansi_term" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -25,17 +33,12 @@ dependencies = [ "scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "arrayref" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "arrayvec" -version = "0.4.7" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -48,89 +51,125 @@ dependencies = [ [[package]] name = "atty" -version = "0.1.2" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "atty" -version = "0.2.10" +name = "autocfg" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "backtrace" -version = "0.3.9" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "backtrace-sys" -version = "0.1.24" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bincode" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bit-set" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "bit-vec" -version = "0.4.4" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bitflags" -version = "1.0.4" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "blake2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "blake2-rfc" version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "block-buffer" -version = "0.3.3" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "block-padding" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "build_const" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bumpalo" +version = "2.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byte-tools" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "byteorder" -version = "1.2.6" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -138,30 +177,30 @@ name = "caseless" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "cc" -version = "1.0.25" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cfg-if" -version = "0.1.3" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "clap" -version = "2.31.2" +version = "2.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -171,7 +210,7 @@ name = "cloudabi" version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -179,6 +218,33 @@ name = "constant_time_eq" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "cpython" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "diff" version = "0.1.11" @@ -186,86 +252,88 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "digest" -version = "0.7.2" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "dirs" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_users 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "docopt" -version = "0.8.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "either" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ena" -version = "0.5.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "env_logger" -version = "0.5.10" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "env_logger" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "failure" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.30 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "failure_derive" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -279,27 +347,23 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "fuchsia-zircon" -version = "0.3.3" +name = "fnv" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" +name = "fuchsia-cprng" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "futures" -version = "0.1.25" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "generic-array" -version = "0.9.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -307,41 +371,81 @@ dependencies = [ [[package]] name = "heck" -version = "0.3.0" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hexf" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hexf-impl" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hexf-parse" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "humantime" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "indexmap" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "itertools" -version = "0.7.8" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "itoa" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "js-sys" -version = "0.3.6" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "keccak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -353,45 +457,23 @@ dependencies = [ [[package]] name = "lalrpop" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lalrpop-snap" -version = "0.15.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ena 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -399,37 +481,37 @@ dependencies = [ [[package]] name = "lalrpop-util" -version = "0.15.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lazy_static" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lexical" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "lexical-core" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.42" +version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -437,96 +519,129 @@ name = "log" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "log" -version = "0.4.2" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "memchr" -version = "2.0.1" +name = "maplit" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "md-5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memchr" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "new_debug_unreachable" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "nix" -version = "0.11.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "nodrop" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nom" -version = "4.1.1" +version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-bigint" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-complex" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-integer" -version = "0.1.39" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-rational" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.2.6" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "numtoa" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "opaque-debug" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -545,19 +660,19 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.7.22" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "phf_shared" -version = "0.7.22" +version = "0.7.24" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -566,21 +681,54 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "proc-macro2" -version = "0.3.8" +name = "proc-macro-hack" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "proc-macro-hack-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" -version = "0.4.27" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pwd" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "python3-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "quick-error" version = "1.2.2" @@ -588,241 +736,368 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "quote" -version = "0.5.2" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "quote" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rand" -version = "0.4.2" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rand" -version = "0.5.5" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rand_core" -version = "0.2.2" +name = "rand_chacha" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rand_core" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] -name = "redox_syscall" -version = "0.1.40" +name = "rand_core" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "redox_termios" -version = "0.1.1" +name = "rand_hc" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "redox_users" -version = "0.2.0" +name = "rand_isaac" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "regex" -version = "0.2.11" +name = "rand_jitter" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "regex" -version = "1.0.3" +name = "rand_os" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "regex-syntax" -version = "0.4.2" +name = "rand_pcg" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] -name = "regex-syntax" -version = "0.5.6" +name = "rand_xorshift" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "regex-syntax" -version = "0.6.0" +name = "rdrand" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rustc-demangle" -version = "0.1.9" +name = "redox_syscall" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "rustc_version" -version = "0.2.3" +name = "redox_termios" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rustc_version_runtime" -version = "0.1.5" +name = "redox_users" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rustpython" -version = "0.0.1" -dependencies = [ - "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustpython_parser 0.0.1", - "rustpython_vm 0.1.0", - "rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +name = "regex" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rustpython_derive" +name = "regex-syntax" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version_runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustpython" +version = "0.0.1" +dependencies = [ + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython-compiler 0.1.0", + "rustpython-parser 0.0.1", + "rustpython-vm 0.1.0", + "rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustpython-bytecode" +version = "0.1.0" +dependencies = [ + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustpython-compiler" version = "0.1.0" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython-bytecode 0.1.0", + "rustpython-parser 0.0.1", ] [[package]] -name = "rustpython_parser" +name = "rustpython-derive" +version = "0.1.0" +dependencies = [ + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython-bytecode 0.1.0", + "rustpython-compiler 0.1.0", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustpython-parser" version = "0.0.1" dependencies = [ - "lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lalrpop-util 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wtf8 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "rustpython_vm" +name = "rustpython-vm" version = "0.1.0" dependencies = [ - "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", - "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "md-5 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-complex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", + "pwd 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rustpython_derive 0.1.0", - "rustpython_parser 0.0.1", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython-bytecode 0.1.0", + "rustpython-compiler 0.1.0", + "rustpython-derive 0.1.0", + "rustpython-parser 0.0.1", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", + "sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unic 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)", + "unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode_categories 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode_names2 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rustpython_wasm" -version = "0.1.0-pre-alpha.1" +version = "0.1.0-pre-alpha.2" dependencies = [ - "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rustpython_parser 0.0.1", - "rustpython_vm 0.1.0", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-futures 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython-compiler 0.1.0", + "rustpython-parser 0.0.1", + "rustpython-vm 0.1.0", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "serde-wasm-bindgen 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-futures 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "web-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rustyline" -version = "2.1.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "ryu" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -845,43 +1120,85 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.66" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde-wasm-bindgen" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "serde_derive" -version = "1.0.66" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.26" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha-1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "sha2" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha3" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "siphasher" -version = "0.2.2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -907,7 +1224,7 @@ name = "statrs" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -915,24 +1232,24 @@ name = "string_cache" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "string_cache_codegen" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -943,42 +1260,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "strsim" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "strsim" -version = "0.7.0" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "subtle" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "syn" -version = "0.14.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "syn" -version = "0.15.29" +version = "0.15.35" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "synstructure" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -993,15 +1323,7 @@ dependencies = [ [[package]] name = "termcolor" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "termcolor" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1009,17 +1331,18 @@ dependencies = [ [[package]] name = "termion" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "textwrap" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1030,7 +1353,7 @@ name = "thread_local" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1040,17 +1363,293 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "ucd-util" -version = "0.1.1" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-bidi 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-emoji 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-idna 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-segment 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-bidi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-bidi 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-char" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-basics 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-char-basics" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-emoji" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-emoji-char" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-idna" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-idna-mapping 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-idna-punycode 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-bidi 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-idna-mapping" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-idna-punycode" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unic-normal" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-ucd-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-ucd-segment 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-ucd-age 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-bidi 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-block 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-case 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-category 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-hangul 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-ident 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-name 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-name_aliases 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-segment 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-age" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-bidi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-block" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-case" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-category" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-hangul" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-ident" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-name" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-hangul 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-name_aliases" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-normal" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-category 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-hangul 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-casing" +version = "0.1.0" +source = "git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb#90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" [[package]] name = "unicode-normalization" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "unicode-segmentation" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1058,11 +1657,26 @@ name = "unicode-width" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode_names2" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unreachable" version = "1.0.0" @@ -1073,7 +1687,7 @@ dependencies = [ [[package]] name = "utf8-ranges" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1086,6 +1700,11 @@ name = "vec_map" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -1093,95 +1712,96 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "wasm-bindgen" -version = "0.2.29" +version = "0.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.29" +version = "0.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "bumpalo 2.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-futures" -version = "0.3.6" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.29" +version = "0.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-macro-support 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.29" +version = "0.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-backend 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-shared 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.29" +version = "0.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "wasm-bindgen-webidl" -version = "0.2.23" +version = "0.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-backend 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", + "weedle 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "web-sys" -version = "0.3.6" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "js-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-bindgen-webidl 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "weedle" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1191,7 +1811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1210,10 +1830,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "winapi-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1223,20 +1843,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "wincolor" -version = "0.1.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "wincolor" -version = "1.0.1" +name = "wtf8" +version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "xdg" @@ -1244,149 +1861,216 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" +"checksum aho-corasick 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "81ce3d38065e618af2d7b77e10c5ad9a069859b4be3c2250f674af3840d9c8a5" +"checksum aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e6f484ae0c99fec2e858eb6134949117399f222608d84cadb3f58c1f97c2364c" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f67b0b6a86dae6e67ff4ca2b6201396074996379fba2b92ff649126f37cb392" -"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f" -"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum arrayvec 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "92c7fb76bc8826a8b33b4ee5bb07a247a81e76764ab4d55e8f73e3a4d8808c71" "checksum ascii-canvas 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b385d69402821a1c254533a011a312531cbcc0e3e24f19bbb4747a5a2daf37e2" -"checksum atty 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d0fd4c0631f06448cc45a6bbb3b710ebb7ff8ccb96a0800c994afe23a70d5df2" -"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" -"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" -"checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" -"checksum bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9bf6104718e80d7b26a68fdbacff3481cfc05df670821affc7e9cbc1884400c" -"checksum bit-vec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" -"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" +"checksum backtrace 0.3.30 (registry+https://github.com/rust-lang/crates.io-index)" = "ada4c783bb7e7443c14e0480f429ae2cc99da95065aeab7ee1b81ada0419404f" +"checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" +"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" +"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" +"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" +"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" +"checksum blake2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "91721a6330935673395a0607df4d49a9cb90ae12d259f1b3e0a3f6e1d486872e" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" -"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" -"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" -"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" +"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +"checksum block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc3af3ee2e12f3e5d224e5e1e3d73668abbeb69e566d361f7d5563a4fdf09" +"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" +"checksum bumpalo 2.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "84dca3afd8e01b9526818b7963e5b4916063b3cdf9f10cf6b73ef0bd0ec37aa5" +"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f" -"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" -"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" -"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum cc 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "39f75544d7bbaf57560d2168f28fd649ff9c76153874db88bdbdfd839b1a7e7d" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" +"checksum cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b489034e723e7f5109fecd19b719e664f89ef925be785885252469e9822fa940" +"checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" "checksum diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "3c2b69f912779fbb121ceb775d74d51e915af17aaebc38d28a592843a2dd0a3a" -"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" -"checksum dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "88972de891f6118092b643d85a0b28e0678e0f948d7f879aa32f2d5aafe97d2a" -"checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" -"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" -"checksum ena 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cabe5a5078ac8c506d3e4430763b1ba9b609b1286913e7d08e581d1c2de9b7e5" -"checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" -"checksum env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "afb070faf94c85d17d50ca44f6ad076bce18ae92f0037d350947240a36e9d42e" -"checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9" -"checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426" +"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +"checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +"checksum docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f525a586d310c87df72ebcd98009e57f1cc030c8c268305287a476beb653969" +"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" +"checksum ena 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f56c93cc076508c549d9bb747f79aa9b4eb098be7b8cad8830c3137ef52d1e00" +"checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" +"checksum env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b61fa891024a945da30a9581546e8cfaf5602c7b3f4c137a2805cf388f92075a" +"checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" +"checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" "checksum fixedbitset 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" -"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" -"checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b" -"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" -"checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" -"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" -"checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" -"checksum itoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5adb58558dcd1d786b5f0bd15f3226ee23486e24b7b58304b60f64dc68e62606" -"checksum js-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58cfec35fd4a94f3cf357d5cb7da71c71cd52720c2f2a7320090a8db5f06f655" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +"checksum futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)" = "a2037ec1c6c1c4f79557762eab1f7eae1f64f6cb418ace90fae88f0942b60139" +"checksum generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c0f28c2f5bfb5960175af447a2da7c18900693738343dc896ffbcabd9839592" +"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +"checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +"checksum hexf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e54653cc32d838771a36532647afad59c4bf7155745eeeec406f71fd5d7e7538" +"checksum hexf-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "22eadcfadba76a730b2764eaa577d045f35e0ef5174b9c5b46adf1ee42b85e12" +"checksum hexf-parse 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79296f72d53a89096cbc9a88c9547ee8dfe793388674620e2207593d370550ac" +"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" +"checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d" +"checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" +"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" +"checksum js-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "9987e7c13a91d9cf0efe59cca48a3a7a70e2b11695d5a4640f85ae71e28f5e73" +"checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lalrpop 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ba451f7bd819b7afc99d4cf4bdcd5a4861e64955ba9680ac70df3a50625ad6cf" -"checksum lalrpop-snap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60013fd6be14317d43f47658b1440956a9ca48a9ed0257e0e0a59aac13e43a1f" -"checksum lalrpop-util 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "60c6c48ba857cd700673ce88907cadcdd7e2cd7783ed02378537c5ffd4f6460c" -"checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1" -"checksum lexical 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4fac65df7e751b57bb3a334c346239cb4ce2601907d698726ceeb82a54ba4ef" -"checksum lexical-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "025babf624c0c2b4bed1373efd684d5d0b2eecd61138d26ec3eec77bf0f2e33d" -"checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" +"checksum lalrpop 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4e2e80bee40b22bca46665b4ef1f3cd88ed0fb043c971407eac17a0712c02572" +"checksum lalrpop-util 0.16.3 (registry+https://github.com/rust-lang/crates.io-index)" = "33b27d8490dbe1f9704b0088d61e8d46edc10d5673a8829836c6ded26a9912c7" +"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" +"checksum lexical 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c106ed999697325a540c43d66a8d5175668cd96d7eb0bdba03a3bd98256cd698" +"checksum lexical-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e82e023e062f1d25f807ad182008fba1b46538e999f908a08cc0c29e084462e" +"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" -"checksum log 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6fddaa003a65722a7fb9e26b0ce95921fe4ba590542ced664d8ce2fa26f9f3ac" -"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" -"checksum new_debug_unreachable 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0cdc457076c78ab54d5e0d6fa7c47981757f1e34dc39ff92787f217dede586c4" -"checksum nix 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d37e713a259ff641624b6cb20e3b12b2952313ba36b6823c0f16e6cfd9e5de17" -"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" -"checksum nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9c349f68f25f596b9f44cf0e7c69752a5c633b0550c3ff849518bfba0233774a" -"checksum num-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "10b8423ea72ec64751198856a853e07b37087cfc9b53a87ecb19bff67b6d1320" -"checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" -"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" -"checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10" -"checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +"checksum log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c84ec4b527950aa83a329754b01dbe3f58361d1c5efacd1f6d68c494d08a17c6" +"checksum maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08cbb6b4fef96b6d77bfc40ec491b1690c779e77b05cd9f07f787ed376fd4c43" +"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +"checksum md-5 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8" +"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" +"checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30" +"checksum nix 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4dbdc256eaac2e3bd236d93ad999d3479ef775c863dbda3068c4006a92eec51b" +"checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" +"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +"checksum num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "57450397855d951f1a41305e54851b1a7b8f5d2e349543a02a2effe25459f718" +"checksum num-complex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a9854fff8af065f715e116e1d61e2d25a41fee956e941b6940f11295199584c" +"checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" +"checksum num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2885278d5fe2adc2f75ced642d52d879bffaceb5a2e0b1d4309ffdfb239b454" +"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" +"checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" +"checksum opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "93f5bb2e8e8dec81642920ccff6b61f1eb94fa3020c5a325c9851ff604152409" "checksum ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a86ed3f5f244b372d6b1a00b72ef7f8876d0bc6a78a4c9985c53614041512063" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" -"checksum phf_generator 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "05a079dd052e7b674d21cb31cbb6c05efd56a2cd2827db7692e2f1a507ebd998" -"checksum phf_shared 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2261d544c2bb6aa3b10022b0be371b9c7c64f762ef28c6f5d4f1ef6d97b5930" +"checksum phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +"checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" "checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" -"checksum proc-macro2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7" -"checksum proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)" = "4d317f9caece796be1980837fd5cb3dfec5613ebdb04ad0956deea83ce168915" +"checksum proc-macro-hack 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b773f824ff2a495833f85fcdddcf85e096949971decada2e93249fa2c6c3d32f" +"checksum proc-macro-hack 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0c1dd4172a1e1f96f709341418f49b11ea6c2d95d53dca08c0f74cbd332d9cf3" +"checksum proc-macro-hack-impl 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f674ccc446da486175527473ec8aa064f980b0966bbf767ee743a5dff6244a7" +"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +"checksum pwd 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5dd32d8bece608e144ca20251e714ed107cdecdabb20c2d383cfc687825106a5" +"checksum python3-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "61e4aac43f833fd637e429506cb2ac9d7df672c4b68f2eaaa163649b7fdc0444" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" -"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" -"checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1" -"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" -"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" -"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" -"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" -"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" +"checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" +"checksum rand 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" +"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +"checksum rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +"checksum redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)" = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum redox_users 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "214a97e49be64fd2c86f568dd0cb2c757d2cc53de95b273b6ad0a1c908482f26" +"checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" "checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" -"checksum regex 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3d8c9f33201f46669484bacc312b00e7541bed6aaf296dffe2bb4e0ac6b8ce2a" -"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" +"checksum regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0b2f0808e7d7e4fb1cb07feb6ff2f4bc827938f24f8c2e6a3beb7370af544bdd" "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" -"checksum regex-syntax 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f1ac0f60d675cc6cf13a20ec076568254472551051ad5dd050364d70671bf6b" -"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" +"checksum regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d76410686f9e3a17f06128962e0ecc5755870bb890c34820c7af7f1db2e1d48" +"checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum rustc_version_runtime 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6de8ecd7fad7731f306f69b6e10ec5a3178c61e464dcc06979427aa4cc891145" -"checksum rustyline 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6010155119d53aac4f5b987cb8f6ea913d0d64d9b237da36f8f96a90cb3f5385" -"checksum ryu 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "eb9e9b8cde282a9fe6a42dd4681319bfb63f121b8a8ee9439c6f4107e58a46f7" +"checksum rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b" +"checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "e9a2d9a9ac5120e0f768801ca2b58ad6eec929dc9d1d616c162f208869c2ce95" -"checksum serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "0a90213fa7e0f5eac3f7afe2d5ff6b088af515052cc7303bd68c7e3b91a3fb79" -"checksum serde_json 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "44dd2cfde475037451fa99b7e5df77aa3cfd1536575fa8e7a538ab36dcde49ae" -"checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" -"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" +"checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be" +"checksum serde-wasm-bindgen 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7ee6f12f7ed0e7ad2e55200da37dbabc2cadeb942355c5b629aa3771f5ac5636" +"checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e" +"checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" +"checksum sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "23962131a91661d643c98940b20fcaffe62d776a823247be80a48fcb8b6fce68" +"checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" +"checksum sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd26bc0e7a2e3a7c959bc494caf58b72ee0c71d67704e9520f736ca7e4853ecf" +"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" +"checksum smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ab606a9c5e214920bb66c458cd7be8ef094f813f20fe77a54cc7dbfff220d4b7" "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" "checksum stackvector 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c049c77bf85fbc036484c97b008276d539d9ebff9dfbde37b632ebcd5b8746b6" "checksum static_assertions 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c19be23126415861cb3a23e501d34a708f7f9b2183c5252d690941c2e69199d5" "checksum statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "10102ac8d55e35db2b3fafc26f81ba8647da2e15879ab686a67e6d19af2685e8" "checksum string_cache 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25d70109977172b127fe834e5449e5ab1740b9ba49fa18a2020f509174f25423" -"checksum string_cache_codegen 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35293b05cf1494e8ddd042a7df6756bf18d07f42d234f32e71dce8a7aabb0191" +"checksum string_cache_codegen 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eea1eee654ef80933142157fdad9dd8bc43cf7c74e999e369263496f04ff4da" "checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" -"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" -"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" -"checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" -"checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2" -"checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7" +"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum strsim 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "032c03039aae92b350aad2e3779c352e104d919cb192ba2fabbd7b831ce4f0f6" +"checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +"checksum syn 0.15.35 (registry+https://github.com/rust-lang/crates.io-index)" = "641e117d55514d6d918490e47102f7e08d096fdde360247e4a10f7a91a8478d3" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +"checksum synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02353edf96d6e4dc81aea2d8490a7e9db177bf8acb0e951c24940bf866cb313f" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" -"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" -"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" -"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" +"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" +"checksum termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" -"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" -"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" -"checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" +"checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" +"checksum unic 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e31748f3e294dc6a9243a44686e8155a162af9a11cd56e07c0ebbc530b2a8a87" +"checksum unic-bidi 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1356b759fb6a82050666f11dce4b6fe3571781f1449f3ef78074e408d468ec09" +"checksum unic-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "af25df79bd134107f088ba725d9c470600f16263205d0be36c75e75b020bac0a" +"checksum unic-char-basics 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "20e5d239bc6394309225a0c1b13e1d059565ff2cfef1a437aff4a5871fa06c4b" +"checksum unic-char-property 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +"checksum unic-char-range 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" +"checksum unic-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" +"checksum unic-emoji 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "74193f32f7966ad20b819e70e29c6f1ac8c386692a9d5e90078eef80ea008bfb" +"checksum unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" +"checksum unic-idna 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "621e9cf526f2094d2c2ced579766458a92f8f422d6bb934c503ba1a95823a62d" +"checksum unic-idna-mapping 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4de70fd4e5331537347a50a0dbc938efb1f127c9f6e5efec980fc90585aa1343" +"checksum unic-idna-punycode 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "06feaedcbf9f1fc259144d833c0d630b8b15207b0486ab817d29258bc89f2f8a" +"checksum unic-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f09d64d33589a94628bc2aeb037f35c2e25f3f049c7348b5aa5580b48e6bba62" +"checksum unic-segment 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +"checksum unic-ucd 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "625b18f7601e1127504a20ae731dc3c7826d0e86d5f7fe3434f8137669240efd" +"checksum unic-ucd-age 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6c8cfdfe71af46b871dc6af2c24fcd360e2f3392ee4c5111877f2947f311671c" +"checksum unic-ucd-bidi 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d1d568b51222484e1f8209ce48caa6b430bf352962b877d592c29ab31fb53d8c" +"checksum unic-ucd-block 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6b2a16f2d7ecd25325a1053ca5a66e7fa1b68911a65c5e97f8d2e1b236b6f1d7" +"checksum unic-ucd-case 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d98d6246a79bac6cf66beee01422bda7c882e11d837fa4969bfaaba5fdea6d3" +"checksum unic-ucd-category 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b8d4591f5fcfe1bd4453baaf803c40e1b1e69ff8455c47620440b46efef91c0" +"checksum unic-ucd-common 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9b78b910beafa1aae5c59bf00877c6cece1c5db28a1241ad801e86cecdff4ad" +"checksum unic-ucd-hangul 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eb1dc690e19010e1523edb9713224cba5ef55b54894fe33424439ec9a40c0054" +"checksum unic-ucd-ident 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" +"checksum unic-ucd-name 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c8fc55a45b2531089dc1773bf60c1f104b38e434b774ffc37b9c29a9b0f492e" +"checksum unic-ucd-name_aliases 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6b7674212643087699ba247a63dd05f1204c7e4880ec9342e545a7cffcc6a46f" +"checksum unic-ucd-normal 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "86aed873b8202d22b13859dda5fe7c001d271412c31d411fd9b827e030569410" +"checksum unic-ucd-segment 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +"checksum unic-ucd-version 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +"checksum unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)" = "" +"checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" +"checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum unicode_categories 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +"checksum unicode_names2 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dc6c5da0c8d7200f9488cc346bd30ba62bcd9f79ef937ea6573132e3d507df9" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9d50aa7650df78abf942826607c62468ce18d9019673d4a2ebe1865dbb96ffde" "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -"checksum wasm-bindgen 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "91f95b8f30407b9ca0c2de157281d3828bbed1fc1f55bea6eb54f40c52ec75ec" -"checksum wasm-bindgen-backend 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "ab7c242ebcb45bae45340986c48d1853eb2c1c52ff551f7724951b62a2c51429" -"checksum wasm-bindgen-futures 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d1784e7401a90119b2a4e8ec9c8d37c3594c3e3bb9ba24533ee1969eebaf0485" -"checksum wasm-bindgen-macro 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "6e353f83716dec9a3597b5719ef88cb6c9e461ec16528f38aa023d3224b4e569" -"checksum wasm-bindgen-macro-support 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "3cc90b65fe69c3dd5a09684517dc79f42b847baa2d479c234d125e0a629d9b0a" -"checksum wasm-bindgen-shared 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "a71a37df4f5845025f96f279d20bbe5b19cbcb77f5410a3a90c6c544d889a162" -"checksum wasm-bindgen-webidl 0.2.23 (registry+https://github.com/rust-lang/crates.io-index)" = "b064b8b2336b5a6bf5f31bc95fc1310842395df29877d910cb6f8f791070f319" -"checksum web-sys 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d7c588c2e5a091bc4532c5a87032955f9133b644e868b54d08ead0185dcc5b9" -"checksum weedle 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26a4c67f132386d965390b8a734d5d10adbcd30eb5cc74bd9229af8b83f10044" +"checksum wasm-bindgen 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "b7ccc7b93cfd13e26700a9e2e41e6305f1951b87e166599069f77d10358100e6" +"checksum wasm-bindgen-backend 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "1953f91b1608eb1522513623c7739f047bb0fed4128ce51a93f08e12cc314645" +"checksum wasm-bindgen-futures 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "fa1af11c73eca3dc8c51c76ea475a4416e912da6402064a49fc6c0214701866d" +"checksum wasm-bindgen-macro 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "0f69da5696545d7ca6607a2e4b1a0edf5a6b36b2c49dbb0f1df6ad1d92884047" +"checksum wasm-bindgen-macro-support 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "2d4246f3bc73223bbb846f4f2430a60725826a96c9389adf715ed1d5af46dec6" +"checksum wasm-bindgen-shared 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "c08381e07e7a79e5e229ad7c60d15833d19033542cc5dd91d085df59d235f4a6" +"checksum wasm-bindgen-webidl 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)" = "1f42ff7adb8102bf5ad8adbc45b1635c520c8175f9fdf6eb2c54479d485d435a" +"checksum web-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "540b8259eb242ff3a566fa0140bda03a4ece4e5c226e1284b5c95dddcd4341f6" +"checksum weedle 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc44aa200daee8b1f3a004beaf16554369746f1b4486f0cf93b0caf8a3c2d1e" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" -"checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" +"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" +"checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" +"checksum wtf8 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d6b9309a86639c488a8eb2b5331cb5127cc9feb0a94a0db4b5d1ab5b84977956" "checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" diff --git a/Cargo.toml b/Cargo.toml index c109e3fea4..b92473d6db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,26 @@ [package] name = "rustpython" version = "0.0.1" -authors = ["Windel Bouwman", "Shing Lyu "] +authors = ["RustPython Team"] edition = "2018" [workspace] -members = [".", "derive", "vm", "wasm/lib", "parser"] +members = [".", "derive", "vm", "wasm/lib", "parser", "compiler", "bytecode"] + +[[bench]] +name = "bench" +path = "./benchmarks/bench.rs" + [dependencies] log="0.4.1" env_logger="0.5.10" clap = "2.31.2" -rustpython_parser = {path = "parser"} -rustpython_vm = {path = "vm"} -rustyline = "2.1.0" +rustpython-compiler = {path = "compiler"} +rustpython-parser = {path = "parser"} +rustpython-vm = {path = "vm"} +rustyline = "4.1.0" xdg = "2.2.0" + +[dev-dependencies.cpython] +version = "0.2" diff --git a/Lib/PSF-LICENSE b/Lib/PSF-LICENSE new file mode 100644 index 0000000000..1afbedba92 --- /dev/null +++ b/Lib/PSF-LICENSE @@ -0,0 +1,254 @@ +A. HISTORY OF THE SOFTWARE +========================== + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations, which became +Zope Corporation. In 2001, the Python Software Foundation (PSF, see +https://www.python.org/psf/) was formed, a non-profit organization +created specifically to own Python-related Intellectual Property. +Zope Corporation was a sponsoring member of the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases. + + Release Derived Year Owner GPL- + from compatible? (1) + + 0.9.0 thru 1.2 1991-1995 CWI yes + 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes + 1.6 1.5.2 2000 CNRI no + 2.0 1.6 2000 BeOpen.com no + 1.6.1 1.6 2001 CNRI yes (2) + 2.1 2.0+1.6.1 2001 PSF no + 2.0.1 2.0+1.6.1 2001 PSF yes + 2.1.1 2.1+2.0.1 2001 PSF yes + 2.1.2 2.1.1 2002 PSF yes + 2.1.3 2.1.2 2002 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, + because its license has a choice of law clause. According to + CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 + is "not incompatible" with the GPL. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/README.md b/Lib/README.md new file mode 100644 index 0000000000..b3aade7bfa --- /dev/null +++ b/Lib/README.md @@ -0,0 +1,10 @@ +Standard Library for RustPython +=============================== + +This directory contains all of the Python files that make up the standard library for RustPython. + +Most of these files are copied over from the CPython repository(the 3.7 branch), with slight modifications to allow them +to work under RustPython. The current goal is to complete the standard library with as few modifications as possible. +Current modifications are just temporary workarounds for bugs/missing feature within the RustPython implementation. + +The first target is to run the ``unittest`` module, so we can leverage the CPython test suite. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py new file mode 100644 index 0000000000..3158a4f16c --- /dev/null +++ b/Lib/_collections_abc.py @@ -0,0 +1,1013 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) for collections, according to PEP 3119. + +Unit tests are in test_collections. +""" + +from abc import ABCMeta, abstractmethod +import sys + +__all__ = ["Awaitable", "Coroutine", + "AsyncIterable", "AsyncIterator", "AsyncGenerator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", + "Sized", "Container", "Callable", "Collection", + "Set", "MutableSet", + "Mapping", "MutableMapping", + "MappingView", "KeysView", "ItemsView", "ValuesView", + "Sequence", "MutableSequence", + "ByteString", + ] + +# This module has been renamed from collections.abc to _collections_abc to +# speed up interpreter startup. Some of the types such as MutableMapping are +# required early but collections module imports a lot of other modules. +# See issue #19218 +__name__ = "collections.abc" + +# Private list of types that we want to register with the various ABCs +# so that they will pass tests like: +# it = iter(somebytearray) +# assert isinstance(it, Iterable) +# Note: in other implementations, these types might not be distinct +# and they may have their own implementation specific types that +# are not included on this list. +bytes_iterator = type(iter(b'')) +bytearray_iterator = type(iter(bytearray())) +#callable_iterator = ??? +dict_keyiterator = type(iter({}.keys())) +dict_valueiterator = type(iter({}.values())) +dict_itemiterator = type(iter({}.items())) +list_iterator = type(iter([])) +# list_reverseiterator = type(iter(reversed([]))) +range_iterator = type(iter(range(0))) +longrange_iterator = type(iter(range(1 << 1000))) +set_iterator = type(iter(set())) +str_iterator = type(iter("")) +tuple_iterator = type(iter(())) +zip_iterator = type(iter(zip())) +## views ## +dict_keys = type({}.keys()) +dict_values = type({}.values()) +dict_items = type({}.items()) +## misc ## +mappingproxy = type(type.__dict__) +# generator = type((lambda: (yield))()) +## coroutine ## +# async def _coro(): pass +# _coro = _coro() +# coroutine = type(_coro) +# _coro.close() # Prevent ResourceWarning +# del _coro +# ## asynchronous generator ## +# async def _ag(): yield +# _ag = _ag() +# async_generator = type(_ag) +# del _ag + + +# ## ONE-TRICK PONIES ### + +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + +class Hashable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + return _check_methods(C, "__hash__") + return NotImplemented + + +# class Awaitable(metaclass=ABCMeta): + +# __slots__ = () + +# @abstractmethod +# def __await__(self): +# yield + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Awaitable: +# return _check_methods(C, "__await__") +# return NotImplemented + + +# class Coroutine(Awaitable): + +# __slots__ = () + +# @abstractmethod +# def send(self, value): +# """Send a value into the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# raise StopIteration + +# @abstractmethod +# def throw(self, typ, val=None, tb=None): +# """Raise an exception in the coroutine. +# Return next yielded value or raise StopIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val + +# def close(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# self.throw(GeneratorExit) +# except (GeneratorExit, StopIteration): +# pass +# else: +# raise RuntimeError("coroutine ignored GeneratorExit") + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is Coroutine: +# return _check_methods(C, '__await__', 'send', 'throw', 'close') +# return NotImplemented + + +# Coroutine.register(coroutine) + + +# class AsyncIterable(metaclass=ABCMeta): + +# __slots__ = () + +# @abstractmethod +# def __aiter__(self): +# return AsyncIterator() + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterable: +# return _check_methods(C, "__aiter__") +# return NotImplemented + + +# class AsyncIterator(AsyncIterable): + +# __slots__ = () + +# @abstractmethod +# async def __anext__(self): +# """Return the next item or raise StopAsyncIteration when exhausted.""" +# raise StopAsyncIteration + +# def __aiter__(self): +# return self + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncIterator: +# return _check_methods(C, "__anext__", "__aiter__") +# return NotImplemented + + +# class AsyncGenerator(AsyncIterator): + +# __slots__ = () + +# async def __anext__(self): +# """Return the next item from the asynchronous generator. +# When exhausted, raise StopAsyncIteration. +# """ +# return await self.asend(None) + +# @abstractmethod +# async def asend(self, value): +# """Send a value into the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# raise StopAsyncIteration + +# @abstractmethod +# async def athrow(self, typ, val=None, tb=None): +# """Raise an exception in the asynchronous generator. +# Return next yielded value or raise StopAsyncIteration. +# """ +# if val is None: +# if tb is None: +# raise typ +# val = typ() +# if tb is not None: +# val = val.with_traceback(tb) +# raise val + +# async def aclose(self): +# """Raise GeneratorExit inside coroutine. +# """ +# try: +# await self.athrow(GeneratorExit) +# except (GeneratorExit, StopAsyncIteration): +# pass +# else: +# raise RuntimeError("asynchronous generator ignored GeneratorExit") + +# @classmethod +# def __subclasshook__(cls, C): +# if cls is AsyncGenerator: +# return _check_methods(C, '__aiter__', '__anext__', +# 'asend', 'athrow', 'aclose') +# return NotImplemented + + +# AsyncGenerator.register(async_generator) + + +class Iterable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + return _check_methods(C, "__iter__") + return NotImplemented + + +class Iterator(Iterable): + + __slots__ = () + + @abstractmethod + def __next__(self): + 'Return the next item from the iterator. When exhausted, raise StopIteration' + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + return _check_methods(C, '__iter__', '__next__') + return NotImplemented + +# Iterator.register(bytes_iterator) +# Iterator.register(bytearray_iterator) +# Iterator.register(callable_iterator) +# Iterator.register(dict_keyiterator) +# Iterator.register(dict_valueiterator) +# Iterator.register(dict_itemiterator) +# Iterator.register(list_iterator) +# Iterator.register(list_reverseiterator) +# Iterator.register(range_iterator) +# Iterator.register(longrange_iterator) +# Iterator.register(set_iterator) +# Iterator.register(str_iterator) +# Iterator.register(tuple_iterator) +# Iterator.register(zip_iterator) + + +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__", "__iter__") + return NotImplemented + + +class Generator(Iterator): + + __slots__ = () + + def __next__(self): + """Return the next item from the generator. + When exhausted, raise StopIteration. + """ + return self.send(None) + + @abstractmethod + def send(self, value): + """Send a value into the generator. + Return next yielded value or raise StopIteration. + """ + raise StopIteration + + @abstractmethod + def throw(self, typ, val=None, tb=None): + """Raise an exception in the generator. + Return next yielded value or raise StopIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + def close(self): + """Raise GeneratorExit inside generator. + """ + try: + self.throw(GeneratorExit) + except (GeneratorExit, StopIteration): + pass + else: + raise RuntimeError("generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is Generator: + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') + return NotImplemented + +# Generator.register(generator) + + +class Sized(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + return _check_methods(C, "__len__") + return NotImplemented + + +class Container(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + return _check_methods(C, "__contains__") + return NotImplemented + +class Collection(Sized, Iterable, Container): + + __slots__ = () + + @classmethod + def __subclasshook__(cls, C): + if cls is Collection: + return _check_methods(C, "__len__", "__iter__", "__contains__") + return NotImplemented + +class Callable(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + return _check_methods(C, "__call__") + return NotImplemented + + +### SETS ### + + +class Set(Collection): + + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), redefine __le__ and __ge__, + then the other operations will automatically follow suit. + """ + + __slots__ = () + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) > len(other) and self.__ge__(other) + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) < len(other): + return False + for elem in other: + if elem not in self: + return False + return True + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + __rand__ = __and__ + + def isdisjoint(self, other): + 'Return True if two sets have a null intersection.' + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + __ror__ = __or__ + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __rsub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in other + if value not in self) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + __rxor__ = __xor__ + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxsize + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + +Set.register(frozenset) + + +class MutableSet(Set): + """A mutable set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__, __len__, + add(), and discard(). + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + __slots__ = () + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError from None + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + +MutableSet.register(set) + + +### MAPPINGS ### + + +class Mapping(Collection): + + __slots__ = () + + """A Mapping is a generic container for associating key/value + pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __iter__, and __len__. + + """ + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return KeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return ItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return ValuesView(self) + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + __reversed__ = None + +Mapping.register(mappingproxy) + + +class MappingView(Sized): + + __slots__ = '_mapping', + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class KeysView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + yield from self._mapping + +KeysView.register(dict_keys) + + +class ItemsView(MappingView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v is value or v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + +ItemsView.register(dict_items) + + +class ValuesView(MappingView, Collection): + + __slots__ = () + + def __contains__(self, value): + for key in self._mapping: + v = self._mapping[key] + if v is value or v == value: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + +ValuesView.register(dict_values) + + +class MutableMapping(Mapping): + + __slots__ = () + + """A MutableMapping is a generic container for associating + key/value pairs. + + This class provides concrete generic implementations of all + methods except for __getitem__, __setitem__, __delitem__, + __iter__, and __len__. + + """ + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + '''D.popitem() -> (k, v), remove and return some (key, value) pair + as a 2-tuple; but raise KeyError if D is empty. + ''' + try: + key = next(iter(self)) + except StopIteration: + raise KeyError from None + value = self[key] + del self[key] + return key, value + + def clear(self): + 'D.clear() -> None. Remove all items from D.' + try: + while True: + self.popitem() + except KeyError: + pass + + def update(*args, **kwds): + ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. + If E present and has a .keys() method, does: for k in E: D[k] = E[k] + If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v + In either case, this is followed by: for k, v in F.items(): D[k] = v + ''' + if not args: + raise TypeError("descriptor 'update' of 'MutableMapping' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('update expected at most 1 arguments, got %d' % + len(args)) + if args: + other = args[0] + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + 'D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D' + try: + return self[key] + except KeyError: + self[key] = default + return default + +MutableMapping.register(dict) + + +### SEQUENCES ### + + +class Sequence(Reversible, Collection): + + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + __slots__ = () + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value, start=0, stop=None): + '''S.index(value, [start, [stop]]) -> integer -- return first index of value. + Raises ValueError if the value is not present. + + Supporting start and stop arguments is optional, but + recommended. + ''' + if start is not None and start < 0: + start = max(len(self) + start, 0) + if stop is not None and stop < 0: + stop += len(self) + + i = start + while stop is None or i < stop: + try: + v = self[i] + if v is value or v == value: + return i + except IndexError: + break + i += 1 + raise ValueError + + def count(self, value): + 'S.count(value) -> integer -- return number of occurrences of value' + return sum(1 for v in self if v is value or v == value) + +Sequence.register(tuple) +Sequence.register(str) +Sequence.register(range) +Sequence.register(memoryview) + + +class ByteString(Sequence): + + """This unifies bytes and bytearray. + + XXX Should add all their methods. + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + + +class MutableSequence(Sequence): + + __slots__ = () + + """All the operations on a read-write sequence. + + Concrete subclasses must provide __new__ or __init__, + __getitem__, __setitem__, __delitem__, __len__, and insert(). + + """ + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + 'S.insert(index, value) -- insert value before index' + raise IndexError + + def append(self, value): + 'S.append(value) -- append value to the end of the sequence' + self.insert(len(self), value) + + def clear(self): + 'S.clear() -> None -- remove all items from S' + try: + while True: + self.pop() + except IndexError: + pass + + def reverse(self): + 'S.reverse() -- reverse *IN PLACE*' + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + if values is self: + values = list(values) + for v in values: + self.append(v) + + def pop(self, index=-1): + '''S.pop([index]) -> item -- remove and return item at index (default last). + Raise IndexError if list is empty or index is out of range. + ''' + v = self[index] + del self[index] + return v + + def remove(self, value): + '''S.remove(value) -- remove first occurrence of value. + Raise ValueError if the value is not present. + ''' + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + +MutableSequence.register(list) +MutableSequence.register(bytearray) # Multiply inheriting, see ByteString diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py new file mode 100644 index 0000000000..07044f2f75 --- /dev/null +++ b/Lib/_py_abc.py @@ -0,0 +1,147 @@ +from _weakrefset import WeakSet + + +def get_cache_token(): + """Returns the current ABC cache token. + + The token is an opaque object (supporting equality testing) identifying the + current version of the ABC cache for virtual subclasses. The token changes + with every call to ``register()`` on any ABC. + """ + return ABCMeta._abc_invalidation_counter + + +class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + # Note: this counter is private. Use `abc.get_cache_token()` for + # external code. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace, **kwargs): + cls = type.__new__(mcls, name, bases, namespace, **kwargs) + # Compute set of abstract method names + abstracts = {name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)} + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = set(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + if not isinstance(subclass, type): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return subclass # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + return subclass + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + for name in cls.__dict__: + if name.startswith("_abc_"): + value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) + print(f"{name}: {value!r}", file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + cls._abc_registry.clear() + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + cls._abc_cache.clear() + cls._abc_negative_cache.clear() + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking + subclass = instance.__class__ + if subclass in cls._abc_cache: + return True + subtype = type(instance) + if subtype is subclass: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subclass in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subclass) + return any(cls.__subclasscheck__(c) for c in (subclass, subtype)) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + if not isinstance(subclass, type): + raise TypeError('issubclass() arg 1 must be a class') + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py new file mode 100644 index 0000000000..304c66f59b --- /dev/null +++ b/Lib/_weakrefset.py @@ -0,0 +1,196 @@ +# Access WeakSet through the weakref module. +# This code is separated-out because it is needed +# by abc.py to load everything else at startup. + +from _weakref import ref + +__all__ = ['WeakSet'] + + +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +class WeakSet: + def __init__(self, data=None): + self.data = set() + def _remove(item, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(item) + else: + self.data.discard(item) + self._remove = _remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + if data is not None: + self.update(data) + + def _commit_removals(self): + l = self._pending_removals + discard = self.data.discard + while l: + discard(l.pop()) + + def __iter__(self): + with _IterationGuard(self): + for itemref in self.data: + item = itemref() + if item is not None: + # Caveat: the iterator will keep a strong reference to + # `item` until it is resumed or closed. + yield item + + def __len__(self): + return len(self.data) - len(self._pending_removals) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __reduce__(self): + return (self.__class__, (list(self),), + getattr(self, '__dict__', None)) + + def add(self, item): + if self._pending_removals: + self._commit_removals() + self.data.add(ref(item, self._remove)) + + def clear(self): + if self._pending_removals: + self._commit_removals() + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + if self._pending_removals: + self._commit_removals() + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError('pop from empty WeakSet') from None + item = itemref() + if item is not None: + return item + + def remove(self, item): + if self._pending_removals: + self._commit_removals() + self.data.remove(ref(item)) + + def discard(self, item): + if self._pending_removals: + self._commit_removals() + self.data.discard(ref(item)) + + def update(self, other): + if self._pending_removals: + self._commit_removals() + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + def difference(self, other): + newset = self.copy() + newset.difference_update(other) + return newset + __sub__ = difference + + def difference_update(self, other): + self.__isub__(other) + def __isub__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + return self + + def intersection(self, other): + return self.__class__(item for item in other if item in self) + __and__ = intersection + + def intersection_update(self, other): + self.__iand__(other) + def __iand__(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + return self + + def issubset(self, other): + return self.data.issubset(ref(item) for item in other) + __le__ = issubset + + def __lt__(self, other): + return self.data < set(map(ref, other)) + + def issuperset(self, other): + return self.data.issuperset(ref(item) for item in other) + __ge__ = issuperset + + def __gt__(self, other): + return self.data > set(map(ref, other)) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.data == set(map(ref, other)) + + def symmetric_difference(self, other): + newset = self.copy() + newset.symmetric_difference_update(other) + return newset + __xor__ = symmetric_difference + + def symmetric_difference_update(self, other): + self.__ixor__(other) + def __ixor__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) + return self + + def union(self, other): + return self.__class__(e for s in (self, other) for e in s) + __or__ = union + + def isdisjoint(self, other): + return len(self.intersection(other)) == 0 diff --git a/Lib/abc.py b/Lib/abc.py new file mode 100644 index 0000000000..7094141277 --- /dev/null +++ b/Lib/abc.py @@ -0,0 +1,170 @@ +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) according to PEP 3119.""" + + +def abstractmethod(funcobj): + """A decorator indicating abstract methods. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract methods are overridden. + The abstract methods can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractmethod + def my_abstract_method(self, ...): + ... + """ + funcobj.__isabstractmethod__ = True + return funcobj + + +class abstractclassmethod(classmethod): + """A decorator indicating abstract classmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractclassmethod + def my_abstract_classmethod(cls, ...): + ... + + 'abstractclassmethod' is deprecated. Use 'classmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractstaticmethod(staticmethod): + """A decorator indicating abstract staticmethods. + + Similar to abstractmethod. + + Usage: + + class C(metaclass=ABCMeta): + @abstractstaticmethod + def my_abstract_staticmethod(...): + ... + + 'abstractstaticmethod' is deprecated. Use 'staticmethod' with + 'abstractmethod' instead. + """ + + __isabstractmethod__ = True + + def __init__(self, callable): + callable.__isabstractmethod__ = True + super().__init__(callable) + + +class abstractproperty(property): + """A decorator indicating abstract properties. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract properties are overridden. + The abstract properties can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C(metaclass=ABCMeta): + @abstractproperty + def my_abstract_property(self): + ... + + This defines a read-only property; you can also define a read-write + abstract property using the 'long' form of property declaration: + + class C(metaclass=ABCMeta): + def getx(self): ... + def setx(self, value): ... + x = abstractproperty(getx, setx) + + 'abstractproperty' is deprecated. Use 'property' with 'abstractmethod' + instead. + """ + + __isabstractmethod__ = True + + +try: + from _abc import (get_cache_token, _abc_init, _abc_register, + _abc_instancecheck, _abc_subclasscheck, _get_dump, + _reset_registry, _reset_caches) +except ImportError: + from _py_abc import ABCMeta, get_cache_token + ABCMeta.__module__ = 'abc' +else: + class ABCMeta(type): + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + """ + def __new__(mcls, name, bases, namespace, **kwargs): + cls = super().__new__(mcls, name, bases, namespace, **kwargs) + _abc_init(cls) + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC. + + Returns the subclass, to allow usage as a class decorator. + """ + return _abc_register(cls, subclass) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + return _abc_instancecheck(cls, instance) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + return _abc_subclasscheck(cls, subclass) + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file) + print(f"Inv. counter: {get_cache_token()}", file=file) + (_abc_registry, _abc_cache, _abc_negative_cache, + _abc_negative_cache_version) = _get_dump(cls) + print(f"_abc_registry: {_abc_registry!r}", file=file) + print(f"_abc_cache: {_abc_cache!r}", file=file) + print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file) + print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}", + file=file) + + def _abc_registry_clear(cls): + """Clear the registry (for debugging or testing).""" + _reset_registry(cls) + + def _abc_caches_clear(cls): + """Clear the caches (for debugging or testing).""" + _reset_caches(cls) + + +class ABC(metaclass=ABCMeta): + """Helper class that provides a standard way to create an ABC using + inheritance. + """ + __slots__ = () diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py new file mode 100644 index 0000000000..9a753db71c --- /dev/null +++ b/Lib/collections/__init__.py @@ -0,0 +1,1279 @@ +'''This module implements specialized container datatypes providing +alternatives to Python's general purpose built-in containers, dict, +list, set, and tuple. + +* namedtuple factory function for creating tuple subclasses with named fields +* deque list-like container with fast appends and pops on either end +* ChainMap dict-like class for creating a single view of multiple mappings +* Counter dict subclass for counting hashable objects +* OrderedDict dict subclass that remembers the order entries were added +* defaultdict dict subclass that calls a factory function to supply missing values +* UserDict wrapper around dictionary objects for easier dict subclassing +* UserList wrapper around list objects for easier list subclassing +* UserString wrapper around string objects for easier string subclassing + +''' + +__all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList', + 'UserString', 'Counter', 'OrderedDict', 'ChainMap'] + +import _collections_abc +from operator import itemgetter as _itemgetter, eq as _eq +from keyword import iskeyword as _iskeyword +import sys as _sys +import heapq as _heapq +from _weakref import proxy as _proxy +from itertools import repeat as _repeat, chain as _chain, starmap as _starmap +from reprlib import recursive_repr as _recursive_repr + +try: + from _collections import deque +except ImportError: + pass +else: + _collections_abc.MutableSequence.register(deque) + +try: + from _collections import defaultdict +except ImportError: + pass + + +def __getattr__(name): + # For backwards compatibility, continue to make the collections ABCs + # through Python 3.6 available through the collections module. + # Note, no new collections ABCs were added in Python 3.7 + if name in _collections_abc.__all__: + obj = getattr(_collections_abc, name) + import warnings + warnings.warn("Using or importing the ABCs from 'collections' instead " + "of from 'collections.abc' is deprecated, " + "and in 3.8 it will stop working", + DeprecationWarning, stacklevel=2) + globals()[name] = obj + return obj + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + +################################################################################ +### OrderedDict +################################################################################ + +class _OrderedDictKeysView(_collections_abc.KeysView): + + def __reversed__(self): + yield from reversed(self._mapping) + +class _OrderedDictItemsView(_collections_abc.ItemsView): + + def __reversed__(self): + for key in reversed(self._mapping): + yield (key, self._mapping[key]) + +class _OrderedDictValuesView(_collections_abc.ValuesView): + + def __reversed__(self): + for key in reversed(self._mapping): + yield self._mapping[key] + +class _Link(object): + __slots__ = 'prev', 'next', 'key', '__weakref__' + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as regular dictionaries. + + # The internal self.__map dict maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # The sentinel is in self.__hardroot with a weakref proxy in self.__root. + # The prev links are weakref proxies (to prevent circular references). + # Individual links are kept alive by the hard reference in self.__map. + # Those hard references disappear when a key is deleted from an OrderedDict. + + def __init__(*args, **kwds): + '''Initialize an ordered dictionary. The signature is the same as + regular dictionaries. Keyword argument order is preserved. + ''' + if not args: + raise TypeError("descriptor '__init__' of 'OrderedDict' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, + dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link at the end of the linked list, + # and the inherited dictionary is updated with the new key/value pair. + if key not in self: + self.__map[key] = link = Link() + root = self.__root + last = root.prev + link.prev, link.next, link.key = last, root, key + last.next = link + root.prev = proxy(link) + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which gets + # removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link = self.__map.pop(key) + link_prev = link.prev + link_next = link.next + link_prev.next = link_next + link_next.prev = link_prev + link.prev = None + link.next = None + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + # Traverse the linked list in order. + root = self.__root + curr = root.next + while curr is not root: + yield curr.key + curr = curr.next + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + # Traverse the linked list in reverse order. + root = self.__root + curr = root.prev + while curr is not root: + yield curr.key + curr = curr.prev + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + root = self.__root + root.prev = root.next = root + self.__map.clear() + dict.clear(self) + + def popitem(self, last=True): + '''Remove and return a (key, value) pair from the dictionary. + + Pairs are returned in LIFO order if last is true or FIFO order if false. + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root.prev + link_prev = link.prev + link_prev.next = root + root.prev = link_prev + else: + link = root.next + link_next = link.next + root.next = link_next + link_next.prev = root + key = link.key + del self.__map[key] + value = dict.pop(self, key) + return key, value + + def move_to_end(self, key, last=True): + '''Move an existing element to the end (or beginning if last is false). + + Raise KeyError if the element does not exist. + ''' + link = self.__map[key] + link_prev = link.prev + link_next = link.next + soft_link = link_next.prev + link_prev.next = link_next + link_next.prev = link_prev + root = self.__root + if last: + last = root.prev + link.prev = last + link.next = root + root.prev = soft_link + last.next = link + else: + first = root.next + link.prev = root + link.next = first + first.prev = soft_link + root.next = link + + def __sizeof__(self): + sizeof = _sys.getsizeof + n = len(self) + 1 # number of links including root + size = sizeof(self.__dict__) # instance dictionary + size += sizeof(self.__map) * 2 # internal dict and inherited dict + size += sizeof(self.__hardroot) * n # link objects + size += sizeof(self.__root) * n # proxy objects + return size + + update = __update = _collections_abc.MutableMapping.update + + def keys(self): + "D.keys() -> a set-like object providing a view on D's keys" + return _OrderedDictKeysView(self) + + def items(self): + "D.items() -> a set-like object providing a view on D's items" + return _OrderedDictItemsView(self) + + def values(self): + "D.values() -> an object providing a view on D's values" + return _OrderedDictValuesView(self) + + __ne__ = _collections_abc.MutableMapping.__ne__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise KeyError + is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + '''Insert key with a value of default if key is not in the dictionary. + + Return the value for key if key is in the dictionary, else default. + ''' + if key in self: + return self[key] + self[key] = default + return default + + @_recursive_repr() + def __repr__(self): + 'od.__repr__() <==> repr(od)' + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, list(self.items())) + + def __reduce__(self): + 'Return state information for pickling' + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + return self.__class__, (), inst_dict or None, None, iter(self.items()) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''Create a new ordered dictionary with keys from iterable and values set to value. + ''' + self = cls() + for key in iterable: + self[key] = value + return self + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return dict.__eq__(self, other) and all(map(_eq, self, other)) + return dict.__eq__(self, other) + + +try: + from _collections import OrderedDict +except ImportError: + # Leave the pure Python version in place. + pass + + +################################################################################ +### namedtuple +################################################################################ + +_nt_itemgetters = {} + +def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', ['x', 'y']) + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessible by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) + + """ + + # Validate the field names. At the user's option, either generate an error + # message or automatically replace the field name with a valid name. + if isinstance(field_names, str): + field_names = field_names.replace(',', ' ').split() + field_names = list(map(str, field_names)) + typename = _sys.intern(str(typename)) + + if rename: + seen = set() + for index, name in enumerate(field_names): + if (not name.isidentifier() + or _iskeyword(name) + or name.startswith('_') + or name in seen): + field_names[index] = f'_{index}' + seen.add(name) + + for name in [typename] + field_names: + if type(name) is not str: + raise TypeError('Type names and field names must be strings') + if not name.isidentifier(): + raise ValueError('Type names and field names must be valid ' + f'identifiers: {name!r}') + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a ' + f'keyword: {name!r}') + + seen = set() + for name in field_names: + if name.startswith('_') and not rename: + raise ValueError('Field names cannot start with an underscore: ' + f'{name!r}') + if name in seen: + raise ValueError(f'Encountered duplicate field name: {name!r}') + seen.add(name) + + field_defaults = {} + if defaults is not None: + defaults = tuple(defaults) + if len(defaults) > len(field_names): + raise TypeError('Got more default values than field names') + field_defaults = dict(reversed(list(zip(reversed(field_names), + reversed(defaults))))) + + # Variables used in the methods and docstrings + field_names = tuple(map(_sys.intern, field_names)) + num_fields = len(field_names) + arg_list = repr(field_names).replace("'", "")[1:-1] + repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' + tuple_new = tuple.__new__ + _len = len + + # Create all the named tuple methods to be added to the class namespace + + s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' + namespace = {'_tuple_new': tuple_new, '__name__': f'namedtuple_{typename}'} + # Note: exec() has the side-effect of interning the field names + exec(s, namespace) + __new__ = namespace['__new__'] + __new__.__doc__ = f'Create new instance of {typename}({arg_list})' + if defaults is not None: + __new__.__defaults__ = defaults + + @classmethod + def _make(cls, iterable): + result = tuple_new(cls, iterable) + if _len(result) != num_fields: + raise TypeError(f'Expected {num_fields} arguments, got {len(result)}') + return result + + _make.__func__.__doc__ = (f'Make a new {typename} object from a sequence ' + 'or iterable') + + def _replace(_self, **kwds): + result = _self._make(map(kwds.pop, field_names, _self)) + if kwds: + raise ValueError(f'Got unexpected field names: {list(kwds)!r}') + return result + + _replace.__doc__ = (f'Return a new {typename} object replacing specified ' + 'fields with new values') + + def __repr__(self): + 'Return a nicely formatted representation string' + return self.__class__.__name__ + repr_fmt % self + + def _asdict(self): + 'Return a new OrderedDict which maps field names to their values.' + return OrderedDict(zip(self._fields, self)) + + def __getnewargs__(self): + 'Return self as a plain tuple. Used by copy and pickle.' + return tuple(self) + + # Modify function metadata to help with introspection and debugging + + for method in (__new__, _make.__func__, _replace, + __repr__, _asdict, __getnewargs__): + method.__qualname__ = f'{typename}.{method.__name__}' + + # Build-up the class namespace dictionary + # and use type() to build the result class + class_namespace = { + '__doc__': f'{typename}({arg_list})', + '__slots__': (), + '_fields': field_names, + '_fields_defaults': field_defaults, + '__new__': __new__, + '_make': _make, + '_replace': _replace, + '__repr__': __repr__, + '_asdict': _asdict, + '__getnewargs__': __getnewargs__, + } + cache = _nt_itemgetters + for index, name in enumerate(field_names): + try: + itemgetter_object, doc = cache[index] + except KeyError: + itemgetter_object = _itemgetter(index) + doc = f'Alias for field number {index}' + cache[index] = itemgetter_object, doc + class_namespace[name] = property(itemgetter_object, doc=doc) + + result = type(typename, (tuple,), class_namespace) + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in environments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython), or where the user has + # specified a particular module. + if module is None: + try: + module = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + result.__module__ = module + + return result + + +######################################################################## +### Counter +######################################################################## + +def _count_elements(mapping, iterable): + 'Tally elements from the iterable.' + mapping_get = mapping.get + for elem in iterable: + mapping[elem] = mapping_get(elem, 0) + 1 + +try: # Load C helper function if available + from _collections import _count_elements +except ImportError: + pass + +class Counter(dict): + '''Dict subclass for counting hashable items. Sometimes called a bag + or multiset. Elements are stored as dictionary keys and their counts + are stored as dictionary values. + + >>> c = Counter('abcdeabcdabcaba') # count elements from a string + + >>> c.most_common(3) # three most common elements + [('a', 5), ('b', 4), ('c', 3)] + >>> sorted(c) # list all unique elements + ['a', 'b', 'c', 'd', 'e'] + >>> ''.join(sorted(c.elements())) # list elements with repetitions + 'aaaaabbbbcccdde' + >>> sum(c.values()) # total of all counts + 15 + + >>> c['a'] # count of letter 'a' + 5 + >>> for elem in 'shazam': # update counts from an iterable + ... c[elem] += 1 # by adding 1 to each element's count + >>> c['a'] # now there are seven 'a' + 7 + >>> del c['b'] # remove all 'b' + >>> c['b'] # now there are zero 'b' + 0 + + >>> d = Counter('simsalabim') # make another counter + >>> c.update(d) # add in the second counter + >>> c['a'] # now there are nine 'a' + 9 + + >>> c.clear() # empty the counter + >>> c + Counter() + + Note: If a count is set to zero or reduced to zero, it will remain + in the counter until the entry is deleted or the counter is cleared: + + >>> c = Counter('aaabbc') + >>> c['b'] -= 2 # reduce the count of 'b' by two + >>> c.most_common() # 'b' is still in, but its count is zero + [('a', 3), ('c', 1), ('b', 0)] + + ''' + # References: + # http://en.wikipedia.org/wiki/Multiset + # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html + # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm + # http://code.activestate.com/recipes/259174/ + # Knuth, TAOCP Vol. II section 4.6.3 + + def __init__(*args, **kwds): + '''Create a new, empty Counter object. And if given, count elements + from an input iterable. Or, initialize the count from another mapping + of elements to their counts. + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping + >>> c = Counter(a=4, b=2) # a new counter from keyword args + + ''' + if not args: + raise TypeError("descriptor '__init__' of 'Counter' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + super(Counter, self).__init__() + self.update(*args, **kwds) + + def __missing__(self, key): + 'The count of elements not in the Counter is zero.' + # Needed so that self[missing_item] does not raise KeyError + return 0 + + def most_common(self, n=None): + '''List the n most common elements and their counts from the most + common to the least. If n is None, then list all element counts. + + >>> Counter('abcdeabcdabcaba').most_common(3) + [('a', 5), ('b', 4), ('c', 3)] + + ''' + # Emulate Bag.sortedByCount from Smalltalk + if n is None: + return sorted(self.items(), key=_itemgetter(1), reverse=True) + return _heapq.nlargest(n, self.items(), key=_itemgetter(1)) + + def elements(self): + '''Iterator over elements repeating each as many times as its count. + + >>> c = Counter('ABCABC') + >>> sorted(c.elements()) + ['A', 'A', 'B', 'B', 'C', 'C'] + + # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1 + >>> prime_factors = Counter({2: 2, 3: 3, 17: 1}) + >>> product = 1 + >>> for factor in prime_factors.elements(): # loop over factors + ... product *= factor # and multiply them + >>> product + 1836 + + Note, if an element's count has been set to zero or is a negative + number, elements() will ignore it. + + ''' + # Emulate Bag.do from Smalltalk and Multiset.begin from C++. + return _chain.from_iterable(_starmap(_repeat, self.items())) + + # Override dict methods where necessary + + @classmethod + def fromkeys(cls, iterable, v=None): + # There is no equivalent method for counters because setting v=1 + # means that no element can have a count greater than one. + raise NotImplementedError( + 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') + + def update(*args, **kwds): + '''Like dict.update() but add counts instead of replacing them. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.update('witch') # add elements from another iterable + >>> d = Counter('watch') + >>> c.update(d) # add elements from another counter + >>> c['h'] # four 'h' in which, witch, and watch + 4 + + ''' + # The regular dict.update() operation makes no sense here because the + # replace behavior results in the some of original untouched counts + # being mixed-in with all of the other counts for a mismash that + # doesn't have a straight-forward interpretation in most counting + # contexts. Instead, we implement straight-addition. Both the inputs + # and outputs are allowed to contain zero and negative counts. + + if not args: + raise TypeError("descriptor 'update' of 'Counter' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + iterable = args[0] if args else None + if iterable is not None: + if isinstance(iterable, _collections_abc.Mapping): + if self: + self_get = self.get + for elem, count in iterable.items(): + self[elem] = count + self_get(elem, 0) + else: + super(Counter, self).update(iterable) # fast path when counter is empty + else: + _count_elements(self, iterable) + if kwds: + self.update(kwds) + + def subtract(*args, **kwds): + '''Like dict.update() but subtracts counts instead of replacing them. + Counts can be reduced below zero. Both the inputs and outputs are + allowed to contain zero and negative counts. + + Source can be an iterable, a dictionary, or another Counter instance. + + >>> c = Counter('which') + >>> c.subtract('witch') # subtract elements from another iterable + >>> c.subtract(Counter('watch')) # subtract elements from another counter + >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch + 0 + >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch + -1 + + ''' + if not args: + raise TypeError("descriptor 'subtract' of 'Counter' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + iterable = args[0] if args else None + if iterable is not None: + self_get = self.get + if isinstance(iterable, _collections_abc.Mapping): + for elem, count in iterable.items(): + self[elem] = self_get(elem, 0) - count + else: + for elem in iterable: + self[elem] = self_get(elem, 0) - 1 + if kwds: + self.subtract(kwds) + + def copy(self): + 'Return a shallow copy.' + return self.__class__(self) + + def __reduce__(self): + return self.__class__, (dict(self),) + + def __delitem__(self, elem): + 'Like dict.__delitem__() but does not raise KeyError for missing values.' + if elem in self: + super().__delitem__(elem) + + def __repr__(self): + if not self: + return '%s()' % self.__class__.__name__ + try: + items = ', '.join(map('%r: %r'.__mod__, self.most_common())) + return '%s({%s})' % (self.__class__.__name__, items) + except TypeError: + # handle case where values are not orderable + return '{0}({1!r})'.format(self.__class__.__name__, dict(self)) + + # Multiset-style mathematical operations discussed in: + # Knuth TAOCP Volume II section 4.6.3 exercise 19 + # and at http://en.wikipedia.org/wiki/Multiset + # + # Outputs guaranteed to only include positive counts. + # + # To strip negative and zero counts, add-in an empty counter: + # c += Counter() + + def __add__(self, other): + '''Add counts from two counters. + + >>> Counter('abbb') + Counter('bcc') + Counter({'b': 4, 'c': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = count + other[elem] + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count + return result + + def __sub__(self, other): + ''' Subtract count, but keep only results with positive counts. + + >>> Counter('abbbc') - Counter('bccd') + Counter({'b': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = count - other[elem] + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count < 0: + result[elem] = 0 - count + return result + + def __or__(self, other): + '''Union is the maximum of value in either of the input counters. + + >>> Counter('abbb') | Counter('bcc') + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + other_count = other[elem] + newcount = other_count if count < other_count else count + if newcount > 0: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count > 0: + result[elem] = count + return result + + def __and__(self, other): + ''' Intersection is the minimum of corresponding counts. + + >>> Counter('abbb') & Counter('bcc') + Counter({'b': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + other_count = other[elem] + newcount = count if count < other_count else other_count + if newcount > 0: + result[elem] = newcount + return result + + def __pos__(self): + 'Adds an empty counter, effectively stripping negative and zero counts' + result = Counter() + for elem, count in self.items(): + if count > 0: + result[elem] = count + return result + + def __neg__(self): + '''Subtracts from an empty counter. Strips positive and zero counts, + and flips the sign on negative counts. + + ''' + result = Counter() + for elem, count in self.items(): + if count < 0: + result[elem] = 0 - count + return result + + def _keep_positive(self): + '''Internal method to strip elements with a negative or zero count''' + nonpositive = [elem for elem, count in self.items() if not count > 0] + for elem in nonpositive: + del self[elem] + return self + + def __iadd__(self, other): + '''Inplace add from another counter, keeping only positive counts. + + >>> c = Counter('abbb') + >>> c += Counter('bcc') + >>> c + Counter({'b': 4, 'c': 2, 'a': 1}) + + ''' + for elem, count in other.items(): + self[elem] += count + return self._keep_positive() + + def __isub__(self, other): + '''Inplace subtract counter, but keep only results with positive counts. + + >>> c = Counter('abbbc') + >>> c -= Counter('bccd') + >>> c + Counter({'b': 2, 'a': 1}) + + ''' + for elem, count in other.items(): + self[elem] -= count + return self._keep_positive() + + def __ior__(self, other): + '''Inplace union is the maximum of value from either counter. + + >>> c = Counter('abbb') + >>> c |= Counter('bcc') + >>> c + Counter({'b': 3, 'c': 2, 'a': 1}) + + ''' + for elem, other_count in other.items(): + count = self[elem] + if other_count > count: + self[elem] = other_count + return self._keep_positive() + + def __iand__(self, other): + '''Inplace intersection is the minimum of corresponding counts. + + >>> c = Counter('abbb') + >>> c &= Counter('bcc') + >>> c + Counter({'b': 1}) + + ''' + for elem, count in self.items(): + other_count = other[elem] + if other_count < count: + self[elem] = other_count + return self._keep_positive() + + +######################################################################## +### ChainMap +######################################################################## + +class ChainMap(_collections_abc.MutableMapping): + ''' A ChainMap groups multiple dicts (or other mappings) together + to create a single, updateable view. + + The underlying mappings are stored in a list. That list is public and can + be accessed or updated using the *maps* attribute. There is no other + state. + + Lookups search the underlying mappings successively until a key is found. + In contrast, writes, updates, and deletions only operate on the first + mapping. + + ''' + + def __init__(self, *maps): + '''Initialize a ChainMap by setting *maps* to the given mappings. + If no mappings are provided, a single empty dictionary is used. + + ''' + self.maps = list(maps) or [{}] # always at least one map + + def __missing__(self, key): + raise KeyError(key) + + def __getitem__(self, key): + for mapping in self.maps: + try: + return mapping[key] # can't use 'key in mapping' with defaultdict + except KeyError: + pass + return self.__missing__(key) # support subclasses that define __missing__ + + def get(self, key, default=None): + return self[key] if key in self else default + + def __len__(self): + return len(set().union(*self.maps)) # reuses stored hash values if possible + + def __iter__(self): + d = {} + for mapping in reversed(self.maps): + d.update(mapping) # reuses stored hash values if possible + return iter(d) + + def __contains__(self, key): + return any(key in m for m in self.maps) + + def __bool__(self): + return any(self.maps) + + @_recursive_repr() + def __repr__(self): + return '{0.__class__.__name__}({1})'.format( + self, ', '.join(map(repr, self.maps))) + + @classmethod + def fromkeys(cls, iterable, *args): + 'Create a ChainMap with a single dict created from the iterable.' + return cls(dict.fromkeys(iterable, *args)) + + def copy(self): + 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]' + return self.__class__(self.maps[0].copy(), *self.maps[1:]) + + __copy__ = copy + + def new_child(self, m=None): # like Django's Context.push() + '''New ChainMap with a new map followed by all previous maps. + If no map is provided, an empty dict is used. + ''' + if m is None: + m = {} + return self.__class__(m, *self.maps) + + @property + def parents(self): # like Django's Context.pop() + 'New ChainMap from maps[1:].' + return self.__class__(*self.maps[1:]) + + def __setitem__(self, key, value): + self.maps[0][key] = value + + def __delitem__(self, key): + try: + del self.maps[0][key] + except KeyError: + raise KeyError('Key not found in the first mapping: {!r}'.format(key)) + + def popitem(self): + 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.' + try: + return self.maps[0].popitem() + except KeyError: + raise KeyError('No keys found in the first mapping.') + + def pop(self, key, *args): + 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].' + try: + return self.maps[0].pop(key, *args) + except KeyError: + raise KeyError('Key not found in the first mapping: {!r}'.format(key)) + + def clear(self): + 'Clear maps[0], leaving maps[1:] intact.' + self.maps[0].clear() + + +################################################################################ +### UserDict +################################################################################ + +class UserDict(_collections_abc.MutableMapping): + + # Start by filling-out the abstract methods + def __init__(*args, **kwargs): + if not args: + raise TypeError("descriptor '__init__' of 'UserDict' object " + "needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + if args: + dict = args[0] + elif 'dict' in kwargs: + dict = kwargs.pop('dict') + import warnings + warnings.warn("Passing 'dict' as keyword argument is deprecated", + DeprecationWarning, stacklevel=2) + else: + dict = None + self.data = {} + if dict is not None: + self.update(dict) + if len(kwargs): + self.update(kwargs) + def __len__(self): return len(self.data) + def __getitem__(self, key): + if key in self.data: + return self.data[key] + if hasattr(self.__class__, "__missing__"): + return self.__class__.__missing__(self, key) + raise KeyError(key) + def __setitem__(self, key, item): self.data[key] = item + def __delitem__(self, key): del self.data[key] + def __iter__(self): + return iter(self.data) + + # Modify __contains__ to work correctly when __missing__ is present + def __contains__(self, key): + return key in self.data + + # Now, add the methods in dicts but not in MutableMapping + def __repr__(self): return repr(self.data) + def copy(self): + if self.__class__ is UserDict: + return UserDict(self.data.copy()) + import copy + data = self.data + try: + self.data = {} + c = copy.copy(self) + finally: + self.data = data + c.update(self) + return c + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + + +################################################################################ +### UserList +################################################################################ + +class UserList(_collections_abc.MutableSequence): + """A more or less complete user-defined wrapper around list objects.""" + def __init__(self, initlist=None): + self.data = [] + if initlist is not None: + # XXX should this accept an arbitrary sequence? + if type(initlist) == type(self.data): + self.data[:] = initlist + elif isinstance(initlist, UserList): + self.data[:] = initlist.data[:] + else: + self.data = list(initlist) + def __repr__(self): return repr(self.data) + def __lt__(self, other): return self.data < self.__cast(other) + def __le__(self, other): return self.data <= self.__cast(other) + def __eq__(self, other): return self.data == self.__cast(other) + def __gt__(self, other): return self.data > self.__cast(other) + def __ge__(self, other): return self.data >= self.__cast(other) + def __cast(self, other): + return other.data if isinstance(other, UserList) else other + def __contains__(self, item): return item in self.data + def __len__(self): return len(self.data) + def __getitem__(self, i): return self.data[i] + def __setitem__(self, i, item): self.data[i] = item + def __delitem__(self, i): del self.data[i] + def __add__(self, other): + if isinstance(other, UserList): + return self.__class__(self.data + other.data) + elif isinstance(other, type(self.data)): + return self.__class__(self.data + other) + return self.__class__(self.data + list(other)) + def __radd__(self, other): + if isinstance(other, UserList): + return self.__class__(other.data + self.data) + elif isinstance(other, type(self.data)): + return self.__class__(other + self.data) + return self.__class__(list(other) + self.data) + def __iadd__(self, other): + if isinstance(other, UserList): + self.data += other.data + elif isinstance(other, type(self.data)): + self.data += other + else: + self.data += list(other) + return self + def __mul__(self, n): + return self.__class__(self.data*n) + __rmul__ = __mul__ + def __imul__(self, n): + self.data *= n + return self + def append(self, item): self.data.append(item) + def insert(self, i, item): self.data.insert(i, item) + def pop(self, i=-1): return self.data.pop(i) + def remove(self, item): self.data.remove(item) + def clear(self): self.data.clear() + def copy(self): return self.__class__(self) + def count(self, item): return self.data.count(item) + def index(self, item, *args): return self.data.index(item, *args) + def reverse(self): self.data.reverse() + def sort(self, *args, **kwds): self.data.sort(*args, **kwds) + def extend(self, other): + if isinstance(other, UserList): + self.data.extend(other.data) + else: + self.data.extend(other) + + + +################################################################################ +### UserString +################################################################################ + +class UserString(_collections_abc.Sequence): + def __init__(self, seq): + if isinstance(seq, str): + self.data = seq + elif isinstance(seq, UserString): + self.data = seq.data[:] + else: + self.data = str(seq) + def __str__(self): return str(self.data) + def __repr__(self): return repr(self.data) + def __int__(self): return int(self.data) + def __float__(self): return float(self.data) + def __complex__(self): return complex(self.data) + def __hash__(self): return hash(self.data) + def __getnewargs__(self): + return (self.data[:],) + + def __eq__(self, string): + if isinstance(string, UserString): + return self.data == string.data + return self.data == string + def __lt__(self, string): + if isinstance(string, UserString): + return self.data < string.data + return self.data < string + def __le__(self, string): + if isinstance(string, UserString): + return self.data <= string.data + return self.data <= string + def __gt__(self, string): + if isinstance(string, UserString): + return self.data > string.data + return self.data > string + def __ge__(self, string): + if isinstance(string, UserString): + return self.data >= string.data + return self.data >= string + + def __contains__(self, char): + if isinstance(char, UserString): + char = char.data + return char in self.data + + def __len__(self): return len(self.data) + def __getitem__(self, index): return self.__class__(self.data[index]) + def __add__(self, other): + if isinstance(other, UserString): + return self.__class__(self.data + other.data) + elif isinstance(other, str): + return self.__class__(self.data + other) + return self.__class__(self.data + str(other)) + def __radd__(self, other): + if isinstance(other, str): + return self.__class__(other + self.data) + return self.__class__(str(other) + self.data) + def __mul__(self, n): + return self.__class__(self.data*n) + __rmul__ = __mul__ + def __mod__(self, args): + return self.__class__(self.data % args) + def __rmod__(self, format): + return self.__class__(format % args) + + # the following methods are defined in alphabetical order: + def capitalize(self): return self.__class__(self.data.capitalize()) + def casefold(self): + return self.__class__(self.data.casefold()) + def center(self, width, *args): + return self.__class__(self.data.center(width, *args)) + def count(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.count(sub, start, end) + def encode(self, encoding=None, errors=None): # XXX improve this? + if encoding: + if errors: + return self.__class__(self.data.encode(encoding, errors)) + return self.__class__(self.data.encode(encoding)) + return self.__class__(self.data.encode()) + def endswith(self, suffix, start=0, end=_sys.maxsize): + return self.data.endswith(suffix, start, end) + def expandtabs(self, tabsize=8): + return self.__class__(self.data.expandtabs(tabsize)) + def find(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.find(sub, start, end) + def format(self, *args, **kwds): + return self.data.format(*args, **kwds) + def format_map(self, mapping): + return self.data.format_map(mapping) + def index(self, sub, start=0, end=_sys.maxsize): + return self.data.index(sub, start, end) + def isalpha(self): return self.data.isalpha() + def isalnum(self): return self.data.isalnum() + def isascii(self): return self.data.isascii() + def isdecimal(self): return self.data.isdecimal() + def isdigit(self): return self.data.isdigit() + def isidentifier(self): return self.data.isidentifier() + def islower(self): return self.data.islower() + def isnumeric(self): return self.data.isnumeric() + def isprintable(self): return self.data.isprintable() + def isspace(self): return self.data.isspace() + def istitle(self): return self.data.istitle() + def isupper(self): return self.data.isupper() + def join(self, seq): return self.data.join(seq) + def ljust(self, width, *args): + return self.__class__(self.data.ljust(width, *args)) + def lower(self): return self.__class__(self.data.lower()) + def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars)) + maketrans = str.maketrans + def partition(self, sep): + return self.data.partition(sep) + def replace(self, old, new, maxsplit=-1): + if isinstance(old, UserString): + old = old.data + if isinstance(new, UserString): + new = new.data + return self.__class__(self.data.replace(old, new, maxsplit)) + def rfind(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data + return self.data.rfind(sub, start, end) + def rindex(self, sub, start=0, end=_sys.maxsize): + return self.data.rindex(sub, start, end) + def rjust(self, width, *args): + return self.__class__(self.data.rjust(width, *args)) + def rpartition(self, sep): + return self.data.rpartition(sep) + def rstrip(self, chars=None): + return self.__class__(self.data.rstrip(chars)) + def split(self, sep=None, maxsplit=-1): + return self.data.split(sep, maxsplit) + def rsplit(self, sep=None, maxsplit=-1): + return self.data.rsplit(sep, maxsplit) + def splitlines(self, keepends=False): return self.data.splitlines(keepends) + def startswith(self, prefix, start=0, end=_sys.maxsize): + return self.data.startswith(prefix, start, end) + def strip(self, chars=None): return self.__class__(self.data.strip(chars)) + def swapcase(self): return self.__class__(self.data.swapcase()) + def title(self): return self.__class__(self.data.title()) + def translate(self, *args): + return self.__class__(self.data.translate(*args)) + def upper(self): return self.__class__(self.data.upper()) + def zfill(self, width): return self.__class__(self.data.zfill(width)) diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py new file mode 100644 index 0000000000..891600d16b --- /dev/null +++ b/Lib/collections/abc.py @@ -0,0 +1,2 @@ +from _collections_abc import * +from _collections_abc import __all__ diff --git a/Lib/difflib.py b/Lib/difflib.py new file mode 100644 index 0000000000..887c3c26ca --- /dev/null +++ b/Lib/difflib.py @@ -0,0 +1,2097 @@ +""" +Module difflib -- helpers for computing deltas between objects. + +Function get_close_matches(word, possibilities, n=3, cutoff=0.6): + Use SequenceMatcher to return list of the best "good enough" matches. + +Function context_diff(a, b): + For two lists of strings, return a delta in context diff format. + +Function ndiff(a, b): + Return a delta: the difference between `a` and `b` (lists of strings). + +Function restore(delta, which): + Return one of the two sequences that generated an ndiff delta. + +Function unified_diff(a, b): + For two lists of strings, return a delta in unified diff format. + +Class SequenceMatcher: + A flexible class for comparing pairs of sequences of any type. + +Class Differ: + For producing human-readable deltas from sequences of lines of text. + +Class HtmlDiff: + For producing HTML side by side comparison with change highlights. +""" + +__all__ = ['get_close_matches', 'ndiff', 'restore', 'SequenceMatcher', + 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', + 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] + +from heapq import nlargest as _nlargest +from collections import namedtuple as _namedtuple + +Match = _namedtuple('Match', 'a b size') + +def _calculate_ratio(matches, length): + if length: + return 2.0 * matches / length + return 1.0 + +class SequenceMatcher: + + """ + SequenceMatcher is a flexible class for comparing pairs of sequences of + any type, so long as the sequence elements are hashable. The basic + algorithm predates, and is a little fancier than, an algorithm + published in the late 1980's by Ratcliff and Obershelp under the + hyperbolic name "gestalt pattern matching". The basic idea is to find + the longest contiguous matching subsequence that contains no "junk" + elements (R-O doesn't address junk). The same idea is then applied + recursively to the pieces of the sequences to the left and to the right + of the matching subsequence. This does not yield minimal edit + sequences, but does tend to yield matches that "look right" to people. + + SequenceMatcher tries to compute a "human-friendly diff" between two + sequences. Unlike e.g. UNIX(tm) diff, the fundamental notion is the + longest *contiguous* & junk-free matching subsequence. That's what + catches peoples' eyes. The Windows(tm) windiff has another interesting + notion, pairing up elements that appear uniquely in each sequence. + That, and the method here, appear to yield more intuitive difference + reports than does diff. This method appears to be the least vulnerable + to synching up on blocks of "junk lines", though (like blank lines in + ordinary text files, or maybe "

" lines in HTML files). That may be + because this is the only method of the 3 that has a *concept* of + "junk" . + + Example, comparing two strings, and considering blanks to be "junk": + + >>> s = SequenceMatcher(lambda x: x == " ", + ... "private Thread currentThread;", + ... "private volatile Thread currentThread;") + >>> + + .ratio() returns a float in [0, 1], measuring the "similarity" of the + sequences. As a rule of thumb, a .ratio() value over 0.6 means the + sequences are close matches: + + >>> print(round(s.ratio(), 3)) + 0.866 + >>> + + If you're only interested in where the sequences match, + .get_matching_blocks() is handy: + + >>> for block in s.get_matching_blocks(): + ... print("a[%d] and b[%d] match for %d elements" % block) + a[0] and b[0] match for 8 elements + a[8] and b[17] match for 21 elements + a[29] and b[38] match for 0 elements + + Note that the last tuple returned by .get_matching_blocks() is always a + dummy, (len(a), len(b), 0), and this is the only case in which the last + tuple element (number of elements matched) is 0. + + If you want to know how to change the first sequence into the second, + use .get_opcodes(): + + >>> for opcode in s.get_opcodes(): + ... print("%6s a[%d:%d] b[%d:%d]" % opcode) + equal a[0:8] b[0:8] + insert a[8:8] b[8:17] + equal a[8:29] b[17:38] + + See the Differ class for a fancy human-friendly file differencer, which + uses SequenceMatcher both to compare sequences of lines, and to compare + sequences of characters within similar (near-matching) lines. + + See also function get_close_matches() in this module, which shows how + simple code building on SequenceMatcher can be used to do useful work. + + Timing: Basic R-O is cubic time worst case and quadratic time expected + case. SequenceMatcher is quadratic time for the worst case and has + expected-case behavior dependent in a complicated way on how many + elements the sequences have in common; best case time is linear. + + Methods: + + __init__(isjunk=None, a='', b='') + Construct a SequenceMatcher. + + set_seqs(a, b) + Set the two sequences to be compared. + + set_seq1(a) + Set the first sequence to be compared. + + set_seq2(b) + Set the second sequence to be compared. + + find_longest_match(alo, ahi, blo, bhi) + Find longest matching block in a[alo:ahi] and b[blo:bhi]. + + get_matching_blocks() + Return list of triples describing matching subsequences. + + get_opcodes() + Return list of 5-tuples describing how to turn a into b. + + ratio() + Return a measure of the sequences' similarity (float in [0,1]). + + quick_ratio() + Return an upper bound on .ratio() relatively quickly. + + real_quick_ratio() + Return an upper bound on ratio() very quickly. + """ + + def __init__(self, isjunk=None, a='', b='', autojunk=True): + """Construct a SequenceMatcher. + + Optional arg isjunk is None (the default), or a one-argument + function that takes a sequence element and returns true iff the + element is junk. None is equivalent to passing "lambda x: 0", i.e. + no elements are considered to be junk. For example, pass + lambda x: x in " \\t" + if you're comparing lines as sequences of characters, and don't + want to synch up on blanks or hard tabs. + + Optional arg a is the first of two sequences to be compared. By + default, an empty string. The elements of a must be hashable. See + also .set_seqs() and .set_seq1(). + + Optional arg b is the second of two sequences to be compared. By + default, an empty string. The elements of b must be hashable. See + also .set_seqs() and .set_seq2(). + + Optional arg autojunk should be set to False to disable the + "automatic junk heuristic" that treats popular elements as junk + (see module documentation for more information). + """ + + # Members: + # a + # first sequence + # b + # second sequence; differences are computed as "what do + # we need to do to 'a' to change it into 'b'?" + # b2j + # for x in b, b2j[x] is a list of the indices (into b) + # at which x appears; junk and popular elements do not appear + # fullbcount + # for x in b, fullbcount[x] == the number of times x + # appears in b; only materialized if really needed (used + # only for computing quick_ratio()) + # matching_blocks + # a list of (i, j, k) triples, where a[i:i+k] == b[j:j+k]; + # ascending & non-overlapping in i and in j; terminated by + # a dummy (len(a), len(b), 0) sentinel + # opcodes + # a list of (tag, i1, i2, j1, j2) tuples, where tag is + # one of + # 'replace' a[i1:i2] should be replaced by b[j1:j2] + # 'delete' a[i1:i2] should be deleted + # 'insert' b[j1:j2] should be inserted + # 'equal' a[i1:i2] == b[j1:j2] + # isjunk + # a user-supplied function taking a sequence element and + # returning true iff the element is "junk" -- this has + # subtle but helpful effects on the algorithm, which I'll + # get around to writing up someday <0.9 wink>. + # DON'T USE! Only __chain_b uses this. Use "in self.bjunk". + # bjunk + # the items in b for which isjunk is True. + # bpopular + # nonjunk items in b treated as junk by the heuristic (if used). + + self.isjunk = isjunk + self.a = self.b = None + self.autojunk = autojunk + self.set_seqs(a, b) + + def set_seqs(self, a, b): + """Set the two sequences to be compared. + + >>> s = SequenceMatcher() + >>> s.set_seqs("abcd", "bcde") + >>> s.ratio() + 0.75 + """ + + self.set_seq1(a) + self.set_seq2(b) + + def set_seq1(self, a): + """Set the first sequence to be compared. + + The second sequence to be compared is not changed. + + >>> s = SequenceMatcher(None, "abcd", "bcde") + >>> s.ratio() + 0.75 + >>> s.set_seq1("bcde") + >>> s.ratio() + 1.0 + >>> + + SequenceMatcher computes and caches detailed information about the + second sequence, so if you want to compare one sequence S against + many sequences, use .set_seq2(S) once and call .set_seq1(x) + repeatedly for each of the other sequences. + + See also set_seqs() and set_seq2(). + """ + + if a is self.a: + return + self.a = a + self.matching_blocks = self.opcodes = None + + def set_seq2(self, b): + """Set the second sequence to be compared. + + The first sequence to be compared is not changed. + + >>> s = SequenceMatcher(None, "abcd", "bcde") + >>> s.ratio() + 0.75 + >>> s.set_seq2("abcd") + >>> s.ratio() + 1.0 + >>> + + SequenceMatcher computes and caches detailed information about the + second sequence, so if you want to compare one sequence S against + many sequences, use .set_seq2(S) once and call .set_seq1(x) + repeatedly for each of the other sequences. + + See also set_seqs() and set_seq1(). + """ + + if b is self.b: + return + self.b = b + self.matching_blocks = self.opcodes = None + self.fullbcount = None + self.__chain_b() + + # For each element x in b, set b2j[x] to a list of the indices in + # b where x appears; the indices are in increasing order; note that + # the number of times x appears in b is len(b2j[x]) ... + # when self.isjunk is defined, junk elements don't show up in this + # map at all, which stops the central find_longest_match method + # from starting any matching block at a junk element ... + # b2j also does not contain entries for "popular" elements, meaning + # elements that account for more than 1 + 1% of the total elements, and + # when the sequence is reasonably large (>= 200 elements); this can + # be viewed as an adaptive notion of semi-junk, and yields an enormous + # speedup when, e.g., comparing program files with hundreds of + # instances of "return NULL;" ... + # note that this is only called when b changes; so for cross-product + # kinds of matches, it's best to call set_seq2 once, then set_seq1 + # repeatedly + + def __chain_b(self): + # Because isjunk is a user-defined (not C) function, and we test + # for junk a LOT, it's important to minimize the number of calls. + # Before the tricks described here, __chain_b was by far the most + # time-consuming routine in the whole module! If anyone sees + # Jim Roskind, thank him again for profile.py -- I never would + # have guessed that. + # The first trick is to build b2j ignoring the possibility + # of junk. I.e., we don't call isjunk at all yet. Throwing + # out the junk later is much cheaper than building b2j "right" + # from the start. + b = self.b + self.b2j = b2j = {} + + for i, elt in enumerate(b): + indices = b2j.setdefault(elt, []) + indices.append(i) + + # Purge junk elements + self.bjunk = junk = set() + isjunk = self.isjunk + if isjunk: + for elt in b2j.keys(): + if isjunk(elt): + junk.add(elt) + for elt in junk: # separate loop avoids separate list of keys + del b2j[elt] + + # Purge popular elements that are not junk + self.bpopular = popular = set() + n = len(b) + if self.autojunk and n >= 200: + ntest = n // 100 + 1 + for elt, idxs in b2j.items(): + if len(idxs) > ntest: + popular.add(elt) + for elt in popular: # ditto; as fast for 1% deletion + del b2j[elt] + + def find_longest_match(self, alo, ahi, blo, bhi): + """Find longest matching block in a[alo:ahi] and b[blo:bhi]. + + If isjunk is not defined: + + Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where + alo <= i <= i+k <= ahi + blo <= j <= j+k <= bhi + and for all (i',j',k') meeting those conditions, + k >= k' + i <= i' + and if i == i', j <= j' + + In other words, of all maximal matching blocks, return one that + starts earliest in a, and of all those maximal matching blocks that + start earliest in a, return the one that starts earliest in b. + + >>> s = SequenceMatcher(None, " abcd", "abcd abcd") + >>> s.find_longest_match(0, 5, 0, 9) + Match(a=0, b=4, size=5) + + If isjunk is defined, first the longest matching block is + determined as above, but with the additional restriction that no + junk element appears in the block. Then that block is extended as + far as possible by matching (only) junk elements on both sides. So + the resulting block never matches on junk except as identical junk + happens to be adjacent to an "interesting" match. + + Here's the same example as before, but considering blanks to be + junk. That prevents " abcd" from matching the " abcd" at the tail + end of the second sequence directly. Instead only the "abcd" can + match, and matches the leftmost "abcd" in the second sequence: + + >>> s = SequenceMatcher(lambda x: x==" ", " abcd", "abcd abcd") + >>> s.find_longest_match(0, 5, 0, 9) + Match(a=1, b=0, size=4) + + If no blocks match, return (alo, blo, 0). + + >>> s = SequenceMatcher(None, "ab", "c") + >>> s.find_longest_match(0, 2, 0, 1) + Match(a=0, b=0, size=0) + """ + + # CAUTION: stripping common prefix or suffix would be incorrect. + # E.g., + # ab + # acab + # Longest matching block is "ab", but if common prefix is + # stripped, it's "a" (tied with "b"). UNIX(tm) diff does so + # strip, so ends up claiming that ab is changed to acab by + # inserting "ca" in the middle. That's minimal but unintuitive: + # "it's obvious" that someone inserted "ac" at the front. + # Windiff ends up at the same place as diff, but by pairing up + # the unique 'b's and then matching the first two 'a's. + + a, b, b2j, isbjunk = self.a, self.b, self.b2j, self.bjunk.__contains__ + besti, bestj, bestsize = alo, blo, 0 + # find longest junk-free match + # during an iteration of the loop, j2len[j] = length of longest + # junk-free match ending with a[i-1] and b[j] + j2len = {} + nothing = [] + for i in range(alo, ahi): + # look at all instances of a[i] in b; note that because + # b2j has no junk keys, the loop is skipped if a[i] is junk + j2lenget = j2len.get + newj2len = {} + for j in b2j.get(a[i], nothing): + # a[i] matches b[j] + if j < blo: + continue + if j >= bhi: + break + k = newj2len[j] = j2lenget(j-1, 0) + 1 + if k > bestsize: + besti, bestj, bestsize = i-k+1, j-k+1, k + j2len = newj2len + + # Extend the best by non-junk elements on each end. In particular, + # "popular" non-junk elements aren't in b2j, which greatly speeds + # the inner loop above, but also means "the best" match so far + # doesn't contain any junk *or* popular non-junk elements. + while besti > alo and bestj > blo and \ + not isbjunk(b[bestj-1]) and \ + a[besti-1] == b[bestj-1]: + besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 + while besti+bestsize < ahi and bestj+bestsize < bhi and \ + not isbjunk(b[bestj+bestsize]) and \ + a[besti+bestsize] == b[bestj+bestsize]: + bestsize += 1 + + # Now that we have a wholly interesting match (albeit possibly + # empty!), we may as well suck up the matching junk on each + # side of it too. Can't think of a good reason not to, and it + # saves post-processing the (possibly considerable) expense of + # figuring out what to do with it. In the case of an empty + # interesting match, this is clearly the right thing to do, + # because no other kind of match is possible in the regions. + while besti > alo and bestj > blo and \ + isbjunk(b[bestj-1]) and \ + a[besti-1] == b[bestj-1]: + besti, bestj, bestsize = besti-1, bestj-1, bestsize+1 + while besti+bestsize < ahi and bestj+bestsize < bhi and \ + isbjunk(b[bestj+bestsize]) and \ + a[besti+bestsize] == b[bestj+bestsize]: + bestsize = bestsize + 1 + + return Match(besti, bestj, bestsize) + + def get_matching_blocks(self): + """Return list of triples describing matching subsequences. + + Each triple is of the form (i, j, n), and means that + a[i:i+n] == b[j:j+n]. The triples are monotonically increasing in + i and in j. New in Python 2.5, it's also guaranteed that if + (i, j, n) and (i', j', n') are adjacent triples in the list, and + the second is not the last triple in the list, then i+n != i' or + j+n != j'. IOW, adjacent triples never describe adjacent equal + blocks. + + The last triple is a dummy, (len(a), len(b), 0), and is the only + triple with n==0. + + >>> s = SequenceMatcher(None, "abxcd", "abcd") + >>> list(s.get_matching_blocks()) + [Match(a=0, b=0, size=2), Match(a=3, b=2, size=2), Match(a=5, b=4, size=0)] + """ + + if self.matching_blocks is not None: + return self.matching_blocks + la, lb = len(self.a), len(self.b) + + # This is most naturally expressed as a recursive algorithm, but + # at least one user bumped into extreme use cases that exceeded + # the recursion limit on their box. So, now we maintain a list + # ('queue`) of blocks we still need to look at, and append partial + # results to `matching_blocks` in a loop; the matches are sorted + # at the end. + queue = [(0, la, 0, lb)] + matching_blocks = [] + while queue: + alo, ahi, blo, bhi = queue.pop() + i, j, k = x = self.find_longest_match(alo, ahi, blo, bhi) + # a[alo:i] vs b[blo:j] unknown + # a[i:i+k] same as b[j:j+k] + # a[i+k:ahi] vs b[j+k:bhi] unknown + if k: # if k is 0, there was no matching block + matching_blocks.append(x) + if alo < i and blo < j: + queue.append((alo, i, blo, j)) + if i+k < ahi and j+k < bhi: + queue.append((i+k, ahi, j+k, bhi)) + matching_blocks.sort() + + # It's possible that we have adjacent equal blocks in the + # matching_blocks list now. Starting with 2.5, this code was added + # to collapse them. + i1 = j1 = k1 = 0 + non_adjacent = [] + for i2, j2, k2 in matching_blocks: + # Is this block adjacent to i1, j1, k1? + if i1 + k1 == i2 and j1 + k1 == j2: + # Yes, so collapse them -- this just increases the length of + # the first block by the length of the second, and the first + # block so lengthened remains the block to compare against. + k1 += k2 + else: + # Not adjacent. Remember the first block (k1==0 means it's + # the dummy we started with), and make the second block the + # new block to compare against. + if k1: + non_adjacent.append((i1, j1, k1)) + i1, j1, k1 = i2, j2, k2 + if k1: + non_adjacent.append((i1, j1, k1)) + + non_adjacent.append( (la, lb, 0) ) + self.matching_blocks = list(map(Match._make, non_adjacent)) + return self.matching_blocks + + def get_opcodes(self): + """Return list of 5-tuples describing how to turn a into b. + + Each tuple is of the form (tag, i1, i2, j1, j2). The first tuple + has i1 == j1 == 0, and remaining tuples have i1 == the i2 from the + tuple preceding it, and likewise for j1 == the previous j2. + + The tags are strings, with these meanings: + + 'replace': a[i1:i2] should be replaced by b[j1:j2] + 'delete': a[i1:i2] should be deleted. + Note that j1==j2 in this case. + 'insert': b[j1:j2] should be inserted at a[i1:i1]. + Note that i1==i2 in this case. + 'equal': a[i1:i2] == b[j1:j2] + + >>> a = "qabxcd" + >>> b = "abycdf" + >>> s = SequenceMatcher(None, a, b) + >>> for tag, i1, i2, j1, j2 in s.get_opcodes(): + ... print(("%7s a[%d:%d] (%s) b[%d:%d] (%s)" % + ... (tag, i1, i2, a[i1:i2], j1, j2, b[j1:j2]))) + delete a[0:1] (q) b[0:0] () + equal a[1:3] (ab) b[0:2] (ab) + replace a[3:4] (x) b[2:3] (y) + equal a[4:6] (cd) b[3:5] (cd) + insert a[6:6] () b[5:6] (f) + """ + + if self.opcodes is not None: + return self.opcodes + i = j = 0 + self.opcodes = answer = [] + for ai, bj, size in self.get_matching_blocks(): + # invariant: we've pumped out correct diffs to change + # a[:i] into b[:j], and the next matching block is + # a[ai:ai+size] == b[bj:bj+size]. So we need to pump + # out a diff to change a[i:ai] into b[j:bj], pump out + # the matching block, and move (i,j) beyond the match + tag = '' + if i < ai and j < bj: + tag = 'replace' + elif i < ai: + tag = 'delete' + elif j < bj: + tag = 'insert' + if tag: + answer.append( (tag, i, ai, j, bj) ) + i, j = ai+size, bj+size + # the list of matching blocks is terminated by a + # sentinel with size 0 + if size: + answer.append( ('equal', ai, i, bj, j) ) + return answer + + def get_grouped_opcodes(self, n=3): + """ Isolate change clusters by eliminating ranges with no changes. + + Return a generator of groups with up to n lines of context. + Each group is in the same format as returned by get_opcodes(). + + >>> from pprint import pprint + >>> a = list(map(str, range(1,40))) + >>> b = a[:] + >>> b[8:8] = ['i'] # Make an insertion + >>> b[20] += 'x' # Make a replacement + >>> b[23:28] = [] # Make a deletion + >>> b[30] += 'y' # Make another replacement + >>> pprint(list(SequenceMatcher(None,a,b).get_grouped_opcodes())) + [[('equal', 5, 8, 5, 8), ('insert', 8, 8, 8, 9), ('equal', 8, 11, 9, 12)], + [('equal', 16, 19, 17, 20), + ('replace', 19, 20, 20, 21), + ('equal', 20, 22, 21, 23), + ('delete', 22, 27, 23, 23), + ('equal', 27, 30, 23, 26)], + [('equal', 31, 34, 27, 30), + ('replace', 34, 35, 30, 31), + ('equal', 35, 38, 31, 34)]] + """ + + codes = self.get_opcodes() + if not codes: + codes = [("equal", 0, 1, 0, 1)] + # Fixup leading and trailing groups if they show no changes. + if codes[0][0] == 'equal': + tag, i1, i2, j1, j2 = codes[0] + codes[0] = tag, max(i1, i2-n), i2, max(j1, j2-n), j2 + if codes[-1][0] == 'equal': + tag, i1, i2, j1, j2 = codes[-1] + codes[-1] = tag, i1, min(i2, i1+n), j1, min(j2, j1+n) + + nn = n + n + group = [] + for tag, i1, i2, j1, j2 in codes: + # End the current group and start a new one whenever + # there is a large range with no changes. + if tag == 'equal' and i2-i1 > nn: + group.append((tag, i1, min(i2, i1+n), j1, min(j2, j1+n))) + yield group + group = [] + i1, j1 = max(i1, i2-n), max(j1, j2-n) + group.append((tag, i1, i2, j1 ,j2)) + if group and not (len(group)==1 and group[0][0] == 'equal'): + yield group + + def ratio(self): + """Return a measure of the sequences' similarity (float in [0,1]). + + Where T is the total number of elements in both sequences, and + M is the number of matches, this is 2.0*M / T. + Note that this is 1 if the sequences are identical, and 0 if + they have nothing in common. + + .ratio() is expensive to compute if you haven't already computed + .get_matching_blocks() or .get_opcodes(), in which case you may + want to try .quick_ratio() or .real_quick_ratio() first to get an + upper bound. + + >>> s = SequenceMatcher(None, "abcd", "bcde") + >>> s.ratio() + 0.75 + >>> s.quick_ratio() + 0.75 + >>> s.real_quick_ratio() + 1.0 + """ + + matches = sum(triple[-1] for triple in self.get_matching_blocks()) + return _calculate_ratio(matches, len(self.a) + len(self.b)) + + def quick_ratio(self): + """Return an upper bound on ratio() relatively quickly. + + This isn't defined beyond that it is an upper bound on .ratio(), and + is faster to compute. + """ + + # viewing a and b as multisets, set matches to the cardinality + # of their intersection; this counts the number of matches + # without regard to order, so is clearly an upper bound + if self.fullbcount is None: + self.fullbcount = fullbcount = {} + for elt in self.b: + fullbcount[elt] = fullbcount.get(elt, 0) + 1 + fullbcount = self.fullbcount + # avail[x] is the number of times x appears in 'b' less the + # number of times we've seen it in 'a' so far ... kinda + avail = {} + availhas, matches = avail.__contains__, 0 + for elt in self.a: + if availhas(elt): + numb = avail[elt] + else: + numb = fullbcount.get(elt, 0) + avail[elt] = numb - 1 + if numb > 0: + matches = matches + 1 + return _calculate_ratio(matches, len(self.a) + len(self.b)) + + def real_quick_ratio(self): + """Return an upper bound on ratio() very quickly. + + This isn't defined beyond that it is an upper bound on .ratio(), and + is faster to compute than either .ratio() or .quick_ratio(). + """ + + la, lb = len(self.a), len(self.b) + # can't have more matches than the number of elements in the + # shorter sequence + return _calculate_ratio(min(la, lb), la + lb) + +def get_close_matches(word, possibilities, n=3, cutoff=0.6): + """Use SequenceMatcher to return list of the best "good enough" matches. + + word is a sequence for which close matches are desired (typically a + string). + + possibilities is a list of sequences against which to match word + (typically a list of strings). + + Optional arg n (default 3) is the maximum number of close matches to + return. n must be > 0. + + Optional arg cutoff (default 0.6) is a float in [0, 1]. Possibilities + that don't score at least that similar to word are ignored. + + The best (no more than n) matches among the possibilities are returned + in a list, sorted by similarity score, most similar first. + + >>> get_close_matches("appel", ["ape", "apple", "peach", "puppy"]) + ['apple', 'ape'] + >>> import keyword as _keyword + >>> get_close_matches("wheel", _keyword.kwlist) + ['while'] + >>> get_close_matches("Apple", _keyword.kwlist) + [] + >>> get_close_matches("accept", _keyword.kwlist) + ['except'] + """ + + if not n > 0: + raise ValueError("n must be > 0: %r" % (n,)) + if not 0.0 <= cutoff <= 1.0: + raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) + result = [] + s = SequenceMatcher() + s.set_seq2(word) + for x in possibilities: + s.set_seq1(x) + if s.real_quick_ratio() >= cutoff and \ + s.quick_ratio() >= cutoff and \ + s.ratio() >= cutoff: + result.append((s.ratio(), x)) + + # Move the best scorers to head of list + result = _nlargest(n, result) + # Strip scores for the best n matches + return [x for score, x in result] + +def _count_leading(line, ch): + """ + Return number of `ch` characters at the start of `line`. + + Example: + + >>> _count_leading(' abc', ' ') + 3 + """ + + i, n = 0, len(line) + while i < n and line[i] == ch: + i += 1 + return i + +class Differ: + r""" + Differ is a class for comparing sequences of lines of text, and + producing human-readable differences or deltas. Differ uses + SequenceMatcher both to compare sequences of lines, and to compare + sequences of characters within similar (near-matching) lines. + + Each line of a Differ delta begins with a two-letter code: + + '- ' line unique to sequence 1 + '+ ' line unique to sequence 2 + ' ' line common to both sequences + '? ' line not present in either input sequence + + Lines beginning with '? ' attempt to guide the eye to intraline + differences, and were not present in either input sequence. These lines + can be confusing if the sequences contain tab characters. + + Note that Differ makes no claim to produce a *minimal* diff. To the + contrary, minimal diffs are often counter-intuitive, because they synch + up anywhere possible, sometimes accidental matches 100 pages apart. + Restricting synch points to contiguous matches preserves some notion of + locality, at the occasional cost of producing a longer diff. + + Example: Comparing two texts. + + First we set up the texts, sequences of individual single-line strings + ending with newlines (such sequences can also be obtained from the + `readlines()` method of file-like objects): + + >>> text1 = ''' 1. Beautiful is better than ugly. + ... 2. Explicit is better than implicit. + ... 3. Simple is better than complex. + ... 4. Complex is better than complicated. + ... '''.splitlines(keepends=True) + >>> len(text1) + 4 + >>> text1[0][-1] + '\n' + >>> text2 = ''' 1. Beautiful is better than ugly. + ... 3. Simple is better than complex. + ... 4. Complicated is better than complex. + ... 5. Flat is better than nested. + ... '''.splitlines(keepends=True) + + Next we instantiate a Differ object: + + >>> d = Differ() + + Note that when instantiating a Differ object we may pass functions to + filter out line and character 'junk'. See Differ.__init__ for details. + + Finally, we compare the two: + + >>> result = list(d.compare(text1, text2)) + + 'result' is a list of strings, so let's pretty-print it: + + >>> from pprint import pprint as _pprint + >>> _pprint(result) + [' 1. Beautiful is better than ugly.\n', + '- 2. Explicit is better than implicit.\n', + '- 3. Simple is better than complex.\n', + '+ 3. Simple is better than complex.\n', + '? ++\n', + '- 4. Complex is better than complicated.\n', + '? ^ ---- ^\n', + '+ 4. Complicated is better than complex.\n', + '? ++++ ^ ^\n', + '+ 5. Flat is better than nested.\n'] + + As a single multi-line string it looks like this: + + >>> print(''.join(result), end="") + 1. Beautiful is better than ugly. + - 2. Explicit is better than implicit. + - 3. Simple is better than complex. + + 3. Simple is better than complex. + ? ++ + - 4. Complex is better than complicated. + ? ^ ---- ^ + + 4. Complicated is better than complex. + ? ++++ ^ ^ + + 5. Flat is better than nested. + + Methods: + + __init__(linejunk=None, charjunk=None) + Construct a text differencer, with optional filters. + + compare(a, b) + Compare two sequences of lines; generate the resulting delta. + """ + + def __init__(self, linejunk=None, charjunk=None): + """ + Construct a text differencer, with optional filters. + + The two optional keyword parameters are for filter functions: + + - `linejunk`: A function that should accept a single string argument, + and return true iff the string is junk. The module-level function + `IS_LINE_JUNK` may be used to filter out lines without visible + characters, except for at most one splat ('#'). It is recommended + to leave linejunk None; the underlying SequenceMatcher class has + an adaptive notion of "noise" lines that's better than any static + definition the author has ever been able to craft. + + - `charjunk`: A function that should accept a string of length 1. The + module-level function `IS_CHARACTER_JUNK` may be used to filter out + whitespace characters (a blank or tab; **note**: bad idea to include + newline in this!). Use of IS_CHARACTER_JUNK is recommended. + """ + + self.linejunk = linejunk + self.charjunk = charjunk + + def compare(self, a, b): + r""" + Compare two sequences of lines; generate the resulting delta. + + Each sequence must contain individual single-line strings ending with + newlines. Such sequences can be obtained from the `readlines()` method + of file-like objects. The delta generated also consists of newline- + terminated strings, ready to be printed as-is via the writeline() + method of a file-like object. + + Example: + + >>> print(''.join(Differ().compare('one\ntwo\nthree\n'.splitlines(True), + ... 'ore\ntree\nemu\n'.splitlines(True))), + ... end="") + - one + ? ^ + + ore + ? ^ + - two + - three + ? - + + tree + + emu + """ + + cruncher = SequenceMatcher(self.linejunk, a, b) + for tag, alo, ahi, blo, bhi in cruncher.get_opcodes(): + if tag == 'replace': + g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + elif tag == 'delete': + g = self._dump('-', a, alo, ahi) + elif tag == 'insert': + g = self._dump('+', b, blo, bhi) + elif tag == 'equal': + g = self._dump(' ', a, alo, ahi) + else: + raise ValueError('unknown tag %r' % (tag,)) + + yield from g + + def _dump(self, tag, x, lo, hi): + """Generate comparison results for a same-tagged range.""" + for i in range(lo, hi): + yield '%s %s' % (tag, x[i]) + + def _plain_replace(self, a, alo, ahi, b, blo, bhi): + assert alo < ahi and blo < bhi + # dump the shorter block first -- reduces the burden on short-term + # memory if the blocks are of very different sizes + if bhi - blo < ahi - alo: + first = self._dump('+', b, blo, bhi) + second = self._dump('-', a, alo, ahi) + else: + first = self._dump('-', a, alo, ahi) + second = self._dump('+', b, blo, bhi) + + for g in first, second: + yield from g + + def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + r""" + When replacing one block of lines with another, search the blocks + for *similar* lines; the best-matching pair (if any) is used as a + synch point, and intraline difference marking is done on the + similar pair. Lots of work, but often worth it. + + Example: + + >>> d = Differ() + >>> results = d._fancy_replace(['abcDefghiJkl\n'], 0, 1, + ... ['abcdefGhijkl\n'], 0, 1) + >>> print(''.join(results), end="") + - abcDefghiJkl + ? ^ ^ ^ + + abcdefGhijkl + ? ^ ^ ^ + """ + + # don't synch up unless the lines have a similarity score of at + # least cutoff; best_ratio tracks the best score seen so far + best_ratio, cutoff = 0.74, 0.75 + cruncher = SequenceMatcher(self.charjunk) + eqi, eqj = None, None # 1st indices of equal lines (if any) + + # search for the pair that matches best without being identical + # (identical lines must be junk lines, & we don't want to synch up + # on junk -- unless we have to) + for j in range(blo, bhi): + bj = b[j] + cruncher.set_seq2(bj) + for i in range(alo, ahi): + ai = a[i] + if ai == bj: + if eqi is None: + eqi, eqj = i, j + continue + cruncher.set_seq1(ai) + # computing similarity is expensive, so use the quick + # upper bounds first -- have seen this speed up messy + # compares by a factor of 3. + # note that ratio() is only expensive to compute the first + # time it's called on a sequence pair; the expensive part + # of the computation is cached by cruncher + if cruncher.real_quick_ratio() > best_ratio and \ + cruncher.quick_ratio() > best_ratio and \ + cruncher.ratio() > best_ratio: + best_ratio, best_i, best_j = cruncher.ratio(), i, j + if best_ratio < cutoff: + # no non-identical "pretty close" pair + if eqi is None: + # no identical pair either -- treat it as a straight replace + yield from self._plain_replace(a, alo, ahi, b, blo, bhi) + return + # no close pair, but an identical pair -- synch up on that + best_i, best_j, best_ratio = eqi, eqj, 1.0 + else: + # there's a close pair, so forget the identical pair (if any) + eqi = None + + # a[best_i] very similar to b[best_j]; eqi is None iff they're not + # identical + + # pump out diffs from before the synch point + yield from self._fancy_helper(a, alo, best_i, b, blo, best_j) + + # do intraline marking on the synch pair + aelt, belt = a[best_i], b[best_j] + if eqi is None: + # pump out a '-', '?', '+', '?' quad for the synched lines + atags = btags = "" + cruncher.set_seqs(aelt, belt) + for tag, ai1, ai2, bj1, bj2 in cruncher.get_opcodes(): + la, lb = ai2 - ai1, bj2 - bj1 + if tag == 'replace': + atags += '^' * la + btags += '^' * lb + elif tag == 'delete': + atags += '-' * la + elif tag == 'insert': + btags += '+' * lb + elif tag == 'equal': + atags += ' ' * la + btags += ' ' * lb + else: + raise ValueError('unknown tag %r' % (tag,)) + yield from self._qformat(aelt, belt, atags, btags) + else: + # the synch pair is identical + yield ' ' + aelt + + # pump out diffs from after the synch point + yield from self._fancy_helper(a, best_i+1, ahi, b, best_j+1, bhi) + + def _fancy_helper(self, a, alo, ahi, b, blo, bhi): + g = [] + if alo < ahi: + if blo < bhi: + g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + else: + g = self._dump('-', a, alo, ahi) + elif blo < bhi: + g = self._dump('+', b, blo, bhi) + + yield from g + + def _qformat(self, aline, bline, atags, btags): + r""" + Format "?" output and deal with leading tabs. + + Example: + + >>> d = Differ() + >>> results = d._qformat('\tabcDefghiJkl\n', '\tabcdefGhijkl\n', + ... ' ^ ^ ^ ', ' ^ ^ ^ ') + >>> for line in results: print(repr(line)) + ... + '- \tabcDefghiJkl\n' + '? \t ^ ^ ^\n' + '+ \tabcdefGhijkl\n' + '? \t ^ ^ ^\n' + """ + + # Can hurt, but will probably help most of the time. + common = min(_count_leading(aline, "\t"), + _count_leading(bline, "\t")) + common = min(common, _count_leading(atags[:common], " ")) + common = min(common, _count_leading(btags[:common], " ")) + atags = atags[common:].rstrip() + btags = btags[common:].rstrip() + + yield "- " + aline + if atags: + yield "? %s%s\n" % ("\t" * common, atags) + + yield "+ " + bline + if btags: + yield "? %s%s\n" % ("\t" * common, btags) + +# With respect to junk, an earlier version of ndiff simply refused to +# *start* a match with a junk element. The result was cases like this: +# before: private Thread currentThread; +# after: private volatile Thread currentThread; +# If you consider whitespace to be junk, the longest contiguous match +# not starting with junk is "e Thread currentThread". So ndiff reported +# that "e volatil" was inserted between the 't' and the 'e' in "private". +# While an accurate view, to people that's absurd. The current version +# looks for matching blocks that are entirely junk-free, then extends the +# longest one of those as far as possible but only with matching junk. +# So now "currentThread" is matched, then extended to suck up the +# preceding blank; then "private" is matched, and extended to suck up the +# following blank; then "Thread" is matched; and finally ndiff reports +# that "volatile " was inserted before "Thread". The only quibble +# remaining is that perhaps it was really the case that " volatile" +# was inserted after "private". I can live with that . + +import re + +def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): + r""" + Return 1 for ignorable line: iff `line` is blank or contains a single '#'. + + Examples: + + >>> IS_LINE_JUNK('\n') + True + >>> IS_LINE_JUNK(' # \n') + True + >>> IS_LINE_JUNK('hello\n') + False + """ + + return pat(line) is not None + +def IS_CHARACTER_JUNK(ch, ws=" \t"): + r""" + Return 1 for ignorable character: iff `ch` is a space or tab. + + Examples: + + >>> IS_CHARACTER_JUNK(' ') + True + >>> IS_CHARACTER_JUNK('\t') + True + >>> IS_CHARACTER_JUNK('\n') + False + >>> IS_CHARACTER_JUNK('x') + False + """ + + return ch in ws + + +######################################################################## +### Unified Diff +######################################################################## + +def _format_range_unified(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if length == 1: + return '{}'.format(beginning) + if not length: + beginning -= 1 # empty ranges begin at line just before the range + return '{},{}'.format(beginning, length) + +def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', + tofiledate='', n=3, lineterm='\n'): + r""" + Compare two sequences of lines; generate the delta as a unified diff. + + Unified diffs are a compact way of showing line changes and a few + lines of context. The number of context lines is set by 'n' which + defaults to three. + + By default, the diff control lines (those with ---, +++, or @@) are + created with a trailing newline. This is helpful so that inputs + created from file.readlines() result in diffs that are suitable for + file.writelines() since both the inputs and outputs have trailing + newlines. + + For inputs that do not have trailing newlines, set the lineterm + argument to "" so that the output will be uniformly newline free. + + The unidiff format normally has a header for filenames and modification + times. Any or all of these may be specified using strings for + 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. + The modification times are normally expressed in the ISO 8601 format. + + Example: + + >>> for line in unified_diff('one two three four'.split(), + ... 'zero one tree four'.split(), 'Original', 'Current', + ... '2005-01-26 23:30:50', '2010-04-02 10:20:52', + ... lineterm=''): + ... print(line) # doctest: +NORMALIZE_WHITESPACE + --- Original 2005-01-26 23:30:50 + +++ Current 2010-04-02 10:20:52 + @@ -1,4 +1,4 @@ + +zero + one + -two + -three + +tree + four + """ + + _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) + started = False + for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): + if not started: + started = True + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) + yield '+++ {}{}{}'.format(tofile, todate, lineterm) + + first, last = group[0], group[-1] + file1_range = _format_range_unified(first[1], last[2]) + file2_range = _format_range_unified(first[3], last[4]) + yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + + for tag, i1, i2, j1, j2 in group: + if tag == 'equal': + for line in a[i1:i2]: + yield ' ' + line + continue + if tag in {'replace', 'delete'}: + for line in a[i1:i2]: + yield '-' + line + if tag in {'replace', 'insert'}: + for line in b[j1:j2]: + yield '+' + line + + +######################################################################## +### Context Diff +######################################################################## + +def _format_range_context(start, stop): + 'Convert range to the "ed" format' + # Per the diff spec at http://www.unix.org/single_unix_specification/ + beginning = start + 1 # lines start numbering with one + length = stop - start + if not length: + beginning -= 1 # empty ranges begin at line just before the range + if length <= 1: + return '{}'.format(beginning) + return '{},{}'.format(beginning, beginning + length - 1) + +# See http://www.unix.org/single_unix_specification/ +def context_diff(a, b, fromfile='', tofile='', + fromfiledate='', tofiledate='', n=3, lineterm='\n'): + r""" + Compare two sequences of lines; generate the delta as a context diff. + + Context diffs are a compact way of showing line changes and a few + lines of context. The number of context lines is set by 'n' which + defaults to three. + + By default, the diff control lines (those with *** or ---) are + created with a trailing newline. This is helpful so that inputs + created from file.readlines() result in diffs that are suitable for + file.writelines() since both the inputs and outputs have trailing + newlines. + + For inputs that do not have trailing newlines, set the lineterm + argument to "" so that the output will be uniformly newline free. + + The context diff format normally has a header for filenames and + modification times. Any or all of these may be specified using + strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. + The modification times are normally expressed in the ISO 8601 format. + If not specified, the strings default to blanks. + + Example: + + >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), + ... 'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')), + ... end="") + *** Original + --- Current + *************** + *** 1,4 **** + one + ! two + ! three + four + --- 1,4 ---- + + zero + one + ! tree + four + """ + + _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) + prefix = dict(insert='+ ', delete='- ', replace='! ', equal=' ') + started = False + for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): + if not started: + started = True + fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' + todate = '\t{}'.format(tofiledate) if tofiledate else '' + yield '*** {}{}{}'.format(fromfile, fromdate, lineterm) + yield '--- {}{}{}'.format(tofile, todate, lineterm) + + first, last = group[0], group[-1] + yield '***************' + lineterm + + file1_range = _format_range_context(first[1], last[2]) + yield '*** {} ****{}'.format(file1_range, lineterm) + + if any(tag in {'replace', 'delete'} for tag, _, _, _, _ in group): + for tag, i1, i2, _, _ in group: + if tag != 'insert': + for line in a[i1:i2]: + yield prefix[tag] + line + + file2_range = _format_range_context(first[3], last[4]) + yield '--- {} ----{}'.format(file2_range, lineterm) + + if any(tag in {'replace', 'insert'} for tag, _, _, _, _ in group): + for tag, _, _, j1, j2 in group: + if tag != 'delete': + for line in b[j1:j2]: + yield prefix[tag] + line + +def _check_types(a, b, *args): + # Checking types is weird, but the alternative is garbled output when + # someone passes mixed bytes and str to {unified,context}_diff(). E.g. + # without this check, passing filenames as bytes results in output like + # --- b'oldfile.txt' + # +++ b'newfile.txt' + # because of how str.format() incorporates bytes objects. + if a and not isinstance(a[0], str): + raise TypeError('lines to compare must be str, not %s (%r)' % + (type(a[0]).__name__, a[0])) + if b and not isinstance(b[0], str): + raise TypeError('lines to compare must be str, not %s (%r)' % + (type(b[0]).__name__, b[0])) + for arg in args: + if not isinstance(arg, str): + raise TypeError('all arguments must be str, not: %r' % (arg,)) + +def diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', + fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n'): + r""" + Compare `a` and `b`, two sequences of lines represented as bytes rather + than str. This is a wrapper for `dfunc`, which is typically either + unified_diff() or context_diff(). Inputs are losslessly converted to + strings so that `dfunc` only has to worry about strings, and encoded + back to bytes on return. This is necessary to compare files with + unknown or inconsistent encoding. All other inputs (except `n`) must be + bytes rather than str. + """ + def decode(s): + try: + return s.decode('ascii', 'surrogateescape') + except AttributeError as err: + msg = ('all arguments must be bytes, not %s (%r)' % + (type(s).__name__, s)) + raise TypeError(msg) from err + a = list(map(decode, a)) + b = list(map(decode, b)) + fromfile = decode(fromfile) + tofile = decode(tofile) + fromfiledate = decode(fromfiledate) + tofiledate = decode(tofiledate) + lineterm = decode(lineterm) + + lines = dfunc(a, b, fromfile, tofile, fromfiledate, tofiledate, n, lineterm) + for line in lines: + yield line.encode('ascii', 'surrogateescape') + +def ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK): + r""" + Compare `a` and `b` (lists of strings); return a `Differ`-style delta. + + Optional keyword parameters `linejunk` and `charjunk` are for filter + functions, or can be None: + + - linejunk: A function that should accept a single string argument and + return true iff the string is junk. The default is None, and is + recommended; the underlying SequenceMatcher class has an adaptive + notion of "noise" lines. + + - charjunk: A function that accepts a character (string of length + 1), and returns true iff the character is junk. The default is + the module-level function IS_CHARACTER_JUNK, which filters out + whitespace characters (a blank or tab; note: it's a bad idea to + include newline in this!). + + Tools/scripts/ndiff.py is a command-line front-end to this function. + + Example: + + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), + ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) + >>> print(''.join(diff), end="") + - one + ? ^ + + ore + ? ^ + - two + - three + ? - + + tree + + emu + """ + return Differ(linejunk, charjunk).compare(a, b) + +def _mdiff(fromlines, tolines, context=None, linejunk=None, + charjunk=IS_CHARACTER_JUNK): + r"""Returns generator yielding marked up from/to side by side differences. + + Arguments: + fromlines -- list of text lines to compared to tolines + tolines -- list of text lines to be compared to fromlines + context -- number of context lines to display on each side of difference, + if None, all from/to text lines will be generated. + linejunk -- passed on to ndiff (see ndiff documentation) + charjunk -- passed on to ndiff (see ndiff documentation) + + This function returns an iterator which returns a tuple: + (from line tuple, to line tuple, boolean flag) + + from/to line tuple -- (line num, line text) + line num -- integer or None (to indicate a context separation) + line text -- original line text with following markers inserted: + '\0+' -- marks start of added text + '\0-' -- marks start of deleted text + '\0^' -- marks start of changed text + '\1' -- marks end of added/deleted/changed text + + boolean flag -- None indicates context separation, True indicates + either "from" or "to" line contains a change, otherwise False. + + This function/iterator was originally developed to generate side by side + file difference for making HTML pages (see HtmlDiff class for example + usage). + + Note, this function utilizes the ndiff function to generate the side by + side difference markup. Optional ndiff arguments may be passed to this + function and they in turn will be passed to ndiff. + """ + import re + + # regular expression for finding intraline change indices + change_re = re.compile(r'(\++|\-+|\^+)') + + # create the difference iterator to generate the differences + diff_lines_iterator = ndiff(fromlines,tolines,linejunk,charjunk) + + def _make_line(lines, format_key, side, num_lines=[0,0]): + """Returns line of text with user's change markup and line formatting. + + lines -- list of lines from the ndiff generator to produce a line of + text from. When producing the line of text to return, the + lines used are removed from this list. + format_key -- '+' return first line in list with "add" markup around + the entire line. + '-' return first line in list with "delete" markup around + the entire line. + '?' return first line in list with add/delete/change + intraline markup (indices obtained from second line) + None return first line in list with no markup + side -- indice into the num_lines list (0=from,1=to) + num_lines -- from/to current line number. This is NOT intended to be a + passed parameter. It is present as a keyword argument to + maintain memory of the current line numbers between calls + of this function. + + Note, this function is purposefully not defined at the module scope so + that data it needs from its parent function (within whose context it + is defined) does not need to be of module scope. + """ + num_lines[side] += 1 + # Handle case where no user markup is to be added, just return line of + # text with user's line format to allow for usage of the line number. + if format_key is None: + return (num_lines[side],lines.pop(0)[2:]) + # Handle case of intraline changes + if format_key == '?': + text, markers = lines.pop(0), lines.pop(0) + # find intraline changes (store change type and indices in tuples) + sub_info = [] + def record_sub_info(match_object,sub_info=sub_info): + sub_info.append([match_object.group(1)[0],match_object.span()]) + return match_object.group(1) + change_re.sub(record_sub_info,markers) + # process each tuple inserting our special marks that won't be + # noticed by an xml/html escaper. + for key,(begin,end) in reversed(sub_info): + text = text[0:begin]+'\0'+key+text[begin:end]+'\1'+text[end:] + text = text[2:] + # Handle case of add/delete entire line + else: + text = lines.pop(0)[2:] + # if line of text is just a newline, insert a space so there is + # something for the user to highlight and see. + if not text: + text = ' ' + # insert marks that won't be noticed by an xml/html escaper. + text = '\0' + format_key + text + '\1' + # Return line of text, first allow user's line formatter to do its + # thing (such as adding the line number) then replace the special + # marks with what the user's change markup. + return (num_lines[side],text) + + def _line_iterator(): + """Yields from/to lines of text with a change indication. + + This function is an iterator. It itself pulls lines from a + differencing iterator, processes them and yields them. When it can + it yields both a "from" and a "to" line, otherwise it will yield one + or the other. In addition to yielding the lines of from/to text, a + boolean flag is yielded to indicate if the text line(s) have + differences in them. + + Note, this function is purposefully not defined at the module scope so + that data it needs from its parent function (within whose context it + is defined) does not need to be of module scope. + """ + lines = [] + num_blanks_pending, num_blanks_to_yield = 0, 0 + while True: + # Load up next 4 lines so we can look ahead, create strings which + # are a concatenation of the first character of each of the 4 lines + # so we can do some very readable comparisons. + while len(lines) < 4: + lines.append(next(diff_lines_iterator, 'X')) + s = ''.join([line[0] for line in lines]) + if s.startswith('X'): + # When no more lines, pump out any remaining blank lines so the + # corresponding add/delete lines get a matching blank line so + # all line pairs get yielded at the next level. + num_blanks_to_yield = num_blanks_pending + elif s.startswith('-?+?'): + # simple intraline change + yield _make_line(lines,'?',0), _make_line(lines,'?',1), True + continue + elif s.startswith('--++'): + # in delete block, add block coming: we do NOT want to get + # caught up on blank lines yet, just process the delete line + num_blanks_pending -= 1 + yield _make_line(lines,'-',0), None, True + continue + elif s.startswith(('--?+', '--+', '- ')): + # in delete block and see an intraline change or unchanged line + # coming: yield the delete line and then blanks + from_line,to_line = _make_line(lines,'-',0), None + num_blanks_to_yield,num_blanks_pending = num_blanks_pending-1,0 + elif s.startswith('-+?'): + # intraline change + yield _make_line(lines,None,0), _make_line(lines,'?',1), True + continue + elif s.startswith('-?+'): + # intraline change + yield _make_line(lines,'?',0), _make_line(lines,None,1), True + continue + elif s.startswith('-'): + # delete FROM line + num_blanks_pending -= 1 + yield _make_line(lines,'-',0), None, True + continue + elif s.startswith('+--'): + # in add block, delete block coming: we do NOT want to get + # caught up on blank lines yet, just process the add line + num_blanks_pending += 1 + yield None, _make_line(lines,'+',1), True + continue + elif s.startswith(('+ ', '+-')): + # will be leaving an add block: yield blanks then add line + from_line, to_line = None, _make_line(lines,'+',1) + num_blanks_to_yield,num_blanks_pending = num_blanks_pending+1,0 + elif s.startswith('+'): + # inside an add block, yield the add line + num_blanks_pending += 1 + yield None, _make_line(lines,'+',1), True + continue + elif s.startswith(' '): + # unchanged text, yield it to both sides + yield _make_line(lines[:],None,0),_make_line(lines,None,1),False + continue + # Catch up on the blank lines so when we yield the next from/to + # pair, they are lined up. + while(num_blanks_to_yield < 0): + num_blanks_to_yield += 1 + yield None,('','\n'),True + while(num_blanks_to_yield > 0): + num_blanks_to_yield -= 1 + yield ('','\n'),None,True + if s.startswith('X'): + return + else: + yield from_line,to_line,True + + def _line_pair_iterator(): + """Yields from/to lines of text with a change indication. + + This function is an iterator. It itself pulls lines from the line + iterator. Its difference from that iterator is that this function + always yields a pair of from/to text lines (with the change + indication). If necessary it will collect single from/to lines + until it has a matching pair from/to pair to yield. + + Note, this function is purposefully not defined at the module scope so + that data it needs from its parent function (within whose context it + is defined) does not need to be of module scope. + """ + line_iterator = _line_iterator() + fromlines,tolines=[],[] + while True: + # Collecting lines of text until we have a from/to pair + while (len(fromlines)==0 or len(tolines)==0): + try: + from_line, to_line, found_diff = next(line_iterator) + except StopIteration: + return + if from_line is not None: + fromlines.append((from_line,found_diff)) + if to_line is not None: + tolines.append((to_line,found_diff)) + # Once we have a pair, remove them from the collection and yield it + from_line, fromDiff = fromlines.pop(0) + to_line, to_diff = tolines.pop(0) + yield (from_line,to_line,fromDiff or to_diff) + + # Handle case where user does not want context differencing, just yield + # them up without doing anything else with them. + line_pair_iterator = _line_pair_iterator() + if context is None: + yield from line_pair_iterator + # Handle case where user wants context differencing. We must do some + # storage of lines until we know for sure that they are to be yielded. + else: + context += 1 + lines_to_write = 0 + while True: + # Store lines up until we find a difference, note use of a + # circular queue because we only need to keep around what + # we need for context. + index, contextLines = 0, [None]*(context) + found_diff = False + while(found_diff is False): + try: + from_line, to_line, found_diff = next(line_pair_iterator) + except StopIteration: + return + i = index % context + contextLines[i] = (from_line, to_line, found_diff) + index += 1 + # Yield lines that we have collected so far, but first yield + # the user's separator. + if index > context: + yield None, None, None + lines_to_write = context + else: + lines_to_write = index + index = 0 + while(lines_to_write): + i = index % context + index += 1 + yield contextLines[i] + lines_to_write -= 1 + # Now yield the context lines after the change + lines_to_write = context-1 + try: + while(lines_to_write): + from_line, to_line, found_diff = next(line_pair_iterator) + # If another change within the context, extend the context + if found_diff: + lines_to_write = context-1 + else: + lines_to_write -= 1 + yield from_line, to_line, found_diff + except StopIteration: + # Catch exception from next() and return normally + return + + +_file_template = """ + + + + + + + + + + + + %(table)s%(legend)s + + +""" + +_styles = """ + table.diff {font-family:Courier; border:medium;} + .diff_header {background-color:#e0e0e0} + td.diff_header {text-align:right} + .diff_next {background-color:#c0c0c0} + .diff_add {background-color:#aaffaa} + .diff_chg {background-color:#ffff77} + .diff_sub {background-color:#ffaaaa}""" + +_table_template = """ + + + + %(header_row)s + +%(data_rows)s +
""" + +_legend = """ + + + + +
Legends
+ + + + +
Colors
 Added 
Changed
Deleted
+ + + + +
Links
(f)irst change
(n)ext change
(t)op
""" + +class HtmlDiff(object): + """For producing HTML side by side comparison with change highlights. + + This class can be used to create an HTML table (or a complete HTML file + containing the table) showing a side by side, line by line comparison + of text with inter-line and intra-line change highlights. The table can + be generated in either full or contextual difference mode. + + The following methods are provided for HTML generation: + + make_table -- generates HTML for a single side by side table + make_file -- generates complete HTML file with a single side by side table + + See tools/scripts/diff.py for an example usage of this class. + """ + + _file_template = _file_template + _styles = _styles + _table_template = _table_template + _legend = _legend + _default_prefix = 0 + + def __init__(self,tabsize=8,wrapcolumn=None,linejunk=None, + charjunk=IS_CHARACTER_JUNK): + """HtmlDiff instance initializer + + Arguments: + tabsize -- tab stop spacing, defaults to 8. + wrapcolumn -- column number where lines are broken and wrapped, + defaults to None where lines are not wrapped. + linejunk,charjunk -- keyword arguments passed into ndiff() (used by + HtmlDiff() to generate the side by side HTML differences). See + ndiff() documentation for argument default values and descriptions. + """ + self._tabsize = tabsize + self._wrapcolumn = wrapcolumn + self._linejunk = linejunk + self._charjunk = charjunk + + def make_file(self, fromlines, tolines, fromdesc='', todesc='', + context=False, numlines=5, *, charset='utf-8'): + """Returns HTML file of side by side comparison with change highlights + + Arguments: + fromlines -- list of "from" lines + tolines -- list of "to" lines + fromdesc -- "from" file column header string + todesc -- "to" file column header string + context -- set to True for contextual differences (defaults to False + which shows full differences). + numlines -- number of context lines. When context is set True, + controls number of lines displayed before and after the change. + When context is False, controls the number of lines to place + the "next" link anchors before the next change (so click of + "next" link jumps to just before the change). + charset -- charset of the HTML document + """ + + return (self._file_template % dict( + styles=self._styles, + legend=self._legend, + table=self.make_table(fromlines, tolines, fromdesc, todesc, + context=context, numlines=numlines), + charset=charset + )).encode(charset, 'xmlcharrefreplace').decode(charset) + + def _tab_newline_replace(self,fromlines,tolines): + """Returns from/to line lists with tabs expanded and newlines removed. + + Instead of tab characters being replaced by the number of spaces + needed to fill in to the next tab stop, this function will fill + the space with tab characters. This is done so that the difference + algorithms can identify changes in a file when tabs are replaced by + spaces and vice versa. At the end of the HTML generation, the tab + characters will be replaced with a nonbreakable space. + """ + def expand_tabs(line): + # hide real spaces + line = line.replace(' ','\0') + # expand tabs into spaces + line = line.expandtabs(self._tabsize) + # replace spaces from expanded tabs back into tab characters + # (we'll replace them with markup after we do differencing) + line = line.replace(' ','\t') + return line.replace('\0',' ').rstrip('\n') + fromlines = [expand_tabs(line) for line in fromlines] + tolines = [expand_tabs(line) for line in tolines] + return fromlines,tolines + + def _split_line(self,data_list,line_num,text): + """Builds list of text lines by splitting text lines at wrap point + + This function will determine if the input text line needs to be + wrapped (split) into separate lines. If so, the first wrap point + will be determined and the first line appended to the output + text line list. This function is used recursively to handle + the second part of the split line to further split it. + """ + # if blank line or context separator, just add it to the output list + if not line_num: + data_list.append((line_num,text)) + return + + # if line text doesn't need wrapping, just add it to the output list + size = len(text) + max = self._wrapcolumn + if (size <= max) or ((size -(text.count('\0')*3)) <= max): + data_list.append((line_num,text)) + return + + # scan text looking for the wrap point, keeping track if the wrap + # point is inside markers + i = 0 + n = 0 + mark = '' + while n < max and i < size: + if text[i] == '\0': + i += 1 + mark = text[i] + i += 1 + elif text[i] == '\1': + i += 1 + mark = '' + else: + i += 1 + n += 1 + + # wrap point is inside text, break it up into separate lines + line1 = text[:i] + line2 = text[i:] + + # if wrap point is inside markers, place end marker at end of first + # line and start marker at beginning of second line because each + # line will have its own table tag markup around it. + if mark: + line1 = line1 + '\1' + line2 = '\0' + mark + line2 + + # tack on first line onto the output list + data_list.append((line_num,line1)) + + # use this routine again to wrap the remaining text + self._split_line(data_list,'>',line2) + + def _line_wrapper(self,diffs): + """Returns iterator that splits (wraps) mdiff text lines""" + + # pull from/to data and flags from mdiff iterator + for fromdata,todata,flag in diffs: + # check for context separators and pass them through + if flag is None: + yield fromdata,todata,flag + continue + (fromline,fromtext),(toline,totext) = fromdata,todata + # for each from/to line split it at the wrap column to form + # list of text lines. + fromlist,tolist = [],[] + self._split_line(fromlist,fromline,fromtext) + self._split_line(tolist,toline,totext) + # yield from/to line in pairs inserting blank lines as + # necessary when one side has more wrapped lines + while fromlist or tolist: + if fromlist: + fromdata = fromlist.pop(0) + else: + fromdata = ('',' ') + if tolist: + todata = tolist.pop(0) + else: + todata = ('',' ') + yield fromdata,todata,flag + + def _collect_lines(self,diffs): + """Collects mdiff output into separate lists + + Before storing the mdiff from/to data into a list, it is converted + into a single line of text with HTML markup. + """ + + fromlist,tolist,flaglist = [],[],[] + # pull from/to data and flags from mdiff style iterator + for fromdata,todata,flag in diffs: + try: + # store HTML markup of the lines into the lists + fromlist.append(self._format_line(0,flag,*fromdata)) + tolist.append(self._format_line(1,flag,*todata)) + except TypeError: + # exceptions occur for lines where context separators go + fromlist.append(None) + tolist.append(None) + flaglist.append(flag) + return fromlist,tolist,flaglist + + def _format_line(self,side,flag,linenum,text): + """Returns HTML markup of "from" / "to" text lines + + side -- 0 or 1 indicating "from" or "to" text + flag -- indicates if difference on line + linenum -- line number (used for line number column) + text -- line text to be marked up + """ + try: + linenum = '%d' % linenum + id = ' id="%s%s"' % (self._prefix[side],linenum) + except TypeError: + # handle blank lines where linenum is '>' or '' + id = '' + # replace those things that would get confused with HTML symbols + text=text.replace("&","&").replace(">",">").replace("<","<") + + # make space non-breakable so they don't get compressed or line wrapped + text = text.replace(' ',' ').rstrip() + + return '%s%s' \ + % (id,linenum,text) + + def _make_prefix(self): + """Create unique anchor prefixes""" + + # Generate a unique anchor prefix so multiple tables + # can exist on the same HTML page without conflicts. + fromprefix = "from%d_" % HtmlDiff._default_prefix + toprefix = "to%d_" % HtmlDiff._default_prefix + HtmlDiff._default_prefix += 1 + # store prefixes so line format method has access + self._prefix = [fromprefix,toprefix] + + def _convert_flags(self,fromlist,tolist,flaglist,context,numlines): + """Makes list of "next" links""" + + # all anchor names will be generated using the unique "to" prefix + toprefix = self._prefix[1] + + # process change flags, generating middle column of next anchors/links + next_id = ['']*len(flaglist) + next_href = ['']*len(flaglist) + num_chg, in_change = 0, False + last = 0 + for i,flag in enumerate(flaglist): + if flag: + if not in_change: + in_change = True + last = i + # at the beginning of a change, drop an anchor a few lines + # (the context lines) before the change for the previous + # link + i = max([0,i-numlines]) + next_id[i] = ' id="difflib_chg_%s_%d"' % (toprefix,num_chg) + # at the beginning of a change, drop a link to the next + # change + num_chg += 1 + next_href[last] = 'n' % ( + toprefix,num_chg) + else: + in_change = False + # check for cases where there is no content to avoid exceptions + if not flaglist: + flaglist = [False] + next_id = [''] + next_href = [''] + last = 0 + if context: + fromlist = [' No Differences Found '] + tolist = fromlist + else: + fromlist = tolist = [' Empty File '] + # if not a change on first line, drop a link + if not flaglist[0]: + next_href[0] = 'f' % toprefix + # redo the last link to link to the top + next_href[last] = 't' % (toprefix) + + return fromlist,tolist,flaglist,next_href,next_id + + def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, + numlines=5): + """Returns HTML table of side by side comparison with change highlights + + Arguments: + fromlines -- list of "from" lines + tolines -- list of "to" lines + fromdesc -- "from" file column header string + todesc -- "to" file column header string + context -- set to True for contextual differences (defaults to False + which shows full differences). + numlines -- number of context lines. When context is set True, + controls number of lines displayed before and after the change. + When context is False, controls the number of lines to place + the "next" link anchors before the next change (so click of + "next" link jumps to just before the change). + """ + + # make unique anchor prefixes so that multiple tables may exist + # on the same page without conflict. + self._make_prefix() + + # change tabs to spaces before it gets more difficult after we insert + # markup + fromlines,tolines = self._tab_newline_replace(fromlines,tolines) + + # create diffs iterator which generates side by side from/to data + if context: + context_lines = numlines + else: + context_lines = None + diffs = _mdiff(fromlines,tolines,context_lines,linejunk=self._linejunk, + charjunk=self._charjunk) + + # set up iterator to wrap lines that exceed desired width + if self._wrapcolumn: + diffs = self._line_wrapper(diffs) + + # collect up from/to lines and flags into lists (also format the lines) + fromlist,tolist,flaglist = self._collect_lines(diffs) + + # process change flags, generating middle column of next anchors/links + fromlist,tolist,flaglist,next_href,next_id = self._convert_flags( + fromlist,tolist,flaglist,context,numlines) + + s = [] + fmt = ' %s%s' + \ + '%s%s\n' + for i in range(len(flaglist)): + if flaglist[i] is None: + # mdiff yields None on separator lines skip the bogus ones + # generated for the first line + if i > 0: + s.append(' \n \n') + else: + s.append( fmt % (next_id[i],next_href[i],fromlist[i], + next_href[i],tolist[i])) + if fromdesc or todesc: + header_row = '%s%s%s%s' % ( + '
', + '%s' % fromdesc, + '
', + '%s' % todesc) + else: + header_row = '' + + table = self._table_template % dict( + data_rows=''.join(s), + header_row=header_row, + prefix=self._prefix[1]) + + return table.replace('\0+',''). \ + replace('\0-',''). \ + replace('\0^',''). \ + replace('\1',''). \ + replace('\t',' ') + +del re + +def restore(delta, which): + r""" + Generate one of the two sequences that generated a delta. + + Given a `delta` produced by `Differ.compare()` or `ndiff()`, extract + lines originating from file 1 or 2 (parameter `which`), stripping off line + prefixes. + + Examples: + + >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), + ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) + >>> diff = list(diff) + >>> print(''.join(restore(diff, 1)), end="") + one + two + three + >>> print(''.join(restore(diff, 2)), end="") + ore + tree + emu + """ + try: + tag = {1: "- ", 2: "+ "}[int(which)] + except KeyError: + raise ValueError('unknown delta choice (must be 1 or 2): %r' + % which) from None + prefixes = (" ", tag) + for line in delta: + if line[:2] in prefixes: + yield line[2:] + +def _test(): + import doctest, difflib + return doctest.testmod(difflib) + +if __name__ == "__main__": + _test() diff --git a/Lib/functools.py b/Lib/functools.py new file mode 100644 index 0000000000..c8b79c2a7c --- /dev/null +++ b/Lib/functools.py @@ -0,0 +1,828 @@ +"""functools.py - Tools for working with functions and callable objects +""" +# Python module wrapper for _functools C module +# to allow utilities written in Python to be added +# to the functools module. +# Written by Nick Coghlan , +# Raymond Hettinger , +# and Łukasz Langa . +# Copyright (C) 2006-2013 Python Software Foundation. +# See C source code for _functools credits/copyright + +__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', + 'total_ordering', 'cmp_to_key', 'lru_cache', 'reduce', 'partial', + 'partialmethod', 'singledispatch'] + +try: + from _functools import reduce +except ImportError: + pass +from abc import get_cache_token +from collections import namedtuple +# import types, weakref # Deferred to single_dispatch() +from reprlib import recursive_repr +from _thread import RLock + + +################################################################################ +### update_wrapper() and wraps() decorator +################################################################################ + +# update_wrapper() and wraps() are tools to help write +# wrapper functions that can handle naive introspection + +WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__qualname__', '__doc__', + '__annotations__') +WRAPPER_UPDATES = ('__dict__',) +def update_wrapper(wrapper, + wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Update a wrapper function to look like the wrapped function + + wrapper is the function to be updated + wrapped is the original function + assigned is a tuple naming the attributes assigned directly + from the wrapped function to the wrapper function (defaults to + functools.WRAPPER_ASSIGNMENTS) + updated is a tuple naming the attributes of the wrapper that + are updated with the corresponding attribute from the wrapped + function (defaults to functools.WRAPPER_UPDATES) + """ + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + pass + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + # Issue #17482: set __wrapped__ last so we don't inadvertently copy it + # from the wrapped function when updating __dict__ + wrapper.__wrapped__ = wrapped + # Return the wrapper so this can be used as a decorator via partial() + return wrapper + +def wraps(wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Decorator factory to apply update_wrapper() to a wrapper function + + Returns a decorator that invokes update_wrapper() with the decorated + function as the wrapper argument and the arguments to wraps() as the + remaining arguments. Default arguments are as for update_wrapper(). + This is a convenience function to simplify applying partial() to + update_wrapper(). + """ + return partial(update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + + +################################################################################ +### total_ordering class decorator +################################################################################ + +# The total ordering functions all invoke the root magic method directly +# rather than using the corresponding operator. This avoids possible +# infinite recursion that could occur when the operator dispatch logic +# detects a NotImplemented result and then calls a reflected method. + +def _gt_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' + op_result = self.__lt__(other) + if op_result is NotImplemented: + return op_result + return not op_result and self != other + +def _le_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' + op_result = self.__lt__(other) + return op_result or self == other + +def _ge_from_lt(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (not a < b).' + op_result = self.__lt__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _ge_from_le(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' + op_result = self.__le__(other) + if op_result is NotImplemented: + return op_result + return not op_result or self == other + +def _lt_from_le(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' + op_result = self.__le__(other) + if op_result is NotImplemented: + return op_result + return op_result and self != other + +def _gt_from_le(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (not a <= b).' + op_result = self.__le__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _lt_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' + op_result = self.__gt__(other) + if op_result is NotImplemented: + return op_result + return not op_result and self != other + +def _ge_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' + op_result = self.__gt__(other) + return op_result or self == other + +def _le_from_gt(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (not a > b).' + op_result = self.__gt__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +def _le_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' + op_result = self.__ge__(other) + if op_result is NotImplemented: + return op_result + return not op_result or self == other + +def _gt_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' + op_result = self.__ge__(other) + if op_result is NotImplemented: + return op_result + return op_result and self != other + +def _lt_from_ge(self, other, NotImplemented=NotImplemented): + 'Return a < b. Computed by @total_ordering from (not a >= b).' + op_result = self.__ge__(other) + if op_result is NotImplemented: + return op_result + return not op_result + +_convert = { + '__lt__': [('__gt__', _gt_from_lt), + ('__le__', _le_from_lt), + ('__ge__', _ge_from_lt)], + '__le__': [('__ge__', _ge_from_le), + ('__lt__', _lt_from_le), + ('__gt__', _gt_from_le)], + '__gt__': [('__lt__', _lt_from_gt), + ('__ge__', _ge_from_gt), + ('__le__', _le_from_gt)], + '__ge__': [('__le__', _le_from_ge), + ('__gt__', _gt_from_ge), + ('__lt__', _lt_from_ge)] +} + +def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + # Find user-defined comparisons (not those inherited from object). + roots = {op for op in _convert if getattr(cls, op, None) is not getattr(object, op, None)} + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in _convert[root]: + if opname not in roots: + opfunc.__name__ = opname + setattr(cls, opname, opfunc) + return cls + + +################################################################################ +### cmp_to_key() function converter +################################################################################ + +def cmp_to_key(mycmp): + """Convert a cmp= function into a key= function""" + class K(object): + __slots__ = ['obj'] + def __init__(self, obj): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + __hash__ = None + return K + +try: + from _functools import cmp_to_key +except ImportError: + pass + + +################################################################################ +### partial() argument application +################################################################################ + +# Purely functional, no descriptor behaviour +class partial: + """New function with partial application of the given arguments + and keywords. + """ + + __slots__ = "func", "args", "keywords", "__dict__", "__weakref__" + + def __new__(*args, **keywords): + if not args: + raise TypeError("descriptor '__new__' of partial needs an argument") + if len(args) < 2: + raise TypeError("type 'partial' takes at least one argument") + cls, func, *args = args + if not callable(func): + raise TypeError("the first argument must be callable") + args = tuple(args) + + if hasattr(func, "func"): + args = func.args + args + tmpkw = func.keywords.copy() + tmpkw.update(keywords) + keywords = tmpkw + del tmpkw + func = func.func + + self = super(partial, cls).__new__(cls) + + self.func = func + self.args = args + self.keywords = keywords + return self + + def __call__(*args, **keywords): + if not args: + raise TypeError("descriptor '__call__' of partial needs an argument") + self, *args = args + newkeywords = self.keywords.copy() + newkeywords.update(keywords) + return self.func(*self.args, *args, **newkeywords) + + @recursive_repr() + def __repr__(self): + qualname = type(self).__qualname__ + args = [repr(self.func)] + args.extend(repr(x) for x in self.args) + args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) + if type(self).__module__ == "functools": + return f"functools.{qualname}({', '.join(args)})" + return f"{qualname}({', '.join(args)})" + + def __reduce__(self): + return type(self), (self.func,), (self.func, self.args, + self.keywords or None, self.__dict__ or None) + + def __setstate__(self, state): + if not isinstance(state, tuple): + raise TypeError("argument to __setstate__ must be a tuple") + if len(state) != 4: + raise TypeError(f"expected 4 items in state, got {len(state)}") + func, args, kwds, namespace = state + if (not callable(func) or not isinstance(args, tuple) or + (kwds is not None and not isinstance(kwds, dict)) or + (namespace is not None and not isinstance(namespace, dict))): + raise TypeError("invalid partial state") + + args = tuple(args) # just in case it's a subclass + if kwds is None: + kwds = {} + elif type(kwds) is not dict: # XXX does it need to be *exactly* dict? + kwds = dict(kwds) + if namespace is None: + namespace = {} + + self.__dict__ = namespace + self.func = func + self.args = args + self.keywords = kwds + +try: + from _functools import partial +except ImportError: + pass + +# Descriptor version +class partialmethod(object): + """Method descriptor with partial application of the given arguments + and keywords. + + Supports wrapping existing descriptors and handles non-descriptor + callables as instance methods. + """ + + def __init__(self, func, *args, **keywords): + if not callable(func) and not hasattr(func, "__get__"): + raise TypeError("{!r} is not callable or a descriptor" + .format(func)) + + # func could be a descriptor like classmethod which isn't callable, + # so we can't inherit from partial (it verifies func is callable) + if isinstance(func, partialmethod): + # flattening is mandatory in order to place cls/self before all + # other arguments + # it's also more efficient since only one function will be called + self.func = func.func + self.args = func.args + args + self.keywords = func.keywords.copy() + self.keywords.update(keywords) + else: + self.func = func + self.args = args + self.keywords = keywords + + def __repr__(self): + args = ", ".join(map(repr, self.args)) + keywords = ", ".join("{}={!r}".format(k, v) + for k, v in self.keywords.items()) + format_string = "{module}.{cls}({func}, {args}, {keywords})" + return format_string.format(module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords) + + def _make_unbound_method(self): + def _method(*args, **keywords): + call_keywords = self.keywords.copy() + call_keywords.update(keywords) + cls_or_self, *rest = args + call_args = (cls_or_self,) + self.args + tuple(rest) + return self.func(*call_args, **call_keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ + _method._partialmethod = self + return _method + + def __get__(self, obj, cls): + get = getattr(self.func, "__get__", None) + result = None + if get is not None: + new_func = get(obj, cls) + if new_func is not self.func: + # Assume __get__ returning something new indicates the + # creation of an appropriate callable + result = partial(new_func, *self.args, **self.keywords) + try: + result.__self__ = new_func.__self__ + except AttributeError: + pass + if result is None: + # If the underlying descriptor didn't do anything, treat this + # like an instance method + result = self._make_unbound_method().__get__(obj, cls) + return result + + @property + def __isabstractmethod__(self): + return getattr(self.func, "__isabstractmethod__", False) + + +################################################################################ +### LRU Cache function decorator +################################################################################ + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + +class _HashedSeq(list): + """ This class guarantees that hash() will be called no more than once + per element. This is important because the lru_cache() will hash + the key multiple times on a cache miss. + + """ + + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + +def _make_key(args, kwds, typed, + kwd_mark = (object(),), + fasttypes = {int, str, frozenset, type(None)}, + tuple=tuple, type=type, len=len): + """Make a cache key from optionally typed positional and keyword arguments + + The key is constructed in a way that is flat as possible rather than + as a nested structure that would take more memory. + + If there is only a single argument and its data type is known to cache + its hash value, then that argument is returned without a wrapper. This + saves space and improves lookup speed. + + """ + # All of code below relies on kwds preserving the order input by the user. + # Formerly, we sorted() the kwds before looping. The new way is *much* + # faster; however, it means that f(x=1, y=2) will now be treated as a + # distinct call from f(y=2, x=1) which will be cached separately. + key = args + if kwds: + key += kwd_mark + for item in kwds.items(): + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for v in kwds.values()) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + +def lru_cache(maxsize=128, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) + with f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + # Early detection of an erroneous call to @lru_cache without any arguments + # resulting in the inner function being passed to maxsize instead of an + # integer or None. + if maxsize is not None and not isinstance(maxsize, int): + raise TypeError('Expected maxsize to be an integer or None') + + def decorating_function(user_function): + wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + return update_wrapper(wrapper, user_function) + + return decorating_function + +def _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo): + # Constants shared by all lru cache instances: + sentinel = object() # unique object used to signal cache misses + make_key = _make_key # build a key from the function arguments + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + cache = {} + hits = misses = 0 + full = False + cache_get = cache.get # bound method to lookup a key or return None + cache_len = cache.__len__ # get cache size without calling len() + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + + if maxsize == 0: + + def wrapper(*args, **kwds): + # No caching -- just a statistics update after a successful call + nonlocal misses + result = user_function(*args, **kwds) + misses += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # Simple caching without ordering or size limit + nonlocal hits, misses + key = make_key(args, kwds, typed) + result = cache_get(key, sentinel) + if result is not sentinel: + hits += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + misses += 1 + return result + + else: + + def wrapper(*args, **kwds): + # Size limited caching that tracks accesses by recency + nonlocal root, hits, misses, full + key = make_key(args, kwds, typed) + with lock: + link = cache_get(key) + if link is not None: + # Move the link to the front of the circular queue + link_prev, link_next, _key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + hits += 1 + return result + result = user_function(*args, **kwds) + with lock: + if key in cache: + # Getting here means that this same key was added to the + # cache while the lock was released. Since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif full: + # Use the old root to store the new key and result. + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # Empty the oldest link and make it the new root. + # Keep a reference to the old key and old result to + # prevent their ref counts from going to zero during the + # update. That will prevent potentially arbitrary object + # clean-up code (i.e. __del__) from running while we're + # still adjusting the links. + root = oldroot[NEXT] + oldkey = root[KEY] + oldresult = root[RESULT] + root[KEY] = root[RESULT] = None + # Now update the cache dictionary. + del cache[oldkey] + # Save the potentially reentrant cache[key] assignment + # for last, after the root and links have been put in + # a consistent state. + cache[key] = oldroot + else: + # Put result in a new link at the front of the queue. + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + # Use the cache_len bound method instead of the len() function + # which could potentially be wrapped in an lru_cache itself. + full = (cache_len() >= maxsize) + misses += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(hits, misses, maxsize, cache_len()) + + def cache_clear(): + """Clear the cache and cache statistics""" + nonlocal hits, misses, full + with lock: + cache.clear() + root[:] = [root, root, None, None] + hits = misses = 0 + full = False + + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return wrapper + +try: + from _functools import _lru_cache_wrapper +except ImportError: + pass + + +################################################################################ +### singledispatch() - single-dispatch generic function decorator +################################################################################ + +def _c3_merge(sequences): + """Merges MROs in *sequences* to a single MRO using the C3 algorithm. + + Adapted from http://www.python.org/download/releases/2.3/mro/. + + """ + result = [] + while True: + sequences = [s for s in sequences if s] # purge empty sequences + if not sequences: + return result + for s1 in sequences: # find merge candidates among seq heads + candidate = s1[0] + for s2 in sequences: + if candidate in s2[1:]: + candidate = None + break # reject the current head, it appears later + else: + break + if candidate is None: + raise RuntimeError("Inconsistent hierarchy") + result.append(candidate) + # remove the chosen candidate + for seq in sequences: + if seq[0] == candidate: + del seq[0] + +def _c3_mro(cls, abcs=None): + """Computes the method resolution order using extended C3 linearization. + + If no *abcs* are given, the algorithm works exactly like the built-in C3 + linearization used for method resolution. + + If given, *abcs* is a list of abstract base classes that should be inserted + into the resulting MRO. Unrelated ABCs are ignored and don't end up in the + result. The algorithm inserts ABCs where their functionality is introduced, + i.e. issubclass(cls, abc) returns True for the class itself but returns + False for all its direct base classes. Implicit ABCs for a given class + (either registered or inferred from the presence of a special method like + __len__) are inserted directly after the last ABC explicitly listed in the + MRO of said class. If two implicit ABCs end up next to each other in the + resulting MRO, their ordering depends on the order of types in *abcs*. + + """ + for i, base in enumerate(reversed(cls.__bases__)): + if hasattr(base, '__abstractmethods__'): + boundary = len(cls.__bases__) - i + break # Bases up to the last explicit ABC are considered first. + else: + boundary = 0 + abcs = list(abcs) if abcs else [] + explicit_bases = list(cls.__bases__[:boundary]) + abstract_bases = [] + other_bases = list(cls.__bases__[boundary:]) + for base in abcs: + if issubclass(cls, base) and not any( + issubclass(b, base) for b in cls.__bases__ + ): + # If *cls* is the class that introduces behaviour described by + # an ABC *base*, insert said ABC to its MRO. + abstract_bases.append(base) + for base in abstract_bases: + abcs.remove(base) + explicit_c3_mros = [_c3_mro(base, abcs=abcs) for base in explicit_bases] + abstract_c3_mros = [_c3_mro(base, abcs=abcs) for base in abstract_bases] + other_c3_mros = [_c3_mro(base, abcs=abcs) for base in other_bases] + return _c3_merge( + [[cls]] + + explicit_c3_mros + abstract_c3_mros + other_c3_mros + + [explicit_bases] + [abstract_bases] + [other_bases] + ) + +def _compose_mro(cls, types): + """Calculates the method resolution order for a given class *cls*. + + Includes relevant abstract base classes (with their respective bases) from + the *types* iterable. Uses a modified C3 linearization algorithm. + + """ + bases = set(cls.__mro__) + # Remove entries which are already present in the __mro__ or unrelated. + def is_related(typ): + return (typ not in bases and hasattr(typ, '__mro__') + and issubclass(cls, typ)) + types = [n for n in types if is_related(n)] + # Remove entries which are strict bases of other entries (they will end up + # in the MRO anyway. + def is_strict_base(typ): + for other in types: + if typ != other and typ in other.__mro__: + return True + return False + types = [n for n in types if not is_strict_base(n)] + # Subclasses of the ABCs in *types* which are also implemented by + # *cls* can be used to stabilize ABC ordering. + type_set = set(types) + mro = [] + for typ in types: + found = [] + for sub in typ.__subclasses__(): + if sub not in bases and issubclass(cls, sub): + found.append([s for s in sub.__mro__ if s in type_set]) + if not found: + mro.append(typ) + continue + # Favor subclasses with the biggest number of useful bases + found.sort(key=len, reverse=True) + for sub in found: + for subcls in sub: + if subcls not in mro: + mro.append(subcls) + return _c3_mro(cls, abcs=mro) + +def _find_impl(cls, registry): + """Returns the best matching implementation from *registry* for type *cls*. + + Where there is no registered implementation for a specific type, its method + resolution order is used to find a more generic implementation. + + Note: if *registry* does not contain an implementation for the base + *object* type, this function may return None. + + """ + mro = _compose_mro(cls, registry.keys()) + match = None + for t in mro: + if match is not None: + # If *match* is an implicit ABC but there is another unrelated, + # equally matching implicit ABC, refuse the temptation to guess. + if (t in registry and t not in cls.__mro__ + and match not in cls.__mro__ + and not issubclass(match, t)): + raise RuntimeError("Ambiguous dispatch: {} or {}".format( + match, t)) + break + if t in registry: + match = t + return registry.get(match) + +def singledispatch(func): + """Single-dispatch generic function decorator. + + Transforms a function into a generic function, which can have different + behaviours depending upon the type of its first argument. The decorated + function acts as the default implementation, and additional + implementations can be registered using the register() attribute of the + generic function. + """ + # There are many programs that use functools without singledispatch, so we + # trade-off making singledispatch marginally slower for the benefit of + # making start-up of such applications slightly faster. + import types, weakref + + registry = {} + dispatch_cache = weakref.WeakKeyDictionary() + cache_token = None + + def dispatch(cls): + """generic_func.dispatch(cls) -> + + Runs the dispatch algorithm to return the best available implementation + for the given *cls* registered on *generic_func*. + + """ + nonlocal cache_token + if cache_token is not None: + current_token = get_cache_token() + if cache_token != current_token: + dispatch_cache.clear() + cache_token = current_token + try: + impl = dispatch_cache[cls] + except KeyError: + try: + impl = registry[cls] + except KeyError: + impl = _find_impl(cls, registry) + dispatch_cache[cls] = impl + return impl + + def register(cls, func=None): + """generic_func.register(cls, func) -> func + + Registers a new implementation for the given *cls* on a *generic_func*. + + """ + nonlocal cache_token + if func is None: + if isinstance(cls, type): + return lambda f: register(cls, f) + ann = getattr(cls, '__annotations__', {}) + if not ann: + raise TypeError( + f"Invalid first argument to `register()`: {cls!r}. " + f"Use either `@register(some_class)` or plain `@register` " + f"on an annotated function." + ) + func = cls + + # only import typing if annotation parsing is necessary + from typing import get_type_hints + argname, cls = next(iter(get_type_hints(func).items())) + assert isinstance(cls, type), ( + f"Invalid annotation for {argname!r}. {cls!r} is not a class." + ) + registry[cls] = func + if cache_token is None and hasattr(cls, '__abstractmethods__'): + cache_token = get_cache_token() + dispatch_cache.clear() + return func + + def wrapper(*args, **kw): + return dispatch(args[0].__class__)(*args, **kw) + + registry[object] = func + wrapper.register = register + wrapper.dispatch = dispatch + wrapper.registry = types.MappingProxyType(registry) + wrapper._clear_cache = dispatch_cache.clear + update_wrapper(wrapper, func) + return wrapper diff --git a/Lib/genericpath.py b/Lib/genericpath.py new file mode 100644 index 0000000000..5dd703d736 --- /dev/null +++ b/Lib/genericpath.py @@ -0,0 +1,151 @@ +""" +Path operations common to more than one OS +Do not use directly. The OS specific modules import the appropriate +functions from this module themselves. +""" +import os +import stat + +__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', + 'getsize', 'isdir', 'isfile', 'samefile', 'sameopenfile', + 'samestat'] + + +# Does a path exist? +# This is false for dangling symbolic links on systems that support them. +def exists(path): + """Test whether a path exists. Returns False for broken symbolic links""" + try: + os.stat(path) + except (OSError, ValueError): + return False + return True + + +# This follows symbolic links, so both islink() and isdir() can be true +# for the same path on systems that support symlinks +def isfile(path): + """Test whether a path is a regular file""" + try: + st = os.stat(path) + except (OSError, ValueError): + return False + return stat.S_ISREG(st.st_mode) + + +# Is a path a directory? +# This follows symbolic links, so both islink() and isdir() +# can be true for the same path on systems that support symlinks +def isdir(s): + """Return true if the pathname refers to an existing directory.""" + try: + st = os.stat(s) + except (OSError, ValueError): + return False + return stat.S_ISDIR(st.st_mode) + + +def getsize(filename): + """Return the size of a file, reported by os.stat().""" + return os.stat(filename).st_size + + +def getmtime(filename): + """Return the last modification time of a file, reported by os.stat().""" + return os.stat(filename).st_mtime + + +def getatime(filename): + """Return the last access time of a file, reported by os.stat().""" + return os.stat(filename).st_atime + + +def getctime(filename): + """Return the metadata change time of a file, reported by os.stat().""" + return os.stat(filename).st_ctime + + +# Return the longest prefix of all list elements. +def commonprefix(m): + "Given a list of pathnames, returns the longest common leading component" + if not m: return '' + # Some people pass in a list of pathname parts to operate in an OS-agnostic + # fashion; don't try to translate in that case as that's an abuse of the + # API and they are already doing what they need to be OS-agnostic and so + # they most likely won't be using an os.PathLike object in the sublists. + if not isinstance(m[0], (list, tuple)): + m = tuple(map(os.fspath, m)) + s1 = min(m) + s2 = max(m) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 + +# Are two stat buffers (obtained from stat, fstat or lstat) +# describing the same file? +def samestat(s1, s2): + """Test whether two stat buffers reference the same file""" + return (s1.st_ino == s2.st_ino and + s1.st_dev == s2.st_dev) + + +# Are two filenames really pointing to the same file? +def samefile(f1, f2): + """Test whether two pathnames reference the same actual file""" + s1 = os.stat(f1) + s2 = os.stat(f2) + return samestat(s1, s2) + + +# Are two open files really referencing the same file? +# (Not necessarily the same file descriptor!) +def sameopenfile(fp1, fp2): + """Test whether two open file objects reference the same file""" + s1 = os.fstat(fp1) + s2 = os.fstat(fp2) + return samestat(s1, s2) + + +# Split a path in root and extension. +# The extension is everything starting at the last dot in the last +# pathname component; the root is everything before that. +# It is always true that root + ext == p. + +# Generic implementation of splitext, to be parametrized with +# the separators +def _splitext(p, sep, altsep, extsep): + """Split the extension from a pathname. + + Extension is everything from the last dot to the end, ignoring + leading dots. Returns "(root, ext)"; ext may be empty.""" + # NOTE: This code must work for text and bytes strings. + + sepIndex = p.rfind(sep) + if altsep: + altsepIndex = p.rfind(altsep) + sepIndex = max(sepIndex, altsepIndex) + + dotIndex = p.rfind(extsep) + if dotIndex > sepIndex: + # skip all leading dots + filenameIndex = sepIndex + 1 + while filenameIndex < dotIndex: + if p[filenameIndex:filenameIndex+1] != extsep: + return p[:dotIndex], p[dotIndex:] + filenameIndex += 1 + + return p, p[:0] + +def _check_arg_types(funcname, *args): + hasstr = hasbytes = False + for s in args: + if isinstance(s, str): + hasstr = True + elif isinstance(s, bytes): + hasbytes = True + else: + raise TypeError('%s() argument must be str or bytes, not %r' % + (funcname, s.__class__.__name__)) from None + if hasstr and hasbytes: + raise TypeError("Can't mix strings and bytes in path components") from None diff --git a/Lib/heapq.py b/Lib/heapq.py new file mode 100644 index 0000000000..0e3555cf91 --- /dev/null +++ b/Lib/heapq.py @@ -0,0 +1,601 @@ +"""Heap queue algorithm (a.k.a. priority queue). + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +Usage: + +heap = [] # creates an empty heap +heappush(heap, item) # pushes a new item on the heap +item = heappop(heap) # pops the smallest item from the heap +item = heap[0] # smallest item on the heap without popping it +heapify(x) # transforms list into a heap, in-place, in linear time +item = heapreplace(heap, item) # pops and returns smallest item, and adds + # new item; the heap size is unchanged + +Our API differs from textbook heap algorithms as follows: + +- We use 0-based indexing. This makes the relationship between the + index for a node and the indexes for its children slightly less + obvious, but is more suitable since Python uses 0-based indexing. + +- Our heappop() method returns the smallest item, not the largest. + +These two make it possible to view the heap as a regular Python list +without surprises: heap[0] is the smallest item, and heap.sort() +maintains the heap invariant! +""" + +# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger + +__about__ = """Heap queues + +[explanation by François Pinard] + +Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for +all k, counting elements from 0. For the sake of comparison, +non-existing elements are considered to be infinite. The interesting +property of a heap is that a[0] is always its smallest element. + +The strange invariant above is meant to be an efficient memory +representation for a tournament. The numbers below are `k', not a[k]: + + 0 + + 1 2 + + 3 4 5 6 + + 7 8 9 10 11 12 13 14 + + 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + + +In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'. In +a usual binary tournament we see in sports, each cell is the winner +over the two cells it tops, and we can trace the winner down the tree +to see all opponents s/he had. However, in many computer applications +of such tournaments, we do not need to trace the history of a winner. +To be more memory efficient, when a winner is promoted, we try to +replace it by something else at a lower level, and the rule becomes +that a cell and the two cells it tops contain three different items, +but the top cell "wins" over the two topped cells. + +If this heap invariant is protected at all time, index 0 is clearly +the overall winner. The simplest algorithmic way to remove it and +find the "next" winner is to move some loser (let's say cell 30 in the +diagram above) into the 0 position, and then percolate this new 0 down +the tree, exchanging values, until the invariant is re-established. +This is clearly logarithmic on the total number of items in the tree. +By iterating over all items, you get an O(n ln n) sort. + +A nice feature of this sort is that you can efficiently insert new +items while the sort is going on, provided that the inserted items are +not "better" than the last 0'th element you extracted. This is +especially useful in simulation contexts, where the tree holds all +incoming events, and the "win" condition means the smallest scheduled +time. When an event schedule other events for execution, they are +scheduled into the future, so they can easily go into the heap. So, a +heap is a good structure for implementing schedulers (this is what I +used for my MIDI sequencer :-). + +Various structures for implementing schedulers have been extensively +studied, and heaps are good for this, as they are reasonably speedy, +the speed is almost constant, and the worst case is not much different +than the average case. However, there are other representations which +are more efficient overall, yet the worst cases might be terrible. + +Heaps are also very useful in big disk sorts. You most probably all +know that a big sort implies producing "runs" (which are pre-sorted +sequences, which size is usually related to the amount of CPU memory), +followed by a merging passes for these runs, which merging is often +very cleverly organised[1]. It is very important that the initial +sort produces the longest runs possible. Tournaments are a good way +to that. If, using all the memory available to hold a tournament, you +replace and percolate items that happen to fit the current run, you'll +produce runs which are twice the size of the memory for random input, +and much better for input fuzzily ordered. + +Moreover, if you output the 0'th item on disk and get an input which +may not fit in the current tournament (because the value "wins" over +the last output value), it cannot fit in the heap, so the size of the +heap decreases. The freed memory could be cleverly reused immediately +for progressively building a second heap, which grows at exactly the +same rate the first heap is melting. When the first heap completely +vanishes, you switch heaps and start a new run. Clever and quite +effective! + +In a word, heaps are useful memory structures to know. I use them in +a few applications, and I think it is good to keep a `heap' module +around. :-) + +-------------------- +[1] The disk balancing algorithms which are current, nowadays, are +more annoying than clever, and this is a consequence of the seeking +capabilities of the disks. On devices which cannot seek, like big +tape drives, the story was quite different, and one had to be very +clever to ensure (far in advance) that each tape movement will be the +most effective possible (that is, will best participate at +"progressing" the merge). Some tapes were even able to read +backwards, and this was also used to avoid the rewinding time. +Believe me, real good tape sorts were quite spectacular to watch! +From all times, sorting has always been a Great Art! :-) +""" + +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', + 'nlargest', 'nsmallest', 'heappushpop'] + +def heappush(heap, item): + """Push item onto heap, maintaining the heap invariant.""" + heap.append(item) + _siftdown(heap, 0, len(heap)-1) + +def heappop(heap): + """Pop the smallest item off the heap, maintaining the heap invariant.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup(heap, 0) + return returnitem + return lastelt + +def heapreplace(heap, item): + """Pop and return the current smallest value, and add the new item. + + This is more efficient than heappop() followed by heappush(), and can be + more appropriate when using a fixed-size heap. Note that the value + returned may be larger than item! That constrains reasonable uses of + this routine unless written as part of a conditional replacement: + + if item > heap[0]: + item = heapreplace(heap, item) + """ + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup(heap, 0) + return returnitem + +def heappushpop(heap, item): + """Fast version of a heappush followed by a heappop.""" + if heap and heap[0] < item: + item, heap[0] = heap[0], item + _siftup(heap, 0) + return item + +def heapify(x): + """Transform list into a heap, in-place, in O(len(x)) time.""" + n = len(x) + # Transform bottom-up. The largest index there's any point to looking at + # is the largest with a child index in-range, so must have 2*i + 1 < n, + # or i < (n-1)/2. If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so + # j-1 is the largest, which is n//2 - 1. If n is odd = 2*j+1, this is + # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1. + for i in reversed(range(n//2)): + _siftup(x, i) + +def _heappop_max(heap): + """Maxheap version of a heappop.""" + lastelt = heap.pop() # raises appropriate IndexError if heap is empty + if heap: + returnitem = heap[0] + heap[0] = lastelt + _siftup_max(heap, 0) + return returnitem + return lastelt + +def _heapreplace_max(heap, item): + """Maxheap version of a heappop followed by a heappush.""" + returnitem = heap[0] # raises appropriate IndexError if heap is empty + heap[0] = item + _siftup_max(heap, 0) + return returnitem + +def _heapify_max(x): + """Transform list into a maxheap, in-place, in O(len(x)) time.""" + n = len(x) + for i in reversed(range(n//2)): + _siftup_max(x, i) + +# 'heap' is a heap at all indices >= startpos, except possibly for pos. pos +# is the index of a leaf with a possibly out-of-order value. Restore the +# heap invariant. +def _siftdown(heap, startpos, pos): + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if newitem < parent: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +# The child indices of heap index pos are already heaps, and we want to make +# a heap at index pos too. We do this by bubbling the smaller child of +# pos up (and so on with that child's children, etc) until hitting a leaf, +# then using _siftdown to move the oddball originally at index pos into place. +# +# We *could* break out of the loop as soon as we find a pos where newitem <= +# both its children, but turns out that's not a good idea, and despite that +# many books write the algorithm that way. During a heap pop, the last array +# element is sifted in, and that tends to be large, so that comparing it +# against values starting from the root usually doesn't pay (= usually doesn't +# get us out of the loop early). See Knuth, Volume 3, where this is +# explained and quantified in an exercise. +# +# Cutting the # of comparisons is important, since these routines have no +# way to extract "the priority" from an array element, so that intelligence +# is likely to be hiding in custom comparison methods, or in array elements +# storing (priority, record) tuples. Comparisons are thus potentially +# expensive. +# +# On random arrays of length 1000, making this change cut the number of +# comparisons made by heapify() a little, and those made by exhaustive +# heappop() a lot, in accord with theory. Here are typical results from 3 +# runs (3 just to demonstrate how small the variance is): +# +# Compares needed by heapify Compares needed by 1000 heappops +# -------------------------- -------------------------------- +# 1837 cut to 1663 14996 cut to 8680 +# 1855 cut to 1659 14966 cut to 8678 +# 1847 cut to 1660 15024 cut to 8703 +# +# Building the heap by using heappush() 1000 times instead required +# 2198, 2148, and 2219 compares: heapify() is more efficient, when +# you can use it. +# +# The total compares needed by list.sort() on the same lists were 8627, +# 8627, and 8632 (this should be compared to the sum of heapify() and +# heappop() compares): list.sort() is (unsurprisingly!) more efficient +# for sorting. + +def _siftup(heap, pos): + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the smaller child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of smaller child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[childpos] < heap[rightpos]: + childpos = rightpos + # Move the smaller child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown(heap, startpos, pos) + +def _siftdown_max(heap, startpos, pos): + 'Maxheap variant of _siftdown' + newitem = heap[pos] + # Follow the path to the root, moving parents down until finding a place + # newitem fits. + while pos > startpos: + parentpos = (pos - 1) >> 1 + parent = heap[parentpos] + if parent < newitem: + heap[pos] = parent + pos = parentpos + continue + break + heap[pos] = newitem + +def _siftup_max(heap, pos): + 'Maxheap variant of _siftup' + endpos = len(heap) + startpos = pos + newitem = heap[pos] + # Bubble up the larger child until hitting a leaf. + childpos = 2*pos + 1 # leftmost child position + while childpos < endpos: + # Set childpos to index of larger child. + rightpos = childpos + 1 + if rightpos < endpos and not heap[rightpos] < heap[childpos]: + childpos = rightpos + # Move the larger child up. + heap[pos] = heap[childpos] + pos = childpos + childpos = 2*pos + 1 + # The leaf at pos is empty now. Put newitem there, and bubble it up + # to its final resting place (by sifting its parents down). + heap[pos] = newitem + _siftdown_max(heap, startpos, pos) + +def merge(*iterables, key=None, reverse=False): + '''Merge multiple sorted inputs into a single sorted output. + + Similar to sorted(itertools.chain(*iterables)) but returns a generator, + does not pull the data into memory all at once, and assumes that each of + the input streams is already sorted (smallest to largest). + + >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25])) + [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25] + + If *key* is not None, applies a key function to each element to determine + its sort order. + + >>> list(merge(['dog', 'horse'], ['cat', 'fish', 'kangaroo'], key=len)) + ['dog', 'cat', 'fish', 'horse', 'kangaroo'] + + ''' + + h = [] + h_append = h.append + + if reverse: + _heapify = _heapify_max + _heappop = _heappop_max + _heapreplace = _heapreplace_max + direction = -1 + else: + _heapify = heapify + _heappop = heappop + _heapreplace = heapreplace + direction = 1 + + if key is None: + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + h_append([next(), order * direction, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + value, order, next = s = h[0] + yield value + s[0] = next() # raises StopIteration when exhausted + _heapreplace(h, s) # restore heap condition + except StopIteration: + _heappop(h) # remove empty iterator + if h: + # fast case when only a single iterator remains + value, order, next = h[0] + yield value + yield from next.__self__ + return + + for order, it in enumerate(map(iter, iterables)): + try: + next = it.__next__ + value = next() + h_append([key(value), order * direction, value, next]) + except StopIteration: + pass + _heapify(h) + while len(h) > 1: + try: + while True: + key_value, order, value, next = s = h[0] + yield value + value = next() + s[0] = key(value) + s[2] = value + _heapreplace(h, s) + except StopIteration: + _heappop(h) + if h: + key_value, order, value, next = h[0] + yield value + yield from next.__self__ + + +# Algorithm notes for nlargest() and nsmallest() +# ============================================== +# +# Make a single pass over the data while keeping the k most extreme values +# in a heap. Memory consumption is limited to keeping k values in a list. +# +# Measured performance for random inputs: +# +# number of comparisons +# n inputs k-extreme values (average of 5 trials) % more than min() +# ------------- ---------------- --------------------- ----------------- +# 1,000 100 3,317 231.7% +# 10,000 100 14,046 40.5% +# 100,000 100 105,749 5.7% +# 1,000,000 100 1,007,751 0.8% +# 10,000,000 100 10,009,401 0.1% +# +# Theoretical number of comparisons for k smallest of n random inputs: +# +# Step Comparisons Action +# ---- -------------------------- --------------------------- +# 1 1.66 * k heapify the first k-inputs +# 2 n - k compare remaining elements to top of heap +# 3 k * (1 + lg2(k)) * ln(n/k) replace the topmost value on the heap +# 4 k * lg2(k) - (k/2) final sort of the k most extreme values +# +# Combining and simplifying for a rough estimate gives: +# +# comparisons = n + k * (log(k, 2) * log(n/k) + log(k, 2) + log(n/k)) +# +# Computing the number of comparisons for step 3: +# ----------------------------------------------- +# * For the i-th new value from the iterable, the probability of being in the +# k most extreme values is k/i. For example, the probability of the 101st +# value seen being in the 100 most extreme values is 100/101. +# * If the value is a new extreme value, the cost of inserting it into the +# heap is 1 + log(k, 2). +# * The probability times the cost gives: +# (k/i) * (1 + log(k, 2)) +# * Summing across the remaining n-k elements gives: +# sum((k/i) * (1 + log(k, 2)) for i in range(k+1, n+1)) +# * This reduces to: +# (H(n) - H(k)) * k * (1 + log(k, 2)) +# * Where H(n) is the n-th harmonic number estimated by: +# gamma = 0.5772156649 +# H(n) = log(n, e) + gamma + 1 / (2 * n) +# http://en.wikipedia.org/wiki/Harmonic_series_(mathematics)#Rate_of_divergence +# * Substituting the H(n) formula: +# comparisons = k * (1 + log(k, 2)) * (log(n/k, e) + (1/n - 1/k) / 2) +# +# Worst-case for step 3: +# ---------------------- +# In the worst case, the input data is reversed sorted so that every new element +# must be inserted in the heap: +# +# comparisons = 1.66 * k + log(k, 2) * (n - k) +# +# Alternative Algorithms +# ---------------------- +# Other algorithms were not used because they: +# 1) Took much more auxiliary memory, +# 2) Made multiple passes over the data. +# 3) Made more comparisons in common cases (small k, large n, semi-random input). +# See the more detailed comparison of approach at: +# http://code.activestate.com/recipes/577573-compare-algorithms-for-heapqsmallest + +def nsmallest(n, iterable, key=None): + """Find the n smallest elements in a dataset. + + Equivalent to: sorted(iterable, key=key)[:n] + """ + + # Short-cut for n==1 is to use min() + if n == 1: + it = iter(iterable) + sentinel = object() + result = min(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + # put the range(n) first so that zip() doesn't + # consume one too many elements from the iterator + result = [(elem, i) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + if elem < top: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order += 1 + result.sort() + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(n), it)] + if not result: + return result + _heapify_max(result) + top = result[0][0] + order = n + _heapreplace = _heapreplace_max + for elem in it: + k = key(elem) + if k < top: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order += 1 + result.sort() + return [elem for (k, order, elem) in result] + +def nlargest(n, iterable, key=None): + """Find the n largest elements in a dataset. + + Equivalent to: sorted(iterable, key=key, reverse=True)[:n] + """ + + # Short-cut for n==1 is to use max() + if n == 1: + it = iter(iterable) + sentinel = object() + result = max(it, default=sentinel, key=key) + return [] if result is sentinel else [result] + + # When n>=size, it's faster to use sorted() + try: + size = len(iterable) + except (TypeError, AttributeError): + pass + else: + if n >= size: + return sorted(iterable, key=key, reverse=True)[:n] + + # When key is none, use simpler decoration + if key is None: + it = iter(iterable) + result = [(elem, i) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + if top < elem: + _heapreplace(result, (elem, order)) + top, _order = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (elem, order) in result] + + # General case, slowest method + it = iter(iterable) + result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)] + if not result: + return result + heapify(result) + top = result[0][0] + order = -n + _heapreplace = heapreplace + for elem in it: + k = key(elem) + if top < k: + _heapreplace(result, (k, order, elem)) + top, _order, _elem = result[0] + order -= 1 + result.sort(reverse=True) + return [elem for (k, order, elem) in result] + +# If available, use C implementation +try: + from _heapq import * +except ImportError: + pass +try: + from _heapq import _heapreplace_max +except ImportError: + pass +try: + from _heapq import _heapify_max +except ImportError: + pass +try: + from _heapq import _heappop_max +except ImportError: + pass + + +if __name__ == "__main__": + + import doctest + print(doctest.testmod()) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py new file mode 100644 index 0000000000..32deef10af --- /dev/null +++ b/Lib/importlib/_bootstrap.py @@ -0,0 +1,1173 @@ +"""Core implementation of import. + +This module is NOT meant to be directly imported! It has been designed such +that it can be bootstrapped into Python as the implementation of import. As +such it requires the injection of specific modules and attributes in order to +work. One should use importlib as the public-facing version of this module. + +""" +# +# IMPORTANT: Whenever making changes to this module, be sure to run a top-level +# `make regen-importlib` followed by `make` in order to get the frozen version +# of the module updated. Not doing so will result in the Makefile to fail for +# all others who don't have a ./python around to freeze the module +# in the early stages of compilation. +# + +# See importlib._setup() for what is injected into the global namespace. + +# When editing this code be aware that code executed at import time CANNOT +# reference any injected objects! This includes not only global code but also +# anything specified at the class level. + +# Bootstrap-related code ###################################################### + +_bootstrap_external = None + +def _wrap(new, old): + """Simple substitute for functools.update_wrapper.""" + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: + if hasattr(old, replace): + setattr(new, replace, getattr(old, replace)) + new.__dict__.update(old.__dict__) + + +def _new_module(name): + return type(sys)(name) + + +# Module-level locking ######################################################## + +# A dict mapping module names to weakrefs of _ModuleLock instances +# Dictionary protected by the global import lock +_module_locks = {} +# A dict mapping thread ids to _ModuleLock instances +_blocking_on = {} + + +class _DeadlockError(RuntimeError): + pass + + +class _ModuleLock: + """A recursive lock implementation which is able to detect deadlocks + (e.g. thread 1 trying to take locks A then B, and thread 2 trying to + take locks B then A). + """ + + def __init__(self, name): + self.lock = _thread.allocate_lock() + self.wakeup = _thread.allocate_lock() + self.name = name + self.owner = None + self.count = 0 + self.waiters = 0 + + def has_deadlock(self): + # Deadlock avoidance for concurrent circular imports. + me = _thread.get_ident() + tid = self.owner + while True: + lock = _blocking_on.get(tid) + if lock is None: + return False + tid = lock.owner + if tid == me: + return True + + def acquire(self): + """ + Acquire the module lock. If a potential deadlock is detected, + a _DeadlockError is raised. + Otherwise, the lock is always acquired and True is returned. + """ + tid = _thread.get_ident() + _blocking_on[tid] = self + try: + while True: + with self.lock: + if self.count == 0 or self.owner == tid: + self.owner = tid + self.count += 1 + return True + if self.has_deadlock(): + raise _DeadlockError('deadlock detected by %r' % self) + if self.wakeup.acquire(False): + self.waiters += 1 + # Wait for a release() call + self.wakeup.acquire() + self.wakeup.release() + finally: + del _blocking_on[tid] + + def release(self): + tid = _thread.get_ident() + with self.lock: + if self.owner != tid: + raise RuntimeError('cannot release un-acquired lock') + assert self.count > 0 + self.count -= 1 + if self.count == 0: + self.owner = None + if self.waiters: + self.waiters -= 1 + self.wakeup.release() + + def __repr__(self): + return '_ModuleLock({!r}) at {}'.format(self.name, id(self)) + + +class _DummyModuleLock: + """A simple _ModuleLock equivalent for Python builds without + multi-threading support.""" + + def __init__(self, name): + self.name = name + self.count = 0 + + def acquire(self): + self.count += 1 + return True + + def release(self): + if self.count == 0: + raise RuntimeError('cannot release un-acquired lock') + self.count -= 1 + + def __repr__(self): + return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self)) + + +class _ModuleLockManager: + + def __init__(self, name): + self._name = name + self._lock = None + + def __enter__(self): + self._lock = _get_module_lock(self._name) + self._lock.acquire() + + def __exit__(self, *args, **kwargs): + self._lock.release() + + +# The following two functions are for consumption by Python/import.c. + +def _get_module_lock(name): + """Get or create the module lock for a given module name. + + Acquire/release internally the global import lock to protect + _module_locks.""" + + _imp.acquire_lock() + try: + try: + lock = _module_locks[name]() + except KeyError: + lock = None + + if lock is None: + if _thread is None: + lock = _DummyModuleLock(name) + else: + lock = _ModuleLock(name) + + def cb(ref, name=name): + _imp.acquire_lock() + try: + # bpo-31070: Check if another thread created a new lock + # after the previous lock was destroyed + # but before the weakref callback was called. + if _module_locks.get(name) is ref: + del _module_locks[name] + finally: + _imp.release_lock() + + _module_locks[name] = _weakref.ref(lock, cb) + finally: + _imp.release_lock() + + return lock + + +def _lock_unlock_module(name): + """Acquires then releases the module lock for a given module name. + + This is used to ensure a module is completely initialized, in the + event it is being imported by another thread. + """ + lock = _get_module_lock(name) + try: + lock.acquire() + except _DeadlockError: + # Concurrent circular import, we'll accept a partially initialized + # module object. + pass + else: + lock.release() + +# Frame stripping magic ############################################### +def _call_with_frames_removed(f, *args, **kwds): + """remove_importlib_frames in import.c will always remove sequences + of importlib frames that end with a call to this function + + Use it instead of a normal call in places where including the importlib + frames introduces unwanted noise into the traceback (e.g. when executing + module code) + """ + return f(*args, **kwds) + + +def _verbose_message(message, *args, verbosity=1): + """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" + if sys.flags.verbose >= verbosity: + if not message.startswith(('#', 'import ')): + message = '# ' + message + print(message.format(*args), file=sys.stderr) + + +def _requires_builtin(fxn): + """Decorator to verify the named module is built-in.""" + def _requires_builtin_wrapper(self, fullname): + if fullname not in sys.builtin_module_names: + raise ImportError('{!r} is not a built-in module'.format(fullname), + name=fullname) + return fxn(self, fullname) + _wrap(_requires_builtin_wrapper, fxn) + return _requires_builtin_wrapper + + +def _requires_frozen(fxn): + """Decorator to verify the named module is frozen.""" + def _requires_frozen_wrapper(self, fullname): + if not _imp.is_frozen(fullname): + raise ImportError('{!r} is not a frozen module'.format(fullname), + name=fullname) + return fxn(self, fullname) + _wrap(_requires_frozen_wrapper, fxn) + return _requires_frozen_wrapper + + +# Typically used by loader classes as a method replacement. +def _load_module_shim(self, fullname): + """Load the specified module into sys.modules and return it. + + This method is deprecated. Use loader.exec_module instead. + + """ + spec = spec_from_loader(fullname, self) + if fullname in sys.modules: + module = sys.modules[fullname] + _exec(spec, module) + return sys.modules[fullname] + else: + return _load(spec) + +# Module specifications ####################################################### + +def _module_repr(module): + # The implementation of ModuleType.__repr__(). + loader = getattr(module, '__loader__', None) + if hasattr(loader, 'module_repr'): + # As soon as BuiltinImporter, FrozenImporter, and NamespaceLoader + # drop their implementations for module_repr. we can add a + # deprecation warning here. + try: + return loader.module_repr(module) + except Exception: + pass + try: + spec = module.__spec__ + except AttributeError: + pass + else: + if spec is not None: + return _module_repr_from_spec(spec) + + # We could use module.__class__.__name__ instead of 'module' in the + # various repr permutations. + try: + name = module.__name__ + except AttributeError: + name = '?' + try: + filename = module.__file__ + except AttributeError: + if loader is None: + return ''.format(name) + else: + return ''.format(name, loader) + else: + return ''.format(name, filename) + + +class ModuleSpec: + """The specification for a module, used for loading. + + A module's spec is the source for information about the module. For + data associated with the module, including source, use the spec's + loader. + + `name` is the absolute name of the module. `loader` is the loader + to use when loading the module. `parent` is the name of the + package the module is in. The parent is derived from the name. + + `is_package` determines if the module is considered a package or + not. On modules this is reflected by the `__path__` attribute. + + `origin` is the specific location used by the loader from which to + load the module, if that information is available. When filename is + set, origin will match. + + `has_location` indicates that a spec's "origin" reflects a location. + When this is True, `__file__` attribute of the module is set. + + `cached` is the location of the cached bytecode file, if any. It + corresponds to the `__cached__` attribute. + + `submodule_search_locations` is the sequence of path entries to + search when importing submodules. If set, is_package should be + True--and False otherwise. + + Packages are simply modules that (may) have submodules. If a spec + has a non-None value in `submodule_search_locations`, the import + system will consider modules loaded from the spec as packages. + + Only finders (see importlib.abc.MetaPathFinder and + importlib.abc.PathEntryFinder) should modify ModuleSpec instances. + + """ + + def __init__(self, name, loader, *, origin=None, loader_state=None, + is_package=None): + self.name = name + self.loader = loader + self.origin = origin + self.loader_state = loader_state + self.submodule_search_locations = [] if is_package else None + + # file-location attributes + self._set_fileattr = False + self._cached = None + + def __repr__(self): + args = ['name={!r}'.format(self.name), + 'loader={!r}'.format(self.loader)] + if self.origin is not None: + args.append('origin={!r}'.format(self.origin)) + if self.submodule_search_locations is not None: + args.append('submodule_search_locations={}' + .format(self.submodule_search_locations)) + return '{}({})'.format(self.__class__.__name__, ', '.join(args)) + + def __eq__(self, other): + smsl = self.submodule_search_locations + try: + return (self.name == other.name and + self.loader == other.loader and + self.origin == other.origin and + smsl == other.submodule_search_locations and + self.cached == other.cached and + self.has_location == other.has_location) + except AttributeError: + return False + + @property + def cached(self): + if self._cached is None: + if self.origin is not None and self._set_fileattr: + if _bootstrap_external is None: + raise NotImplementedError + self._cached = _bootstrap_external._get_cached(self.origin) + return self._cached + + @cached.setter + def cached(self, cached): + self._cached = cached + + @property + def parent(self): + """The name of the module's parent.""" + if self.submodule_search_locations is None: + return self.name.rpartition('.')[0] + else: + return self.name + + @property + def has_location(self): + return self._set_fileattr + + @has_location.setter + def has_location(self, value): + self._set_fileattr = bool(value) + + +def spec_from_loader(name, loader, *, origin=None, is_package=None): + """Return a module spec based on various loader methods.""" + if hasattr(loader, 'get_filename'): + if _bootstrap_external is None: + raise NotImplementedError + spec_from_file_location = _bootstrap_external.spec_from_file_location + + if is_package is None: + return spec_from_file_location(name, loader=loader) + search = [] if is_package else None + return spec_from_file_location(name, loader=loader, + submodule_search_locations=search) + + if is_package is None: + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + is_package = None # aka, undefined + else: + # the default + is_package = False + + return ModuleSpec(name, loader, origin=origin, is_package=is_package) + + +def _spec_from_module(module, loader=None, origin=None): + # This function is meant for use in _setup(). + try: + spec = module.__spec__ + except AttributeError: + pass + else: + if spec is not None: + return spec + + name = module.__name__ + if loader is None: + try: + loader = module.__loader__ + except AttributeError: + # loader will stay None. + pass + try: + location = module.__file__ + except AttributeError: + location = None + if origin is None: + if location is None: + try: + origin = loader._ORIGIN + except AttributeError: + origin = None + else: + origin = location + try: + cached = module.__cached__ + except AttributeError: + cached = None + try: + submodule_search_locations = list(module.__path__) + except AttributeError: + submodule_search_locations = None + + spec = ModuleSpec(name, loader, origin=origin) + spec._set_fileattr = False if location is None else True + spec.cached = cached + spec.submodule_search_locations = submodule_search_locations + return spec + + +def _init_module_attrs(spec, module, *, override=False): + # The passed-in module may be not support attribute assignment, + # in which case we simply don't set the attributes. + # __name__ + if (override or getattr(module, '__name__', None) is None): + try: + module.__name__ = spec.name + except AttributeError: + pass + # __loader__ + if override or getattr(module, '__loader__', None) is None: + loader = spec.loader + if loader is None: + # A backward compatibility hack. + if spec.submodule_search_locations is not None: + if _bootstrap_external is None: + raise NotImplementedError + _NamespaceLoader = _bootstrap_external._NamespaceLoader + + loader = _NamespaceLoader.__new__(_NamespaceLoader) + loader._path = spec.submodule_search_locations + spec.loader = loader + # While the docs say that module.__file__ is not set for + # built-in modules, and the code below will avoid setting it if + # spec.has_location is false, this is incorrect for namespace + # packages. Namespace packages have no location, but their + # __spec__.origin is None, and thus their module.__file__ + # should also be None for consistency. While a bit of a hack, + # this is the best place to ensure this consistency. + # + # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module + # and bpo-32305 + module.__file__ = None + try: + module.__loader__ = loader + except AttributeError: + pass + # __package__ + if override or getattr(module, '__package__', None) is None: + try: + module.__package__ = spec.parent + except AttributeError: + pass + # __spec__ + try: + module.__spec__ = spec + except AttributeError: + pass + # __path__ + if override or getattr(module, '__path__', None) is None: + if spec.submodule_search_locations is not None: + try: + module.__path__ = spec.submodule_search_locations + except AttributeError: + pass + # __file__/__cached__ + if spec.has_location: + if override or getattr(module, '__file__', None) is None: + try: + module.__file__ = spec.origin + except AttributeError: + pass + + if override or getattr(module, '__cached__', None) is None: + if spec.cached is not None: + try: + module.__cached__ = spec.cached + except AttributeError: + pass + return module + + +def module_from_spec(spec): + """Create a module based on the provided spec.""" + # Typically loaders will not implement create_module(). + module = None + if hasattr(spec.loader, 'create_module'): + # If create_module() returns `None` then it means default + # module creation should be used. + module = spec.loader.create_module(spec) + elif hasattr(spec.loader, 'exec_module'): + raise ImportError('loaders that define exec_module() ' + 'must also define create_module()') + if module is None: + module = _new_module(spec.name) + _init_module_attrs(spec, module) + return module + + +def _module_repr_from_spec(spec): + """Return the repr to use for the module.""" + # We mostly replicate _module_repr() using the spec attributes. + name = '?' if spec.name is None else spec.name + if spec.origin is None: + if spec.loader is None: + return ''.format(name) + else: + return ''.format(name, spec.loader) + else: + if spec.has_location: + return ''.format(name, spec.origin) + else: + return ''.format(spec.name, spec.origin) + + +# Used by importlib.reload() and _load_module_shim(). +def _exec(spec, module): + """Execute the spec's specified module in an existing module's namespace.""" + name = spec.name + with _ModuleLockManager(name): + if sys.modules.get(name) is not module: + msg = 'module {!r} not in sys.modules'.format(name) + raise ImportError(msg, name=name) + try: + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # Namespace package. + _init_module_attrs(spec, module, override=True) + else: + _init_module_attrs(spec, module, override=True) + if not hasattr(spec.loader, 'exec_module'): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + spec.loader.load_module(name) + else: + spec.loader.exec_module(module) + finally: + # Update the order of insertion into sys.modules for module + # clean-up at shutdown. + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + return module + + +def _load_backward_compatible(spec): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + try: + spec.loader.load_module(spec.name) + except: + if spec.name in sys.modules: + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + raise + # The module must be in sys.modules at this point! + # Move it to the end of sys.modules. + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + if getattr(module, '__loader__', None) is None: + try: + module.__loader__ = spec.loader + except AttributeError: + pass + if getattr(module, '__package__', None) is None: + try: + # Since module.__path__ may not line up with + # spec.submodule_search_paths, we can't necessarily rely + # on spec.parent here. + module.__package__ = module.__name__ + if not hasattr(module, '__path__'): + module.__package__ = spec.name.rpartition('.')[0] + except AttributeError: + pass + if getattr(module, '__spec__', None) is None: + try: + module.__spec__ = spec + except AttributeError: + pass + return module + +def _load_unlocked(spec): + # A helper for direct use by the import system. + if spec.loader is not None: + # Not a namespace package. + if not hasattr(spec.loader, 'exec_module'): + return _load_backward_compatible(spec) + + module = module_from_spec(spec) + + # This must be done before putting the module in sys.modules + # (otherwise an optimization shortcut in import.c becomes + # wrong). + spec._initializing = True + try: + sys.modules[spec.name] = module + try: + if spec.loader is None: + if spec.submodule_search_locations is None: + raise ImportError('missing loader', name=spec.name) + # A namespace package so do nothing. + else: + spec.loader.exec_module(module) + except: + try: + del sys.modules[spec.name] + except KeyError: + pass + raise + # Move the module to the end of sys.modules. + # We don't ensure that the import-related module attributes get + # set in the sys.modules replacement case. Such modules are on + # their own. + module = sys.modules.pop(spec.name) + sys.modules[spec.name] = module + _verbose_message('import {!r} # {!r}', spec.name, spec.loader) + finally: + spec._initializing = False + + return module + +# A method used during testing of _load_unlocked() and by +# _load_module_shim(). +def _load(spec): + """Return a new module object, loaded by the spec's loader. + + The module is not added to its parent. + + If a module is already in sys.modules, that existing module gets + clobbered. + + """ + with _ModuleLockManager(spec.name): + return _load_unlocked(spec) + + +# Loaders ##################################################################### + +class BuiltinImporter: + + """Meta path import for built-in modules. + + All methods are either class or static methods to avoid the need to + instantiate the class. + + """ + + @staticmethod + def module_repr(module): + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return ''.format(module.__name__) + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + if path is not None: + return None + if _imp.is_builtin(fullname): + return spec_from_loader(fullname, cls, origin='built-in') + else: + return None + + @classmethod + def find_module(cls, fullname, path=None): + """Find the built-in module. + + If 'path' is ever specified then the search is considered a failure. + + This method is deprecated. Use find_spec() instead. + + """ + spec = cls.find_spec(fullname, path) + return spec.loader if spec is not None else None + + @classmethod + def create_module(self, spec): + """Create a built-in module""" + if spec.name not in sys.builtin_module_names: + raise ImportError('{!r} is not a built-in module'.format(spec.name), + name=spec.name) + return _call_with_frames_removed(_imp.create_builtin, spec) + + @classmethod + def exec_module(self, module): + """Exec a built-in module""" + _call_with_frames_removed(_imp.exec_builtin, module) + + @classmethod + @_requires_builtin + def get_code(cls, fullname): + """Return None as built-in modules do not have code objects.""" + return None + + @classmethod + @_requires_builtin + def get_source(cls, fullname): + """Return None as built-in modules do not have source code.""" + return None + + @classmethod + @_requires_builtin + def is_package(cls, fullname): + """Return False as built-in modules are never packages.""" + return False + + load_module = classmethod(_load_module_shim) + + +class FrozenImporter: + + """Meta path import for frozen modules. + + All methods are either class or static methods to avoid the need to + instantiate the class. + + """ + + _ORIGIN = "frozen" + + @staticmethod + def module_repr(m): + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return ''.format(m.__name__, FrozenImporter._ORIGIN) + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + if _imp.is_frozen(fullname): + return spec_from_loader(fullname, cls, origin=cls._ORIGIN) + else: + return None + + @classmethod + def find_module(cls, fullname, path=None): + """Find a frozen module. + + This method is deprecated. Use find_spec() instead. + + """ + return cls if _imp.is_frozen(fullname) else None + + @classmethod + def create_module(cls, spec): + """Use default semantics for module creation.""" + + @staticmethod + def exec_module(module): + name = module.__spec__.name + if not _imp.is_frozen(name): + raise ImportError('{!r} is not a frozen module'.format(name), + name=name) + code = _call_with_frames_removed(_imp.get_frozen_object, name) + exec(code, module.__dict__) + + @classmethod + def load_module(cls, fullname): + """Load a frozen module. + + This method is deprecated. Use exec_module() instead. + + """ + return _load_module_shim(cls, fullname) + + @classmethod + @_requires_frozen + def get_code(cls, fullname): + """Return the code object for the frozen module.""" + return _imp.get_frozen_object(fullname) + + @classmethod + @_requires_frozen + def get_source(cls, fullname): + """Return None as frozen modules do not have source code.""" + return None + + @classmethod + @_requires_frozen + def is_package(cls, fullname): + """Return True if the frozen module is a package.""" + return _imp.is_frozen_package(fullname) + + +# Import itself ############################################################### + +class _ImportLockContext: + + """Context manager for the import lock.""" + + def __enter__(self): + """Acquire the import lock.""" + _imp.acquire_lock() + + def __exit__(self, exc_type, exc_value, exc_traceback): + """Release the import lock regardless of any raised exceptions.""" + _imp.release_lock() + + +def _resolve_name(name, package, level): + """Resolve a relative module name to an absolute one.""" + bits = package.rsplit('.', level - 1) + if len(bits) < level: + raise ValueError('attempted relative import beyond top-level package') + base = bits[0] + return '{}.{}'.format(base, name) if name else base + + +def _find_spec_legacy(finder, name, path): + # This would be a good place for a DeprecationWarning if + # we ended up going that route. + loader = finder.find_module(name, path) + if loader is None: + return None + return spec_from_loader(name, loader) + + +def _find_spec(name, path, target=None): + """Find a module's spec.""" + meta_path = sys.meta_path + if meta_path is None: + # PyImport_Cleanup() is running or has been called. + raise ImportError("sys.meta_path is None, Python is likely " + "shutting down") + + if not meta_path: + _warnings.warn('sys.meta_path is empty', ImportWarning) + + # We check sys.modules here for the reload case. While a passed-in + # target will usually indicate a reload there is no guarantee, whereas + # sys.modules provides one. + is_reload = name in sys.modules + for finder in meta_path: + with _ImportLockContext(): + try: + find_spec = finder.find_spec + except AttributeError: + spec = _find_spec_legacy(finder, name, path) + if spec is None: + continue + else: + spec = find_spec(name, path, target) + if spec is not None: + # The parent import may have already imported this module. + if not is_reload and name in sys.modules: + module = sys.modules[name] + try: + __spec__ = module.__spec__ + except AttributeError: + # We use the found spec since that is the one that + # we would have used if the parent module hadn't + # beaten us to the punch. + return spec + else: + if __spec__ is None: + return spec + else: + return __spec__ + else: + return spec + else: + return None + + +def _sanity_check(name, package, level): + """Verify arguments are "sane".""" + if not isinstance(name, str): + raise TypeError('module name must be str, not {}'.format(type(name))) + if level < 0: + raise ValueError('level must be >= 0') + if level > 0: + if not isinstance(package, str): + raise TypeError('__package__ not set to a string') + elif not package: + raise ImportError('attempted relative import with no known parent ' + 'package') + if not name and level == 0: + raise ValueError('Empty module name') + + +_ERR_MSG_PREFIX = 'No module named ' +_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' + +def _find_and_load_unlocked(name, import_): + path = None + parent = name.rpartition('.')[0] + if parent: + if parent not in sys.modules: + _call_with_frames_removed(import_, parent) + # Crazy side-effects! + if name in sys.modules: + return sys.modules[name] + parent_module = sys.modules[parent] + try: + path = parent_module.__path__ + except AttributeError: + msg = (_ERR_MSG + '; {!r} is not a package').format(name, parent) + raise ModuleNotFoundError(msg, name=name) from None + spec = _find_spec(name, path) + if spec is None: + raise ModuleNotFoundError(_ERR_MSG.format(name), name=name) + else: + module = _load_unlocked(spec) + if parent: + # Set the module as an attribute on its parent. + parent_module = sys.modules[parent] + setattr(parent_module, name.rpartition('.')[2], module) + return module + + +_NEEDS_LOADING = object() + + +def _find_and_load(name, import_): + """Find and load the module.""" + with _ModuleLockManager(name): + module = sys.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + return _find_and_load_unlocked(name, import_) + + if module is None: + message = ('import of {} halted; ' + 'None in sys.modules'.format(name)) + raise ModuleNotFoundError(message, name=name) + + _lock_unlock_module(name) + return module + + +def _gcd_import(name, package=None, level=0): + """Import and return the module based on its name, the package the call is + being made from, and the level adjustment. + + This function represents the greatest common denominator of functionality + between import_module and __import__. This includes setting __package__ if + the loader did not. + + """ + _sanity_check(name, package, level) + if level > 0: + name = _resolve_name(name, package, level) + return _find_and_load(name, _gcd_import) + + +def _handle_fromlist(module, fromlist, import_, *, recursive=False): + """Figure out what __import__ should return. + + The import_ parameter is a callable which takes the name of module to + import. It is required to decouple the function from assuming importlib's + import implementation is desired. + + """ + # The hell that is fromlist ... + # If a package was imported, try to import stuff from fromlist. + for x in fromlist: + if not isinstance(x, str): + if recursive: + where = module.__name__ + '.__all__' + else: + where = "``from list''" + raise TypeError(f"Item in {where} must be str, " + f"not {type(x).__name__}") + elif x == '*': + if not recursive and hasattr(module, '__all__'): + _handle_fromlist(module, module.__all__, import_, + recursive=True) + elif not hasattr(module, x): + from_name = '{}.{}'.format(module.__name__, x) + try: + _call_with_frames_removed(import_, from_name) + except ModuleNotFoundError as exc: + # Backwards-compatibility dictates we ignore failed + # imports triggered by fromlist for modules that don't + # exist. + if (exc.name == from_name and + sys.modules.get(from_name, _NEEDS_LOADING) is not None): + continue + raise + return module + + +def _calc___package__(globals): + """Calculate what __package__ should be. + + __package__ is not guaranteed to be defined or could be set to None + to represent that its proper value is unknown. + + """ + package = globals.get('__package__') + spec = globals.get('__spec__') + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn("__package__ != __spec__.parent " + f"({package!r} != {spec.parent!r})", + ImportWarning, stacklevel=3) + return package + elif spec is not None: + return spec.parent + else: + _warnings.warn("can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, stacklevel=3) + package = globals['__name__'] + if '__path__' not in globals: + package = package.rpartition('.')[0] + return package + + +def __import__(name, globals=None, locals=None, fromlist=(), level=0): + """Import a module. + + The 'globals' argument is used to infer where the import is occurring from + to handle relative imports. The 'locals' argument is ignored. The + 'fromlist' argument specifies what should exist as attributes on the module + being imported (e.g. ``from module import ``). The 'level' + argument represents the package location to import from in a relative + import (e.g. ``from ..pkg import mod`` would have a 'level' of 2). + + """ + if level == 0: + module = _gcd_import(name) + else: + globals_ = globals if globals is not None else {} + package = _calc___package__(globals_) + module = _gcd_import(name, package, level) + if not fromlist: + # Return up to the first dot in 'name'. This is complicated by the fact + # that 'name' may be relative. + if level == 0: + return _gcd_import(name.partition('.')[0]) + elif not name: + return module + else: + # Figure out where to slice the module's name up to the first dot + # in 'name'. + cut_off = len(name) - len(name.partition('.')[0]) + # Slice end needs to be positive to alleviate need to special-case + # when ``'.' not in name``. + return sys.modules[module.__name__[:len(module.__name__)-cut_off]] + elif hasattr(module, '__path__'): + return _handle_fromlist(module, fromlist, _gcd_import) + else: + return module + + +def _builtin_from_name(name): + spec = BuiltinImporter.find_spec(name) + if spec is None: + raise ImportError('no built-in module named ' + name) + return _load_unlocked(spec) + + +def _setup(sys_module, _imp_module): + """Setup importlib by importing needed built-in modules and injecting them + into the global namespace. + + As sys is needed for sys.modules access and _imp is needed to load built-in + modules, those two modules must be explicitly passed in. + + """ + global _imp, sys + _imp = _imp_module + sys = sys_module + + # Set up the spec for existing builtin/frozen modules. + module_type = type(sys) + for name, module in sys.modules.items(): + if isinstance(module, module_type): + if name in sys.builtin_module_names: + loader = BuiltinImporter + elif _imp.is_frozen(name): + loader = FrozenImporter + else: + continue + spec = _spec_from_module(module, loader) + _init_module_attrs(spec, module) + + # Directly load built-in modules needed during bootstrap. + self_module = sys.modules[__name__] + for builtin_name in ('_thread', '_warnings', '_weakref'): + if builtin_name not in sys.modules: + builtin_module = _builtin_from_name(builtin_name) + else: + builtin_module = sys.modules[builtin_name] + setattr(self_module, builtin_name, builtin_module) + + +def _install(sys_module, _imp_module): + """Install importers for builtin and frozen modules""" + _setup(sys_module, _imp_module) + + sys.meta_path.append(BuiltinImporter) + sys.meta_path.append(FrozenImporter) + + +def _install_external_importers(): + """Install importers that require external filesystem access""" + global _bootstrap_external + import _frozen_importlib_external + _bootstrap_external = _frozen_importlib_external + _frozen_importlib_external._install(sys.modules[__name__]) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py new file mode 100644 index 0000000000..124948ef48 --- /dev/null +++ b/Lib/importlib/_bootstrap_external.py @@ -0,0 +1,1616 @@ +"""Core implementation of path-based import. + +This module is NOT meant to be directly imported! It has been designed such +that it can be bootstrapped into Python as the implementation of import. As +such it requires the injection of specific modules and attributes in order to +work. One should use importlib as the public-facing version of this module. + +""" +# IMPORTANT: Whenever making changes to this module, be sure to run a top-level +# `make regen-importlib` followed by `make` in order to get the frozen version +# of the module updated. Not doing so will result in the Makefile to fail for +# all others who don't have a ./python around to freeze the module in the early +# stages of compilation. +# + +# See importlib._setup() for what is injected into the global namespace. + +# When editing this code be aware that code executed at import time CANNOT +# reference any injected objects! This includes not only global code but also +# anything specified at the class level. + +# Bootstrap-related code ###################################################### +_CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) + + +def _make_relax_case(): + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY): + key = 'PYTHONCASEOK' + else: + key = b'PYTHONCASEOK' + + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return key in _os.environ + else: + def _relax_case(): + """True if filenames must be checked case-insensitively.""" + return False + return _relax_case + + +def _pack_uint32(x): + """Convert a 32-bit integer to little-endian.""" + return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') + + +def _unpack_uint32(data): + """Convert 4 bytes in little-endian to an integer.""" + assert len(data) == 4 + return int.from_bytes(data, 'little') + +def _unpack_uint16(data): + """Convert 2 bytes in little-endian to an integer.""" + assert len(data) == 2 + return int.from_bytes(data, 'little') + + +def _path_join(*path_parts): + """Replacement for os.path.join().""" + return path_sep.join([part.rstrip(path_separators) + for part in path_parts if part]) + + +def _path_split(path): + """Replacement for os.path.split().""" + if len(path_separators) == 1: + front, _, tail = path.rpartition(path_sep) + return front, tail + for x in reversed(path): + if x in path_separators: + front, tail = path.rsplit(x, maxsplit=1) + return front, tail + return '', path + + +def _path_stat(path): + """Stat the path. + + Made a separate function to make it easier to override in experiments + (e.g. cache stat results). + + """ + return _os.stat(path) + + +def _path_is_mode_type(path, mode): + """Test whether the path is the specified mode type.""" + try: + stat_info = _path_stat(path) + except OSError: + return False + return (stat_info.st_mode & 0o170000) == mode + + +def _path_isfile(path): + """Replacement for os.path.isfile.""" + return _path_is_mode_type(path, 0o100000) + + +def _path_isdir(path): + """Replacement for os.path.isdir.""" + if not path: + path = _os.getcwd() + return _path_is_mode_type(path, 0o040000) + + +def _path_isabs(path): + """Replacement for os.path.isabs. + + Considers a Windows drive-relative path (no drive, but starts with slash) to + still be "absolute". + """ + return path.startswith(path_separators) or path[1:3] in _pathseps_with_colon + + +def _write_atomic(path, data, mode=0o666): + """Best-effort function to write data to a path atomically. + Be prepared to handle a FileExistsError if concurrent writing of the + temporary file is attempted.""" + # id() is used to generate a pseudo-random filename. + path_tmp = '{}.{}'.format(path, id(path)) + fd = _os.open(path_tmp, + _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666) + try: + # We first write data to a temporary file, and then use os.replace() to + # perform an atomic rename. + with _io.FileIO(fd, 'wb') as file: + file.write(data) + _os.replace(path_tmp, path) + except OSError: + try: + _os.unlink(path_tmp) + except OSError: + pass + raise + + +_code_type = type(_write_atomic.__code__) + + +# Finder/loader utility code ############################################### + +# Magic word to reject .pyc files generated by other Python versions. +# It should change for each incompatible change to the bytecode. +# +# The value of CR and LF is incorporated so if you ever read or write +# a .pyc file in text mode the magic number will be wrong; also, the +# Apple MPW compiler swaps their values, botching string constants. +# +# There were a variety of old schemes for setting the magic number. +# The current working scheme is to increment the previous value by +# 10. +# +# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic +# number also includes a new "magic tag", i.e. a human readable string used +# to represent the magic number in __pycache__ directories. When you change +# the magic number, you must also set a new unique magic tag. Generally this +# can be named after the Python major version of the magic number bump, but +# it can really be anything, as long as it's different than anything else +# that's come before. The tags are included in the following table, starting +# with Python 3.2a0. +# +# Known values: +# Python 1.5: 20121 +# Python 1.5.1: 20121 +# Python 1.5.2: 20121 +# Python 1.6: 50428 +# Python 2.0: 50823 +# Python 2.0.1: 50823 +# Python 2.1: 60202 +# Python 2.1.1: 60202 +# Python 2.1.2: 60202 +# Python 2.2: 60717 +# Python 2.3a0: 62011 +# Python 2.3a0: 62021 +# Python 2.3a0: 62011 (!) +# Python 2.4a0: 62041 +# Python 2.4a3: 62051 +# Python 2.4b1: 62061 +# Python 2.5a0: 62071 +# Python 2.5a0: 62081 (ast-branch) +# Python 2.5a0: 62091 (with) +# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) +# Python 2.5b3: 62101 (fix wrong code: for x, in ...) +# Python 2.5b3: 62111 (fix wrong code: x += yield) +# Python 2.5c1: 62121 (fix wrong lnotab with for loops and +# storing constants that should have been removed) +# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) +# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) +# Python 2.6a1: 62161 (WITH_CLEANUP optimization) +# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) +# Python 2.7a0: 62181 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 2.7a0 62191 (introduce SETUP_WITH) +# Python 2.7a0 62201 (introduce BUILD_SET) +# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) +# Python 3000: 3000 +# 3010 (removed UNARY_CONVERT) +# 3020 (added BUILD_SET) +# 3030 (added keyword-only parameters) +# 3040 (added signature annotations) +# 3050 (print becomes a function) +# 3060 (PEP 3115 metaclass syntax) +# 3061 (string literals become unicode) +# 3071 (PEP 3109 raise changes) +# 3081 (PEP 3137 make __file__ and __name__ unicode) +# 3091 (kill str8 interning) +# 3101 (merge from 2.6a0, see 62151) +# 3103 (__file__ points to source file) +# Python 3.0a4: 3111 (WITH_CLEANUP optimization). +# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT + #3021) +# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: +# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) +# Python 3.1a1: 3151 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE + #4715) +# Python 3.2a1: 3160 (add SETUP_WITH #6101) +# tag: cpython-32 +# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) +# tag: cpython-32 +# Python 3.2a3 3180 (add DELETE_DEREF #4617) +# Python 3.3a1 3190 (__class__ super closure changed) +# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) +# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) +# Python 3.3a2 3220 (changed PEP 380 implementation #14230) +# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) +# Python 3.4a1 3250 (evaluate positional default arguments before +# keyword-only defaults #16967) +# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override +# free vars #17853) +# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) +# Python 3.4a1 3280 (remove implicit class argument) +# Python 3.4a4 3290 (changes to __qualname__ computation #19301) +# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) +# Python 3.4rc2 3310 (alter __qualname__ computation #20625) +# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) +# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) +# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) +# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) +# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) +# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) +# Python 3.6a2 3370 (16 bit wordcode #26647) +# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) +# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE +# #27095) +# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) +# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes +# #27985) +# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL + #27213) +# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) +# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) +# Python 3.6rc1 3379 (more thorough __class__ validation #23722) +# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) +# Python 3.7a2 3391 (update GET_AITER #31709) +# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) +# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) +# Python 3.7b5 3394 (restored docstring as the firts stmt in the body; +# this might affected the first line number #32911) +# Python 3.8a1 3400 (move frame block handling to compiler #17611) +# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) +# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) +# +# MAGIC must change whenever the bytecode emitted by the compiler may no +# longer be understood by older implementations of the eval loop (usually +# due to the addition of new opcodes). +# +# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array +# in PC/launcher.c must also be updated. + +MAGIC_NUMBER = (3410).to_bytes(2, 'little') + b'\r\n' +_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c + +_PYCACHE = '__pycache__' +_OPT = 'opt-' + +SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed. + +BYTECODE_SUFFIXES = ['.pyc'] +# Deprecated. +DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES + +def cache_from_source(path, debug_override=None, *, optimization=None): + """Given the path to a .py file, return the path to its .pyc file. + + The .py file does not need to exist; this simply returns the path to the + .pyc file calculated as if the .py file were imported. + + The 'optimization' parameter controls the presumed optimization level of + the bytecode file. If 'optimization' is not None, the string representation + of the argument is taken and verified to be alphanumeric (else ValueError + is raised). + + The debug_override parameter is deprecated. If debug_override is not None, + a True value is the same as setting 'optimization' to the empty string + while a False value is equivalent to setting 'optimization' to '1'. + + If sys.implementation.cache_tag is None then NotImplementedError is raised. + + """ + if debug_override is not None: + _warnings.warn('the debug_override parameter is deprecated; use ' + "'optimization' instead", DeprecationWarning) + if optimization is not None: + message = 'debug_override or optimization must be set to None' + raise TypeError(message) + optimization = '' if debug_override else 1 + path = _os.fspath(path) + head, tail = _path_split(path) + base, sep, rest = tail.rpartition('.') + tag = sys.implementation.cache_tag + if tag is None: + raise NotImplementedError('sys.implementation.cache_tag is None') + almost_filename = ''.join([(base if base else rest), sep, tag]) + if optimization is None: + if sys.flags.optimize == 0: + optimization = '' + else: + optimization = sys.flags.optimize + optimization = str(optimization) + if optimization != '': + if not optimization.isalnum(): + raise ValueError('{!r} is not alphanumeric'.format(optimization)) + almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization) + filename = almost_filename + BYTECODE_SUFFIXES[0] + if sys.pycache_prefix is not None: + # We need an absolute path to the py file to avoid the possibility of + # collisions within sys.pycache_prefix, if someone has two different + # `foo/bar.py` on their system and they import both of them using the + # same sys.pycache_prefix. Let's say sys.pycache_prefix is + # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first + # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative + # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an + # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`. + if not _path_isabs(head): + head = _path_join(_os.getcwd(), head) + + # Strip initial drive from a Windows path. We know we have an absolute + # path here, so the second part of the check rules out a POSIX path that + # happens to contain a colon at the second character. + if head[1] == ':' and head[0] not in path_separators: + head = head[2:] + + # Strip initial path separator from `head` to complete the conversion + # back to a root-relative path before joining. + return _path_join( + sys.pycache_prefix, + head.lstrip(path_separators), + filename, + ) + return _path_join(head, _PYCACHE, filename) + + +def source_from_cache(path): + """Given the path to a .pyc. file, return the path to its .py file. + + The .pyc file does not need to exist; this simply returns the path to + the .py file calculated to correspond to the .pyc file. If path does + not conform to PEP 3147/488 format, ValueError will be raised. If + sys.implementation.cache_tag is None then NotImplementedError is raised. + + """ + if sys.implementation.cache_tag is None: + raise NotImplementedError('sys.implementation.cache_tag is None') + path = _os.fspath(path) + head, pycache_filename = _path_split(path) + found_in_pycache_prefix = False + if sys.pycache_prefix is not None: + stripped_path = sys.pycache_prefix.rstrip(path_separators) + if head.startswith(stripped_path + path_sep): + head = head[len(stripped_path):] + found_in_pycache_prefix = True + if not found_in_pycache_prefix: + head, pycache = _path_split(head) + if pycache != _PYCACHE: + raise ValueError(f'{_PYCACHE} not bottom-level directory in ' + f'{path!r}') + dot_count = pycache_filename.count('.') + if dot_count not in {2, 3}: + raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}') + elif dot_count == 3: + optimization = pycache_filename.rsplit('.', 2)[-2] + if not optimization.startswith(_OPT): + raise ValueError("optimization portion of filename does not start " + f"with {_OPT!r}") + opt_level = optimization[len(_OPT):] + if not opt_level.isalnum(): + raise ValueError(f"optimization level {optimization!r} is not an " + "alphanumeric value") + base_filename = pycache_filename.partition('.')[0] + return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) + + +def _get_sourcefile(bytecode_path): + """Convert a bytecode file path to a source path (if possible). + + This function exists purely for backwards-compatibility for + PyImport_ExecCodeModuleWithFilenames() in the C API. + + """ + if len(bytecode_path) == 0: + return None + rest, _, extension = bytecode_path.rpartition('.') + if not rest or extension.lower()[-3:-1] != 'py': + return bytecode_path + try: + source_path = source_from_cache(bytecode_path) + except (NotImplementedError, ValueError): + source_path = bytecode_path[:-1] + return source_path if _path_isfile(source_path) else bytecode_path + + +def _get_cached(filename): + if filename.endswith(tuple(SOURCE_SUFFIXES)): + try: + return cache_from_source(filename) + except NotImplementedError: + pass + elif filename.endswith(tuple(BYTECODE_SUFFIXES)): + return filename + else: + return None + + +def _calc_mode(path): + """Calculate the mode permissions for a bytecode file.""" + try: + mode = _path_stat(path).st_mode + except OSError: + mode = 0o666 + # We always ensure write access so we can update cached files + # later even when the source files are read-only on Windows (#6074) + mode |= 0o200 + return mode + + +def _check_name(method): + """Decorator to verify that the module being requested matches the one the + loader can handle. + + The first argument (self) must define _name which the second argument is + compared against. If the comparison fails then ImportError is raised. + + """ + def _check_name_wrapper(self, name=None, *args, **kwargs): + if name is None: + name = self.name + elif self.name != name: + raise ImportError('loader for %s cannot handle %s' % + (self.name, name), name=name) + return method(self, name, *args, **kwargs) + try: + _wrap = _bootstrap._wrap + except NameError: + # XXX yuck + def _wrap(new, old): + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: + if hasattr(old, replace): + setattr(new, replace, getattr(old, replace)) + new.__dict__.update(old.__dict__) + _wrap(_check_name_wrapper, method) + return _check_name_wrapper + + +def _find_module_shim(self, fullname): + """Try to find a loader for the specified module by delegating to + self.find_loader(). + + This method is deprecated in favor of finder.find_spec(). + + """ + # Call find_loader(). If it returns a string (indicating this + # is a namespace package portion), generate a warning and + # return None. + loader, portions = self.find_loader(fullname) + if loader is None and len(portions): + msg = 'Not importing directory {}: missing __init__' + _warnings.warn(msg.format(portions[0]), ImportWarning) + return loader + + +def _classify_pyc(data, name, exc_details): + """Perform basic validity checking of a pyc header and return the flags field, + which determines how the pyc should be further validated against the source. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required, though.) + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + ImportError is raised when the magic number is incorrect or when the flags + field is invalid. EOFError is raised when the data is found to be truncated. + + """ + magic = data[:4] + if magic != MAGIC_NUMBER: + message = f'bad magic number in {name!r}: {magic!r}' + _bootstrap._verbose_message('{}', message) + raise ImportError(message, **exc_details) + if len(data) < 16: + message = f'reached EOF while reading pyc header of {name!r}' + _bootstrap._verbose_message('{}', message) + raise EOFError(message) + flags = _unpack_uint32(data[4:8]) + # Only the first two flags are defined. + if flags & ~0b11: + message = f'invalid flags {flags!r} in {name!r}' + raise ImportError(message, **exc_details) + return flags + + +def _validate_timestamp_pyc(data, source_mtime, source_size, name, + exc_details): + """Validate a pyc against the source last-modified time. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required.) + + *source_mtime* is the last modified timestamp of the source file. + + *source_size* is None or the size of the source file in bytes. + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + An ImportError is raised if the bytecode is stale. + + """ + if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF): + message = f'bytecode is stale for {name!r}' + _bootstrap._verbose_message('{}', message) + raise ImportError(message, **exc_details) + if (source_size is not None and + _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)): + raise ImportError(f'bytecode is stale for {name!r}', **exc_details) + + +def _validate_hash_pyc(data, source_hash, name, exc_details): + """Validate a hash-based pyc by checking the real source hash against the one in + the pyc header. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required.) + + *source_hash* is the importlib.util.source_hash() of the source file. + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + An ImportError is raised if the bytecode is stale. + + """ + if data[8:16] != source_hash: + raise ImportError( + f'hash in bytecode doesn\'t match hash of source {name!r}', + **exc_details, + ) + + +def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): + """Compile bytecode as found in a pyc.""" + code = marshal.loads(data) + if isinstance(code, _code_type): + _bootstrap._verbose_message('code object from {!r}', bytecode_path) + if source_path is not None: + _imp._fix_co_filename(code, source_path) + return code + else: + raise ImportError('Non-code object in {!r}'.format(bytecode_path), + name=name, path=bytecode_path) + + +def _code_to_timestamp_pyc(code, mtime=0, source_size=0): + "Produce the data for a timestamp-based pyc." + data = bytearray(MAGIC_NUMBER) + data.extend(_pack_uint32(0)) + data.extend(_pack_uint32(mtime)) + data.extend(_pack_uint32(source_size)) + data.extend(marshal.dumps(code)) + return data + + +def _code_to_hash_pyc(code, source_hash, checked=True): + "Produce the data for a hash-based pyc." + data = bytearray(MAGIC_NUMBER) + flags = 0b1 | checked << 1 + data.extend(_pack_uint32(flags)) + assert len(source_hash) == 8 + data.extend(source_hash) + data.extend(marshal.dumps(code)) + return data + + +def decode_source(source_bytes): + """Decode bytes representing source code and return the string. + + Universal newline support is used in the decoding. + """ + import tokenize # To avoid bootstrap issues. + source_bytes_readline = _io.BytesIO(source_bytes).readline + encoding = tokenize.detect_encoding(source_bytes_readline) + newline_decoder = _io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) + + +# Module specifications ####################################################### + +_POPULATE = object() + + +def spec_from_file_location(name, location=None, *, loader=None, + submodule_search_locations=_POPULATE): + """Return a module spec based on a file location. + + To indicate that the module is a package, set + submodule_search_locations to a list of directory paths. An + empty list is sufficient, though its not otherwise useful to the + import system. + + The loader must take a spec as its only __init__() arg. + + """ + if location is None: + # The caller may simply want a partially populated location- + # oriented spec. So we set the location to a bogus value and + # fill in as much as we can. + location = '' + if hasattr(loader, 'get_filename'): + # ExecutionLoader + try: + location = loader.get_filename(name) + except ImportError: + pass + else: + location = _os.fspath(location) + + # If the location is on the filesystem, but doesn't actually exist, + # we could return None here, indicating that the location is not + # valid. However, we don't have a good way of testing since an + # indirect location (e.g. a zip file or URL) will look like a + # non-existent file relative to the filesystem. + + spec = _bootstrap.ModuleSpec(name, loader, origin=location) + spec._set_fileattr = True + + # Pick a loader if one wasn't provided. + if loader is None: + for loader_class, suffixes in _get_supported_file_loaders(): + if location.endswith(tuple(suffixes)): + loader = loader_class(name, location) + spec.loader = loader + break + else: + return None + + # Set submodule_search_paths appropriately. + if submodule_search_locations is _POPULATE: + # Check the loader. + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + pass + else: + if is_package: + spec.submodule_search_locations = [] + else: + spec.submodule_search_locations = submodule_search_locations + if spec.submodule_search_locations == []: + if location: + dirname = _path_split(location)[0] + spec.submodule_search_locations.append(dirname) + + return spec + + +# Loaders ##################################################################### + +class WindowsRegistryFinder: + + """Meta path finder for modules declared in the Windows registry.""" + + REGISTRY_KEY = ( + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}') + REGISTRY_KEY_DEBUG = ( + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}\\Debug') + DEBUG_BUILD = False # Changed in _setup() + + @classmethod + def _open_registry(cls, key): + try: + return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) + except OSError: + return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) + + @classmethod + def _search_registry(cls, fullname): + if cls.DEBUG_BUILD: + registry_key = cls.REGISTRY_KEY_DEBUG + else: + registry_key = cls.REGISTRY_KEY + key = registry_key.format(fullname=fullname, + sys_version='%d.%d' % sys.version_info[:2]) + try: + with cls._open_registry(key) as hkey: + filepath = _winreg.QueryValue(hkey, '') + except OSError: + return None + return filepath + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + filepath = cls._search_registry(fullname) + if filepath is None: + return None + try: + _path_stat(filepath) + except OSError: + return None + for loader, suffixes in _get_supported_file_loaders(): + if filepath.endswith(tuple(suffixes)): + spec = _bootstrap.spec_from_loader(fullname, + loader(fullname, filepath), + origin=filepath) + return spec + + @classmethod + def find_module(cls, fullname, path=None): + """Find module named in the registry. + + This method is deprecated. Use exec_module() instead. + + """ + spec = cls.find_spec(fullname, path) + if spec is not None: + return spec.loader + else: + return None + + +class _LoaderBasics: + + """Base class of common code needed by both SourceLoader and + SourcelessFileLoader.""" + + def is_package(self, fullname): + """Concrete implementation of InspectLoader.is_package by checking if + the path returned by get_filename has a filename of '__init__.py'.""" + filename = _path_split(self.get_filename(fullname))[1] + filename_base = filename.rsplit('.', 1)[0] + tail_name = fullname.rpartition('.')[2] + return filename_base == '__init__' and tail_name != '__init__' + + def create_module(self, spec): + """Use default semantics for module creation.""" + + def exec_module(self, module): + """Execute the module.""" + code = self.get_code(module.__name__) + if code is None: + raise ImportError('cannot load module {!r} when get_code() ' + 'returns None'.format(module.__name__)) + _bootstrap._call_with_frames_removed(exec, code, module.__dict__) + + def load_module(self, fullname): + """This module is deprecated.""" + return _bootstrap._load_module_shim(self, fullname) + + +class SourceLoader(_LoaderBasics): + + def path_mtime(self, path): + """Optional method that returns the modification time (an int) for the + specified path (a str). + + Raises OSError when the path cannot be handled. + """ + raise OSError + + def path_stats(self, path): + """Optional method returning a metadata dict for the specified + path (a str). + + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + + Implementing this method allows the loader to read bytecode files. + Raises OSError when the path cannot be handled. + """ + return {'mtime': self.path_mtime(path)} + + def _cache_bytecode(self, source_path, cache_path, data): + """Optional method which writes data (bytes) to a file path (a str). + + Implementing this method allows for the writing of bytecode files. + + The source path is needed in order to correctly transfer permissions + """ + # For backwards compatibility, we delegate to set_data() + return self.set_data(cache_path, data) + + def set_data(self, path, data): + """Optional method which writes data (bytes) to a file path (a str). + + Implementing this method allows for the writing of bytecode files. + """ + + + def get_source(self, fullname): + """Concrete implementation of InspectLoader.get_source.""" + path = self.get_filename(fullname) + try: + source_bytes = self.get_data(path) + except OSError as exc: + raise ImportError('source not available through get_data()', + name=fullname) from exc + return decode_source(source_bytes) + + def source_to_code(self, data, path, *, _optimize=-1): + """Return the code object compiled from source. + + The 'data' argument can be any object type that compile() supports. + """ + return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', + dont_inherit=True, optimize=_optimize) + + def get_code(self, fullname): + """Concrete implementation of InspectLoader.get_code. + + Reading of bytecode requires path_stats to be implemented. To write + bytecode, set_data must also be implemented. + + """ + source_path = self.get_filename(fullname) + source_mtime = None + source_bytes = None + source_hash = None + hash_based = False + check_source = True + try: + bytecode_path = cache_from_source(source_path) + except NotImplementedError: + bytecode_path = None + else: + try: + st = self.path_stats(source_path) + except OSError: + pass + else: + source_mtime = int(st['mtime']) + try: + data = self.get_data(bytecode_path) + except OSError: + pass + else: + exc_details = { + 'name': fullname, + 'path': bytecode_path, + } + try: + flags = _classify_pyc(data, fullname, exc_details) + bytes_data = memoryview(data)[16:] + hash_based = flags & 0b1 != 0 + if hash_based: + check_source = flags & 0b10 != 0 + if (_imp.check_hash_based_pycs != 'never' and + (check_source or + _imp.check_hash_based_pycs == 'always')): + source_bytes = self.get_data(source_path) + source_hash = _imp.source_hash( + _RAW_MAGIC_NUMBER, + source_bytes, + ) + _validate_hash_pyc(data, source_hash, fullname, + exc_details) + else: + _validate_timestamp_pyc( + data, + source_mtime, + st['size'], + fullname, + exc_details, + ) + except (ImportError, EOFError): + pass + else: + _bootstrap._verbose_message('{} matches {}', bytecode_path, + source_path) + return _compile_bytecode(bytes_data, name=fullname, + bytecode_path=bytecode_path, + source_path=source_path) + if source_bytes is None: + source_bytes = self.get_data(source_path) + code_object = self.source_to_code(source_bytes, source_path) + _bootstrap._verbose_message('code object from {}', source_path) + if (not sys.dont_write_bytecode and bytecode_path is not None and + source_mtime is not None): + if hash_based: + if source_hash is None: + source_hash = _imp.source_hash(source_bytes) + data = _code_to_hash_pyc(code_object, source_hash, check_source) + else: + data = _code_to_timestamp_pyc(code_object, source_mtime, + len(source_bytes)) + try: + self._cache_bytecode(source_path, bytecode_path, data) + except NotImplementedError: + pass + return code_object + + +class FileLoader: + + """Base file loader class which implements the loader protocol methods that + require file system usage.""" + + def __init__(self, fullname, path): + """Cache the module name and the path to the file found by the + finder.""" + self.name = fullname + self.path = path + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) + + def __hash__(self): + return hash(self.name) ^ hash(self.path) + + @_check_name + def load_module(self, fullname): + """Load a module from a file. + + This method is deprecated. Use exec_module() instead. + + """ + # The only reason for this method is for the name check. + # Issue #14857: Avoid the zero-argument form of super so the implementation + # of that form can be updated without breaking the frozen module + return super(FileLoader, self).load_module(fullname) + + @_check_name + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self.path + + def get_data(self, path): + """Return the data from path as raw bytes.""" + with _io.FileIO(path, 'r') as file: + return file.read() + + # ResourceReader ABC API. + + @_check_name + def get_resource_reader(self, module): + if self.is_package(module): + return self + return None + + def open_resource(self, resource): + path = _path_join(_path_split(self.path)[0], resource) + return _io.FileIO(path, 'r') + + def resource_path(self, resource): + if not self.is_resource(resource): + raise FileNotFoundError + path = _path_join(_path_split(self.path)[0], resource) + return path + + def is_resource(self, name): + if path_sep in name: + return False + path = _path_join(_path_split(self.path)[0], name) + return _path_isfile(path) + + def contents(self): + return iter(_os.listdir(_path_split(self.path)[0])) + + +class SourceFileLoader(FileLoader, SourceLoader): + + """Concrete implementation of SourceLoader using the file system.""" + + def path_stats(self, path): + """Return the metadata for the path.""" + st = _path_stat(path) + return {'mtime': st.st_mtime, 'size': st.st_size} + + def _cache_bytecode(self, source_path, bytecode_path, data): + # Adapt between the two APIs + mode = _calc_mode(source_path) + return self.set_data(bytecode_path, data, _mode=mode) + + def set_data(self, path, data, *, _mode=0o666): + """Write bytes data to a file.""" + parent, filename = _path_split(path) + path_parts = [] + # Figure out what directories are missing. + while parent and not _path_isdir(parent): + parent, part = _path_split(parent) + path_parts.append(part) + # Create needed directories. + for part in reversed(path_parts): + parent = _path_join(parent, part) + try: + _os.mkdir(parent) + except FileExistsError: + # Probably another Python process already created the dir. + continue + except OSError as exc: + # Could be a permission error, read-only filesystem: just forget + # about writing the data. + _bootstrap._verbose_message('could not create {!r}: {!r}', + parent, exc) + return + try: + _write_atomic(path, data, _mode) + _bootstrap._verbose_message('created {!r}', path) + except OSError as exc: + # Same as above: just don't write the bytecode. + _bootstrap._verbose_message('could not create {!r}: {!r}', path, + exc) + + +class SourcelessFileLoader(FileLoader, _LoaderBasics): + + """Loader which handles sourceless file imports.""" + + def get_code(self, fullname): + path = self.get_filename(fullname) + data = self.get_data(path) + # Call _classify_pyc to do basic validation of the pyc but ignore the + # result. There's no source to check against. + exc_details = { + 'name': fullname, + 'path': path, + } + _classify_pyc(data, fullname, exc_details) + return _compile_bytecode( + memoryview(data)[16:], + name=fullname, + bytecode_path=path, + ) + + def get_source(self, fullname): + """Return None as there is no source code.""" + return None + + +# Filled in by _setup(). +EXTENSION_SUFFIXES = [] + + +class ExtensionFileLoader(FileLoader, _LoaderBasics): + + """Loader for extension modules. + + The constructor is designed to work with FileFinder. + + """ + + def __init__(self, name, path): + self.name = name + self.path = path + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) + + def __hash__(self): + return hash(self.name) ^ hash(self.path) + + def create_module(self, spec): + """Create an unitialized extension module""" + module = _bootstrap._call_with_frames_removed( + _imp.create_dynamic, spec) + _bootstrap._verbose_message('extension module {!r} loaded from {!r}', + spec.name, self.path) + return module + + def exec_module(self, module): + """Initialize an extension module""" + _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) + _bootstrap._verbose_message('extension module {!r} executed from {!r}', + self.name, self.path) + + def is_package(self, fullname): + """Return True if the extension module is a package.""" + file_name = _path_split(self.path)[1] + return any(file_name == '__init__' + suffix + for suffix in EXTENSION_SUFFIXES) + + def get_code(self, fullname): + """Return None as an extension module cannot create a code object.""" + return None + + def get_source(self, fullname): + """Return None as extension modules have no source code.""" + return None + + @_check_name + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self.path + + +class _NamespacePath: + """Represents a namespace package's path. It uses the module name + to find its parent module, and from there it looks up the parent's + __path__. When this changes, the module's own path is recomputed, + using path_finder. For top-level modules, the parent module's path + is sys.path.""" + + def __init__(self, name, path, path_finder): + self._name = name + self._path = path + self._last_parent_path = tuple(self._get_parent_path()) + self._path_finder = path_finder + + def _find_parent_path_names(self): + """Returns a tuple of (parent-module-name, parent-path-attr-name)""" + parent, dot, me = self._name.rpartition('.') + if dot == '': + # This is a top-level module. sys.path contains the parent path. + return 'sys', 'path' + # Not a top-level module. parent-module.__path__ contains the + # parent path. + return parent, '__path__' + + def _get_parent_path(self): + parent_module_name, path_attr_name = self._find_parent_path_names() + return getattr(sys.modules[parent_module_name], path_attr_name) + + def _recalculate(self): + # If the parent's path has changed, recalculate _path + parent_path = tuple(self._get_parent_path()) # Make a copy + if parent_path != self._last_parent_path: + spec = self._path_finder(self._name, parent_path) + # Note that no changes are made if a loader is returned, but we + # do remember the new parent path + if spec is not None and spec.loader is None: + if spec.submodule_search_locations: + self._path = spec.submodule_search_locations + self._last_parent_path = parent_path # Save the copy + return self._path + + def __iter__(self): + return iter(self._recalculate()) + + def __getitem__(self, index): + return self._recalculate()[index] + + def __setitem__(self, index, path): + self._path[index] = path + + def __len__(self): + return len(self._recalculate()) + + def __repr__(self): + return '_NamespacePath({!r})'.format(self._path) + + def __contains__(self, item): + return item in self._recalculate() + + def append(self, item): + self._path.append(item) + + +# We use this exclusively in module_from_spec() for backward-compatibility. +class _NamespaceLoader: + def __init__(self, name, path, path_finder): + self._path = _NamespacePath(name, path, path_finder) + + @classmethod + def module_repr(cls, module): + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return ''.format(module.__name__) + + def is_package(self, fullname): + return True + + def get_source(self, fullname): + return '' + + def get_code(self, fullname): + return compile('', '', 'exec', dont_inherit=True) + + def create_module(self, spec): + """Use default semantics for module creation.""" + + def exec_module(self, module): + pass + + def load_module(self, fullname): + """Load a namespace module. + + This method is deprecated. Use exec_module() instead. + + """ + # The import system never calls this method. + _bootstrap._verbose_message('namespace module loaded with path {!r}', + self._path) + return _bootstrap._load_module_shim(self, fullname) + + +# Finders ##################################################################### + +class PathFinder: + + """Meta path finder for sys.path and package __path__ attributes.""" + + @classmethod + def invalidate_caches(cls): + """Call the invalidate_caches() method on all path entry finders + stored in sys.path_importer_caches (where implemented).""" + for name, finder in list(sys.path_importer_cache.items()): + if finder is None: + del sys.path_importer_cache[name] + elif hasattr(finder, 'invalidate_caches'): + finder.invalidate_caches() + + @classmethod + def _path_hooks(cls, path): + """Search sys.path_hooks for a finder for 'path'.""" + if sys.path_hooks is not None and not sys.path_hooks: + _warnings.warn('sys.path_hooks is empty', ImportWarning) + for hook in sys.path_hooks: + try: + return hook(path) + except ImportError: + continue + else: + return None + + @classmethod + def _path_importer_cache(cls, path): + """Get the finder for the path entry from sys.path_importer_cache. + + If the path entry is not in the cache, find the appropriate finder + and cache it. If no finder is available, store None. + + """ + if path == '': + try: + path = _os.getcwd() + except FileNotFoundError: + # Don't cache the failure as the cwd can easily change to + # a valid directory later on. + return None + try: + finder = sys.path_importer_cache[path] + except KeyError: + finder = cls._path_hooks(path) + sys.path_importer_cache[path] = finder + return finder + + @classmethod + def _legacy_get_spec(cls, fullname, finder): + # This would be a good place for a DeprecationWarning if + # we ended up going that route. + if hasattr(finder, 'find_loader'): + loader, portions = finder.find_loader(fullname) + else: + loader = finder.find_module(fullname) + portions = [] + if loader is not None: + return _bootstrap.spec_from_loader(fullname, loader) + spec = _bootstrap.ModuleSpec(fullname, None) + spec.submodule_search_locations = portions + return spec + + @classmethod + def _get_spec(cls, fullname, path, target=None): + """Find the loader or namespace_path for this module/package name.""" + # If this ends up being a namespace package, namespace_path is + # the list of paths that will become its __path__ + namespace_path = [] + for entry in path: + if not isinstance(entry, (str, bytes)): + continue + finder = cls._path_importer_cache(entry) + if finder is not None: + if hasattr(finder, 'find_spec'): + spec = finder.find_spec(fullname, target) + else: + spec = cls._legacy_get_spec(fullname, finder) + if spec is None: + continue + if spec.loader is not None: + return spec + portions = spec.submodule_search_locations + if portions is None: + raise ImportError('spec missing loader') + # This is possibly part of a namespace package. + # Remember these path entries (if any) for when we + # create a namespace package, and continue iterating + # on path. + namespace_path.extend(portions) + else: + spec = _bootstrap.ModuleSpec(fullname, None) + spec.submodule_search_locations = namespace_path + return spec + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + """Try to find a spec for 'fullname' on sys.path or 'path'. + + The search is based on sys.path_hooks and sys.path_importer_cache. + """ + if path is None: + path = sys.path + spec = cls._get_spec(fullname, path, target) + if spec is None: + return None + elif spec.loader is None: + namespace_path = spec.submodule_search_locations + if namespace_path: + # We found at least one namespace path. Return a spec which + # can create the namespace package. + spec.origin = None + spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) + return spec + else: + return None + else: + return spec + + @classmethod + def find_module(cls, fullname, path=None): + """find the module on sys.path or 'path' based on sys.path_hooks and + sys.path_importer_cache. + + This method is deprecated. Use find_spec() instead. + + """ + spec = cls.find_spec(fullname, path) + if spec is None: + return None + return spec.loader + + +class FileFinder: + + """File-based finder. + + Interactions with the file system are cached for performance, being + refreshed when the directory the finder is handling has been modified. + + """ + + def __init__(self, path, *loader_details): + """Initialize with the path to search on and a variable number of + 2-tuples containing the loader and the file suffixes the loader + recognizes.""" + loaders = [] + for loader, suffixes in loader_details: + loaders.extend((suffix, loader) for suffix in suffixes) + self._loaders = loaders + # Base (directory) path + self.path = path or '.' + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + def invalidate_caches(self): + """Invalidate the directory mtime.""" + self._path_mtime = -1 + + find_module = _find_module_shim + + def find_loader(self, fullname): + """Try to find a loader for the specified module, or the namespace + package portions. Returns (loader, list-of-portions). + + This method is deprecated. Use find_spec() instead. + + """ + spec = self.find_spec(fullname) + if spec is None: + return None, [] + return spec.loader, spec.submodule_search_locations or [] + + def _get_spec(self, loader_class, fullname, path, smsl, target): + loader = loader_class(fullname, path) + return spec_from_file_location(fullname, path, loader=loader, + submodule_search_locations=smsl) + + def find_spec(self, fullname, target=None): + """Try to find a spec for the specified module. + + Returns the matching spec, or None if not found. + """ + is_namespace = False + tail_module = fullname.rpartition('.')[2] + try: + mtime = _path_stat(self.path or _os.getcwd()).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + # tail_module keeps the original casing, for __file__ and friends + if _relax_case(): + cache = self._relaxed_path_cache + cache_module = tail_module.lower() + else: + cache = self._path_cache + cache_module = tail_module + # Check if the module is the name of a directory (and thus a package). + if cache_module in cache: + base_path = _path_join(self.path, tail_module) + for suffix, loader_class in self._loaders: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, [base_path], target) + else: + # If a namespace package, return the path if we don't + # find a module in the next section. + is_namespace = _path_isdir(base_path) + # Check for a file w/ a proper suffix exists. + for suffix, loader_class in self._loaders: + full_path = _path_join(self.path, tail_module + suffix) + _bootstrap._verbose_message('trying {}', full_path, verbosity=2) + if cache_module + suffix in cache: + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, + None, target) + if is_namespace: + _bootstrap._verbose_message('possible namespace for {}', base_path) + spec = _bootstrap.ModuleSpec(fullname, None) + spec.submodule_search_locations = [base_path] + return spec + return None + + def _fill_cache(self): + """Fill the cache of potential modules and packages for this directory.""" + path = self.path + try: + contents = _os.listdir(path or _os.getcwd()) + except (FileNotFoundError, PermissionError, NotADirectoryError): + # Directory has either been removed, turned into a file, or made + # unreadable. + contents = [] + # We store two cached versions, to handle runtime changes of the + # PYTHONCASEOK environment variable. + if not sys.platform.startswith('win'): + self._path_cache = set(contents) + else: + # Windows users can import modules with case-insensitive file + # suffixes (for legacy reasons). Make the suffix lowercase here + # so it's done once instead of for every import. This is safe as + # the specified suffixes to check against are always specified in a + # case-sensitive manner. + lower_suffix_contents = set() + for item in contents: + name, dot, suffix = item.partition('.') + if dot: + new_name = '{}.{}'.format(name, suffix.lower()) + else: + new_name = name + lower_suffix_contents.add(new_name) + self._path_cache = lower_suffix_contents + if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): + self._relaxed_path_cache = {fn.lower() for fn in contents} + + @classmethod + def path_hook(cls, *loader_details): + """A class method which returns a closure to use on sys.path_hook + which will return an instance using the specified loaders and the path + called on the closure. + + If the path called on the closure is not a directory, ImportError is + raised. + + """ + def path_hook_for_FileFinder(path): + """Path hook for importlib.machinery.FileFinder.""" + if not _path_isdir(path): + raise ImportError('only directories are supported', path=path) + return cls(path, *loader_details) + + return path_hook_for_FileFinder + + def __repr__(self): + return 'FileFinder({!r})'.format(self.path) + + +# Import setup ############################################################### + +def _fix_up_module(ns, name, pathname, cpathname=None): + # This function is used by PyImport_ExecCodeModuleObject(). + loader = ns.get('__loader__') + spec = ns.get('__spec__') + if not loader: + if spec: + loader = spec.loader + elif pathname == cpathname: + loader = SourcelessFileLoader(name, pathname) + else: + loader = SourceFileLoader(name, pathname) + if not spec: + spec = spec_from_file_location(name, pathname, loader=loader) + try: + ns['__spec__'] = spec + ns['__loader__'] = loader + ns['__file__'] = pathname + ns['__cached__'] = cpathname + except Exception: + # Not important enough to report. + pass + + +def _get_supported_file_loaders(): + """Returns a list of file-based module loaders. + + Each item is a tuple (loader, suffixes). + """ + extensions = ExtensionFileLoader, _imp.extension_suffixes() + source = SourceFileLoader, SOURCE_SUFFIXES + bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES + return [extensions, source, bytecode] + + +def _setup(_bootstrap_module): + """Setup the path-based importers for importlib by importing needed + built-in modules and injecting them into the global namespace. + + Other components are extracted from the core bootstrap module. + + """ + global sys, _imp, _bootstrap + _bootstrap = _bootstrap_module + sys = _bootstrap.sys + _imp = _bootstrap._imp + + # Directly load built-in modules needed during bootstrap. + self_module = sys.modules[__name__] + for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'): + if builtin_name not in sys.modules: + builtin_module = _bootstrap._builtin_from_name(builtin_name) + else: + builtin_module = sys.modules[builtin_name] + setattr(self_module, builtin_name, builtin_module) + + # Directly load the os module (needed during bootstrap). + os_details = ('_os', ['/']), ('_os', ['\\', '/']) # XXX Changed to fit RustPython!!! + for builtin_os, path_separators in os_details: + # Assumption made in _path_join() + assert all(len(sep) == 1 for sep in path_separators) + path_sep = path_separators[0] + if builtin_os in sys.modules: + os_module = sys.modules[builtin_os] + break + else: + try: + os_module = _bootstrap._builtin_from_name(builtin_os) + break + except ImportError: + continue + else: + raise ImportError('importlib requires posix or nt') + setattr(self_module, '_os', os_module) + setattr(self_module, 'path_sep', path_sep) + setattr(self_module, 'path_separators', ''.join(path_separators)) + setattr(self_module, '_pathseps_with_colon', {f':{s}' for s in path_separators}) + + # Directly load the _thread module (needed during bootstrap). + thread_module = _bootstrap._builtin_from_name('_thread') + setattr(self_module, '_thread', thread_module) + + # Directly load the _weakref module (needed during bootstrap). + weakref_module = _bootstrap._builtin_from_name('_weakref') + setattr(self_module, '_weakref', weakref_module) + + # Directly load the winreg module (needed during bootstrap). + if builtin_os == 'nt': + winreg_module = _bootstrap._builtin_from_name('winreg') + setattr(self_module, '_winreg', winreg_module) + + # Constants + setattr(self_module, '_relax_case', _make_relax_case()) + EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) + if builtin_os == 'nt': + SOURCE_SUFFIXES.append('.pyw') + if '_d.pyd' in EXTENSION_SUFFIXES: + WindowsRegistryFinder.DEBUG_BUILD = True + + +def _install(_bootstrap_module): + """Install the path-based import components.""" + _setup(_bootstrap_module) + supported_loaders = _get_supported_file_loaders() + sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) + sys.meta_path.append(PathFinder) diff --git a/Lib/io.py b/Lib/io.py new file mode 100644 index 0000000000..5536a308c3 --- /dev/null +++ b/Lib/io.py @@ -0,0 +1 @@ +from _io import * diff --git a/Lib/linecache.py b/Lib/linecache.py new file mode 100644 index 0000000000..3afcce1f0a --- /dev/null +++ b/Lib/linecache.py @@ -0,0 +1,177 @@ +"""Cache lines from Python source files. + +This is intended to read lines from modules imported -- hence if a filename +is not found, it will look down the module search path for a file by +that name. +""" + +import functools +import sys +import os +import tokenize + +__all__ = ["getline", "clearcache", "checkcache"] + +def getline(filename, lineno, module_globals=None): + lines = getlines(filename, module_globals) + if 1 <= lineno <= len(lines): + return lines[lineno-1] + else: + return '' + + +# The cache + +# The cache. Maps filenames to either a thunk which will provide source code, +# or a tuple (size, mtime, lines, fullname) once loaded. +cache = {} + + +def clearcache(): + """Clear the cache entirely.""" + + global cache + cache = {} + + +def getlines(filename, module_globals=None): + """Get the lines for a Python source file from the cache. + Update the cache if it doesn't contain an entry for this file already.""" + + if filename in cache: + entry = cache[filename] + if len(entry) != 1: + return cache[filename][2] + + try: + return updatecache(filename, module_globals) + except MemoryError: + clearcache() + return [] + + +def checkcache(filename=None): + """Discard cache entries that are out of date. + (This is not checked upon each call!)""" + + if filename is None: + filenames = list(cache.keys()) + else: + if filename in cache: + filenames = [filename] + else: + return + + for filename in filenames: + entry = cache[filename] + if len(entry) == 1: + # lazy cache entry, leave it lazy. + continue + size, mtime, lines, fullname = entry + if mtime is None: + continue # no-op for files loaded via a __loader__ + try: + stat = os.stat(fullname) + except OSError: + del cache[filename] + continue + if size != stat.st_size or mtime != stat.st_mtime: + del cache[filename] + + +def updatecache(filename, module_globals=None): + """Update a cache entry and return its list of lines. + If something's wrong, print a message, discard the cache entry, + and return an empty list.""" + + if filename in cache: + if len(cache[filename]) != 1: + del cache[filename] + if not filename or (filename.startswith('<') and filename.endswith('>')): + return [] + + fullname = filename + try: + stat = os.stat(fullname) + except OSError: + basename = filename + + # Realise a lazy loader based lookup if there is one + # otherwise try to lookup right now. + if lazycache(filename, module_globals): + try: + data = cache[filename][0]() + except (ImportError, OSError): + pass + else: + if data is None: + # No luck, the PEP302 loader cannot find the source + # for this module. + return [] + cache[filename] = ( + len(data), None, + [line+'\n' for line in data.splitlines()], fullname + ) + return cache[filename][2] + + # Try looking through the module search path, which is only useful + # when handling a relative filename. + if os.path.isabs(filename): + return [] + + for dirname in sys.path: + try: + fullname = os.path.join(dirname, basename) + except (TypeError, AttributeError): + # Not sufficiently string-like to do anything useful with. + continue + try: + stat = os.stat(fullname) + break + except OSError: + pass + else: + return [] + try: + with tokenize.open(fullname) as fp: + lines = fp.readlines() + except OSError: + return [] + if lines and not lines[-1].endswith('\n'): + lines[-1] += '\n' + size, mtime = stat.st_size, stat.st_mtime + cache[filename] = size, mtime, lines, fullname + return lines + + +def lazycache(filename, module_globals): + """Seed the cache for filename with module_globals. + + The module loader will be asked for the source only when getlines is + called, not immediately. + + If there is an entry in the cache already, it is not altered. + + :return: True if a lazy load is registered in the cache, + otherwise False. To register such a load a module loader with a + get_source method must be found, the filename must be a cachable + filename, and the filename must not be already cached. + """ + if filename in cache: + if len(cache[filename]) == 1: + return True + else: + return False + if not filename or (filename.startswith('<') and filename.endswith('>')): + return False + # Try for a __loader__, if available + if module_globals and '__loader__' in module_globals: + name = module_globals.get('__name__') + loader = module_globals['__loader__'] + get_source = getattr(loader, 'get_source', None) + + if name and get_source: + get_lines = functools.partial(get_source, name) + cache[filename] = (get_lines,) + return True + return False diff --git a/Lib/ntpath.py b/Lib/ntpath.py new file mode 100644 index 0000000000..f3cfabf023 --- /dev/null +++ b/Lib/ntpath.py @@ -0,0 +1,661 @@ +# Module 'ntpath' -- common operations on WinNT/Win95 pathnames +"""Common pathname manipulations, WindowsNT/95 version. + +Instead of importing this module directly, import os and refer to this +module as os.path. +""" + +# strings representing various path-related bits and pieces +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = '.' +pardir = '..' +extsep = '.' +sep = '\\' +pathsep = ';' +altsep = '/' +defpath = '.;C:\\bin' +devnull = 'nul' + +import os +import sys +import stat +import genericpath +from genericpath import * + +__all__ = ["normcase","isabs","join","splitdrive","split","splitext", + "basename","dirname","commonprefix","getsize","getmtime", + "getatime","getctime", "islink","exists","lexists","isdir","isfile", + "ismount", "expanduser","expandvars","normpath","abspath", + "curdir","pardir","sep","pathsep","defpath","altsep", + "extsep","devnull","realpath","supports_unicode_filenames","relpath", + "samefile", "sameopenfile", "samestat", "commonpath"] + +def _get_bothseps(path): + if isinstance(path, bytes): + return b'\\/' + else: + return '\\/' + +# Normalize the case of a pathname and map slashes to backslashes. +# Other normalizations (such as optimizing '../' away) are not done +# (this is done by normpath). + +def normcase(s): + """Normalize case of pathname. + + Makes all characters lowercase and all slashes into backslashes.""" + s = os.fspath(s) + if isinstance(s, bytes): + return s.replace(b'/', b'\\').lower() + else: + return s.replace('/', '\\').lower() + + +# Return whether a path is absolute. +# Trivial in Posix, harder on Windows. +# For Windows it is absolute if it starts with a slash or backslash (current +# volume), or if a pathname after the volume-letter-and-colon or UNC-resource +# starts with a slash or backslash. + +def isabs(s): + """Test whether a path is absolute""" + s = os.fspath(s) + s = splitdrive(s)[1] + return len(s) > 0 and s[0] in _get_bothseps(s) + + +# Join two (or more) paths. +def join(path, *paths): + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + seps = b'\\/' + colon = b':' + else: + sep = '\\' + seps = '\\/' + colon = ':' + try: + if not paths: + path[:0] + sep #23780: Ensure compatible data type even if p is null. + result_drive, result_path = splitdrive(path) + for p in map(os.fspath, paths): + p_drive, p_path = splitdrive(p) + if p_path and p_path[0] in seps: + # Second path is absolute + if p_drive or not result_drive: + result_drive = p_drive + result_path = p_path + continue + elif p_drive and p_drive != result_drive: + if p_drive.lower() != result_drive.lower(): + # Different drives => ignore the first path entirely + result_drive = p_drive + result_path = p_path + continue + # Same drive in different case + result_drive = p_drive + # Second path is relative to the first + if result_path and result_path[-1] not in seps: + result_path = result_path + sep + result_path = result_path + p_path + ## add separator between UNC and non-absolute path + if (result_path and result_path[0] not in seps and + result_drive and result_drive[-1:] != colon): + return result_drive + sep + result_path + return result_drive + result_path + except (TypeError, AttributeError, BytesWarning): + genericpath._check_arg_types('join', path, *paths) + raise + + +# Split a path in a drive specification (a drive letter followed by a +# colon) and the path specification. +# It is always true that drivespec + pathspec == p +def splitdrive(p): + """Split a pathname into drive/UNC sharepoint and relative path specifiers. + Returns a 2-tuple (drive_or_unc, path); either part may be empty. + + If you assign + result = splitdrive(p) + It is always true that: + result[0] + result[1] == p + + If the path contained a drive letter, drive_or_unc will contain everything + up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") + + If the path contained a UNC path, the drive_or_unc will contain the host name + and share up to but not including the fourth directory separator character. + e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") + + Paths cannot contain both a drive letter and a UNC path. + + """ + p = os.fspath(p) + if len(p) >= 2: + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + else: + sep = '\\' + altsep = '/' + colon = ':' + normp = p.replace(altsep, sep) + if (normp[0:2] == sep*2) and (normp[2:3] != sep): + # is a UNC path: + # vvvvvvvvvvvvvvvvvvvv drive letter or UNC path + # \\machine\mountpoint\directory\etc\... + # directory ^^^^^^^^^^^^^^^ + index = normp.find(sep, 2) + if index == -1: + return p[:0], p + index2 = normp.find(sep, index + 1) + # a UNC path can't have two slashes in a row + # (after the initial two) + if index2 == index + 1: + return p[:0], p + if index2 == -1: + index2 = len(p) + return p[:index2], p[index2:] + if normp[1:2] == colon: + return p[:2], p[2:] + return p[:0], p + + +# Split a path in head (everything up to the last '/') and tail (the +# rest). After the trailing '/' is stripped, the invariant +# join(head, tail) == p holds. +# The resulting head won't end in '/' unless it is the root. + +def split(p): + """Split a pathname. + + Return tuple (head, tail) where tail is everything after the final slash. + Either part may be empty.""" + p = os.fspath(p) + seps = _get_bothseps(p) + d, p = splitdrive(p) + # set i to index beyond p's last slash + i = len(p) + while i and p[i-1] not in seps: + i -= 1 + head, tail = p[:i], p[i:] # now tail has no slashes + # remove trailing slashes from head, unless it's all slashes + head = head.rstrip(seps) or head + return d + head, tail + + +# Split a path in root and extension. +# The extension is everything starting at the last dot in the last +# pathname component; the root is everything before that. +# It is always true that root + ext == p. + +def splitext(p): + p = os.fspath(p) + if isinstance(p, bytes): + return genericpath._splitext(p, b'\\', b'/', b'.') + else: + return genericpath._splitext(p, '\\', '/', '.') +splitext.__doc__ = genericpath._splitext.__doc__ + + +# Return the tail (basename) part of a path. + +def basename(p): + """Returns the final component of a pathname""" + return split(p)[1] + + +# Return the head (dirname) part of a path. + +def dirname(p): + """Returns the directory component of a pathname""" + return split(p)[0] + +# Is a path a symbolic link? +# This will always return false on systems where os.lstat doesn't exist. + +def islink(path): + """Test whether a path is a symbolic link. + This will always return false for Windows prior to 6.0. + """ + try: + st = os.lstat(path) + except (OSError, ValueError, AttributeError): + return False + return stat.S_ISLNK(st.st_mode) + +# Being true for dangling symbolic links is also useful. + +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + st = os.lstat(path) + except (OSError, ValueError): + return False + return True + +# Is a path a mount point? +# Any drive letter root (eg c:\) +# Any share UNC (eg \\server\share) +# Any volume mounted on a filesystem folder +# +# No one method detects all three situations. Historically we've lexically +# detected drive letter roots and share UNCs. The canonical approach to +# detecting mounted volumes (querying the reparse tag) fails for the most +# common case: drive letter roots. The alternative which uses GetVolumePathName +# fails if the drive letter is the result of a SUBST. +try: + from nt import _getvolumepathname +except ImportError: + _getvolumepathname = None +def ismount(path): + """Test whether a path is a mount point (a drive root, the root of a + share, or a mounted volume)""" + path = os.fspath(path) + seps = _get_bothseps(path) + path = abspath(path) + root, rest = splitdrive(path) + if root and root[0] in seps: + return (not rest) or (rest in seps) + if rest in seps: + return True + + if _getvolumepathname: + return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps) + else: + return False + + +# Expand paths beginning with '~' or '~user'. +# '~' means $HOME; '~user' means that user's home directory. +# If the path doesn't begin with '~', or if the user or $HOME is unknown, +# the path is returned unchanged (leaving error reporting to whatever +# function is called with the expanded path as argument). +# See also module 'glob' for expansion of *, ? and [...] in pathnames. +# (A function should also be defined to do full *sh-style environment +# variable expansion.) + +def expanduser(path): + """Expand ~ and ~user constructs. + + If user or $HOME is unknown, do nothing.""" + path = os.fspath(path) + if isinstance(path, bytes): + tilde = b'~' + else: + tilde = '~' + if not path.startswith(tilde): + return path + i, n = 1, len(path) + while i < n and path[i] not in _get_bothseps(path): + i += 1 + + if 'USERPROFILE' in os.environ: + userhome = os.environ['USERPROFILE'] + elif not 'HOMEPATH' in os.environ: + return path + else: + try: + drive = os.environ['HOMEDRIVE'] + except KeyError: + drive = '' + userhome = join(drive, os.environ['HOMEPATH']) + + if isinstance(path, bytes): + userhome = os.fsencode(userhome) + + if i != 1: #~user + userhome = join(dirname(userhome), path[1:i]) + + return userhome + path[i:] + + +# Expand paths containing shell variable substitutions. +# The following rules apply: +# - no expansion within single quotes +# - '$$' is translated into '$' +# - '%%' is translated into '%' if '%%' are not seen in %var1%%var2% +# - ${varname} is accepted. +# - $varname is accepted. +# - %varname% is accepted. +# - varnames can be made out of letters, digits and the characters '_-' +# (though is not verified in the ${varname} and %varname% cases) +# XXX With COMMAND.COM you can use any characters in a variable name, +# XXX except '^|<>='. + +def expandvars(path): + """Expand shell variables of the forms $var, ${var} and %var%. + + Unknown variables are left unchanged.""" + path = os.fspath(path) + if isinstance(path, bytes): + if b'$' not in path and b'%' not in path: + return path + import string + varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') + quote = b'\'' + percent = b'%' + brace = b'{' + rbrace = b'}' + dollar = b'$' + environ = getattr(os, 'environb', None) + else: + if '$' not in path and '%' not in path: + return path + import string + varchars = string.ascii_letters + string.digits + '_-' + quote = '\'' + percent = '%' + brace = '{' + rbrace = '}' + dollar = '$' + environ = os.environ + res = path[:0] + index = 0 + pathlen = len(path) + while index < pathlen: + c = path[index:index+1] + if c == quote: # no expansion within single quotes + path = path[index + 1:] + pathlen = len(path) + try: + index = path.index(c) + res += c + path[:index + 1] + except ValueError: + res += c + path + index = pathlen - 1 + elif c == percent: # variable or '%' + if path[index + 1:index + 2] == percent: + res += c + index += 1 + else: + path = path[index+1:] + pathlen = len(path) + try: + index = path.index(percent) + except ValueError: + res += percent + path + index = pathlen - 1 + else: + var = path[:index] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(var)]) + else: + value = environ[var] + except KeyError: + value = percent + var + percent + res += value + elif c == dollar: # variable or '$$' + if path[index + 1:index + 2] == dollar: + res += c + index += 1 + elif path[index + 1:index + 2] == brace: + path = path[index+2:] + pathlen = len(path) + try: + index = path.index(rbrace) + except ValueError: + res += dollar + brace + path + index = pathlen - 1 + else: + var = path[:index] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(var)]) + else: + value = environ[var] + except KeyError: + value = dollar + brace + var + rbrace + res += value + else: + var = path[:0] + index += 1 + c = path[index:index + 1] + while c and c in varchars: + var += c + index += 1 + c = path[index:index + 1] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(var)]) + else: + value = environ[var] + except KeyError: + value = dollar + var + res += value + if c: + index -= 1 + else: + res += c + index += 1 + return res + + +# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. +# Previously, this function also truncated pathnames to 8+3 format, +# but as this module is called "ntpath", that's obviously wrong! + +def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + pardir = b'..' + special_prefixes = (b'\\\\.\\', b'\\\\?\\') + else: + sep = '\\' + altsep = '/' + curdir = '.' + pardir = '..' + special_prefixes = ('\\\\.\\', '\\\\?\\') + if path.startswith(special_prefixes): + # in the case of paths with these prefixes: + # \\.\ -> device names + # \\?\ -> literal paths + # do not do any normalization, but return the path unchanged + return path + path = path.replace(altsep, sep) + prefix, path = splitdrive(path) + + # collapse initial backslashes + if path.startswith(sep): + prefix += sep + path = path.lstrip(sep) + + comps = path.split(sep) + i = 0 + while i < len(comps): + if not comps[i] or comps[i] == curdir: + del comps[i] + elif comps[i] == pardir: + if i > 0 and comps[i-1] != pardir: + del comps[i-1:i+1] + i -= 1 + elif i == 0 and prefix.endswith(sep): + del comps[i] + else: + i += 1 + else: + i += 1 + # If the path is now empty, substitute '.' + if not prefix and not comps: + comps.append(curdir) + return prefix + sep.join(comps) + +def _abspath_fallback(path): + """Return the absolute version of a path as a fallback function in case + `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for + more. + + """ + + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) + +# Return an absolute path. +try: + from nt import _getfullpathname + +except ImportError: # not running on Windows - mock up something sensible + abspath = _abspath_fallback + +else: # use native Windows method on Windows + def abspath(path): + """Return the absolute version of a path.""" + try: + return normpath(_getfullpathname(path)) + except (OSError, ValueError): + return _abspath_fallback(path) + +# realpath is a no-op on systems without islink support +realpath = abspath +# Win9x family and earlier have no Unicode filename support. +supports_unicode_filenames = (hasattr(sys, "getwindowsversion") and + sys.getwindowsversion()[3] >= 2) + +def relpath(path, start=None): + """Return a relative version of a path""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'\\' + curdir = b'.' + pardir = b'..' + else: + sep = '\\' + curdir = '.' + pardir = '..' + + if start is None: + start = curdir + + if not path: + raise ValueError("no path specified") + + start = os.fspath(start) + try: + start_abs = abspath(normpath(start)) + path_abs = abspath(normpath(path)) + start_drive, start_rest = splitdrive(start_abs) + path_drive, path_rest = splitdrive(path_abs) + if normcase(start_drive) != normcase(path_drive): + raise ValueError("path is on mount %r, start on mount %r" % ( + path_drive, start_drive)) + + start_list = [x for x in start_rest.split(sep) if x] + path_list = [x for x in path_rest.split(sep) if x] + # Work out how much of the filepath is shared by start and path. + i = 0 + for e1, e2 in zip(start_list, path_list): + if normcase(e1) != normcase(e2): + break + i += 1 + + rel_list = [pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return curdir + return join(*rel_list) + except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): + genericpath._check_arg_types('relpath', path, start) + raise + + +# Return the longest common sub-path of the sequence of paths given as input. +# The function is case-insensitive and 'separator-insensitive', i.e. if the +# only difference between two paths is the use of '\' versus '/' as separator, +# they are deemed to be equal. +# +# However, the returned path will have the standard '\' separator (even if the +# given paths had the alternative '/' separator) and will have the case of the +# first path given in the sequence. Additionally, any trailing separator is +# stripped from the returned path. + +def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError('commonpath() arg is an empty sequence') + + paths = tuple(map(os.fspath, paths)) + if isinstance(paths[0], bytes): + sep = b'\\' + altsep = b'/' + curdir = b'.' + else: + sep = '\\' + altsep = '/' + curdir = '.' + + try: + drivesplits = [splitdrive(p.replace(altsep, sep).lower()) for p in paths] + split_paths = [p.split(sep) for d, p in drivesplits] + + try: + isabs, = set(p[:1] == sep for d, p in drivesplits) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") from None + + # Check that all drive letters or UNC paths match. The check is made only + # now otherwise type errors for mixing strings and bytes would not be + # caught. + if len(set(d for d, p in drivesplits)) != 1: + raise ValueError("Paths don't have the same drive") + + drive, path = splitdrive(paths[0].replace(altsep, sep)) + common = path.split(sep) + common = [c for c in common if c and c != curdir] + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + for i, c in enumerate(s1): + if c != s2[i]: + common = common[:i] + break + else: + common = common[:len(s1)] + + prefix = drive + sep if isabs else drive + return prefix + sep.join(common) + except (TypeError, AttributeError): + genericpath._check_arg_types('commonpath', *paths) + raise + + +# determine if two files are in fact the same file +try: + # GetFinalPathNameByHandle is available starting with Windows 6.0. + # Windows XP and non-Windows OS'es will mock _getfinalpathname. + if sys.getwindowsversion()[:2] >= (6, 0): + from nt import _getfinalpathname + else: + raise ImportError +except (AttributeError, ImportError): + # On Windows XP and earlier, two files are the same if their absolute + # pathnames are the same. + # Non-Windows operating systems fake this method with an XP + # approximation. + def _getfinalpathname(f): + return normcase(abspath(f)) + + +try: + # The genericpath.isdir implementation uses os.stat and checks the mode + # attribute to tell whether or not the path is a directory. + # This is overkill on Windows - just pass the path to GetFileAttributes + # and check the attribute from there. + from nt import _isdir as isdir +except ImportError: + # Use genericpath.isdir as imported above. + pass diff --git a/Lib/operator.py b/Lib/operator.py index f219200641..0e2e53efc6 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -1,15 +1,464 @@ +""" +Operator Interface +This module exports a set of functions corresponding to the intrinsic +operators of Python. For example, operator.add(x, y) is equivalent +to the expression x+y. The function names are those used for special +methods; variants without leading and trailing '__' are also provided +for convenience. -# Comparison operations: +This is the pure Python implementation of the module. +""" + +__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', + 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', + 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', + 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', + 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', + 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', + 'setitem', 'sub', 'truediv', 'truth', 'xor'] + +from builtins import abs as _abs + + +# Comparison Operations *******************************************************# + +def lt(a, b): + "Same as a < b." + return a < b + +def le(a, b): + "Same as a <= b." + return a <= b def eq(a, b): + "Same as a == b." return a == b - def ne(a, b): + "Same as a != b." return a != b - def ge(a, b): + "Same as a >= b." return a >= b +def gt(a, b): + "Same as a > b." + return a > b + +# Logical Operations **********************************************************# + +def not_(a): + "Same as not a." + return not a + +def truth(a): + "Return True if a is true, False otherwise." + return True if a else False + +def is_(a, b): + "Same as a is b." + return a is b + +def is_not(a, b): + "Same as a is not b." + return a is not b + +# Mathematical/Bitwise Operations *********************************************# + +def abs(a): + "Same as abs(a)." + return _abs(a) + +def add(a, b): + "Same as a + b." + return a + b + +def and_(a, b): + "Same as a & b." + return a & b + +def floordiv(a, b): + "Same as a // b." + return a // b + +def index(a): + "Same as a.__index__()." + return a.__index__() + +def inv(a): + "Same as ~a." + return ~a +invert = inv + +def lshift(a, b): + "Same as a << b." + return a << b + +def mod(a, b): + "Same as a % b." + return a % b + +def mul(a, b): + "Same as a * b." + return a * b + +def matmul(a, b): + "Same as a @ b." + return a @ b + +def neg(a): + "Same as -a." + return -a + +def or_(a, b): + "Same as a | b." + return a | b + +def pos(a): + "Same as +a." + return +a + +def pow(a, b): + "Same as a ** b." + return a ** b + +def rshift(a, b): + "Same as a >> b." + return a >> b + +def sub(a, b): + "Same as a - b." + return a - b + +def truediv(a, b): + "Same as a / b." + return a / b + +def xor(a, b): + "Same as a ^ b." + return a ^ b + +# Sequence Operations *********************************************************# + +def concat(a, b): + "Same as a + b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + return a + b + +def contains(a, b): + "Same as b in a (note reversed operands)." + return b in a + +def countOf(a, b): + "Return the number of times b occurs in a." + count = 0 + for i in a: + if i == b: + count += 1 + return count + +def delitem(a, b): + "Same as del a[b]." + del a[b] + +def getitem(a, b): + "Same as a[b]." + return a[b] + +def indexOf(a, b): + "Return the first index of b in a." + for i, j in enumerate(a): + if j == b: + return i + else: + raise ValueError('sequence.index(x): x not in sequence') + +def setitem(a, b, c): + "Same as a[b] = c." + a[b] = c + +def length_hint(obj, default=0): + """ + Return an estimate of the number of items in obj. + This is useful for presizing containers when building from an iterable. + + If the object supports len(), the result will be exact. Otherwise, it may + over- or under-estimate by an arbitrary amount. The result will be an + integer >= 0. + """ + if not isinstance(default, int): + msg = ("'%s' object cannot be interpreted as an integer" % + type(default).__name__) + raise TypeError(msg) + + try: + return len(obj) + except TypeError: + pass + + try: + hint = type(obj).__length_hint__ + except AttributeError: + return default + + try: + val = hint(obj) + except TypeError: + return default + if val is NotImplemented: + return default + if not isinstance(val, int): + msg = ('__length_hint__ must be integer, not %s' % + type(val).__name__) + raise TypeError(msg) + if val < 0: + msg = '__length_hint__() should return >= 0' + raise ValueError(msg) + return val + +# Generalized Lookup Objects **************************************************# + +class attrgetter: + """ + Return a callable object that fetches the given attribute(s) from its operand. + After f = attrgetter('name'), the call f(r) returns r.name. + After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date). + After h = attrgetter('name.first', 'name.last'), the call h(r) returns + (r.name.first, r.name.last). + """ + __slots__ = ('_attrs', '_call') + + def __init__(self, attr, *attrs): + if not attrs: + if not isinstance(attr, str): + raise TypeError('attribute name must be a string') + self._attrs = (attr,) + names = attr.split('.') + def func(obj): + for name in names: + obj = getattr(obj, name) + return obj + self._call = func + else: + self._attrs = (attr,) + attrs + getters = tuple(map(attrgetter, self._attrs)) + def func(obj): + return tuple(getter(obj) for getter in getters) + self._call = func + + def __call__(self, obj): + return self._call(obj) + + def __repr__(self): + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__qualname__, + ', '.join(map(repr, self._attrs))) + + def __reduce__(self): + return self.__class__, self._attrs + +class itemgetter: + """ + Return a callable object that fetches the given item(s) from its operand. + After f = itemgetter(2), the call f(r) returns r[2]. + After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3]) + """ + __slots__ = ('_items', '_call') + + def __init__(self, item, *items): + if not items: + self._items = (item,) + def func(obj): + return obj[item] + self._call = func + else: + self._items = items = (item,) + items + def func(obj): + return tuple(obj[i] for i in items) + self._call = func + + def __call__(self, obj): + return self._call(obj) + + def __repr__(self): + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(map(repr, self._items))) + + def __reduce__(self): + return self.__class__, self._items + +class methodcaller: + """ + Return a callable object that calls the given method on its operand. + After f = methodcaller('name'), the call f(r) returns r.name(). + After g = methodcaller('name', 'date', foo=1), the call g(r) returns + r.name('date', foo=1). + """ + __slots__ = ('_name', '_args', '_kwargs') + + def __init__(*args, **kwargs): + if len(args) < 2: + msg = "methodcaller needs at least one argument, the method name" + raise TypeError(msg) + self = args[0] + self._name = args[1] + if not isinstance(self._name, str): + raise TypeError('method name must be a string') + self._args = args[2:] + self._kwargs = kwargs + + def __call__(self, obj): + return getattr(obj, self._name)(*self._args, **self._kwargs) + + def __repr__(self): + args = [repr(self._name)] + args.extend(map(repr, self._args)) + args.extend('%s=%r' % (k, v) for k, v in self._kwargs.items()) + return '%s.%s(%s)' % (self.__class__.__module__, + self.__class__.__name__, + ', '.join(args)) + + def __reduce__(self): + if not self._kwargs: + return self.__class__, (self._name,) + self._args + else: + from functools import partial + return partial(self.__class__, self._name, **self._kwargs), self._args + + +# In-place Operations *********************************************************# + +def iadd(a, b): + "Same as a += b." + a += b + return a + +def iand(a, b): + "Same as a &= b." + a &= b + return a + +def iconcat(a, b): + "Same as a += b, for a and b sequences." + if not hasattr(a, '__getitem__'): + msg = "'%s' object can't be concatenated" % type(a).__name__ + raise TypeError(msg) + a += b + return a + +def ifloordiv(a, b): + "Same as a //= b." + a //= b + return a + +def ilshift(a, b): + "Same as a <<= b." + a <<= b + return a + +def imod(a, b): + "Same as a %= b." + a %= b + return a + +def imul(a, b): + "Same as a *= b." + a *= b + return a + +def imatmul(a, b): + "Same as a @= b." + a @= b + return a + +def ior(a, b): + "Same as a |= b." + a |= b + return a + +def ipow(a, b): + "Same as a **= b." + a **=b + return a + +def irshift(a, b): + "Same as a >>= b." + a >>= b + return a + +def isub(a, b): + "Same as a -= b." + a -= b + return a + +def itruediv(a, b): + "Same as a /= b." + a /= b + return a + +def ixor(a, b): + "Same as a ^= b." + a ^= b + return a + + +try: + from _operator import * +except ImportError: + pass +else: + from _operator import __doc__ + +# All of these "__func__ = func" assignments have to happen after importing +# from _operator to make sure they're set to the right function +__lt__ = lt +__le__ = le +__eq__ = eq +__ne__ = ne +__ge__ = ge +__gt__ = gt +__not__ = not_ +__abs__ = abs +__add__ = add +__and__ = and_ +__floordiv__ = floordiv +__index__ = index +__inv__ = inv +__invert__ = invert +__lshift__ = lshift +__mod__ = mod +__mul__ = mul +__matmul__ = matmul +__neg__ = neg +__or__ = or_ +__pos__ = pos +__pow__ = pow +__rshift__ = rshift +__sub__ = sub +__truediv__ = truediv +__xor__ = xor +__concat__ = concat +__contains__ = contains +__delitem__ = delitem +__getitem__ = getitem +__setitem__ = setitem +__iadd__ = iadd +__iand__ = iand +__iconcat__ = iconcat +__ifloordiv__ = ifloordiv +__ilshift__ = ilshift +__imod__ = imod +__imul__ = imul +__imatmul__ = imatmul +__ior__ = ior +__ipow__ = ipow +__irshift__ = irshift +__isub__ = isub +__itruediv__ = itruediv +__ixor__ = ixor diff --git a/Lib/os.py b/Lib/os.py new file mode 100644 index 0000000000..cb49481dff --- /dev/null +++ b/Lib/os.py @@ -0,0 +1,135 @@ +import sys + +from _os import * + + +if name == 'nt': + linesep = '\r\n' + import ntpath as path +else: + linesep = '\n' + import posixpath as path + + +sys.modules['os.path'] = path +from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, altsep, + devnull) + +# Change environ to automatically call putenv(), unsetenv if they exist. +from _collections_abc import MutableMapping + +class _Environ(MutableMapping): + def __init__(self, data, encodekey, decodekey, encodevalue, decodevalue, putenv, unsetenv): + self.encodekey = encodekey + self.decodekey = decodekey + self.encodevalue = encodevalue + self.decodevalue = decodevalue + self.putenv = putenv + self.unsetenv = unsetenv + self._data = data + + def __getitem__(self, key): + try: + value = self._data[self.encodekey(key)] + except KeyError: + # raise KeyError with the original key value + raise KeyError(key) from None + + return self.decodevalue(value) + + def __setitem__(self, key, value): + key = self.encodekey(key) + value = self.encodevalue(value) + self.putenv(key, value) + self._data[key] = value + + def __delitem__(self, key): + encodedkey = self.encodekey(key) + self.unsetenv(encodedkey) + try: + del self._data[encodedkey] + except KeyError: + # raise KeyError with the original key value + raise KeyError(key) from None + + def __iter__(self): + # list() from dict object is an atomic operation + keys = list(self._data) + for key in keys: + yield self.decodekey(key) + + def __len__(self): + return len(self._data) + + def __repr__(self): + return 'environ({{{}}})'.format(', '.join( + ('{!r}: {!r}'.format(self.decodekey(key), self.decodevalue(value)) + for key, value in self._data.items()))) + + def copy(self): + return dict(self) + + def setdefault(self, key, value): + if key not in self: + self[key] = value + return self[key] + +try: + _putenv = putenv +except NameError: + _putenv = lambda key, value: None +# else: +# if "putenv" not in __all__: +# __all__.append("putenv") + +try: + _unsetenv = unsetenv +except NameError: + _unsetenv = lambda key: _putenv(key, "") +# else: +# if "unsetenv" not in __all__: +# __all__.append("unsetenv") + +def _createenviron(): + # if name == 'nt': + # # Where Env Var Names Must Be UPPERCASE + # def check_str(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value + # encode = check_str + # decode = str + # def encodekey(key): + # return encode(key).upper() + # data = {} + # for key, value in environ.items(): + # data[encodekey(key)] = value + # else: + # # Where Env Var Names Can Be Mixed Case + # encoding = sys.getfilesystemencoding() + # def encode(value): + # if not isinstance(value, str): + # raise TypeError("str expected, not %s" % type(value).__name__) + # return value.encode(encoding, 'surrogateescape') + # def decode(value): + # return value.decode(encoding, 'surrogateescape') + # encodekey = encode + decode = str + encode = str + encodekey = encode + data = environ + return _Environ(data, + encodekey, decode, + encode, decode, + _putenv, _unsetenv) + +# unicode environ +environ = _createenviron() +del _createenviron + + +def getenv(key, default=None): + """Get an environment variable, return None if it doesn't exist. + The optional second argument can specify an alternate default. + key, default and the result are str.""" + return environ.get(key, default) diff --git a/Lib/posixpath.py b/Lib/posixpath.py new file mode 100644 index 0000000000..ecb4e5a8f7 --- /dev/null +++ b/Lib/posixpath.py @@ -0,0 +1,525 @@ +"""Common operations on Posix pathnames. + +Instead of importing this module directly, import os and refer to +this module as os.path. The "os.path" name is an alias for this +module on Posix systems; on other systems (e.g. Windows), +os.path provides the same operations in a manner specific to that +platform, and is an alias to another module (e.g. ntpath). + +Some of this can actually be useful on non-Posix systems too, e.g. +for manipulation of the pathname component of URLs. +""" + +# Strings representing various path-related bits and pieces. +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = '.' +pardir = '..' +extsep = '.' +sep = '/' +pathsep = ':' +defpath = '/bin:/usr/bin' +altsep = None +devnull = '/dev/null' + +import os +import sys +import stat +import genericpath +from genericpath import * + +__all__ = ["normcase","isabs","join","splitdrive","split","splitext", + "basename","dirname","commonprefix","getsize","getmtime", + "getatime","getctime","islink","exists","lexists","isdir","isfile", + "ismount", "expanduser","expandvars","normpath","abspath", + "samefile","sameopenfile","samestat", + "curdir","pardir","sep","pathsep","defpath","altsep","extsep", + "devnull","realpath","supports_unicode_filenames","relpath", + "commonpath"] + + +def _get_sep(path): + if isinstance(path, bytes): + return b'/' + else: + return '/' + +# Normalize the case of a pathname. Trivial in Posix, string.lower on Mac. +# On MS-DOS this may also turn slashes into backslashes; however, other +# normalizations (such as optimizing '../' away) are not allowed +# (another function should be defined to do that). + +def normcase(s): + """Normalize case of pathname. Has no effect under Posix""" + return os.fspath(s) + + +# Return whether a path is absolute. +# Trivial in Posix, harder on the Mac or MS-DOS. + +def isabs(s): + """Test whether a path is absolute""" + s = os.fspath(s) + sep = _get_sep(s) + return s.startswith(sep) + + +# Join pathnames. +# Ignore the previous parts if a part is absolute. +# Insert a '/' unless the first part is empty or already ends in '/'. + +def join(a, *p): + """Join two or more pathname components, inserting '/' as needed. + If any component is an absolute path, all previous path components + will be discarded. An empty last part will result in a path that + ends with a separator.""" + a = os.fspath(a) + sep = _get_sep(a) + path = a + try: + if not p: + path[:0] + sep #23780: Ensure compatible data type even if p is null. + for b in map(os.fspath, p): + if b.startswith(sep): + path = b + elif not path or path.endswith(sep): + path += b + else: + path += sep + b + except (TypeError, AttributeError, BytesWarning): + genericpath._check_arg_types('join', a, *p) + raise + return path + + +# Split a path in head (everything up to the last '/') and tail (the +# rest). If the path ends in '/', tail will be empty. If there is no +# '/' in the path, head will be empty. +# Trailing '/'es are stripped from head unless it is the root. + +def split(p): + """Split a pathname. Returns tuple "(head, tail)" where "tail" is + everything after the final slash. Either part may be empty.""" + p = os.fspath(p) + sep = _get_sep(p) + i = p.rfind(sep) + 1 + head, tail = p[:i], p[i:] + if head and head != sep*len(head): + head = head.rstrip(sep) + return head, tail + + +# Split a path in root and extension. +# The extension is everything starting at the last dot in the last +# pathname component; the root is everything before that. +# It is always true that root + ext == p. + +def splitext(p): + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + extsep = b'.' + else: + sep = '/' + extsep = '.' + return genericpath._splitext(p, sep, None, extsep) +splitext.__doc__ = genericpath._splitext.__doc__ + +# Split a pathname into a drive specification and the rest of the +# path. Useful on DOS/Windows/NT; on Unix, the drive is always empty. + +def splitdrive(p): + """Split a pathname into drive and path. On Posix, drive is always + empty.""" + p = os.fspath(p) + return p[:0], p + + +# Return the tail (basename) part of a path, same as split(path)[1]. + +def basename(p): + """Returns the final component of a pathname""" + p = os.fspath(p) + sep = _get_sep(p) + i = p.rfind(sep) + 1 + return p[i:] + + +# Return the head (dirname) part of a path, same as split(path)[0]. + +def dirname(p): + """Returns the directory component of a pathname""" + p = os.fspath(p) + sep = _get_sep(p) + i = p.rfind(sep) + 1 + head = p[:i] + if head and head != sep*len(head): + head = head.rstrip(sep) + return head + + +# Is a path a symbolic link? +# This will always return false on systems where os.lstat doesn't exist. + +def islink(path): + """Test whether a path is a symbolic link""" + try: + st = os.lstat(path) + except (OSError, ValueError, AttributeError): + return False + return stat.S_ISLNK(st.st_mode) + +# Being true for dangling symbolic links is also useful. + +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + + +# Is a path a mount point? +# (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) + +def ismount(path): + """Test whether a path is a mount point""" + try: + s1 = os.lstat(path) + except (OSError, ValueError): + # It doesn't exist -- so not a mount point. :-) + return False + else: + # A symlink can never be a mount point + if stat.S_ISLNK(s1.st_mode): + return False + + if isinstance(path, bytes): + parent = join(path, b'..') + else: + parent = join(path, '..') + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except (OSError, ValueError): + return False + + dev1 = s1.st_dev + dev2 = s2.st_dev + if dev1 != dev2: + return True # path/.. on a different device as path + ino1 = s1.st_ino + ino2 = s2.st_ino + if ino1 == ino2: + return True # path/.. is the same i-node as path + return False + + +# Expand paths beginning with '~' or '~user'. +# '~' means $HOME; '~user' means that user's home directory. +# If the path doesn't begin with '~', or if the user or $HOME is unknown, +# the path is returned unchanged (leaving error reporting to whatever +# function is called with the expanded path as argument). +# See also module 'glob' for expansion of *, ? and [...] in pathnames. +# (A function should also be defined to do full *sh-style environment +# variable expansion.) + +def expanduser(path): + """Expand ~ and ~user constructions. If user or $HOME is unknown, + do nothing.""" + path = os.fspath(path) + if isinstance(path, bytes): + tilde = b'~' + else: + tilde = '~' + if not path.startswith(tilde): + return path + sep = _get_sep(path) + i = path.find(sep, 1) + if i < 0: + i = len(path) + if i == 1: + if 'HOME' not in os.environ: + import pwd + try: + userhome = pwd.getpwuid(os.getuid()).pw_dir + except KeyError: + # bpo-10496: if the current user identifier doesn't exist in the + # password database, return the path unchanged + return path + else: + userhome = os.environ['HOME'] + else: + import pwd + name = path[1:i] + if isinstance(name, bytes): + name = str(name, 'ASCII') + try: + pwent = pwd.getpwnam(name) + except KeyError: + # bpo-10496: if the user name from the path doesn't exist in the + # password database, return the path unchanged + return path + userhome = pwent.pw_dir + if isinstance(path, bytes): + userhome = os.fsencode(userhome) + root = b'/' + else: + root = '/' + userhome = userhome.rstrip(root) + return (userhome + path[i:]) or root + + +# Expand paths containing shell variable substitutions. +# This expands the forms $variable and ${variable} only. +# Non-existent variables are left unchanged. + +_varprog = None +_varprogb = None + +def expandvars(path): + """Expand shell variables of form $var and ${var}. Unknown variables + are left unchanged.""" + path = os.fspath(path) + global _varprog, _varprogb + if isinstance(path, bytes): + if b'$' not in path: + return path + if not _varprogb: + import re + _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) + search = _varprogb.search + start = b'{' + end = b'}' + environ = getattr(os, 'environb', None) + else: + if '$' not in path: + return path + if not _varprog: + import re + _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) + search = _varprog.search + start = '{' + end = '}' + environ = os.environ + i = 0 + while True: + m = search(path, i) + if not m: + break + i, j = m.span(0) + name = m.group(1) + if name.startswith(start) and name.endswith(end): + name = name[1:-1] + try: + if environ is None: + value = os.fsencode(os.environ[os.fsdecode(name)]) + else: + value = environ[name] + except KeyError: + i = j + else: + tail = path[j:] + path = path[:i] + value + i = len(path) + path += tail + return path + + +# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. +# It should be understood that this may change the meaning of the path +# if it contains symbolic links! + +def normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = os.fspath(path) + if isinstance(path, bytes): + sep = b'/' + empty = b'' + dot = b'.' + dotdot = b'..' + else: + sep = '/' + empty = '' + dot = '.' + dotdot = '..' + if path == empty: + return dot + initial_slashes = path.startswith(sep) + # POSIX allows one or two initial slashes, but treats three or more + # as single slash. + if (initial_slashes and + path.startswith(sep*2) and not path.startswith(sep*3)): + initial_slashes = 2 + comps = path.split(sep) + new_comps = [] + for comp in comps: + if comp in (empty, dot): + continue + if (comp != dotdot or (not initial_slashes and not new_comps) or + (new_comps and new_comps[-1] == dotdot)): + new_comps.append(comp) + elif new_comps: + new_comps.pop() + comps = new_comps + path = sep.join(comps) + if initial_slashes: + path = sep*initial_slashes + path + return path or dot + + +def abspath(path): + """Return an absolute path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) + + +# Return a canonical path (i.e. the absolute location of a file on the +# filesystem). + +def realpath(filename): + """Return the canonical path of the specified filename, eliminating any +symbolic links encountered in the path.""" + filename = os.fspath(filename) + path, ok = _joinrealpath(filename[:0], filename, {}) + return abspath(path) + +# Join two paths, normalizing and eliminating any symbolic links +# encountered in the second path. +def _joinrealpath(path, rest, seen): + if isinstance(path, bytes): + sep = b'/' + curdir = b'.' + pardir = b'..' + else: + sep = '/' + curdir = '.' + pardir = '..' + + if isabs(rest): + rest = rest[1:] + path = sep + + while rest: + name, _, rest = rest.partition(sep) + if not name or name == curdir: + # current dir + continue + if name == pardir: + # parent dir + if path: + path, name = split(path) + if name == pardir: + path = join(path, pardir, pardir) + else: + path = pardir + continue + newpath = join(path, name) + if not islink(newpath): + path = newpath + continue + # Resolve the symbolic link + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + # Return already resolved part + rest of the path unchanged. + return join(newpath, rest), False + seen[newpath] = None # not resolved symlink + path, ok = _joinrealpath(path, os.readlink(newpath), seen) + if not ok: + return join(path, rest), False + seen[newpath] = path # resolved symlink + + return path, True + + +supports_unicode_filenames = (sys.platform == 'darwin') + +def relpath(path, start=None): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + + path = os.fspath(path) + if isinstance(path, bytes): + curdir = b'.' + sep = b'/' + pardir = b'..' + else: + curdir = '.' + sep = '/' + pardir = '..' + + if start is None: + start = curdir + else: + start = os.fspath(start) + + try: + start_list = [x for x in abspath(start).split(sep) if x] + path_list = [x for x in abspath(path).split(sep) if x] + # Work out how much of the filepath is shared by start and path. + i = len(commonprefix([start_list, path_list])) + + rel_list = [pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return curdir + return join(*rel_list) + except (TypeError, AttributeError, BytesWarning, DeprecationWarning): + genericpath._check_arg_types('relpath', path, start) + raise + + +# Return the longest common sub-path of the sequence of paths given as input. +# The paths are not normalized before comparing them (this is the +# responsibility of the caller). Any trailing separator is stripped from the +# returned path. + +def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError('commonpath() arg is an empty sequence') + + paths = tuple(map(os.fspath, paths)) + if isinstance(paths[0], bytes): + sep = b'/' + curdir = b'.' + else: + sep = '/' + curdir = '.' + + try: + split_paths = [path.split(sep) for path in paths] + + try: + isabs, = set(p[:1] == sep for p in paths) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") from None + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + common = s1 + for i, c in enumerate(s1): + if c != s2[i]: + common = s1[:i] + break + + prefix = sep if isabs else sep[:0] + return prefix + sep.join(common) + except (TypeError, AttributeError): + genericpath._check_arg_types('commonpath', *paths) + raise diff --git a/Lib/re.py b/Lib/re.py deleted file mode 100644 index 8511222be3..0000000000 --- a/Lib/re.py +++ /dev/null @@ -1,11 +0,0 @@ - -""" Regular expressions """ - - -def match(pattern, string, flags=0): - return _compile(pattern, flags).match(string) - - -def _compile(pattern, flags): - p = sre_compile.compile(pattern, flags) - return p diff --git a/Lib/reprlib.py b/Lib/reprlib.py new file mode 100644 index 0000000000..616b3439b5 --- /dev/null +++ b/Lib/reprlib.py @@ -0,0 +1,161 @@ +"""Redo the builtin repr() (representation) but with limits on most sizes.""" + +__all__ = ["Repr", "repr", "recursive_repr"] + +import builtins +from itertools import islice +from _thread import get_ident + +def recursive_repr(fillvalue='...'): + 'Decorator to make a repr function return fillvalue for a recursive call' + + def decorating_function(user_function): + repr_running = set() + + def wrapper(self): + key = id(self), get_ident() + if key in repr_running: + return fillvalue + repr_running.add(key) + try: + result = user_function(self) + finally: + repr_running.discard(key) + return result + + # Can't use functools.wraps() here because of bootstrap issues + wrapper.__module__ = getattr(user_function, '__module__') + wrapper.__doc__ = getattr(user_function, '__doc__') + wrapper.__name__ = getattr(user_function, '__name__') + wrapper.__qualname__ = getattr(user_function, '__qualname__') + wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + return wrapper + + return decorating_function + +class Repr: + + def __init__(self): + self.maxlevel = 6 + self.maxtuple = 6 + self.maxlist = 6 + self.maxarray = 5 + self.maxdict = 4 + self.maxset = 6 + self.maxfrozenset = 6 + self.maxdeque = 6 + self.maxstring = 30 + self.maxlong = 40 + self.maxother = 30 + + def repr(self, x): + return self.repr1(x, self.maxlevel) + + def repr1(self, x, level): + typename = type(x).__name__ + if ' ' in typename: + parts = typename.split() + typename = '_'.join(parts) + if hasattr(self, 'repr_' + typename): + return getattr(self, 'repr_' + typename)(x, level) + else: + return self.repr_instance(x, level) + + def _repr_iterable(self, x, level, left, right, maxiter, trail=''): + n = len(x) + if level <= 0 and n: + s = '...' + else: + newlevel = level - 1 + repr1 = self.repr1 + pieces = [repr1(elem, newlevel) for elem in islice(x, maxiter)] + if n > maxiter: pieces.append('...') + s = ', '.join(pieces) + if n == 1 and trail: right = trail + right + return '%s%s%s' % (left, s, right) + + def repr_tuple(self, x, level): + return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',') + + def repr_list(self, x, level): + return self._repr_iterable(x, level, '[', ']', self.maxlist) + + def repr_array(self, x, level): + if not x: + return "array('%s')" % x.typecode + header = "array('%s', [" % x.typecode + return self._repr_iterable(x, level, header, '])', self.maxarray) + + def repr_set(self, x, level): + if not x: + return 'set()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, '{', '}', self.maxset) + + def repr_frozenset(self, x, level): + if not x: + return 'frozenset()' + x = _possibly_sorted(x) + return self._repr_iterable(x, level, 'frozenset({', '})', + self.maxfrozenset) + + def repr_deque(self, x, level): + return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque) + + def repr_dict(self, x, level): + n = len(x) + if n == 0: return '{}' + if level <= 0: return '{...}' + newlevel = level - 1 + repr1 = self.repr1 + pieces = [] + for key in islice(_possibly_sorted(x), self.maxdict): + keyrepr = repr1(key, newlevel) + valrepr = repr1(x[key], newlevel) + pieces.append('%s: %s' % (keyrepr, valrepr)) + if n > self.maxdict: pieces.append('...') + s = ', '.join(pieces) + return '{%s}' % (s,) + + def repr_str(self, x, level): + s = builtins.repr(x[:self.maxstring]) + if len(s) > self.maxstring: + i = max(0, (self.maxstring-3)//2) + j = max(0, self.maxstring-3-i) + s = builtins.repr(x[:i] + x[len(x)-j:]) + s = s[:i] + '...' + s[len(s)-j:] + return s + + def repr_int(self, x, level): + s = builtins.repr(x) # XXX Hope this isn't too slow... + if len(s) > self.maxlong: + i = max(0, (self.maxlong-3)//2) + j = max(0, self.maxlong-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + + def repr_instance(self, x, level): + try: + s = builtins.repr(x) + # Bugs in x.__repr__() can cause arbitrary + # exceptions -- then make up something + except Exception: + return '<%s instance at %#x>' % (x.__class__.__name__, id(x)) + if len(s) > self.maxother: + i = max(0, (self.maxother-3)//2) + j = max(0, self.maxother-3-i) + s = s[:i] + '...' + s[len(s)-j:] + return s + + +def _possibly_sorted(x): + # Since not all sequences of items can be sorted and comparison + # functions may raise arbitrary exceptions, return an unsorted + # sequence in that case. + try: + return sorted(x) + except Exception: + return list(x) + +aRepr = Repr() +repr = aRepr.repr diff --git a/Lib/stat.py b/Lib/stat.py new file mode 100644 index 0000000000..a9c678ec03 --- /dev/null +++ b/Lib/stat.py @@ -0,0 +1,179 @@ +"""Constants/functions for interpreting results of os.stat() and os.lstat(). + +Suggested usage: from stat import * +""" + +# Indices for stat struct members in the tuple returned by os.stat() + +ST_MODE = 0 +ST_INO = 1 +ST_DEV = 2 +ST_NLINK = 3 +ST_UID = 4 +ST_GID = 5 +ST_SIZE = 6 +ST_ATIME = 7 +ST_MTIME = 8 +ST_CTIME = 9 + +# Extract bits from the mode + +def S_IMODE(mode): + """Return the portion of the file's mode that can be set by + os.chmod(). + """ + return mode & 0o7777 + +def S_IFMT(mode): + """Return the portion of the file's mode that describes the + file type. + """ + return mode & 0o170000 + +# Constants used as S_IFMT() for various file types +# (not all are implemented on all systems) + +S_IFDIR = 0o040000 # directory +S_IFCHR = 0o020000 # character device +S_IFBLK = 0o060000 # block device +S_IFREG = 0o100000 # regular file +S_IFIFO = 0o010000 # fifo (named pipe) +S_IFLNK = 0o120000 # symbolic link +S_IFSOCK = 0o140000 # socket file + +# Functions to test for each file type + +def S_ISDIR(mode): + """Return True if mode is from a directory.""" + return S_IFMT(mode) == S_IFDIR + +def S_ISCHR(mode): + """Return True if mode is from a character special device file.""" + return S_IFMT(mode) == S_IFCHR + +def S_ISBLK(mode): + """Return True if mode is from a block special device file.""" + return S_IFMT(mode) == S_IFBLK + +def S_ISREG(mode): + """Return True if mode is from a regular file.""" + return S_IFMT(mode) == S_IFREG + +def S_ISFIFO(mode): + """Return True if mode is from a FIFO (named pipe).""" + return S_IFMT(mode) == S_IFIFO + +def S_ISLNK(mode): + """Return True if mode is from a symbolic link.""" + return S_IFMT(mode) == S_IFLNK + +def S_ISSOCK(mode): + """Return True if mode is from a socket.""" + return S_IFMT(mode) == S_IFSOCK + +# Names for permission bits + +S_ISUID = 0o4000 # set UID bit +S_ISGID = 0o2000 # set GID bit +S_ENFMT = S_ISGID # file locking enforcement +S_ISVTX = 0o1000 # sticky bit +S_IREAD = 0o0400 # Unix V7 synonym for S_IRUSR +S_IWRITE = 0o0200 # Unix V7 synonym for S_IWUSR +S_IEXEC = 0o0100 # Unix V7 synonym for S_IXUSR +S_IRWXU = 0o0700 # mask for owner permissions +S_IRUSR = 0o0400 # read by owner +S_IWUSR = 0o0200 # write by owner +S_IXUSR = 0o0100 # execute by owner +S_IRWXG = 0o0070 # mask for group permissions +S_IRGRP = 0o0040 # read by group +S_IWGRP = 0o0020 # write by group +S_IXGRP = 0o0010 # execute by group +S_IRWXO = 0o0007 # mask for others (not in group) permissions +S_IROTH = 0o0004 # read by others +S_IWOTH = 0o0002 # write by others +S_IXOTH = 0o0001 # execute by others + +# Names for file flags + +UF_NODUMP = 0x00000001 # do not dump file +UF_IMMUTABLE = 0x00000002 # file may not be changed +UF_APPEND = 0x00000004 # file may only be appended to +UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack +UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted +UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed +UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +SF_ARCHIVED = 0x00010000 # file may be archived +SF_IMMUTABLE = 0x00020000 # file may not be changed +SF_APPEND = 0x00040000 # file may only be appended to +SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted +SF_SNAPSHOT = 0x00200000 # file is a snapshot file + + +_filemode_table = ( + ((S_IFLNK, "l"), + (S_IFSOCK, "s"), # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR + (S_IFREG, "-"), + (S_IFBLK, "b"), + (S_IFDIR, "d"), + (S_IFCHR, "c"), + (S_IFIFO, "p")), + + ((S_IRUSR, "r"),), + ((S_IWUSR, "w"),), + ((S_IXUSR|S_ISUID, "s"), + (S_ISUID, "S"), + (S_IXUSR, "x")), + + ((S_IRGRP, "r"),), + ((S_IWGRP, "w"),), + ((S_IXGRP|S_ISGID, "s"), + (S_ISGID, "S"), + (S_IXGRP, "x")), + + ((S_IROTH, "r"),), + ((S_IWOTH, "w"),), + ((S_IXOTH|S_ISVTX, "t"), + (S_ISVTX, "T"), + (S_IXOTH, "x")) +) + +def filemode(mode): + """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" + perm = [] + for table in _filemode_table: + for bit, char in table: + if mode & bit == bit: + perm.append(char) + break + else: + perm.append("-") + return "".join(perm) + + +# Windows FILE_ATTRIBUTE constants for interpreting os.stat()'s +# "st_file_attributes" member + +FILE_ATTRIBUTE_ARCHIVE = 32 +FILE_ATTRIBUTE_COMPRESSED = 2048 +FILE_ATTRIBUTE_DEVICE = 64 +FILE_ATTRIBUTE_DIRECTORY = 16 +FILE_ATTRIBUTE_ENCRYPTED = 16384 +FILE_ATTRIBUTE_HIDDEN = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768 +FILE_ATTRIBUTE_NORMAL = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072 +FILE_ATTRIBUTE_OFFLINE = 4096 +FILE_ATTRIBUTE_READONLY = 1 +FILE_ATTRIBUTE_REPARSE_POINT = 1024 +FILE_ATTRIBUTE_SPARSE_FILE = 512 +FILE_ATTRIBUTE_SYSTEM = 4 +FILE_ATTRIBUTE_TEMPORARY = 256 +FILE_ATTRIBUTE_VIRTUAL = 65536 + + +# If available, use C implementation +try: + from _stat import * +except ImportError: + pass diff --git a/Lib/this.py b/Lib/this.py new file mode 100644 index 0000000000..e68dd3ff39 --- /dev/null +++ b/Lib/this.py @@ -0,0 +1,28 @@ +s = """Gur Mra bs Clguba, ol Gvz Crgref + +Ornhgvshy vf orggre guna htyl. +Rkcyvpvg vf orggre guna vzcyvpvg. +Fvzcyr vf orggre guna pbzcyrk. +Pbzcyrk vf orggre guna pbzcyvpngrq. +Syng vf orggre guna arfgrq. +Fcnefr vf orggre guna qrafr. +Ernqnovyvgl pbhagf. +Fcrpvny pnfrf nera'g fcrpvny rabhtu gb oernx gur ehyrf. +Nygubhtu cenpgvpnyvgl orngf chevgl. +Reebef fubhyq arire cnff fvyragyl. +Hayrff rkcyvpvgyl fvyraprq. +Va gur snpr bs nzovthvgl, ershfr gur grzcgngvba gb thrff. +Gurer fubhyq or bar-- naq cersrenoyl bayl bar --boivbhf jnl gb qb vg. +Nygubhtu gung jnl znl abg or boivbhf ng svefg hayrff lbh'er Qhgpu. +Abj vf orggre guna arire. +Nygubhtu arire vf bsgra orggre guna *evtug* abj. +Vs gur vzcyrzragngvba vf uneq gb rkcynva, vg'f n onq vqrn. +Vs gur vzcyrzragngvba vf rnfl gb rkcynva, vg znl or n tbbq vqrn. +Anzrfcnprf ner bar ubaxvat terng vqrn -- yrg'f qb zber bs gubfr!""" + +d = {} +for c in (65, 97): + for i in range(26): + d[chr(i+c)] = chr((i+13) % 26 + c) + +print("".join([d.get(c, c) for c in s])) diff --git a/Lib/types.py b/Lib/types.py new file mode 100644 index 0000000000..a7034901c6 --- /dev/null +++ b/Lib/types.py @@ -0,0 +1,295 @@ +""" +Define names for built-in types that aren't directly accessible as a builtin. +""" +import sys + +# Iterators in Python aren't a matter of type but of protocol. A large +# and changing number of builtin types implement *some* flavor of +# iterator. Don't check the type! Use hasattr to check for both +# "__iter__" and "__next__" attributes instead. + +def _f(): pass +FunctionType = type(_f) +LambdaType = type(lambda: None) # Same as FunctionType +CodeType = type(_f.__code__) +MappingProxyType = type(type.__dict__) +SimpleNamespace = type(sys.implementation) + +def _g(): + yield 1 +GeneratorType = type(_g()) + +# async def _c(): pass +# _c = _c() +# CoroutineType = type(_c) +# _c.close() # Prevent ResourceWarning + +# async def _ag(): +# yield +# _ag = _ag() +# AsyncGeneratorType = type(_ag) + +class _C: + def _m(self): pass +MethodType = type(_C()._m) + +BuiltinFunctionType = type(len) +BuiltinMethodType = type([].append) # Same as BuiltinFunctionType + +WrapperDescriptorType = type(object.__init__) +MethodWrapperType = type(object().__str__) +MethodDescriptorType = type(str.join) +ClassMethodDescriptorType = type(dict.__dict__['fromkeys']) + +ModuleType = type(sys) + +# try: +# raise TypeError +# except TypeError: +# tb = sys.exc_info()[2] +# TracebackType = type(tb) +# FrameType = type(tb.tb_frame) +# tb = None; del tb + +# For Jython, the following two types are identical +GetSetDescriptorType = type(FunctionType.__code__) +# MemberDescriptorType = type(FunctionType.__globals__) + +del sys, _f, _g, _C # Not for export + + +# Provide a PEP 3115 compliant mechanism for class creation +def new_class(name, bases=(), kwds=None, exec_body=None): + """Create a class object dynamically using the appropriate metaclass.""" + resolved_bases = resolve_bases(bases) + meta, ns, kwds = prepare_class(name, resolved_bases, kwds) + if exec_body is not None: + exec_body(ns) + if resolved_bases is not bases: + ns['__orig_bases__'] = bases + return meta(name, resolved_bases, ns, **kwds) + +def resolve_bases(bases): + """Resolve MRO entries dynamically as specified by PEP 560.""" + new_bases = list(bases) + updated = False + shift = 0 + for i, base in enumerate(bases): + if isinstance(base, type): + continue + if not hasattr(base, "__mro_entries__"): + continue + new_base = base.__mro_entries__(bases) + updated = True + if not isinstance(new_base, tuple): + raise TypeError("__mro_entries__ must return a tuple") + else: + new_bases[i+shift:i+shift+1] = new_base + shift += len(new_base) - 1 + if not updated: + return bases + return tuple(new_bases) + +def prepare_class(name, bases=(), kwds=None): + """Call the __prepare__ method of the appropriate metaclass. + + Returns (metaclass, namespace, kwds) as a 3-tuple + + *metaclass* is the appropriate metaclass + *namespace* is the prepared class namespace + *kwds* is an updated copy of the passed in kwds argument with any + 'metaclass' entry removed. If no kwds argument is passed in, this will + be an empty dict. + """ + if kwds is None: + kwds = {} + else: + kwds = dict(kwds) # Don't alter the provided mapping + if 'metaclass' in kwds: + meta = kwds.pop('metaclass') + else: + if bases: + meta = type(bases[0]) + else: + meta = type + if isinstance(meta, type): + # when meta is a type, we first determine the most-derived metaclass + # instead of invoking the initial candidate directly + meta = _calculate_meta(meta, bases) + if hasattr(meta, '__prepare__'): + ns = meta.__prepare__(name, bases, **kwds) + else: + ns = {} + return meta, ns, kwds + +def _calculate_meta(meta, bases): + """Calculate the most derived metaclass.""" + winner = meta + for base in bases: + base_meta = type(base) + if issubclass(winner, base_meta): + continue + if issubclass(base_meta, winner): + winner = base_meta + continue + # else: + raise TypeError("metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases") + return winner + +class DynamicClassAttribute: + """Route attribute access on a class to __getattr__. + + This is a descriptor, used to define attributes that act differently when + accessed through an instance and through a class. Instance access remains + normal, but access to an attribute through a class will be routed to the + class's __getattr__ method; this is done by raising AttributeError. + + This allows one to have properties active on an instance, and have virtual + attributes on the class with the same name (see Enum for an example). + + """ + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + # next two lines make DynamicClassAttribute act the same as property + self.__doc__ = doc or fget.__doc__ + self.overwrite_doc = doc is None + # support for abstract methods + self.__isabstractmethod__ = bool(getattr(fget, '__isabstractmethod__', False)) + + def __get__(self, instance, ownerclass=None): + if instance is None: + if self.__isabstractmethod__: + return self + raise AttributeError() + elif self.fget is None: + raise AttributeError("unreadable attribute") + return self.fget(instance) + + def __set__(self, instance, value): + if self.fset is None: + raise AttributeError("can't set attribute") + self.fset(instance, value) + + def __delete__(self, instance): + if self.fdel is None: + raise AttributeError("can't delete attribute") + self.fdel(instance) + + def getter(self, fget): + fdoc = fget.__doc__ if self.overwrite_doc else None + result = type(self)(fget, self.fset, self.fdel, fdoc or self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def setter(self, fset): + result = type(self)(self.fget, fset, self.fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + def deleter(self, fdel): + result = type(self)(self.fget, self.fset, fdel, self.__doc__) + result.overwrite_doc = self.overwrite_doc + return result + + +class _GeneratorWrapper: + # TODO: Implement this in C. + def __init__(self, gen): + self.__wrapped = gen + self.__isgen = gen.__class__ is GeneratorType + self.__name__ = getattr(gen, '__name__', None) + self.__qualname__ = getattr(gen, '__qualname__', None) + def send(self, val): + return self.__wrapped.send(val) + def throw(self, tp, *rest): + return self.__wrapped.throw(tp, *rest) + def close(self): + return self.__wrapped.close() + @property + def gi_code(self): + return self.__wrapped.gi_code + @property + def gi_frame(self): + return self.__wrapped.gi_frame + @property + def gi_running(self): + return self.__wrapped.gi_running + @property + def gi_yieldfrom(self): + return self.__wrapped.gi_yieldfrom + cr_code = gi_code + cr_frame = gi_frame + cr_running = gi_running + cr_await = gi_yieldfrom + def __next__(self): + return next(self.__wrapped) + def __iter__(self): + if self.__isgen: + return self.__wrapped + return self + __await__ = __iter__ + +def coroutine(func): + """Convert regular generator function to a coroutine.""" + + if not callable(func): + raise TypeError('types.coroutine() expects a callable') + + if (func.__class__ is FunctionType and + getattr(func, '__code__', None).__class__ is CodeType): + + co_flags = func.__code__.co_flags + + # Check if 'func' is a coroutine function. + # (0x180 == CO_COROUTINE | CO_ITERABLE_COROUTINE) + if co_flags & 0x180: + return func + + # Check if 'func' is a generator function. + # (0x20 == CO_GENERATOR) + if co_flags & 0x20: + # TODO: Implement this in C. + co = func.__code__ + func.__code__ = CodeType( + co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, + co.co_stacksize, + co.co_flags | 0x100, # 0x100 == CO_ITERABLE_COROUTINE + co.co_code, + co.co_consts, co.co_names, co.co_varnames, co.co_filename, + co.co_name, co.co_firstlineno, co.co_lnotab, co.co_freevars, + co.co_cellvars) + return func + + # The following code is primarily to support functions that + # return generator-like objects (for instance generators + # compiled with Cython). + + # Delay functools and _collections_abc import for speeding up types import. + import functools + import _collections_abc + @functools.wraps(func) + def wrapped(*args, **kwargs): + coro = func(*args, **kwargs) + if (coro.__class__ is CoroutineType or + coro.__class__ is GeneratorType and coro.gi_code.co_flags & 0x100): + # 'coro' is a native coroutine object or an iterable coroutine + return coro + if (isinstance(coro, _collections_abc.Generator) and + not isinstance(coro, _collections_abc.Coroutine)): + # 'coro' is either a pure Python generator iterator, or it + # implements collections.abc.Generator (and does not implement + # collections.abc.Coroutine). + return _GeneratorWrapper(coro) + # 'coro' is either an instance of collections.abc.Coroutine or + # some other object -- pass it through. + return coro + + return wrapped + + +__all__ = [n for n in globals() if n[:1] != '_'] diff --git a/Lib/warnings.py b/Lib/warnings.py new file mode 100644 index 0000000000..81f9864778 --- /dev/null +++ b/Lib/warnings.py @@ -0,0 +1,554 @@ +"""Python part of the warnings subsystem.""" + +import sys + + +__all__ = ["warn", "warn_explicit", "showwarning", + "formatwarning", "filterwarnings", "simplefilter", + "resetwarnings", "catch_warnings"] + +def showwarning(message, category, filename, lineno, file=None, line=None): + """Hook to write a warning to a file; replace if you like.""" + msg = WarningMessage(message, category, filename, lineno, file, line) + _showwarnmsg_impl(msg) + +def formatwarning(message, category, filename, lineno, line=None): + """Function to format a warning the standard way.""" + msg = WarningMessage(message, category, filename, lineno, None, line) + return _formatwarnmsg_impl(msg) + +def _showwarnmsg_impl(msg): + file = msg.file + if file is None: + file = sys.stderr + if file is None: + # sys.stderr is None when run with pythonw.exe: + # warnings get lost + return + text = _formatwarnmsg(msg) + try: + file.write(text) + except OSError: + # the file (probably stderr) is invalid - this warning gets lost. + pass + +def _formatwarnmsg_impl(msg): + s = ("%s:%s: %s: %s\n" + % (msg.filename, msg.lineno, msg.category.__name__, + msg.message)) + + if msg.line is None: + try: + import linecache + line = linecache.getline(msg.filename, msg.lineno) + except Exception: + # When a warning is logged during Python shutdown, linecache + # and the import machinery don't work anymore + line = None + linecache = None + else: + line = msg.line + if line: + line = line.strip() + s += " %s\n" % line + + if msg.source is not None: + try: + import tracemalloc + tb = tracemalloc.get_object_traceback(msg.source) + except Exception: + # When a warning is logged during Python shutdown, tracemalloc + # and the import machinery don't work anymore + tb = None + + if tb is not None: + s += 'Object allocated at (most recent call last):\n' + for frame in tb: + s += (' File "%s", lineno %s\n' + % (frame.filename, frame.lineno)) + + try: + if linecache is not None: + line = linecache.getline(frame.filename, frame.lineno) + else: + line = None + except Exception: + line = None + if line: + line = line.strip() + s += ' %s\n' % line + return s + +# Keep a reference to check if the function was replaced +_showwarning_orig = showwarning + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + try: + sw = showwarning + except NameError: + pass + else: + if sw is not _showwarning_orig: + # warnings.showwarning() was replaced + if not callable(sw): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + sw(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + _showwarnmsg_impl(msg) + +# Keep a reference to check if the function was replaced +_formatwarning_orig = formatwarning + +def _formatwarnmsg(msg): + """Function to format a warning the standard way.""" + try: + fw = formatwarning + except NameError: + pass + else: + if fw is not _formatwarning_orig: + # warnings.formatwarning() was replaced + return fw(msg.message, msg.category, + msg.filename, msg.lineno, line=msg.line) + return _formatwarnmsg_impl(msg) + +def filterwarnings(action, message="", category=Warning, module="", lineno=0, + append=False): + """Insert an entry into the list of warnings filters (at the front). + + 'action' -- one of "error", "ignore", "always", "default", "module", + or "once" + 'message' -- a regex that the warning message must match + 'category' -- a class that the warning must be a subclass of + 'module' -- a regex that the module name must match + 'lineno' -- an integer line number, 0 matches all warnings + 'append' -- if true, append to the list of filters + """ + assert action in ("error", "ignore", "always", "default", "module", + "once"), "invalid action: %r" % (action,) + assert isinstance(message, str), "message must be a string" + assert isinstance(category, type), "category must be a class" + assert issubclass(category, Warning), "category must be a Warning subclass" + assert isinstance(module, str), "module must be a string" + assert isinstance(lineno, int) and lineno >= 0, \ + "lineno must be an int >= 0" + + if message or module: + import re + + if message: + message = re.compile(message, re.I) + else: + message = None + if module: + module = re.compile(module) + else: + module = None + + _add_filter(action, message, category, module, lineno, append=append) + +def simplefilter(action, category=Warning, lineno=0, append=False): + """Insert a simple entry into the list of warnings filters (at the front). + + A simple filter matches all modules and messages. + 'action' -- one of "error", "ignore", "always", "default", "module", + or "once" + 'category' -- a class that the warning must be a subclass of + 'lineno' -- an integer line number, 0 matches all warnings + 'append' -- if true, append to the list of filters + """ + assert action in ("error", "ignore", "always", "default", "module", + "once"), "invalid action: %r" % (action,) + assert isinstance(lineno, int) and lineno >= 0, \ + "lineno must be an int >= 0" + _add_filter(action, None, category, None, lineno, append=append) + +def _add_filter(*item, append): + # Remove possible duplicate filters, so new one will be placed + # in correct place. If append=True and duplicate exists, do nothing. + if not append: + try: + filters.remove(item) + except ValueError: + pass + filters.insert(0, item) + else: + if item not in filters: + filters.append(item) + _filters_mutated() + +def resetwarnings(): + """Clear the list of warning filters, so that no filters are active.""" + filters[:] = [] + _filters_mutated() + +class _OptionError(Exception): + """Exception used by option processing helpers.""" + pass + +# Helper to process -W options passed via sys.warnoptions +def _processoptions(args): + for arg in args: + try: + _setoption(arg) + except _OptionError as msg: + print("Invalid -W option ignored:", msg, file=sys.stderr) + +# Helper for _processoptions() +def _setoption(arg): + import re + parts = arg.split(':') + if len(parts) > 5: + raise _OptionError("too many fields (max 5): %r" % (arg,)) + while len(parts) < 5: + parts.append('') + action, message, category, module, lineno = [s.strip() + for s in parts] + action = _getaction(action) + message = re.escape(message) + category = _getcategory(category) + module = re.escape(module) + if module: + module = module + '$' + if lineno: + try: + lineno = int(lineno) + if lineno < 0: + raise ValueError + except (ValueError, OverflowError): + raise _OptionError("invalid lineno %r" % (lineno,)) from None + else: + lineno = 0 + filterwarnings(action, message, category, module, lineno) + +# Helper for _setoption() +def _getaction(action): + if not action: + return "default" + if action == "all": return "always" # Alias + for a in ('default', 'always', 'ignore', 'module', 'once', 'error'): + if a.startswith(action): + return a + raise _OptionError("invalid action: %r" % (action,)) + +# Helper for _setoption() +def _getcategory(category): + import re + if not category: + return Warning + if re.match("^[a-zA-Z0-9_]+$", category): + try: + cat = eval(category) + except NameError: + raise _OptionError("unknown warning category: %r" % (category,)) from None + else: + i = category.rfind(".") + module = category[:i] + klass = category[i+1:] + try: + m = __import__(module, None, None, [klass]) + except ImportError: + raise _OptionError("invalid module name: %r" % (module,)) from None + try: + cat = getattr(m, klass) + except AttributeError: + raise _OptionError("unknown warning category: %r" % (category,)) from None + if not issubclass(cat, Warning): + raise _OptionError("invalid warning category: %r" % (category,)) + return cat + + +def _is_internal_frame(frame): + """Signal whether the frame is an internal CPython implementation detail.""" + filename = frame.f_code.co_filename + return 'importlib' in filename and '_bootstrap' in filename + + +def _next_external_frame(frame): + """Find the next frame that doesn't involve CPython internals.""" + frame = frame.f_back + while frame is not None and _is_internal_frame(frame): + frame = frame.f_back + return frame + + +# Code typically replaced by _warnings +def warn(message, category=None, stacklevel=1, source=None): + """Issue a warning, or maybe ignore it or raise an exception.""" + # Check if message is already a Warning object + if isinstance(message, Warning): + category = message.__class__ + # Check category argument + if category is None: + category = UserWarning + if not (isinstance(category, type) and issubclass(category, Warning)): + raise TypeError("category must be a Warning subclass, " + "not '{:s}'".format(type(category).__name__)) + # Get context information + try: + if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): + # If frame is too small to care or if the warning originated in + # internal code, then do not try to hide any frames. + frame = sys._getframe(stacklevel) + else: + frame = sys._getframe(1) + # Look for one frame less since the above line starts us off. + for x in range(stacklevel-1): + frame = _next_external_frame(frame) + if frame is None: + raise ValueError + except ValueError: + globals = sys.__dict__ + lineno = 1 + else: + globals = frame.f_globals + lineno = frame.f_lineno + if '__name__' in globals: + module = globals['__name__'] + else: + module = "" + filename = globals.get('__file__') + if filename: + fnl = filename.lower() + if fnl.endswith(".pyc"): + filename = filename[:-1] + else: + if module == "__main__": + try: + filename = sys.argv[0] + except AttributeError: + # embedded interpreters don't have sys.argv, see bug #839151 + filename = '__main__' + if not filename: + filename = module + registry = globals.setdefault("__warningregistry__", {}) + warn_explicit(message, category, filename, lineno, module, registry, + globals, source) + +def warn_explicit(message, category, filename, lineno, + module=None, registry=None, module_globals=None, + source=None): + lineno = int(lineno) + if module is None: + module = filename or "" + if module[-3:].lower() == ".py": + module = module[:-3] # XXX What about leading pathname? + if registry is None: + registry = {} + if registry.get('version', 0) != _filters_version: + registry.clear() + registry['version'] = _filters_version + if isinstance(message, Warning): + text = str(message) + category = message.__class__ + else: + text = message + message = category(message) + key = (text, category, lineno) + # Quick test for common case + if registry.get(key): + return + # Search the filters + for item in filters: + action, msg, cat, mod, ln = item + if ((msg is None or msg.match(text)) and + issubclass(category, cat) and + (mod is None or mod.match(module)) and + (ln == 0 or lineno == ln)): + break + else: + action = defaultaction + # Early exit actions + if action == "ignore": + return + + # Prime the linecache for formatting, in case the + # "file" is actually in a zipfile or something. + import linecache + linecache.getlines(filename, module_globals) + + if action == "error": + raise message + # Other actions + if action == "once": + registry[key] = 1 + oncekey = (text, category) + if onceregistry.get(oncekey): + return + onceregistry[oncekey] = 1 + elif action == "always": + pass + elif action == "module": + registry[key] = 1 + altkey = (text, category, 0) + if registry.get(altkey): + return + registry[altkey] = 1 + elif action == "default": + registry[key] = 1 + else: + # Unrecognized actions are errors + raise RuntimeError( + "Unrecognized action (%r) in warnings.filters:\n %s" % + (action, item)) + # Print message and context + msg = WarningMessage(message, category, filename, lineno, source) + _showwarnmsg(msg) + + +class WarningMessage(object): + + _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", + "line", "source") + + def __init__(self, message, category, filename, lineno, file=None, + line=None, source=None): + self.message = message + self.category = category + self.filename = filename + self.lineno = lineno + self.file = file + self.line = line + self.source = source + self._category_name = category.__name__ if category else None + + def __str__(self): + return ("{message : %r, category : %r, filename : %r, lineno : %s, " + "line : %r}" % (self.message, self._category_name, + self.filename, self.lineno, self.line)) + + +class catch_warnings(object): + + """A context manager that copies and restores the warnings filter upon + exiting the context. + + The 'record' argument specifies whether warnings should be captured by a + custom implementation of warnings.showwarning() and be appended to a list + returned by the context manager. Otherwise None is returned by the context + manager. The objects appended to the list are arguments whose attributes + mirror the arguments to showwarning(). + + The 'module' argument is to specify an alternative module to the module + named 'warnings' and imported under that name. This argument is only useful + when testing the warnings module itself. + + """ + + def __init__(self, *, record=False, module=None): + """Specify whether to record warnings and if an alternative module + should be used other than sys.modules['warnings']. + + For compatibility with Python 3.0, please consider all arguments to be + keyword-only. + + """ + self._record = record + self._module = sys.modules['warnings'] if module is None else module + self._entered = False + + def __repr__(self): + args = [] + if self._record: + args.append("record=True") + if self._module is not sys.modules['warnings']: + args.append("module=%r" % self._module) + name = type(self).__name__ + return "%s(%s)" % (name, ", ".join(args)) + + def __enter__(self): + if self._entered: + raise RuntimeError("Cannot enter %r twice" % self) + self._entered = True + self._filters = self._module.filters + self._module.filters = self._filters[:] + self._module._filters_mutated() + self._showwarning = self._module.showwarning + self._showwarnmsg_impl = self._module._showwarnmsg_impl + if self._record: + log = [] + self._module._showwarnmsg_impl = log.append + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig + return log + else: + return None + + def __exit__(self, *exc_info): + if not self._entered: + raise RuntimeError("Cannot exit %r without entering first" % self) + self._module.filters = self._filters + self._module._filters_mutated() + self._module.showwarning = self._showwarning + self._module._showwarnmsg_impl = self._showwarnmsg_impl + + +# Private utility function called by _PyErr_WarnUnawaitedCoroutine +def _warn_unawaited_coroutine(coro): + msg_lines = [ + f"coroutine '{coro.__qualname__}' was never awaited\n" + ] + if coro.cr_origin is not None: + import linecache, traceback + def extract(): + for filename, lineno, funcname in reversed(coro.cr_origin): + line = linecache.getline(filename, lineno) + yield (filename, lineno, funcname, line) + msg_lines.append("Coroutine created at (most recent call last)\n") + msg_lines += traceback.format_list(list(extract())) + msg = "".join(msg_lines).rstrip("\n") + # Passing source= here means that if the user happens to have tracemalloc + # enabled and tracking where the coroutine was created, the warning will + # contain that traceback. This does mean that if they have *both* + # coroutine origin tracking *and* tracemalloc enabled, they'll get two + # partially-redundant tracebacks. If we wanted to be clever we could + # probably detect this case and avoid it, but for now we don't bother. + warn(msg, category=RuntimeWarning, stacklevel=2, source=coro) + + +# filters contains a sequence of filter 5-tuples +# The components of the 5-tuple are: +# - an action: error, ignore, always, default, module, or once +# - a compiled regex that must match the warning message +# - a class representing the warning category +# - a compiled regex that must match the module that is being warned +# - a line number for the line being warning, or 0 to mean any line +# If either if the compiled regexs are None, match anything. +try: + from _warnings import (filters, _defaultaction, _onceregistry, + warn, warn_explicit, _filters_mutated) + defaultaction = _defaultaction + onceregistry = _onceregistry + _warnings_defaults = True +except ImportError: + filters = [] + defaultaction = "default" + onceregistry = {} + + _filters_version = 1 + + def _filters_mutated(): + global _filters_version + _filters_version += 1 + + _warnings_defaults = False + + +# Module initialization +_processoptions(sys.warnoptions) +if not _warnings_defaults: + # Several warning categories are ignored by default in regular builds + if not hasattr(sys, 'gettotalrefcount'): + filterwarnings("default", category=DeprecationWarning, + module="__main__", append=1) + simplefilter("ignore", category=DeprecationWarning, append=1) + simplefilter("ignore", category=PendingDeprecationWarning, append=1) + simplefilter("ignore", category=ImportWarning, append=1) + simplefilter("ignore", category=ResourceWarning, append=1) + +del _warnings_defaults diff --git a/Lib/weakref.py b/Lib/weakref.py new file mode 100644 index 0000000000..99de2eab74 --- /dev/null +++ b/Lib/weakref.py @@ -0,0 +1,632 @@ +"""Weak reference support for Python. + +This module is an implementation of PEP 205: + +http://www.python.org/dev/peps/pep-0205/ +""" + +# Naming convention: Variables named "wr" are weak reference objects; +# they are called this instead of "ref" to avoid name collisions with +# the module-global ref() function imported from _weakref. + +from _weakref import ( + getweakrefcount, + getweakrefs, + ref, + proxy, + CallableProxyType, + ProxyType, + ReferenceType, + _remove_dead_weakref) + +from _weakrefset import WeakSet, _IterationGuard + +import _collections_abc # Import after _weakref to avoid circular import. +import sys +import itertools + +ProxyTypes = (ProxyType, CallableProxyType) + +__all__ = ["ref", "proxy", "getweakrefcount", "getweakrefs", + "WeakKeyDictionary", "ReferenceType", "ProxyType", + "CallableProxyType", "ProxyTypes", "WeakValueDictionary", + "WeakSet", "WeakMethod", "finalize"] + + +class WeakMethod(ref): + """ + A custom `weakref.ref` subclass which simulates a weak reference to + a bound method, working around the lifetime problem of bound methods. + """ + + __slots__ = "_func_ref", "_meth_type", "_alive", "__weakref__" + + def __new__(cls, meth, callback=None): + try: + obj = meth.__self__ + func = meth.__func__ + except AttributeError: + raise TypeError("argument should be a bound method, not {}" + .format(type(meth))) from None + def _cb(arg): + # The self-weakref trick is needed to avoid creating a reference + # cycle. + self = self_wr() + if self._alive: + self._alive = False + if callback is not None: + callback(self) + self = ref.__new__(cls, obj, _cb) + self._func_ref = ref(func, _cb) + self._meth_type = type(meth) + self._alive = True + self_wr = ref(self) + return self + + def __call__(self): + obj = super().__call__() + func = self._func_ref() + if obj is None or func is None: + return None + return self._meth_type(func, obj) + + def __eq__(self, other): + if isinstance(other, WeakMethod): + if not self._alive or not other._alive: + return self is other + return ref.__eq__(self, other) and self._func_ref == other._func_ref + return False + + def __ne__(self, other): + if isinstance(other, WeakMethod): + if not self._alive or not other._alive: + return self is not other + return ref.__ne__(self, other) or self._func_ref != other._func_ref + return True + + __hash__ = ref.__hash__ + + +class WeakValueDictionary(_collections_abc.MutableMapping): + """Mapping class that references values weakly. + + Entries in the dictionary will be discarded when no strong + reference to the value exists anymore + """ + # We inherit the constructor without worrying about the input + # dictionary; since it uses our .update() method, we get the right + # checks (if the other dictionary is a WeakValueDictionary, + # objects are unwrapped on the way out, and we always wrap on the + # way in). + + def __init__(*args, **kw): + if not args: + raise TypeError("descriptor '__init__' of 'WeakValueDictionary' " + "object needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(wr.key) + else: + # Atomic removal is necessary since this function + # can be called asynchronously by the GC + _atomic_removal(d, wr.key) + self._remove = remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + self.data = d = {} + self.update(*args, **kw) + + def _commit_removals(self): + l = self._pending_removals + d = self.data + # We shouldn't encounter any KeyError, because this method should + # always be called *before* mutating the dict. + while l: + key = l.pop() + _remove_dead_weakref(d, key) + + def __getitem__(self, key): + if self._pending_removals: + self._commit_removals() + o = self.data[key]() + if o is None: + raise KeyError(key) + else: + return o + + def __delitem__(self, key): + if self._pending_removals: + self._commit_removals() + del self.data[key] + + def __len__(self): + if self._pending_removals: + self._commit_removals() + return len(self.data) + + def __contains__(self, key): + if self._pending_removals: + self._commit_removals() + try: + o = self.data[key]() + except KeyError: + return False + return o is not None + + def __repr__(self): + return "<%s at %#x>" % (self.__class__.__name__, id(self)) + + def __setitem__(self, key, value): + if self._pending_removals: + self._commit_removals() + self.data[key] = KeyedRef(value, self._remove, key) + + def copy(self): + if self._pending_removals: + self._commit_removals() + new = WeakValueDictionary() + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[key] = o + return new + + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + if self._pending_removals: + self._commit_removals() + new = self.__class__() + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o + return new + + def get(self, key, default=None): + if self._pending_removals: + self._commit_removals() + try: + wr = self.data[key] + except KeyError: + return default + else: + o = wr() + if o is None: + # This should only happen + return default + else: + return o + + def items(self): + if self._pending_removals: + self._commit_removals() + with _IterationGuard(self): + for k, wr in self.data.items(): + v = wr() + if v is not None: + yield k, v + + def keys(self): + if self._pending_removals: + self._commit_removals() + with _IterationGuard(self): + for k, wr in self.data.items(): + if wr() is not None: + yield k + + __iter__ = keys + + def itervaluerefs(self): + """Return an iterator that yields the weak references to the values. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the values around longer than needed. + + """ + if self._pending_removals: + self._commit_removals() + with _IterationGuard(self): + yield from self.data.values() + + def values(self): + if self._pending_removals: + self._commit_removals() + with _IterationGuard(self): + for wr in self.data.values(): + obj = wr() + if obj is not None: + yield obj + + def popitem(self): + if self._pending_removals: + self._commit_removals() + while True: + key, wr = self.data.popitem() + o = wr() + if o is not None: + return key, o + + def pop(self, key, *args): + if self._pending_removals: + self._commit_removals() + try: + o = self.data.pop(key)() + except KeyError: + o = None + if o is None: + if args: + return args[0] + else: + raise KeyError(key) + else: + return o + + def setdefault(self, key, default=None): + try: + o = self.data[key]() + except KeyError: + o = None + if o is None: + if self._pending_removals: + self._commit_removals() + self.data[key] = KeyedRef(default, self._remove, key) + return default + else: + return o + + def update(*args, **kwargs): + if not args: + raise TypeError("descriptor 'update' of 'WeakValueDictionary' " + "object needs an argument") + self, *args = args + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + dict = args[0] if args else None + if self._pending_removals: + self._commit_removals() + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, o in dict.items(): + d[key] = KeyedRef(o, self._remove, key) + if len(kwargs): + self.update(kwargs) + + def valuerefs(self): + """Return a list of weak references to the values. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the values around longer than needed. + + """ + if self._pending_removals: + self._commit_removals() + return list(self.data.values()) + + +class KeyedRef(ref): + """Specialized reference that includes a key corresponding to the value. + + This is used in the WeakValueDictionary to avoid having to create + a function object for each key stored in the mapping. A shared + callback object can use the 'key' attribute of a KeyedRef instead + of getting a reference to the key from an enclosing scope. + + """ + + __slots__ = "key", + + def __new__(type, ob, callback, key): + self = ref.__new__(type, ob, callback) + self.key = key + return self + + def __init__(self, ob, callback, key): + super().__init__(ob, callback) + + +class WeakKeyDictionary(_collections_abc.MutableMapping): + """ Mapping class that references keys weakly. + + Entries in the dictionary will be discarded when there is no + longer a strong reference to the key. This can be used to + associate additional data with an object owned by other parts of + an application without adding attributes to those objects. This + can be especially useful with objects that override attribute + accesses. + """ + + def __init__(self, dict=None): + self.data = {} + def remove(k, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(k) + else: + del self.data[k] + self._remove = remove + # A list of dead weakrefs (keys to be removed) + self._pending_removals = [] + self._iterating = set() + self._dirty_len = False + if dict is not None: + self.update(dict) + + def _commit_removals(self): + # NOTE: We don't need to call this method before mutating the dict, + # because a dead weakref never compares equal to a live weakref, + # even if they happened to refer to equal objects. + # However, it means keys may already have been removed. + l = self._pending_removals + d = self.data + while l: + try: + del d[l.pop()] + except KeyError: + pass + + def _scrub_removals(self): + d = self.data + self._pending_removals = [k for k in self._pending_removals if k in d] + self._dirty_len = False + + def __delitem__(self, key): + self._dirty_len = True + del self.data[ref(key)] + + def __getitem__(self, key): + return self.data[ref(key)] + + def __len__(self): + if self._dirty_len and self._pending_removals: + # self._pending_removals may still contain keys which were + # explicitly removed, we have to scrub them (see issue #21173). + self._scrub_removals() + return len(self.data) - len(self._pending_removals) + + def __repr__(self): + return "<%s at %#x>" % (self.__class__.__name__, id(self)) + + def __setitem__(self, key, value): + self.data[ref(key, self._remove)] = value + + def copy(self): + new = WeakKeyDictionary() + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = value + return new + + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + new = self.__class__() + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) + return new + + def get(self, key, default=None): + return self.data.get(ref(key),default) + + def __contains__(self, key): + try: + wr = ref(key) + except TypeError: + return False + return wr in self.data + + def items(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + key = wr() + if key is not None: + yield key, value + + def keys(self): + with _IterationGuard(self): + for wr in self.data: + obj = wr() + if obj is not None: + yield obj + + __iter__ = keys + + def values(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + if wr() is not None: + yield value + + def keyrefs(self): + """Return a list of weak references to the keys. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the keys around longer than needed. + + """ + return list(self.data) + + def popitem(self): + self._dirty_len = True + while True: + key, value = self.data.popitem() + o = key() + if o is not None: + return o, value + + def pop(self, key, *args): + self._dirty_len = True + return self.data.pop(ref(key), *args) + + def setdefault(self, key, default=None): + return self.data.setdefault(ref(key, self._remove),default) + + def update(self, dict=None, **kwargs): + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, value in dict.items(): + d[ref(key, self._remove)] = value + if len(kwargs): + self.update(kwargs) + + +class finalize: + """Class for finalization of weakrefable objects + + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info: + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + atexit.register(self._exitfunc) + finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) + else: + return '<%s object at %#x; for %r at %#x>' % \ + (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item:item[1].index) + return [f for (f,i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or finalize._dirty: + pending = cls._select_for_exit() + finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + finalize._shutdown = True + if reenable_gc: + gc.enable() diff --git a/Lib/xdrlib.py b/Lib/xdrlib.py new file mode 100644 index 0000000000..d6e1aeb527 --- /dev/null +++ b/Lib/xdrlib.py @@ -0,0 +1,241 @@ +"""Implements (a subset of) Sun XDR -- eXternal Data Representation. + +See: RFC 1014 + +""" + +import struct +from io import BytesIO +from functools import wraps + +__all__ = ["Error", "Packer", "Unpacker", "ConversionError"] + +# exceptions +class Error(Exception): + """Exception class for this module. Use: + + except xdrlib.Error as var: + # var has the Error instance for the exception + + Public ivars: + msg -- contains the message + + """ + def __init__(self, msg): + self.msg = msg + def __repr__(self): + return repr(self.msg) + def __str__(self): + return str(self.msg) + + +class ConversionError(Error): + pass + +def raise_conversion_error(function): + """ Wrap any raised struct.errors in a ConversionError. """ + + @wraps(function) + def result(self, value): + try: + return function(self, value) + except struct.error as e: + raise ConversionError(e.args[0]) from None + return result + + +class Packer: + """Pack various data representations into a buffer.""" + + def __init__(self): + self.reset() + + def reset(self): + self.__buf = BytesIO() + + def get_buffer(self): + return self.__buf.getvalue() + # backwards compatibility + get_buf = get_buffer + + @raise_conversion_error + def pack_uint(self, x): + self.__buf.write(struct.pack('>L', x)) + + @raise_conversion_error + def pack_int(self, x): + self.__buf.write(struct.pack('>l', x)) + + pack_enum = pack_int + + def pack_bool(self, x): + if x: self.__buf.write(b'\0\0\0\1') + else: self.__buf.write(b'\0\0\0\0') + + def pack_uhyper(self, x): + try: + self.pack_uint(x>>32 & 0xffffffff) + except (TypeError, struct.error) as e: + raise ConversionError(e.args[0]) from None + try: + self.pack_uint(x & 0xffffffff) + except (TypeError, struct.error) as e: + raise ConversionError(e.args[0]) from None + + pack_hyper = pack_uhyper + + @raise_conversion_error + def pack_float(self, x): + self.__buf.write(struct.pack('>f', x)) + + @raise_conversion_error + def pack_double(self, x): + self.__buf.write(struct.pack('>d', x)) + + def pack_fstring(self, n, s): + if n < 0: + raise ValueError('fstring size must be nonnegative') + data = s[:n] + n = ((n+3)//4)*4 + data = data + (n - len(data)) * b'\0' + self.__buf.write(data) + + pack_fopaque = pack_fstring + + def pack_string(self, s): + n = len(s) + self.pack_uint(n) + self.pack_fstring(n, s) + + pack_opaque = pack_string + pack_bytes = pack_string + + def pack_list(self, list, pack_item): + for item in list: + self.pack_uint(1) + pack_item(item) + self.pack_uint(0) + + def pack_farray(self, n, list, pack_item): + if len(list) != n: + raise ValueError('wrong array size') + for item in list: + pack_item(item) + + def pack_array(self, list, pack_item): + n = len(list) + self.pack_uint(n) + self.pack_farray(n, list, pack_item) + + + +class Unpacker: + """Unpacks various data representations from the given buffer.""" + + def __init__(self, data): + self.reset(data) + + def reset(self, data): + self.__buf = data + self.__pos = 0 + + def get_position(self): + return self.__pos + + def set_position(self, position): + self.__pos = position + + def get_buffer(self): + return self.__buf + + def done(self): + if self.__pos < len(self.__buf): + raise Error('unextracted data remains') + + def unpack_uint(self): + i = self.__pos + self.__pos = j = i+4 + data = self.__buf[i:j] + if len(data) < 4: + raise EOFError + return struct.unpack('>L', data)[0] + + def unpack_int(self): + i = self.__pos + self.__pos = j = i+4 + data = self.__buf[i:j] + if len(data) < 4: + raise EOFError + return struct.unpack('>l', data)[0] + + unpack_enum = unpack_int + + def unpack_bool(self): + return bool(self.unpack_int()) + + def unpack_uhyper(self): + hi = self.unpack_uint() + lo = self.unpack_uint() + return int(hi)<<32 | lo + + def unpack_hyper(self): + x = self.unpack_uhyper() + if x >= 0x8000000000000000: + x = x - 0x10000000000000000 + return x + + def unpack_float(self): + i = self.__pos + self.__pos = j = i+4 + data = self.__buf[i:j] + if len(data) < 4: + raise EOFError + return struct.unpack('>f', data)[0] + + def unpack_double(self): + i = self.__pos + self.__pos = j = i+8 + data = self.__buf[i:j] + if len(data) < 8: + raise EOFError + return struct.unpack('>d', data)[0] + + def unpack_fstring(self, n): + if n < 0: + raise ValueError('fstring size must be nonnegative') + i = self.__pos + j = i + (n+3)//4*4 + if j > len(self.__buf): + raise EOFError + self.__pos = j + return self.__buf[i:i+n] + + unpack_fopaque = unpack_fstring + + def unpack_string(self): + n = self.unpack_uint() + return self.unpack_fstring(n) + + unpack_opaque = unpack_string + unpack_bytes = unpack_string + + def unpack_list(self, unpack_item): + list = [] + while 1: + x = self.unpack_uint() + if x == 0: break + if x != 1: + raise ConversionError('0 or 1 expected, got %r' % (x,)) + item = unpack_item() + list.append(item) + return list + + def unpack_farray(self, n, unpack_item): + list = [] + for i in range(n): + list.append(unpack_item()) + return list + + def unpack_array(self, unpack_item): + n = self.unpack_uint() + return self.unpack_farray(n, unpack_item) diff --git a/README.md b/README.md index 41bde43e07..b27e2b2313 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + # RustPython A Python-3 (CPython >= 3.5.0) Interpreter written in Rust :snake: :scream: :metal:. @@ -83,7 +85,7 @@ Another approach is to checkout the source code: builtin functions and object me and easiest way to contribute. You can also simply run -`cargo run tests/snippets/whats_left_to_implement.py` to assist in finding any +`./whats_left.sh` to assist in finding any unimplemented method. # Testing @@ -116,7 +118,7 @@ To do this, follow this method: ```shell $ cd ~/GIT $ git clone git@github.com:pybee/ouroboros.git -$ export PYTHONPATH=~/GIT/ouroboros/ouroboros +$ export RUSTPYTHONPATH=~/GIT/ouroboros/ouroboros $ cd RustPython $ cargo run -- -c 'import statistics' ``` @@ -195,7 +197,7 @@ crate and webpack. # Code style -The code style used is the default rustfmt codestyle. Please format your code accordingly. +The code style used is the default [rustfmt](https://github.com/rust-lang/rustfmt) codestyle. Please format your code accordingly. # Community diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000..464dc65f21 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,19 @@ + +# Benchmarking + +These are some files to determine performance of rustpython. + +# Usage + +Install pytest and pytest-benchmark: + + $ pip install pytest-benchmark + +Then run: + + $ pytest + +# Benchmark source + +- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html + diff --git a/benchmarks/bench.rs b/benchmarks/bench.rs new file mode 100644 index 0000000000..55f2763a5e --- /dev/null +++ b/benchmarks/bench.rs @@ -0,0 +1,125 @@ +#![feature(test)] + +extern crate cpython; +extern crate rustpython_parser; +extern crate rustpython_vm; +extern crate test; + +use rustpython_vm::pyobject::PyResult; +use rustpython_vm::{compile, VirtualMachine}; + +#[bench] +fn bench_tokenization(b: &mut test::Bencher) { + use rustpython_parser::lexer::{make_tokenizer, Tok}; + + let source = include_str!("./benchmarks/minidom.py"); + + b.bytes = source.len() as _; + b.iter(|| { + let lexer = make_tokenizer(source); + for res in lexer { + let _token: Tok = res.unwrap().1; + } + }) +} + +#[bench] +fn bench_rustpy_parse_to_ast(b: &mut test::Bencher) { + use rustpython_parser::parser::parse_program; + + let source = include_str!("./benchmarks/minidom.py"); + + b.bytes = source.len() as _; + b.iter(|| parse_program(source).unwrap()) +} + +#[bench] +fn bench_cpython_parse_to_ast(b: &mut test::Bencher) { + let source = include_str!("./benchmarks/minidom.py"); + + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = cpython::PyDict::new(python); + + locals.set_item(python, "SOURCE_CODE", source).unwrap(); + + let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")"; + + b.bytes = source.len() as _; + b.iter(|| { + let res: cpython::PyResult = python.eval(code, globals, Some(&locals)); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_cpython_nbody(b: &mut test::Bencher) { + let source = include_str!("./benchmarks/nbody.py"); + + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = None; + + b.iter(|| { + let res: cpython::PyResult<()> = python.run(source, globals, locals); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_cpython_mandelbrot(b: &mut test::Bencher) { + let source = include_str!("./benchmarks/mandelbrot.py"); + + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = None; + + b.iter(|| { + let res: cpython::PyResult<()> = python.run(source, globals, locals); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_rustpy_nbody(b: &mut test::Bencher) { + // NOTE: Take long time. + let source = include_str!("./benchmarks/nbody.py"); + + let vm = VirtualMachine::new(); + + let code = match vm.compile(source, &compile::Mode::Single, "".to_string()) { + Ok(code) => code, + Err(e) => panic!("{:?}", e), + }; + + b.iter(|| { + let scope = vm.new_scope_with_builtins(); + let res: PyResult = vm.run_code_obj(code.clone(), scope); + assert!(res.is_ok()); + }) +} + +#[bench] +fn bench_rustpy_mandelbrot(b: &mut test::Bencher) { + // NOTE: Take long time. + let source = include_str!("./benchmarks/mandelbrot.py"); + + let vm = VirtualMachine::new(); + + let code = match vm.compile(source, &compile::Mode::Single, "".to_string()) { + Ok(code) => code, + Err(e) => panic!("{:?}", e), + }; + + b.iter(|| { + let scope = vm.new_scope_with_builtins(); + let res: PyResult = vm.run_code_obj(code.clone(), scope); + assert!(res.is_ok()); + }) +} diff --git a/benchmarks/benchmarks/mandelbrot.py b/benchmarks/benchmarks/mandelbrot.py new file mode 100644 index 0000000000..88ae04101a --- /dev/null +++ b/benchmarks/benchmarks/mandelbrot.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +w = 50.0 +h = 50.0 + +y = 0.0 +while y < h: + x = 0.0 + while x < w: + Zr, Zi, Tr, Ti = 0.0, 0.0, 0.0, 0.0 + Cr = 2*x/w - 1.5 + Ci = 2*y/h - 1.0 + + i = 0 + while i < 50 and Tr+Ti <= 4: + Zi = 2*Zr*Zi + Ci + Zr = Tr - Ti + Cr + Tr = Zr * Zr + Ti = Zi * Zi + i = i+1 + + if Tr+Ti <= 4: + # print('*', end='') + pass + else: + # print('·', end='') + pass + + x = x+1 + + # print() + y = y+1 diff --git a/benchmarks/benchmarks/minidom.py b/benchmarks/benchmarks/minidom.py new file mode 100644 index 0000000000..24957ea14f --- /dev/null +++ b/benchmarks/benchmarks/minidom.py @@ -0,0 +1,1981 @@ +"""Simple implementation of the Level 1 DOM. + +Namespaces and other minor Level 2 features are also supported. + +parse("foo.xml") + +parseString("") + +Todo: +===== + * convenience methods for getting elements and text. + * more testing + * bring some of the writer and linearizer code into conformance with this + interface + * SAX 2 namespaces +""" + +import io +import xml.dom + +from xml.dom import EMPTY_NAMESPACE, EMPTY_PREFIX, XMLNS_NAMESPACE, domreg +from xml.dom.minicompat import * +from xml.dom.xmlbuilder import DOMImplementationLS, DocumentLS + +# This is used by the ID-cache invalidation checks; the list isn't +# actually complete, since the nodes being checked will never be the +# DOCUMENT_NODE or DOCUMENT_FRAGMENT_NODE. (The node being checked is +# the node being added or removed, not the node being modified.) +# +_nodeTypes_with_children = (xml.dom.Node.ELEMENT_NODE, + xml.dom.Node.ENTITY_REFERENCE_NODE) + + +class Node(xml.dom.Node): + namespaceURI = None # this is non-null only for elements and attributes + parentNode = None + ownerDocument = None + nextSibling = None + previousSibling = None + + prefix = EMPTY_PREFIX # non-null only for NS elements and attributes + + def __bool__(self): + return True + + def toxml(self, encoding=None): + return self.toprettyxml("", "", encoding) + + def toprettyxml(self, indent="\t", newl="\n", encoding=None): + if encoding is None: + writer = io.StringIO() + else: + writer = io.TextIOWrapper(io.BytesIO(), + encoding=encoding, + errors="xmlcharrefreplace", + newline='\n') + if self.nodeType == Node.DOCUMENT_NODE: + # Can pass encoding only to document, to put it into XML header + self.writexml(writer, "", indent, newl, encoding) + else: + self.writexml(writer, "", indent, newl) + if encoding is None: + return writer.getvalue() + else: + return writer.detach().getvalue() + + def hasChildNodes(self): + return bool(self.childNodes) + + def _get_childNodes(self): + return self.childNodes + + def _get_firstChild(self): + if self.childNodes: + return self.childNodes[0] + + def _get_lastChild(self): + if self.childNodes: + return self.childNodes[-1] + + def insertBefore(self, newChild, refChild): + if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: + for c in tuple(newChild.childNodes): + self.insertBefore(c, refChild) + ### The DOM does not clearly specify what to return in this case + return newChild + if newChild.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(newChild), repr(self))) + if newChild.parentNode is not None: + newChild.parentNode.removeChild(newChild) + if refChild is None: + self.appendChild(newChild) + else: + try: + index = self.childNodes.index(refChild) + except ValueError: + raise xml.dom.NotFoundErr() + if newChild.nodeType in _nodeTypes_with_children: + _clear_id_cache(self) + self.childNodes.insert(index, newChild) + newChild.nextSibling = refChild + refChild.previousSibling = newChild + if index: + node = self.childNodes[index-1] + node.nextSibling = newChild + newChild.previousSibling = node + else: + newChild.previousSibling = None + newChild.parentNode = self + return newChild + + def appendChild(self, node): + if node.nodeType == self.DOCUMENT_FRAGMENT_NODE: + for c in tuple(node.childNodes): + self.appendChild(c) + ### The DOM does not clearly specify what to return in this case + return node + if node.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(node), repr(self))) + elif node.nodeType in _nodeTypes_with_children: + _clear_id_cache(self) + if node.parentNode is not None: + node.parentNode.removeChild(node) + _append_child(self, node) + node.nextSibling = None + return node + + def replaceChild(self, newChild, oldChild): + if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: + refChild = oldChild.nextSibling + self.removeChild(oldChild) + return self.insertBefore(newChild, refChild) + if newChild.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(newChild), repr(self))) + if newChild is oldChild: + return + if newChild.parentNode is not None: + newChild.parentNode.removeChild(newChild) + try: + index = self.childNodes.index(oldChild) + except ValueError: + raise xml.dom.NotFoundErr() + self.childNodes[index] = newChild + newChild.parentNode = self + oldChild.parentNode = None + if (newChild.nodeType in _nodeTypes_with_children + or oldChild.nodeType in _nodeTypes_with_children): + _clear_id_cache(self) + newChild.nextSibling = oldChild.nextSibling + newChild.previousSibling = oldChild.previousSibling + oldChild.nextSibling = None + oldChild.previousSibling = None + if newChild.previousSibling: + newChild.previousSibling.nextSibling = newChild + if newChild.nextSibling: + newChild.nextSibling.previousSibling = newChild + return oldChild + + def removeChild(self, oldChild): + try: + self.childNodes.remove(oldChild) + except ValueError: + raise xml.dom.NotFoundErr() + if oldChild.nextSibling is not None: + oldChild.nextSibling.previousSibling = oldChild.previousSibling + if oldChild.previousSibling is not None: + oldChild.previousSibling.nextSibling = oldChild.nextSibling + oldChild.nextSibling = oldChild.previousSibling = None + if oldChild.nodeType in _nodeTypes_with_children: + _clear_id_cache(self) + + oldChild.parentNode = None + return oldChild + + def normalize(self): + L = [] + for child in self.childNodes: + if child.nodeType == Node.TEXT_NODE: + if not child.data: + # empty text node; discard + if L: + L[-1].nextSibling = child.nextSibling + if child.nextSibling: + child.nextSibling.previousSibling = child.previousSibling + child.unlink() + elif L and L[-1].nodeType == child.nodeType: + # collapse text node + node = L[-1] + node.data = node.data + child.data + node.nextSibling = child.nextSibling + if child.nextSibling: + child.nextSibling.previousSibling = node + child.unlink() + else: + L.append(child) + else: + L.append(child) + if child.nodeType == Node.ELEMENT_NODE: + child.normalize() + self.childNodes[:] = L + + def cloneNode(self, deep): + return _clone_node(self, deep, self.ownerDocument or self) + + def isSupported(self, feature, version): + return self.ownerDocument.implementation.hasFeature(feature, version) + + def _get_localName(self): + # Overridden in Element and Attr where localName can be Non-Null + return None + + # Node interfaces from Level 3 (WD 9 April 2002) + + def isSameNode(self, other): + return self is other + + def getInterface(self, feature): + if self.isSupported(feature, None): + return self + else: + return None + + # The "user data" functions use a dictionary that is only present + # if some user data has been set, so be careful not to assume it + # exists. + + def getUserData(self, key): + try: + return self._user_data[key][0] + except (AttributeError, KeyError): + return None + + def setUserData(self, key, data, handler): + old = None + try: + d = self._user_data + except AttributeError: + d = {} + self._user_data = d + if key in d: + old = d[key][0] + if data is None: + # ignore handlers passed for None + handler = None + if old is not None: + del d[key] + else: + d[key] = (data, handler) + return old + + def _call_user_data_handler(self, operation, src, dst): + if hasattr(self, "_user_data"): + for key, (data, handler) in list(self._user_data.items()): + if handler is not None: + handler.handle(operation, key, data, src, dst) + + # minidom-specific API: + + def unlink(self): + self.parentNode = self.ownerDocument = None + if self.childNodes: + for child in self.childNodes: + child.unlink() + self.childNodes = NodeList() + self.previousSibling = None + self.nextSibling = None + + # A Node is its own context manager, to ensure that an unlink() call occurs. + # This is similar to how a file object works. + def __enter__(self): + return self + + def __exit__(self, et, ev, tb): + self.unlink() + +defproperty(Node, "firstChild", doc="First child node, or None.") +defproperty(Node, "lastChild", doc="Last child node, or None.") +defproperty(Node, "localName", doc="Namespace-local name of this node.") + + +def _append_child(self, node): + # fast path with less checks; usable by DOM builders if careful + childNodes = self.childNodes + if childNodes: + last = childNodes[-1] + node.previousSibling = last + last.nextSibling = node + childNodes.append(node) + node.parentNode = self + +def _in_document(node): + # return True iff node is part of a document tree + while node is not None: + if node.nodeType == Node.DOCUMENT_NODE: + return True + node = node.parentNode + return False + +def _write_data(writer, data): + "Writes datachars to writer." + if data: + data = data.replace("&", "&").replace("<", "<"). \ + replace("\"", """).replace(">", ">") + writer.write(data) + +def _get_elements_by_tagName_helper(parent, name, rc): + for node in parent.childNodes: + if node.nodeType == Node.ELEMENT_NODE and \ + (name == "*" or node.tagName == name): + rc.append(node) + _get_elements_by_tagName_helper(node, name, rc) + return rc + +def _get_elements_by_tagName_ns_helper(parent, nsURI, localName, rc): + for node in parent.childNodes: + if node.nodeType == Node.ELEMENT_NODE: + if ((localName == "*" or node.localName == localName) and + (nsURI == "*" or node.namespaceURI == nsURI)): + rc.append(node) + _get_elements_by_tagName_ns_helper(node, nsURI, localName, rc) + return rc + +class DocumentFragment(Node): + nodeType = Node.DOCUMENT_FRAGMENT_NODE + nodeName = "#document-fragment" + nodeValue = None + attributes = None + parentNode = None + _child_node_types = (Node.ELEMENT_NODE, + Node.TEXT_NODE, + Node.CDATA_SECTION_NODE, + Node.ENTITY_REFERENCE_NODE, + Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, + Node.NOTATION_NODE) + + def __init__(self): + self.childNodes = NodeList() + + +class Attr(Node): + __slots__=('_name', '_value', 'namespaceURI', + '_prefix', 'childNodes', '_localName', 'ownerDocument', 'ownerElement') + nodeType = Node.ATTRIBUTE_NODE + attributes = None + specified = False + _is_id = False + + _child_node_types = (Node.TEXT_NODE, Node.ENTITY_REFERENCE_NODE) + + def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, + prefix=None): + self.ownerElement = None + self._name = qName + self.namespaceURI = namespaceURI + self._prefix = prefix + self.childNodes = NodeList() + + # Add the single child node that represents the value of the attr + self.childNodes.append(Text()) + + # nodeValue and value are set elsewhere + + def _get_localName(self): + try: + return self._localName + except AttributeError: + return self.nodeName.split(":", 1)[-1] + + def _get_specified(self): + return self.specified + + def _get_name(self): + return self._name + + def _set_name(self, value): + self._name = value + if self.ownerElement is not None: + _clear_id_cache(self.ownerElement) + + nodeName = name = property(_get_name, _set_name) + + def _get_value(self): + return self._value + + def _set_value(self, value): + self._value = value + self.childNodes[0].data = value + if self.ownerElement is not None: + _clear_id_cache(self.ownerElement) + self.childNodes[0].data = value + + nodeValue = value = property(_get_value, _set_value) + + def _get_prefix(self): + return self._prefix + + def _set_prefix(self, prefix): + nsuri = self.namespaceURI + if prefix == "xmlns": + if nsuri and nsuri != XMLNS_NAMESPACE: + raise xml.dom.NamespaceErr( + "illegal use of 'xmlns' prefix for the wrong namespace") + self._prefix = prefix + if prefix is None: + newName = self.localName + else: + newName = "%s:%s" % (prefix, self.localName) + if self.ownerElement: + _clear_id_cache(self.ownerElement) + self.name = newName + + prefix = property(_get_prefix, _set_prefix) + + def unlink(self): + # This implementation does not call the base implementation + # since most of that is not needed, and the expense of the + # method call is not warranted. We duplicate the removal of + # children, but that's all we needed from the base class. + elem = self.ownerElement + if elem is not None: + del elem._attrs[self.nodeName] + del elem._attrsNS[(self.namespaceURI, self.localName)] + if self._is_id: + self._is_id = False + elem._magic_id_nodes -= 1 + self.ownerDocument._magic_id_count -= 1 + for child in self.childNodes: + child.unlink() + del self.childNodes[:] + + def _get_isId(self): + if self._is_id: + return True + doc = self.ownerDocument + elem = self.ownerElement + if doc is None or elem is None: + return False + + info = doc._get_elem_info(elem) + if info is None: + return False + if self.namespaceURI: + return info.isIdNS(self.namespaceURI, self.localName) + else: + return info.isId(self.nodeName) + + def _get_schemaType(self): + doc = self.ownerDocument + elem = self.ownerElement + if doc is None or elem is None: + return _no_type + + info = doc._get_elem_info(elem) + if info is None: + return _no_type + if self.namespaceURI: + return info.getAttributeTypeNS(self.namespaceURI, self.localName) + else: + return info.getAttributeType(self.nodeName) + +defproperty(Attr, "isId", doc="True if this attribute is an ID.") +defproperty(Attr, "localName", doc="Namespace-local name of this attribute.") +defproperty(Attr, "schemaType", doc="Schema type for this attribute.") + + +class NamedNodeMap(object): + """The attribute list is a transient interface to the underlying + dictionaries. Mutations here will change the underlying element's + dictionary. + + Ordering is imposed artificially and does not reflect the order of + attributes as found in an input document. + """ + + __slots__ = ('_attrs', '_attrsNS', '_ownerElement') + + def __init__(self, attrs, attrsNS, ownerElement): + self._attrs = attrs + self._attrsNS = attrsNS + self._ownerElement = ownerElement + + def _get_length(self): + return len(self._attrs) + + def item(self, index): + try: + return self[list(self._attrs.keys())[index]] + except IndexError: + return None + + def items(self): + L = [] + for node in self._attrs.values(): + L.append((node.nodeName, node.value)) + return L + + def itemsNS(self): + L = [] + for node in self._attrs.values(): + L.append(((node.namespaceURI, node.localName), node.value)) + return L + + def __contains__(self, key): + if isinstance(key, str): + return key in self._attrs + else: + return key in self._attrsNS + + def keys(self): + return self._attrs.keys() + + def keysNS(self): + return self._attrsNS.keys() + + def values(self): + return self._attrs.values() + + def get(self, name, value=None): + return self._attrs.get(name, value) + + __len__ = _get_length + + def _cmp(self, other): + if self._attrs is getattr(other, "_attrs", None): + return 0 + else: + return (id(self) > id(other)) - (id(self) < id(other)) + + def __eq__(self, other): + return self._cmp(other) == 0 + + def __ge__(self, other): + return self._cmp(other) >= 0 + + def __gt__(self, other): + return self._cmp(other) > 0 + + def __le__(self, other): + return self._cmp(other) <= 0 + + def __lt__(self, other): + return self._cmp(other) < 0 + + def __getitem__(self, attname_or_tuple): + if isinstance(attname_or_tuple, tuple): + return self._attrsNS[attname_or_tuple] + else: + return self._attrs[attname_or_tuple] + + # same as set + def __setitem__(self, attname, value): + if isinstance(value, str): + try: + node = self._attrs[attname] + except KeyError: + node = Attr(attname) + node.ownerDocument = self._ownerElement.ownerDocument + self.setNamedItem(node) + node.value = value + else: + if not isinstance(value, Attr): + raise TypeError("value must be a string or Attr object") + node = value + self.setNamedItem(node) + + def getNamedItem(self, name): + try: + return self._attrs[name] + except KeyError: + return None + + def getNamedItemNS(self, namespaceURI, localName): + try: + return self._attrsNS[(namespaceURI, localName)] + except KeyError: + return None + + def removeNamedItem(self, name): + n = self.getNamedItem(name) + if n is not None: + _clear_id_cache(self._ownerElement) + del self._attrs[n.nodeName] + del self._attrsNS[(n.namespaceURI, n.localName)] + if hasattr(n, 'ownerElement'): + n.ownerElement = None + return n + else: + raise xml.dom.NotFoundErr() + + def removeNamedItemNS(self, namespaceURI, localName): + n = self.getNamedItemNS(namespaceURI, localName) + if n is not None: + _clear_id_cache(self._ownerElement) + del self._attrsNS[(n.namespaceURI, n.localName)] + del self._attrs[n.nodeName] + if hasattr(n, 'ownerElement'): + n.ownerElement = None + return n + else: + raise xml.dom.NotFoundErr() + + def setNamedItem(self, node): + if not isinstance(node, Attr): + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(node), repr(self))) + old = self._attrs.get(node.name) + if old: + old.unlink() + self._attrs[node.name] = node + self._attrsNS[(node.namespaceURI, node.localName)] = node + node.ownerElement = self._ownerElement + _clear_id_cache(node.ownerElement) + return old + + def setNamedItemNS(self, node): + return self.setNamedItem(node) + + def __delitem__(self, attname_or_tuple): + node = self[attname_or_tuple] + _clear_id_cache(node.ownerElement) + node.unlink() + + def __getstate__(self): + return self._attrs, self._attrsNS, self._ownerElement + + def __setstate__(self, state): + self._attrs, self._attrsNS, self._ownerElement = state + +defproperty(NamedNodeMap, "length", + doc="Number of nodes in the NamedNodeMap.") + +AttributeList = NamedNodeMap + + +class TypeInfo(object): + __slots__ = 'namespace', 'name' + + def __init__(self, namespace, name): + self.namespace = namespace + self.name = name + + def __repr__(self): + if self.namespace: + return "<%s %r (from %r)>" % (self.__class__.__name__, self.name, + self.namespace) + else: + return "<%s %r>" % (self.__class__.__name__, self.name) + + def _get_name(self): + return self.name + + def _get_namespace(self): + return self.namespace + +_no_type = TypeInfo(None, None) + +class Element(Node): + __slots__=('ownerDocument', 'parentNode', 'tagName', 'nodeName', 'prefix', + 'namespaceURI', '_localName', 'childNodes', '_attrs', '_attrsNS', + 'nextSibling', 'previousSibling') + nodeType = Node.ELEMENT_NODE + nodeValue = None + schemaType = _no_type + + _magic_id_nodes = 0 + + _child_node_types = (Node.ELEMENT_NODE, + Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, + Node.TEXT_NODE, + Node.CDATA_SECTION_NODE, + Node.ENTITY_REFERENCE_NODE) + + def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None, + localName=None): + self.parentNode = None + self.tagName = self.nodeName = tagName + self.prefix = prefix + self.namespaceURI = namespaceURI + self.childNodes = NodeList() + self.nextSibling = self.previousSibling = None + + # Attribute dictionaries are lazily created + # attributes are double-indexed: + # tagName -> Attribute + # URI,localName -> Attribute + # in the future: consider lazy generation + # of attribute objects this is too tricky + # for now because of headaches with + # namespaces. + self._attrs = None + self._attrsNS = None + + def _ensure_attributes(self): + if self._attrs is None: + self._attrs = {} + self._attrsNS = {} + + def _get_localName(self): + try: + return self._localName + except AttributeError: + return self.tagName.split(":", 1)[-1] + + def _get_tagName(self): + return self.tagName + + def unlink(self): + if self._attrs is not None: + for attr in list(self._attrs.values()): + attr.unlink() + self._attrs = None + self._attrsNS = None + Node.unlink(self) + + def getAttribute(self, attname): + if self._attrs is None: + return "" + try: + return self._attrs[attname].value + except KeyError: + return "" + + def getAttributeNS(self, namespaceURI, localName): + if self._attrsNS is None: + return "" + try: + return self._attrsNS[(namespaceURI, localName)].value + except KeyError: + return "" + + def setAttribute(self, attname, value): + attr = self.getAttributeNode(attname) + if attr is None: + attr = Attr(attname) + attr.value = value # also sets nodeValue + attr.ownerDocument = self.ownerDocument + self.setAttributeNode(attr) + elif value != attr.value: + attr.value = value + if attr.isId: + _clear_id_cache(self) + + def setAttributeNS(self, namespaceURI, qualifiedName, value): + prefix, localname = _nssplit(qualifiedName) + attr = self.getAttributeNodeNS(namespaceURI, localname) + if attr is None: + attr = Attr(qualifiedName, namespaceURI, localname, prefix) + attr.value = value + attr.ownerDocument = self.ownerDocument + self.setAttributeNode(attr) + else: + if value != attr.value: + attr.value = value + if attr.isId: + _clear_id_cache(self) + if attr.prefix != prefix: + attr.prefix = prefix + attr.nodeName = qualifiedName + + def getAttributeNode(self, attrname): + if self._attrs is None: + return None + return self._attrs.get(attrname) + + def getAttributeNodeNS(self, namespaceURI, localName): + if self._attrsNS is None: + return None + return self._attrsNS.get((namespaceURI, localName)) + + def setAttributeNode(self, attr): + if attr.ownerElement not in (None, self): + raise xml.dom.InuseAttributeErr("attribute node already owned") + self._ensure_attributes() + old1 = self._attrs.get(attr.name, None) + if old1 is not None: + self.removeAttributeNode(old1) + old2 = self._attrsNS.get((attr.namespaceURI, attr.localName), None) + if old2 is not None and old2 is not old1: + self.removeAttributeNode(old2) + _set_attribute_node(self, attr) + + if old1 is not attr: + # It might have already been part of this node, in which case + # it doesn't represent a change, and should not be returned. + return old1 + if old2 is not attr: + return old2 + + setAttributeNodeNS = setAttributeNode + + def removeAttribute(self, name): + if self._attrsNS is None: + raise xml.dom.NotFoundErr() + try: + attr = self._attrs[name] + except KeyError: + raise xml.dom.NotFoundErr() + self.removeAttributeNode(attr) + + def removeAttributeNS(self, namespaceURI, localName): + if self._attrsNS is None: + raise xml.dom.NotFoundErr() + try: + attr = self._attrsNS[(namespaceURI, localName)] + except KeyError: + raise xml.dom.NotFoundErr() + self.removeAttributeNode(attr) + + def removeAttributeNode(self, node): + if node is None: + raise xml.dom.NotFoundErr() + try: + self._attrs[node.name] + except KeyError: + raise xml.dom.NotFoundErr() + _clear_id_cache(self) + node.unlink() + # Restore this since the node is still useful and otherwise + # unlinked + node.ownerDocument = self.ownerDocument + + removeAttributeNodeNS = removeAttributeNode + + def hasAttribute(self, name): + if self._attrs is None: + return False + return name in self._attrs + + def hasAttributeNS(self, namespaceURI, localName): + if self._attrsNS is None: + return False + return (namespaceURI, localName) in self._attrsNS + + def getElementsByTagName(self, name): + return _get_elements_by_tagName_helper(self, name, NodeList()) + + def getElementsByTagNameNS(self, namespaceURI, localName): + return _get_elements_by_tagName_ns_helper( + self, namespaceURI, localName, NodeList()) + + def __repr__(self): + return "" % (self.tagName, id(self)) + + def writexml(self, writer, indent="", addindent="", newl=""): + # indent = current indentation + # addindent = indentation to add to higher levels + # newl = newline string + writer.write(indent+"<" + self.tagName) + + attrs = self._get_attributes() + a_names = sorted(attrs.keys()) + + for a_name in a_names: + writer.write(" %s=\"" % a_name) + _write_data(writer, attrs[a_name].value) + writer.write("\"") + if self.childNodes: + writer.write(">") + if (len(self.childNodes) == 1 and + self.childNodes[0].nodeType == Node.TEXT_NODE): + self.childNodes[0].writexml(writer, '', '', '') + else: + writer.write(newl) + for node in self.childNodes: + node.writexml(writer, indent+addindent, addindent, newl) + writer.write(indent) + writer.write("%s" % (self.tagName, newl)) + else: + writer.write("/>%s"%(newl)) + + def _get_attributes(self): + self._ensure_attributes() + return NamedNodeMap(self._attrs, self._attrsNS, self) + + def hasAttributes(self): + if self._attrs: + return True + else: + return False + + # DOM Level 3 attributes, based on the 22 Oct 2002 draft + + def setIdAttribute(self, name): + idAttr = self.getAttributeNode(name) + self.setIdAttributeNode(idAttr) + + def setIdAttributeNS(self, namespaceURI, localName): + idAttr = self.getAttributeNodeNS(namespaceURI, localName) + self.setIdAttributeNode(idAttr) + + def setIdAttributeNode(self, idAttr): + if idAttr is None or not self.isSameNode(idAttr.ownerElement): + raise xml.dom.NotFoundErr() + if _get_containing_entref(self) is not None: + raise xml.dom.NoModificationAllowedErr() + if not idAttr._is_id: + idAttr._is_id = True + self._magic_id_nodes += 1 + self.ownerDocument._magic_id_count += 1 + _clear_id_cache(self) + +defproperty(Element, "attributes", + doc="NamedNodeMap of attributes on the element.") +defproperty(Element, "localName", + doc="Namespace-local name of this element.") + + +def _set_attribute_node(element, attr): + _clear_id_cache(element) + element._ensure_attributes() + element._attrs[attr.name] = attr + element._attrsNS[(attr.namespaceURI, attr.localName)] = attr + + # This creates a circular reference, but Element.unlink() + # breaks the cycle since the references to the attribute + # dictionaries are tossed. + attr.ownerElement = element + +class Childless: + """Mixin that makes childless-ness easy to implement and avoids + the complexity of the Node methods that deal with children. + """ + __slots__ = () + + attributes = None + childNodes = EmptyNodeList() + firstChild = None + lastChild = None + + def _get_firstChild(self): + return None + + def _get_lastChild(self): + return None + + def appendChild(self, node): + raise xml.dom.HierarchyRequestErr( + self.nodeName + " nodes cannot have children") + + def hasChildNodes(self): + return False + + def insertBefore(self, newChild, refChild): + raise xml.dom.HierarchyRequestErr( + self.nodeName + " nodes do not have children") + + def removeChild(self, oldChild): + raise xml.dom.NotFoundErr( + self.nodeName + " nodes do not have children") + + def normalize(self): + # For childless nodes, normalize() has nothing to do. + pass + + def replaceChild(self, newChild, oldChild): + raise xml.dom.HierarchyRequestErr( + self.nodeName + " nodes do not have children") + + +class ProcessingInstruction(Childless, Node): + nodeType = Node.PROCESSING_INSTRUCTION_NODE + __slots__ = ('target', 'data') + + def __init__(self, target, data): + self.target = target + self.data = data + + # nodeValue is an alias for data + def _get_nodeValue(self): + return self.data + def _set_nodeValue(self, value): + self.data = value + nodeValue = property(_get_nodeValue, _set_nodeValue) + + # nodeName is an alias for target + def _get_nodeName(self): + return self.target + def _set_nodeName(self, value): + self.target = value + nodeName = property(_get_nodeName, _set_nodeName) + + def writexml(self, writer, indent="", addindent="", newl=""): + writer.write("%s%s" % (indent,self.target, self.data, newl)) + + +class CharacterData(Childless, Node): + __slots__=('_data', 'ownerDocument','parentNode', 'previousSibling', 'nextSibling') + + def __init__(self): + self.ownerDocument = self.parentNode = None + self.previousSibling = self.nextSibling = None + self._data = '' + Node.__init__(self) + + def _get_length(self): + return len(self.data) + __len__ = _get_length + + def _get_data(self): + return self._data + def _set_data(self, data): + self._data = data + + data = nodeValue = property(_get_data, _set_data) + + def __repr__(self): + data = self.data + if len(data) > 10: + dotdotdot = "..." + else: + dotdotdot = "" + return '' % ( + self.__class__.__name__, data[0:10], dotdotdot) + + def substringData(self, offset, count): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if count < 0: + raise xml.dom.IndexSizeErr("count cannot be negative") + return self.data[offset:offset+count] + + def appendData(self, arg): + self.data = self.data + arg + + def insertData(self, offset, arg): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if arg: + self.data = "%s%s%s" % ( + self.data[:offset], arg, self.data[offset:]) + + def deleteData(self, offset, count): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if count < 0: + raise xml.dom.IndexSizeErr("count cannot be negative") + if count: + self.data = self.data[:offset] + self.data[offset+count:] + + def replaceData(self, offset, count, arg): + if offset < 0: + raise xml.dom.IndexSizeErr("offset cannot be negative") + if offset >= len(self.data): + raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") + if count < 0: + raise xml.dom.IndexSizeErr("count cannot be negative") + if count: + self.data = "%s%s%s" % ( + self.data[:offset], arg, self.data[offset+count:]) + +defproperty(CharacterData, "length", doc="Length of the string data.") + + +class Text(CharacterData): + __slots__ = () + + nodeType = Node.TEXT_NODE + nodeName = "#text" + attributes = None + + def splitText(self, offset): + if offset < 0 or offset > len(self.data): + raise xml.dom.IndexSizeErr("illegal offset value") + newText = self.__class__() + newText.data = self.data[offset:] + newText.ownerDocument = self.ownerDocument + next = self.nextSibling + if self.parentNode and self in self.parentNode.childNodes: + if next is None: + self.parentNode.appendChild(newText) + else: + self.parentNode.insertBefore(newText, next) + self.data = self.data[:offset] + return newText + + def writexml(self, writer, indent="", addindent="", newl=""): + _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + + # DOM Level 3 (WD 9 April 2002) + + def _get_wholeText(self): + L = [self.data] + n = self.previousSibling + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + L.insert(0, n.data) + n = n.previousSibling + else: + break + n = self.nextSibling + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + L.append(n.data) + n = n.nextSibling + else: + break + return ''.join(L) + + def replaceWholeText(self, content): + # XXX This needs to be seriously changed if minidom ever + # supports EntityReference nodes. + parent = self.parentNode + n = self.previousSibling + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + next = n.previousSibling + parent.removeChild(n) + n = next + else: + break + n = self.nextSibling + if not content: + parent.removeChild(self) + while n is not None: + if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): + next = n.nextSibling + parent.removeChild(n) + n = next + else: + break + if content: + self.data = content + return self + else: + return None + + def _get_isWhitespaceInElementContent(self): + if self.data.strip(): + return False + elem = _get_containing_element(self) + if elem is None: + return False + info = self.ownerDocument._get_elem_info(elem) + if info is None: + return False + else: + return info.isElementContent() + +defproperty(Text, "isWhitespaceInElementContent", + doc="True iff this text node contains only whitespace" + " and is in element content.") +defproperty(Text, "wholeText", + doc="The text of all logically-adjacent text nodes.") + + +def _get_containing_element(node): + c = node.parentNode + while c is not None: + if c.nodeType == Node.ELEMENT_NODE: + return c + c = c.parentNode + return None + +def _get_containing_entref(node): + c = node.parentNode + while c is not None: + if c.nodeType == Node.ENTITY_REFERENCE_NODE: + return c + c = c.parentNode + return None + + +class Comment(CharacterData): + nodeType = Node.COMMENT_NODE + nodeName = "#comment" + + def __init__(self, data): + CharacterData.__init__(self) + self._data = data + + def writexml(self, writer, indent="", addindent="", newl=""): + if "--" in self.data: + raise ValueError("'--' is not allowed in a comment node") + writer.write("%s%s" % (indent, self.data, newl)) + + +class CDATASection(Text): + __slots__ = () + + nodeType = Node.CDATA_SECTION_NODE + nodeName = "#cdata-section" + + def writexml(self, writer, indent="", addindent="", newl=""): + if self.data.find("]]>") >= 0: + raise ValueError("']]>' not allowed in a CDATA section") + writer.write("" % self.data) + + +class ReadOnlySequentialNamedNodeMap(object): + __slots__ = '_seq', + + def __init__(self, seq=()): + # seq should be a list or tuple + self._seq = seq + + def __len__(self): + return len(self._seq) + + def _get_length(self): + return len(self._seq) + + def getNamedItem(self, name): + for n in self._seq: + if n.nodeName == name: + return n + + def getNamedItemNS(self, namespaceURI, localName): + for n in self._seq: + if n.namespaceURI == namespaceURI and n.localName == localName: + return n + + def __getitem__(self, name_or_tuple): + if isinstance(name_or_tuple, tuple): + node = self.getNamedItemNS(*name_or_tuple) + else: + node = self.getNamedItem(name_or_tuple) + if node is None: + raise KeyError(name_or_tuple) + return node + + def item(self, index): + if index < 0: + return None + try: + return self._seq[index] + except IndexError: + return None + + def removeNamedItem(self, name): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def removeNamedItemNS(self, namespaceURI, localName): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def setNamedItem(self, node): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def setNamedItemNS(self, node): + raise xml.dom.NoModificationAllowedErr( + "NamedNodeMap instance is read-only") + + def __getstate__(self): + return [self._seq] + + def __setstate__(self, state): + self._seq = state[0] + +defproperty(ReadOnlySequentialNamedNodeMap, "length", + doc="Number of entries in the NamedNodeMap.") + + +class Identified: + """Mix-in class that supports the publicId and systemId attributes.""" + + __slots__ = 'publicId', 'systemId' + + def _identified_mixin_init(self, publicId, systemId): + self.publicId = publicId + self.systemId = systemId + + def _get_publicId(self): + return self.publicId + + def _get_systemId(self): + return self.systemId + +class DocumentType(Identified, Childless, Node): + nodeType = Node.DOCUMENT_TYPE_NODE + nodeValue = None + name = None + publicId = None + systemId = None + internalSubset = None + + def __init__(self, qualifiedName): + self.entities = ReadOnlySequentialNamedNodeMap() + self.notations = ReadOnlySequentialNamedNodeMap() + if qualifiedName: + prefix, localname = _nssplit(qualifiedName) + self.name = localname + self.nodeName = self.name + + def _get_internalSubset(self): + return self.internalSubset + + def cloneNode(self, deep): + if self.ownerDocument is None: + # it's ok + clone = DocumentType(None) + clone.name = self.name + clone.nodeName = self.name + operation = xml.dom.UserDataHandler.NODE_CLONED + if deep: + clone.entities._seq = [] + clone.notations._seq = [] + for n in self.notations._seq: + notation = Notation(n.nodeName, n.publicId, n.systemId) + clone.notations._seq.append(notation) + n._call_user_data_handler(operation, n, notation) + for e in self.entities._seq: + entity = Entity(e.nodeName, e.publicId, e.systemId, + e.notationName) + entity.actualEncoding = e.actualEncoding + entity.encoding = e.encoding + entity.version = e.version + clone.entities._seq.append(entity) + e._call_user_data_handler(operation, e, entity) + self._call_user_data_handler(operation, self, clone) + return clone + else: + return None + + def writexml(self, writer, indent="", addindent="", newl=""): + writer.write(""+newl) + +class Entity(Identified, Node): + attributes = None + nodeType = Node.ENTITY_NODE + nodeValue = None + + actualEncoding = None + encoding = None + version = None + + def __init__(self, name, publicId, systemId, notation): + self.nodeName = name + self.notationName = notation + self.childNodes = NodeList() + self._identified_mixin_init(publicId, systemId) + + def _get_actualEncoding(self): + return self.actualEncoding + + def _get_encoding(self): + return self.encoding + + def _get_version(self): + return self.version + + def appendChild(self, newChild): + raise xml.dom.HierarchyRequestErr( + "cannot append children to an entity node") + + def insertBefore(self, newChild, refChild): + raise xml.dom.HierarchyRequestErr( + "cannot insert children below an entity node") + + def removeChild(self, oldChild): + raise xml.dom.HierarchyRequestErr( + "cannot remove children from an entity node") + + def replaceChild(self, newChild, oldChild): + raise xml.dom.HierarchyRequestErr( + "cannot replace children of an entity node") + +class Notation(Identified, Childless, Node): + nodeType = Node.NOTATION_NODE + nodeValue = None + + def __init__(self, name, publicId, systemId): + self.nodeName = name + self._identified_mixin_init(publicId, systemId) + + +class DOMImplementation(DOMImplementationLS): + _features = [("core", "1.0"), + ("core", "2.0"), + ("core", None), + ("xml", "1.0"), + ("xml", "2.0"), + ("xml", None), + ("ls-load", "3.0"), + ("ls-load", None), + ] + + def hasFeature(self, feature, version): + if version == "": + version = None + return (feature.lower(), version) in self._features + + def createDocument(self, namespaceURI, qualifiedName, doctype): + if doctype and doctype.parentNode is not None: + raise xml.dom.WrongDocumentErr( + "doctype object owned by another DOM tree") + doc = self._create_document() + + add_root_element = not (namespaceURI is None + and qualifiedName is None + and doctype is None) + + if not qualifiedName and add_root_element: + # The spec is unclear what to raise here; SyntaxErr + # would be the other obvious candidate. Since Xerces raises + # InvalidCharacterErr, and since SyntaxErr is not listed + # for createDocument, that seems to be the better choice. + # XXX: need to check for illegal characters here and in + # createElement. + + # DOM Level III clears this up when talking about the return value + # of this function. If namespaceURI, qName and DocType are + # Null the document is returned without a document element + # Otherwise if doctype or namespaceURI are not None + # Then we go back to the above problem + raise xml.dom.InvalidCharacterErr("Element with no name") + + if add_root_element: + prefix, localname = _nssplit(qualifiedName) + if prefix == "xml" \ + and namespaceURI != "http://www.w3.org/XML/1998/namespace": + raise xml.dom.NamespaceErr("illegal use of 'xml' prefix") + if prefix and not namespaceURI: + raise xml.dom.NamespaceErr( + "illegal use of prefix without namespaces") + element = doc.createElementNS(namespaceURI, qualifiedName) + if doctype: + doc.appendChild(doctype) + doc.appendChild(element) + + if doctype: + doctype.parentNode = doctype.ownerDocument = doc + + doc.doctype = doctype + doc.implementation = self + return doc + + def createDocumentType(self, qualifiedName, publicId, systemId): + doctype = DocumentType(qualifiedName) + doctype.publicId = publicId + doctype.systemId = systemId + return doctype + + # DOM Level 3 (WD 9 April 2002) + + def getInterface(self, feature): + if self.hasFeature(feature, None): + return self + else: + return None + + # internal + def _create_document(self): + return Document() + +class ElementInfo(object): + """Object that represents content-model information for an element. + + This implementation is not expected to be used in practice; DOM + builders should provide implementations which do the right thing + using information available to it. + + """ + + __slots__ = 'tagName', + + def __init__(self, name): + self.tagName = name + + def getAttributeType(self, aname): + return _no_type + + def getAttributeTypeNS(self, namespaceURI, localName): + return _no_type + + def isElementContent(self): + return False + + def isEmpty(self): + """Returns true iff this element is declared to have an EMPTY + content model.""" + return False + + def isId(self, aname): + """Returns true iff the named attribute is a DTD-style ID.""" + return False + + def isIdNS(self, namespaceURI, localName): + """Returns true iff the identified attribute is a DTD-style ID.""" + return False + + def __getstate__(self): + return self.tagName + + def __setstate__(self, state): + self.tagName = state + +def _clear_id_cache(node): + if node.nodeType == Node.DOCUMENT_NODE: + node._id_cache.clear() + node._id_search_stack = None + elif _in_document(node): + node.ownerDocument._id_cache.clear() + node.ownerDocument._id_search_stack= None + +class Document(Node, DocumentLS): + __slots__ = ('_elem_info', 'doctype', + '_id_search_stack', 'childNodes', '_id_cache') + _child_node_types = (Node.ELEMENT_NODE, Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, Node.DOCUMENT_TYPE_NODE) + + implementation = DOMImplementation() + nodeType = Node.DOCUMENT_NODE + nodeName = "#document" + nodeValue = None + attributes = None + parentNode = None + previousSibling = nextSibling = None + + + # Document attributes from Level 3 (WD 9 April 2002) + + actualEncoding = None + encoding = None + standalone = None + version = None + strictErrorChecking = False + errorHandler = None + documentURI = None + + _magic_id_count = 0 + + def __init__(self): + self.doctype = None + self.childNodes = NodeList() + # mapping of (namespaceURI, localName) -> ElementInfo + # and tagName -> ElementInfo + self._elem_info = {} + self._id_cache = {} + self._id_search_stack = None + + def _get_elem_info(self, element): + if element.namespaceURI: + key = element.namespaceURI, element.localName + else: + key = element.tagName + return self._elem_info.get(key) + + def _get_actualEncoding(self): + return self.actualEncoding + + def _get_doctype(self): + return self.doctype + + def _get_documentURI(self): + return self.documentURI + + def _get_encoding(self): + return self.encoding + + def _get_errorHandler(self): + return self.errorHandler + + def _get_standalone(self): + return self.standalone + + def _get_strictErrorChecking(self): + return self.strictErrorChecking + + def _get_version(self): + return self.version + + def appendChild(self, node): + if node.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(node), repr(self))) + if node.parentNode is not None: + # This needs to be done before the next test since this + # may *be* the document element, in which case it should + # end up re-ordered to the end. + node.parentNode.removeChild(node) + + if node.nodeType == Node.ELEMENT_NODE \ + and self._get_documentElement(): + raise xml.dom.HierarchyRequestErr( + "two document elements disallowed") + return Node.appendChild(self, node) + + def removeChild(self, oldChild): + try: + self.childNodes.remove(oldChild) + except ValueError: + raise xml.dom.NotFoundErr() + oldChild.nextSibling = oldChild.previousSibling = None + oldChild.parentNode = None + if self.documentElement is oldChild: + self.documentElement = None + + return oldChild + + def _get_documentElement(self): + for node in self.childNodes: + if node.nodeType == Node.ELEMENT_NODE: + return node + + def unlink(self): + if self.doctype is not None: + self.doctype.unlink() + self.doctype = None + Node.unlink(self) + + def cloneNode(self, deep): + if not deep: + return None + clone = self.implementation.createDocument(None, None, None) + clone.encoding = self.encoding + clone.standalone = self.standalone + clone.version = self.version + for n in self.childNodes: + childclone = _clone_node(n, deep, clone) + assert childclone.ownerDocument.isSameNode(clone) + clone.childNodes.append(childclone) + if childclone.nodeType == Node.DOCUMENT_NODE: + assert clone.documentElement is None + elif childclone.nodeType == Node.DOCUMENT_TYPE_NODE: + assert clone.doctype is None + clone.doctype = childclone + childclone.parentNode = clone + self._call_user_data_handler(xml.dom.UserDataHandler.NODE_CLONED, + self, clone) + return clone + + def createDocumentFragment(self): + d = DocumentFragment() + d.ownerDocument = self + return d + + def createElement(self, tagName): + e = Element(tagName) + e.ownerDocument = self + return e + + def createTextNode(self, data): + if not isinstance(data, str): + raise TypeError("node contents must be a string") + t = Text() + t.data = data + t.ownerDocument = self + return t + + def createCDATASection(self, data): + if not isinstance(data, str): + raise TypeError("node contents must be a string") + c = CDATASection() + c.data = data + c.ownerDocument = self + return c + + def createComment(self, data): + c = Comment(data) + c.ownerDocument = self + return c + + def createProcessingInstruction(self, target, data): + p = ProcessingInstruction(target, data) + p.ownerDocument = self + return p + + def createAttribute(self, qName): + a = Attr(qName) + a.ownerDocument = self + a.value = "" + return a + + def createElementNS(self, namespaceURI, qualifiedName): + prefix, localName = _nssplit(qualifiedName) + e = Element(qualifiedName, namespaceURI, prefix) + e.ownerDocument = self + return e + + def createAttributeNS(self, namespaceURI, qualifiedName): + prefix, localName = _nssplit(qualifiedName) + a = Attr(qualifiedName, namespaceURI, localName, prefix) + a.ownerDocument = self + a.value = "" + return a + + # A couple of implementation-specific helpers to create node types + # not supported by the W3C DOM specs: + + def _create_entity(self, name, publicId, systemId, notationName): + e = Entity(name, publicId, systemId, notationName) + e.ownerDocument = self + return e + + def _create_notation(self, name, publicId, systemId): + n = Notation(name, publicId, systemId) + n.ownerDocument = self + return n + + def getElementById(self, id): + if id in self._id_cache: + return self._id_cache[id] + if not (self._elem_info or self._magic_id_count): + return None + + stack = self._id_search_stack + if stack is None: + # we never searched before, or the cache has been cleared + stack = [self.documentElement] + self._id_search_stack = stack + elif not stack: + # Previous search was completed and cache is still valid; + # no matching node. + return None + + result = None + while stack: + node = stack.pop() + # add child elements to stack for continued searching + stack.extend([child for child in node.childNodes + if child.nodeType in _nodeTypes_with_children]) + # check this node + info = self._get_elem_info(node) + if info: + # We have to process all ID attributes before + # returning in order to get all the attributes set to + # be IDs using Element.setIdAttribute*(). + for attr in node.attributes.values(): + if attr.namespaceURI: + if info.isIdNS(attr.namespaceURI, attr.localName): + self._id_cache[attr.value] = node + if attr.value == id: + result = node + elif not node._magic_id_nodes: + break + elif info.isId(attr.name): + self._id_cache[attr.value] = node + if attr.value == id: + result = node + elif not node._magic_id_nodes: + break + elif attr._is_id: + self._id_cache[attr.value] = node + if attr.value == id: + result = node + elif node._magic_id_nodes == 1: + break + elif node._magic_id_nodes: + for attr in node.attributes.values(): + if attr._is_id: + self._id_cache[attr.value] = node + if attr.value == id: + result = node + if result is not None: + break + return result + + def getElementsByTagName(self, name): + return _get_elements_by_tagName_helper(self, name, NodeList()) + + def getElementsByTagNameNS(self, namespaceURI, localName): + return _get_elements_by_tagName_ns_helper( + self, namespaceURI, localName, NodeList()) + + def isSupported(self, feature, version): + return self.implementation.hasFeature(feature, version) + + def importNode(self, node, deep): + if node.nodeType == Node.DOCUMENT_NODE: + raise xml.dom.NotSupportedErr("cannot import document nodes") + elif node.nodeType == Node.DOCUMENT_TYPE_NODE: + raise xml.dom.NotSupportedErr("cannot import document type nodes") + return _clone_node(node, deep, self) + + def writexml(self, writer, indent="", addindent="", newl="", encoding=None): + if encoding is None: + writer.write(''+newl) + else: + writer.write('%s' % ( + encoding, newl)) + for node in self.childNodes: + node.writexml(writer, indent, addindent, newl) + + # DOM Level 3 (WD 9 April 2002) + + def renameNode(self, n, namespaceURI, name): + if n.ownerDocument is not self: + raise xml.dom.WrongDocumentErr( + "cannot rename nodes from other documents;\n" + "expected %s,\nfound %s" % (self, n.ownerDocument)) + if n.nodeType not in (Node.ELEMENT_NODE, Node.ATTRIBUTE_NODE): + raise xml.dom.NotSupportedErr( + "renameNode() only applies to element and attribute nodes") + if namespaceURI != EMPTY_NAMESPACE: + if ':' in name: + prefix, localName = name.split(':', 1) + if ( prefix == "xmlns" + and namespaceURI != xml.dom.XMLNS_NAMESPACE): + raise xml.dom.NamespaceErr( + "illegal use of 'xmlns' prefix") + else: + if ( name == "xmlns" + and namespaceURI != xml.dom.XMLNS_NAMESPACE + and n.nodeType == Node.ATTRIBUTE_NODE): + raise xml.dom.NamespaceErr( + "illegal use of the 'xmlns' attribute") + prefix = None + localName = name + else: + prefix = None + localName = None + if n.nodeType == Node.ATTRIBUTE_NODE: + element = n.ownerElement + if element is not None: + is_id = n._is_id + element.removeAttributeNode(n) + else: + element = None + n.prefix = prefix + n._localName = localName + n.namespaceURI = namespaceURI + n.nodeName = name + if n.nodeType == Node.ELEMENT_NODE: + n.tagName = name + else: + # attribute node + n.name = name + if element is not None: + element.setAttributeNode(n) + if is_id: + element.setIdAttributeNode(n) + # It's not clear from a semantic perspective whether we should + # call the user data handlers for the NODE_RENAMED event since + # we're re-using the existing node. The draft spec has been + # interpreted as meaning "no, don't call the handler unless a + # new node is created." + return n + +defproperty(Document, "documentElement", + doc="Top-level element of this document.") + + +def _clone_node(node, deep, newOwnerDocument): + """ + Clone a node and give it the new owner document. + Called by Node.cloneNode and Document.importNode + """ + if node.ownerDocument.isSameNode(newOwnerDocument): + operation = xml.dom.UserDataHandler.NODE_CLONED + else: + operation = xml.dom.UserDataHandler.NODE_IMPORTED + if node.nodeType == Node.ELEMENT_NODE: + clone = newOwnerDocument.createElementNS(node.namespaceURI, + node.nodeName) + for attr in node.attributes.values(): + clone.setAttributeNS(attr.namespaceURI, attr.nodeName, attr.value) + a = clone.getAttributeNodeNS(attr.namespaceURI, attr.localName) + a.specified = attr.specified + + if deep: + for child in node.childNodes: + c = _clone_node(child, deep, newOwnerDocument) + clone.appendChild(c) + + elif node.nodeType == Node.DOCUMENT_FRAGMENT_NODE: + clone = newOwnerDocument.createDocumentFragment() + if deep: + for child in node.childNodes: + c = _clone_node(child, deep, newOwnerDocument) + clone.appendChild(c) + + elif node.nodeType == Node.TEXT_NODE: + clone = newOwnerDocument.createTextNode(node.data) + elif node.nodeType == Node.CDATA_SECTION_NODE: + clone = newOwnerDocument.createCDATASection(node.data) + elif node.nodeType == Node.PROCESSING_INSTRUCTION_NODE: + clone = newOwnerDocument.createProcessingInstruction(node.target, + node.data) + elif node.nodeType == Node.COMMENT_NODE: + clone = newOwnerDocument.createComment(node.data) + elif node.nodeType == Node.ATTRIBUTE_NODE: + clone = newOwnerDocument.createAttributeNS(node.namespaceURI, + node.nodeName) + clone.specified = True + clone.value = node.value + elif node.nodeType == Node.DOCUMENT_TYPE_NODE: + assert node.ownerDocument is not newOwnerDocument + operation = xml.dom.UserDataHandler.NODE_IMPORTED + clone = newOwnerDocument.implementation.createDocumentType( + node.name, node.publicId, node.systemId) + clone.ownerDocument = newOwnerDocument + if deep: + clone.entities._seq = [] + clone.notations._seq = [] + for n in node.notations._seq: + notation = Notation(n.nodeName, n.publicId, n.systemId) + notation.ownerDocument = newOwnerDocument + clone.notations._seq.append(notation) + if hasattr(n, '_call_user_data_handler'): + n._call_user_data_handler(operation, n, notation) + for e in node.entities._seq: + entity = Entity(e.nodeName, e.publicId, e.systemId, + e.notationName) + entity.actualEncoding = e.actualEncoding + entity.encoding = e.encoding + entity.version = e.version + entity.ownerDocument = newOwnerDocument + clone.entities._seq.append(entity) + if hasattr(e, '_call_user_data_handler'): + e._call_user_data_handler(operation, e, entity) + else: + # Note the cloning of Document and DocumentType nodes is + # implementation specific. minidom handles those cases + # directly in the cloneNode() methods. + raise xml.dom.NotSupportedErr("Cannot clone node %s" % repr(node)) + + # Check for _call_user_data_handler() since this could conceivably + # used with other DOM implementations (one of the FourThought + # DOMs, perhaps?). + if hasattr(node, '_call_user_data_handler'): + node._call_user_data_handler(operation, node, clone) + return clone + + +def _nssplit(qualifiedName): + fields = qualifiedName.split(':', 1) + if len(fields) == 2: + return fields + else: + return (None, fields[0]) + + +def _do_pulldom_parse(func, args, kwargs): + events = func(*args, **kwargs) + toktype, rootNode = events.getEvent() + events.expandNode(rootNode) + events.clear() + return rootNode + +def parse(file, parser=None, bufsize=None): + """Parse a file into a DOM by filename or file object.""" + if parser is None and not bufsize: + from xml.dom import expatbuilder + return expatbuilder.parse(file) + else: + from xml.dom import pulldom + return _do_pulldom_parse(pulldom.parse, (file,), + {'parser': parser, 'bufsize': bufsize}) + +def parseString(string, parser=None): + """Parse a file into a DOM from a string.""" + if parser is None: + from xml.dom import expatbuilder + return expatbuilder.parseString(string) + else: + from xml.dom import pulldom + return _do_pulldom_parse(pulldom.parseString, (string,), + {'parser': parser}) + +def getDOMImplementation(features=None): + if features: + if isinstance(features, str): + features = domreg._parse_feature_string(features) + for f, v in features: + if not Document.implementation.hasFeature(f, v): + return None + return Document.implementation diff --git a/benchmarks/benchmarks/nbody.py b/benchmarks/benchmarks/nbody.py new file mode 100644 index 0000000000..b9ab142df9 --- /dev/null +++ b/benchmarks/benchmarks/nbody.py @@ -0,0 +1,110 @@ + +# The Computer Language Benchmarks Game +# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ +# +# originally by Kevin Carson +# modified by Tupteq, Fredrik Johansson, and Daniel Nanz +# modified by Maciej Fijalkowski +# 2to3 +# modified by Andriy Misyura + +from math import sqrt + +def combinations(l): + result = [] + for x in range(len(l) - 1): + ls = l[x+1:] + for y in ls: + result.append((l[x][0],l[x][1],l[x][2],y[0],y[1],y[2])) + return result + +PI = 3.14159265358979323 +SOLAR_MASS = 4 * PI * PI +DAYS_PER_YEAR = 365.24 + +BODIES = { + 'sun': ([0.0, 0.0, 0.0], [0.0, 0.0, 0.0], SOLAR_MASS), + + 'jupiter': ([4.84143144246472090e+00, + -1.16032004402742839e+00, + -1.03622044471123109e-01], + [1.66007664274403694e-03 * DAYS_PER_YEAR, + 7.69901118419740425e-03 * DAYS_PER_YEAR, + -6.90460016972063023e-05 * DAYS_PER_YEAR], + 9.54791938424326609e-04 * SOLAR_MASS), + + 'saturn': ([8.34336671824457987e+00, + 4.12479856412430479e+00, + -4.03523417114321381e-01], + [-2.76742510726862411e-03 * DAYS_PER_YEAR, + 4.99852801234917238e-03 * DAYS_PER_YEAR, + 2.30417297573763929e-05 * DAYS_PER_YEAR], + 2.85885980666130812e-04 * SOLAR_MASS), + + 'uranus': ([1.28943695621391310e+01, + -1.51111514016986312e+01, + -2.23307578892655734e-01], + [2.96460137564761618e-03 * DAYS_PER_YEAR, + 2.37847173959480950e-03 * DAYS_PER_YEAR, + -2.96589568540237556e-05 * DAYS_PER_YEAR], + 4.36624404335156298e-05 * SOLAR_MASS), + + 'neptune': ([1.53796971148509165e+01, + -2.59193146099879641e+01, + 1.79258772950371181e-01], + [2.68067772490389322e-03 * DAYS_PER_YEAR, + 1.62824170038242295e-03 * DAYS_PER_YEAR, + -9.51592254519715870e-05 * DAYS_PER_YEAR], + 5.15138902046611451e-05 * SOLAR_MASS) } + +SYSTEM = tuple(BODIES.values()) +PAIRS = tuple(combinations(SYSTEM)) + +def advance(dt, n, bodies=SYSTEM, pairs=PAIRS): + for i in range(n): + for ([x1, y1, z1], v1, m1, [x2, y2, z2], v2, m2) in pairs: + dx = x1 - x2 + dy = y1 - y2 + dz = z1 - z2 + dist = sqrt(dx * dx + dy * dy + dz * dz); + mag = dt / (dist*dist*dist) + b1m = m1 * mag + b2m = m2 * mag + v1[0] -= dx * b2m + v1[1] -= dy * b2m + v1[2] -= dz * b2m + v2[2] += dz * b1m + v2[1] += dy * b1m + v2[0] += dx * b1m + for (r, [vx, vy, vz], m) in bodies: + r[0] += dt * vx + r[1] += dt * vy + r[2] += dt * vz + +def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0): + for ((x1, y1, z1), v1, m1, (x2, y2, z2), v2, m2) in pairs: + dx = x1 - x2 + dy = y1 - y2 + dz = z1 - z2 + e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5) + for (r, [vx, vy, vz], m) in bodies: + e += m * (vx * vx + vy * vy + vz * vz) / 2. + # print(f"{e}") + +def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0): + for (r, [vx, vy, vz], m) in bodies: + px -= vx * m + py -= vy * m + pz -= vz * m + (r, v, m) = ref + v[0] = px / m + v[1] = py / m + v[2] = pz / m + +def main(n, ref='sun'): + offset_momentum(BODIES[ref]) + report_energy() + advance(0.01, n) + report_energy() + +main(500) diff --git a/benchmarks/test_benchmarks.py b/benchmarks/test_benchmarks.py new file mode 100644 index 0000000000..571785e60f --- /dev/null +++ b/benchmarks/test_benchmarks.py @@ -0,0 +1,31 @@ + +import time +import sys + +import pytest +import subprocess + +from benchmarks import nbody + +# Interpreters: +rustpython_exe = '../target/release/rustpython' +cpython_exe = sys.executable +pythons = [ + cpython_exe, + rustpython_exe +] + +# Benchmark scripts: +benchmarks = [ + ['benchmarks/nbody.py'], + ['benchmarks/mandelbrot.py'], +] + +@pytest.mark.parametrize('exe', pythons) +@pytest.mark.parametrize('args', benchmarks) +def test_bench(exe, args, benchmark): + def bench(): + subprocess.run([exe] + args) + + benchmark(bench) + diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml new file mode 100644 index 0000000000..fad46c1757 --- /dev/null +++ b/bytecode/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rustpython-bytecode" +version = "0.1.0" +authors = ["RustPython Team"] +edition = "2018" + +[dependencies] +bitflags = "1.1" +num-bigint = { version = "0.2", features = ["serde"] } +num-complex = { version = "0.2", features = ["serde"] } +serde = { version = "1.0", features = ["derive"] } diff --git a/vm/src/bytecode.rs b/bytecode/src/bytecode.rs similarity index 80% rename from vm/src/bytecode.rs rename to bytecode/src/bytecode.rs index 2fe485b63b..0f08e22bb7 100644 --- a/vm/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -1,23 +1,42 @@ //! Implement python as a virtual machine with bytecodes. This module //! implements bytecode structure. -/* - * Primitive instruction type, which can be encoded and decoded. - */ - +use bitflags::bitflags; use num_bigint::BigInt; use num_complex::Complex64; -use rustpython_parser::ast; +use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fmt; +/// Sourcode location. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Location { + row: usize, + column: usize, +} + +impl Location { + pub fn new(row: usize, column: usize) -> Self { + Location { row, column } + } + + pub fn row(&self) -> usize { + self.row + } + + pub fn column(&self) -> usize { + self.column + } +} + /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct CodeObject { pub instructions: Vec, + /// Jump targets. pub label_map: HashMap, - pub locations: Vec, + pub locations: Vec, pub arg_names: Vec, // Names of positional arguments pub varargs: Varargs, // *args or * pub kwonlyarg_names: Vec, @@ -29,29 +48,53 @@ pub struct CodeObject { } bitflags! { + #[derive(Serialize, Deserialize)] pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; + const HAS_KW_ONLY_DEFAULTS = 0x02; const HAS_ANNOTATIONS = 0x04; } } pub type Label = usize; +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum NameScope { + Local, + NonLocal, + Global, +} + +/// Transforms a value prior to formatting it. +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum ConversionFlag { + /// Converts by calling `str()`. + Str, + /// Converts by calling `ascii()`. + Ascii, + /// Converts by calling `repr()`. + Repr, +} + /// A Single bytecode instruction. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Instruction { Import { name: String, - symbol: Option, + symbols: Vec, + level: usize, }, ImportStar { name: String, + level: usize, }, LoadName { name: String, + scope: NameScope, }, StoreName { name: String, + scope: NameScope, }, DeleteName { name: String, @@ -169,21 +212,22 @@ pub enum Instruction { }, Unpack, FormatValue { - conversion: Option, + conversion: Option, spec: String, }, + PopException, } use self::Instruction::*; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum CallType { Positional(usize), Keyword(usize), Ex(bool), } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Constant { Integer { value: BigInt }, Float { value: f64 }, @@ -197,7 +241,7 @@ pub enum Constant { Ellipsis, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum ComparisonOperator { Greater, GreaterOrEqual, @@ -211,7 +255,7 @@ pub enum ComparisonOperator { IsNot, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum BinaryOperator { Power, Multiply, @@ -229,7 +273,7 @@ pub enum BinaryOperator { Or, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum UnaryOperator { Not, Invert, @@ -237,7 +281,7 @@ pub enum UnaryOperator { Plus, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Varargs { None, Unnamed, @@ -316,13 +360,27 @@ impl Instruction { ($variant:ident, $var1:expr, $var2:expr) => { write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2) }; + ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => { + write!( + f, + "{:20} ({}, {}, {})\n", + stringify!($variant), + $var1, + $var2, + $var3 + ) + }; } match self { - Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)), - ImportStar { name } => w!(ImportStar, name), - LoadName { name } => w!(LoadName, name), - StoreName { name } => w!(StoreName, name), + Import { + name, + symbols, + level, + } => w!(Import, name, format!("{:?}", symbols), level), + ImportStar { name, level } => w!(ImportStar, name, level), + LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)), + StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)), DeleteName { name } => w!(DeleteName, name), StoreSubscript => w!(StoreSubscript), DeleteSubscript => w!(DeleteSubscript), @@ -370,6 +428,7 @@ impl Instruction { UnpackEx { before, after } => w!(UnpackEx, before, after), Unpack => w!(Unpack), FormatValue { spec, .. } => w!(FormatValue, spec), // TODO: write conversion + PopException => w!(PopException), } } } @@ -408,23 +467,3 @@ impl fmt::Debug for CodeObject { ) } } - -impl From for Varargs { - fn from(varargs: ast::Varargs) -> Varargs { - match varargs { - ast::Varargs::None => Varargs::None, - ast::Varargs::Unnamed => Varargs::Unnamed, - ast::Varargs::Named(param) => Varargs::Named(param.arg), - } - } -} - -impl<'a> From<&'a ast::Varargs> for Varargs { - fn from(varargs: &'a ast::Varargs) -> Varargs { - match varargs { - ast::Varargs::None => Varargs::None, - ast::Varargs::Unnamed => Varargs::Unnamed, - ast::Varargs::Named(ref param) => Varargs::Named(param.arg.clone()), - } - } -} diff --git a/bytecode/src/lib.rs b/bytecode/src/lib.rs new file mode 100644 index 0000000000..c6e3077a91 --- /dev/null +++ b/bytecode/src/lib.rs @@ -0,0 +1 @@ +pub mod bytecode; diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml new file mode 100644 index 0000000000..8fdb37871a --- /dev/null +++ b/compiler/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rustpython-compiler" +version = "0.1.0" +authors = ["RustPython Team"] +edition = "2018" + +[dependencies] +rustpython-bytecode = { path = "../bytecode" } +rustpython-parser = { path = "../parser" } +num-complex = { version = "0.2", features = ["serde"] } +log = "0.3" diff --git a/vm/src/compile.rs b/compiler/src/compile.rs similarity index 72% rename from vm/src/compile.rs rename to compiler/src/compile.rs index 1742392cdd..877200f809 100644 --- a/vm/src/compile.rs +++ b/compiler/src/compile.rs @@ -5,51 +5,83 @@ //! https://github.com/python/cpython/blob/master/Python/compile.c //! https://github.com/micropython/micropython/blob/master/py/compile.c -use crate::bytecode::{self, CallType, CodeObject, Instruction, Varargs}; -use crate::error::CompileError; -use crate::obj::objcode; -use crate::pyobject::{PyObject, PyObjectRef}; +use crate::error::{CompileError, CompileErrorType}; +use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope}; use num_complex::Complex64; +use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs}; use rustpython_parser::{ast, parser}; struct Compiler { code_object_stack: Vec, + scope_stack: Vec, nxt_label: usize, source_path: Option, current_source_location: ast::Location, + current_qualified_path: Option, in_loop: bool, in_function_def: bool, } /// Compile a given sourcecode into a bytecode object. -pub fn compile( - source: &str, - mode: &Mode, - source_path: String, - code_type: PyObjectRef, -) -> Result { - let mut compiler = Compiler::new(); - compiler.source_path = Some(source_path); - compiler.push_new_code_object("".to_string()); - +pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result { match mode { Mode::Exec => { - let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - compiler.compile_program(&ast) + let ast = parser::parse_program(source)?; + compile_program(ast, source_path) } Mode::Eval => { - let statement = parser::parse_statement(source).map_err(CompileError::Parse)?; - compiler.compile_statement_eval(&statement) + let statement = parser::parse_statement(source)?; + compile_statement_eval(statement, source_path) } Mode::Single => { - let ast = parser::parse_program(source).map_err(CompileError::Parse)?; - compiler.compile_program_single(&ast) + let ast = parser::parse_program(source)?; + compile_program_single(ast, source_path) } - }?; + } +} +/// A helper function for the shared code of the different compile functions +fn with_compiler( + source_path: String, + f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>, +) -> Result { + let mut compiler = Compiler::new(); + compiler.source_path = Some(source_path); + compiler.push_new_code_object("".to_string()); + f(&mut compiler)?; let code = compiler.pop_code_object(); trace!("Compilation completed: {:?}", code); - Ok(PyObject::new(objcode::PyCode::new(code), code_type)) + Ok(code) +} + +/// Compile a standard Python program to bytecode +pub fn compile_program(ast: ast::Program, source_path: String) -> Result { + with_compiler(source_path, |compiler| { + let symbol_table = make_symbol_table(&ast)?; + compiler.compile_program(&ast, symbol_table) + }) +} + +/// Compile a single Python expression to bytecode +pub fn compile_statement_eval( + statement: Vec, + source_path: String, +) -> Result { + with_compiler(source_path, |compiler| { + let symbol_table = statements_to_symbol_table(&statement)?; + compiler.compile_statement_eval(&statement, symbol_table) + }) +} + +/// Compile a Python program to bytecode for the context of a REPL +pub fn compile_program_single( + ast: ast::Program, + source_path: String, +) -> Result { + with_compiler(source_path, |compiler| { + let symbol_table = make_symbol_table(&ast)?; + compiler.compile_program_single(&ast, symbol_table) + }) } pub enum Mode { @@ -70,9 +102,11 @@ impl Compiler { fn new() -> Self { Compiler { code_object_stack: Vec::new(), + scope_stack: Vec::new(), nxt_label: 0, source_path: None, current_source_location: ast::Location::default(), + current_qualified_path: None, in_loop: false, in_function_def: false, } @@ -92,11 +126,17 @@ impl Compiler { } fn pop_code_object(&mut self) -> CodeObject { + // self.scope_stack.pop().unwrap(); self.code_object_stack.pop().unwrap() } - fn compile_program(&mut self, program: &ast::Program) -> Result<(), CompileError> { + fn compile_program( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { let size_before = self.code_object_stack.len(); + self.scope_stack.push(symbol_scope); self.compile_statements(&program.statements)?; assert!(self.code_object_stack.len() == size_before); @@ -108,34 +148,63 @@ impl Compiler { Ok(()) } - fn compile_program_single(&mut self, program: &ast::Program) -> Result<(), CompileError> { - for statement in &program.statements { + fn compile_program_single( + &mut self, + program: &ast::Program, + symbol_scope: SymbolScope, + ) -> Result<(), CompileError> { + self.scope_stack.push(symbol_scope); + + let mut emitted_return = false; + + for (i, statement) in program.statements.iter().enumerate() { + let is_last = i == program.statements.len() - 1; + if let ast::Statement::Expression { ref expression } = statement.node { self.compile_expression(expression)?; - self.emit(Instruction::PrintExpr); + + if is_last { + self.emit(Instruction::Duplicate); + self.emit(Instruction::PrintExpr); + self.emit(Instruction::ReturnValue); + emitted_return = true; + } else { + self.emit(Instruction::PrintExpr); + } } else { self.compile_statement(&statement)?; } } - self.emit(Instruction::LoadConst { - value: bytecode::Constant::None, - }); - self.emit(Instruction::ReturnValue); + + if !emitted_return { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::None, + }); + self.emit(Instruction::ReturnValue); + } + Ok(()) } // Compile statement in eval mode: fn compile_statement_eval( &mut self, - statement: &ast::LocatedStatement, + statements: &[ast::LocatedStatement], + symbol_table: SymbolScope, ) -> Result<(), CompileError> { - if let ast::Statement::Expression { ref expression } = statement.node { - self.compile_expression(expression)?; - self.emit(Instruction::ReturnValue); - Ok(()) - } else { - Err(CompileError::ExpectExpr) + self.scope_stack.push(symbol_table); + for statement in statements { + if let ast::Statement::Expression { ref expression } = statement.node { + self.compile_expression(expression)?; + } else { + return Err(CompileError { + error: CompileErrorType::ExpectExpr, + location: statement.location.clone(), + }); + } } + self.emit(Instruction::ReturnValue); + Ok(()) } fn compile_statements( @@ -148,6 +217,31 @@ impl Compiler { Ok(()) } + fn scope_for_name(&self, name: &str) -> bytecode::NameScope { + let role = self.lookup_name(name); + match role { + SymbolRole::Global => bytecode::NameScope::Global, + SymbolRole::Nonlocal => bytecode::NameScope::NonLocal, + _ => bytecode::NameScope::Local, + } + } + + fn load_name(&mut self, name: &str) { + let scope = self.scope_for_name(name); + self.emit(Instruction::LoadName { + name: name.to_string(), + scope, + }); + } + + fn store_name(&mut self, name: &str) { + let scope = self.scope_for_name(name); + self.emit(Instruction::StoreName { + name: name.to_string(), + scope, + }); + } + fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> { trace!("Compiling {:?}", statement); self.set_source_location(&statement.location); @@ -156,30 +250,56 @@ impl Compiler { ast::Statement::Import { import_parts } => { for ast::SingleImport { module, - symbol, + symbols, alias, + level, } in import_parts { - match symbol { - Some(name) if name == "*" => { + let level = *level; + if let Some(alias) = alias { + // import module as alias + self.emit(Instruction::Import { + name: module.clone(), + symbols: vec![], + level, + }); + self.store_name(&alias); + } else if symbols.is_empty() { + // import module + self.emit(Instruction::Import { + name: module.clone(), + symbols: vec![], + level, + }); + self.store_name(&module.clone()); + } else { + let import_star = symbols + .iter() + .any(|import_symbol| import_symbol.symbol == "*"); + if import_star { + // from module import * self.emit(Instruction::ImportStar { name: module.clone(), + level, }); - } - _ => { + } else { + // from module import symbol + // from module import symbol as alias + let (names, symbols_strings): (Vec, Vec) = symbols + .iter() + .map(|ast::ImportSymbol { symbol, alias }| { + ( + alias.clone().unwrap_or_else(|| symbol.to_string()), + symbol.to_string(), + ) + }) + .unzip(); self.emit(Instruction::Import { name: module.clone(), - symbol: symbol.clone(), - }); - self.emit(Instruction::StoreName { - name: match alias { - Some(alias) => alias.clone(), - None => match symbol { - Some(symbol) => symbol.clone(), - None => module.clone(), - }, - }, + symbols: symbols_strings, + level, }); + names.iter().rev().for_each(|name| self.store_name(&name)); } } } @@ -190,11 +310,8 @@ impl Compiler { // Pop result of stack, since we not use it: self.emit(Instruction::Pop); } - ast::Statement::Global { names } => { - unimplemented!("global {:?}", names); - } - ast::Statement::Nonlocal { names } => { - unimplemented!("nonlocal {:?}", names); + ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => { + // Handled during symbol table construction. } ast::Statement::If { test, body, orelse } => { let end_label = self.new_label(); @@ -273,6 +390,9 @@ impl Compiler { body, orelse, } => self.compile_for(target, iter, body, orelse)?, + ast::Statement::AsyncFor { .. } => { + unimplemented!("async for"); + } ast::Statement::Raise { exception, cause } => match exception { Some(value) => { self.compile_expression(value)?; @@ -303,6 +423,9 @@ impl Compiler { decorator_list, returns, } => self.compile_function_def(name, args, body, decorator_list, returns)?, + ast::Statement::AsyncFunctionDef { .. } => { + unimplemented!("async def"); + } ast::Statement::ClassDef { name, body, @@ -317,6 +440,7 @@ impl Compiler { self.compile_test(test, Some(end_label), None, EvalContext::Statement)?; self.emit(Instruction::LoadName { name: String::from("AssertionError"), + scope: bytecode::NameScope::Local, }); match msg { Some(e) => { @@ -336,34 +460,32 @@ impl Compiler { } ast::Statement::Break => { if !self.in_loop { - return Err(CompileError::InvalidBreak); + return Err(CompileError { + error: CompileErrorType::InvalidBreak, + location: statement.location.clone(), + }); } self.emit(Instruction::Break); } ast::Statement::Continue => { if !self.in_loop { - return Err(CompileError::InvalidContinue); + return Err(CompileError { + error: CompileErrorType::InvalidContinue, + location: statement.location.clone(), + }); } self.emit(Instruction::Continue); } ast::Statement::Return { value } => { if !self.in_function_def { - return Err(CompileError::InvalidReturn); + return Err(CompileError { + error: CompileErrorType::InvalidReturn, + location: statement.location.clone(), + }); } match value { - Some(e) => { - let size = e.len(); - for v in e { - self.compile_expression(v)?; - } - - // If we have more than 1 return value, make it a tuple: - if size > 1 { - self.emit(Instruction::BuildTuple { - size, - unpack: false, - }); - } + Some(v) => { + self.compile_expression(v)?; } None => { self.emit(Instruction::LoadConst { @@ -394,27 +516,7 @@ impl Compiler { } ast::Statement::Delete { targets } => { for target in targets { - match target { - ast::Expression::Identifier { name } => { - self.emit(Instruction::DeleteName { - name: name.to_string(), - }); - } - ast::Expression::Attribute { value, name } => { - self.compile_expression(value)?; - self.emit(Instruction::DeleteAttr { - name: name.to_string(), - }); - } - ast::Expression::Subscript { a, b } => { - self.compile_expression(a)?; - self.compile_expression(b)?; - self.emit(Instruction::DeleteSubscript); - } - _ => { - return Err(CompileError::Delete(target.name())); - } - } + self.compile_delete(target)?; } } ast::Statement::Pass => { @@ -424,13 +526,46 @@ impl Compiler { Ok(()) } + fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> { + match expression { + ast::Expression::Identifier { name } => { + self.emit(Instruction::DeleteName { + name: name.to_string(), + }); + } + ast::Expression::Attribute { value, name } => { + self.compile_expression(value)?; + self.emit(Instruction::DeleteAttr { + name: name.to_string(), + }); + } + ast::Expression::Subscript { a, b } => { + self.compile_expression(a)?; + self.compile_expression(b)?; + self.emit(Instruction::DeleteSubscript); + } + ast::Expression::Tuple { elements } => { + for element in elements { + self.compile_delete(element)?; + } + } + _ => { + return Err(CompileError { + error: CompileErrorType::Delete(expression.name()), + location: self.current_source_location.clone(), + }); + } + } + Ok(()) + } + fn enter_function( &mut self, name: &str, args: &ast::Parameters, ) -> Result { - let have_kwargs = !args.defaults.is_empty(); - if have_kwargs { + let have_defaults = !args.defaults.is_empty(); + if have_defaults { // Construct a tuple: let size = args.defaults.len(); for element in &args.defaults { @@ -442,21 +577,44 @@ impl Compiler { }); } + let mut num_kw_only_defaults = 0; + for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) { + if let Some(default) = default { + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: kw.arg.clone(), + }, + }); + self.compile_expression(default)?; + num_kw_only_defaults += 1; + } + } + if num_kw_only_defaults > 0 { + self.emit(Instruction::BuildMap { + size: num_kw_only_defaults, + unpack: false, + }); + } + let line_number = self.get_source_line_number(); self.code_object_stack.push(CodeObject::new( args.args.iter().map(|a| a.arg.clone()).collect(), - Varargs::from(&args.vararg), + compile_varargs(&args.vararg), args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(), - Varargs::from(&args.kwarg), + compile_varargs(&args.kwarg), self.source_path.clone().unwrap(), line_number, name.to_string(), )); + self.enter_scope(); let mut flags = bytecode::FunctionOpArg::empty(); - if have_kwargs { + if have_defaults { flags |= bytecode::FunctionOpArg::HAS_DEFAULTS; } + if num_kw_only_defaults > 0 { + flags |= bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS; + } Ok(flags) } @@ -512,6 +670,7 @@ impl Compiler { // Check exception type: self.emit(Instruction::LoadName { name: String::from("isinstance"), + scope: bytecode::NameScope::Local, }); self.emit(Instruction::Rotate { amount: 2 }); self.compile_expression(exc_type)?; @@ -526,9 +685,7 @@ impl Compiler { // We have a match, store in name (except x as y) if let Some(alias) = &handler.name { - self.emit(Instruction::StoreName { - name: alias.clone(), - }); + self.store_name(alias); } else { // Drop exception from top of stack: self.emit(Instruction::Pop); @@ -541,6 +698,7 @@ impl Compiler { // Handler code: self.compile_statements(&handler.body)?; + self.emit(Instruction::PopException); self.emit(Instruction::Jump { target: finally_label, }); @@ -561,7 +719,7 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - self.emit(Instruction::Raise { argc: 1 }); + self.emit(Instruction::Raise { argc: 0 }); // We successfully ran the try block: // else: @@ -575,7 +733,6 @@ impl Compiler { if let Some(statements) = finalbody { self.compile_statements(statements)?; } - // unimplemented!(); Ok(()) } @@ -594,8 +751,18 @@ impl Compiler { let was_in_function_def = self.in_function_def; self.in_loop = false; self.in_function_def = true; + + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(self.create_qualified_name(name, ".")); + + self.prepare_decorators(decorator_list)?; + let mut flags = self.enter_function(name, args)?; - self.compile_statements(body)?; + + let (new_body, doc_str) = get_doc(body); + + self.compile_statements(new_body)?; // Emit None at end: self.emit(Instruction::LoadConst { @@ -603,8 +770,7 @@ impl Compiler { }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); - - self.prepare_decorators(decorator_list)?; + self.leave_scope(); // Prepare type annotations: let mut num_annotations = 0; @@ -649,17 +815,18 @@ impl Compiler { }); self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); // Turn code object into function object: self.emit(Instruction::MakeFunction { flags }); + self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); + + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; self.in_function_def = was_in_function_def; Ok(()) @@ -675,6 +842,11 @@ impl Compiler { ) -> Result<(), CompileError> { let was_in_loop = self.in_loop; self.in_loop = false; + + let old_qualified_path = self.current_qualified_path.clone(); + let qualified_name = self.create_qualified_name(name, ""); + self.current_qualified_path = Some(qualified_name.clone()); + self.prepare_decorators(decorator_list)?; self.emit(Instruction::LoadBuildClass); let line_number = self.get_source_line_number(); @@ -687,13 +859,27 @@ impl Compiler { line_number, name.to_string(), )); - self.compile_statements(body)?; + self.enter_scope(); + + let (new_body, doc_str) = get_doc(body); + + self.emit(Instruction::LoadName { + name: "__name__".to_string(), + scope: bytecode::NameScope::Local, + }); + self.emit(Instruction::StoreName { + name: "__module__".to_string(), + scope: bytecode::NameScope::Local, + }); + self.compile_statements(new_body)?; self.emit(Instruction::LoadConst { value: bytecode::Constant::None, }); self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); + self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -712,7 +898,7 @@ impl Compiler { self.emit(Instruction::LoadConst { value: bytecode::Constant::String { - value: name.to_string(), + value: qualified_name, }, }); @@ -748,19 +934,38 @@ impl Compiler { }); } + self.store_docstring(doc_str); self.apply_decorators(decorator_list); - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); + self.current_qualified_path = old_qualified_path; self.in_loop = was_in_loop; Ok(()) } + fn store_docstring(&mut self, doc_str: Option) { + if let Some(doc_string) = doc_str { + // Duplicate top of stack (the function or class object) + self.emit(Instruction::Duplicate); + + // Doc string value: + self.emit(Instruction::LoadConst { + value: bytecode::Constant::String { + value: doc_string.to_string(), + }, + }); + + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::StoreAttr { + name: "__doc__".to_string(), + }); + } + } + fn compile_for( &mut self, target: &ast::Expression, - iter: &[ast::Expression], + iter: &ast::Expression, body: &[ast::LocatedStatement], orelse: &Option>, ) -> Result<(), CompileError> { @@ -774,9 +979,7 @@ impl Compiler { }); // The thing iterated: - for i in iter { - self.compile_expression(i)?; - } + self.compile_expression(iter)?; // Retrieve Iterator self.emit(Instruction::GetIter); @@ -804,12 +1007,83 @@ impl Compiler { Ok(()) } + fn compile_chained_comparison( + &mut self, + vals: &[ast::Expression], + ops: &[ast::Comparison], + ) -> Result<(), CompileError> { + assert!(!ops.is_empty()); + assert_eq!(vals.len(), ops.len() + 1); + + let to_operator = |op: &ast::Comparison| match op { + ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, + ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, + ast::Comparison::Less => bytecode::ComparisonOperator::Less, + ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, + ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, + ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, + ast::Comparison::In => bytecode::ComparisonOperator::In, + ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, + ast::Comparison::Is => bytecode::ComparisonOperator::Is, + ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, + }; + + // a == b == c == d + // compile into (pseudocode): + // result = a == b + // if result: + // result = b == c + // if result: + // result = c == d + + // initialize lhs outside of loop + self.compile_expression(&vals[0])?; + + let break_label = self.new_label(); + let last_label = self.new_label(); + + // for all comparisons except the last (as the last one doesn't need a conditional jump) + let ops_slice = &ops[0..ops.len()]; + let vals_slice = &vals[1..ops.len()]; + for (op, val) in ops_slice.iter().zip(vals_slice.iter()) { + self.compile_expression(val)?; + // store rhs for the next comparison in chain + self.emit(Instruction::Duplicate); + self.emit(Instruction::Rotate { amount: 3 }); + + self.emit(Instruction::CompareOperation { + op: to_operator(op), + }); + + // if comparison result is false, we break with this value; if true, try the next one. + // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP) + self.emit(Instruction::Duplicate); + self.emit(Instruction::JumpIfFalse { + target: break_label, + }); + self.emit(Instruction::Pop); + } + + // handle the last comparison + self.compile_expression(vals.last().unwrap())?; + self.emit(Instruction::CompareOperation { + op: to_operator(ops.last().unwrap()), + }); + self.emit(Instruction::Jump { target: last_label }); + + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.set_label(break_label); + self.emit(Instruction::Rotate { amount: 2 }); + self.emit(Instruction::Pop); + + self.set_label(last_label); + Ok(()) + } + fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> { match target { ast::Expression::Identifier { name } => { - self.emit(Instruction::StoreName { - name: name.to_string(), - }); + self.store_name(name); } ast::Expression::Subscript { a, b } => { self.compile_expression(a)?; @@ -822,14 +1096,17 @@ impl Compiler { name: name.to_string(), }); } - ast::Expression::Tuple { elements } => { + ast::Expression::List { elements } | ast::Expression::Tuple { elements } => { let mut seen_star = false; // Scan for star args: for (i, element) in elements.iter().enumerate() { if let ast::Expression::Starred { .. } = element { if seen_star { - return Err(CompileError::StarArgs); + return Err(CompileError { + error: CompileErrorType::StarArgs, + location: self.current_source_location.clone(), + }); } else { seen_star = true; self.emit(Instruction::UnpackEx { @@ -855,7 +1132,10 @@ impl Compiler { } } _ => { - return Err(CompileError::Assign(target.name())); + return Err(CompileError { + error: CompileErrorType::Assign(target.name()), + location: self.current_source_location.clone(), + }); } } @@ -986,24 +1266,8 @@ impl Compiler { name: name.to_string(), }); } - ast::Expression::Compare { a, op, b } => { - self.compile_expression(a)?; - self.compile_expression(b)?; - - let i = match op { - ast::Comparison::Equal => bytecode::ComparisonOperator::Equal, - ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual, - ast::Comparison::Less => bytecode::ComparisonOperator::Less, - ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual, - ast::Comparison::Greater => bytecode::ComparisonOperator::Greater, - ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual, - ast::Comparison::In => bytecode::ComparisonOperator::In, - ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn, - ast::Comparison::Is => bytecode::ComparisonOperator::Is, - ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot, - }; - let i = Instruction::CompareOperation { op: i }; - self.emit(i); + ast::Expression::Compare { vals, ops } => { + self.compile_chained_comparison(vals, ops)?; } ast::Expression::Number { value } => { let const_value = match value { @@ -1043,13 +1307,25 @@ impl Compiler { } ast::Expression::Dict { elements } => { let size = elements.len(); + let has_double_star = elements.iter().any(|e| e.0.is_none()); for (key, value) in elements { - self.compile_expression(key)?; - self.compile_expression(value)?; + if let Some(key) = key { + self.compile_expression(key)?; + self.compile_expression(value)?; + if has_double_star { + self.emit(Instruction::BuildMap { + size: 1, + unpack: false, + }); + } + } else { + // dict unpacking + self.compile_expression(value)?; + } } self.emit(Instruction::BuildMap { size, - unpack: false, + unpack: has_double_star, }); } ast::Expression::Slice { elements } => { @@ -1061,7 +1337,10 @@ impl Compiler { } ast::Expression::Yield { value } => { if !self.in_function_def { - return Err(CompileError::InvalidYield); + return Err(CompileError { + error: CompileErrorType::InvalidYield, + location: self.current_source_location.clone(), + }); } self.mark_generator(); match value { @@ -1072,6 +1351,9 @@ impl Compiler { }; self.emit(Instruction::YieldValue); } + ast::Expression::Await { .. } => { + unimplemented!("await"); + } ast::Expression::YieldFrom { value } => { self.mark_generator(); self.compile_expression(value)?; @@ -1112,9 +1394,7 @@ impl Compiler { }); } ast::Expression::Identifier { name } => { - self.emit(Instruction::LoadName { - name: name.to_string(), - }); + self.load_name(name); } ast::Expression::Lambda { args, body } => { let name = "".to_string(); @@ -1123,6 +1403,7 @@ impl Compiler { self.compile_expression(body)?; self.emit(Instruction::ReturnValue); let code = self.pop_code_object(); + self.leave_scope(); self.emit(Instruction::LoadConst { value: bytecode::Constant::Code { code: Box::new(code), @@ -1145,11 +1426,15 @@ impl Compiler { ast::Expression::IfExpression { test, body, orelse } => { let no_label = self.new_label(); let end_label = self.new_label(); - self.compile_test(test, None, Some(no_label), EvalContext::Expression)?; + self.compile_test(test, None, None, EvalContext::Expression)?; + self.emit(Instruction::JumpIfFalse { target: no_label }); + // True case self.compile_expression(body)?; self.emit(Instruction::Jump { target: end_label }); + // False case self.set_label(no_label); self.compile_expression(orelse)?; + // End self.set_label(end_label); } } @@ -1330,6 +1615,7 @@ impl Compiler { // Load iterator onto stack (passed as first argument): self.emit(Instruction::LoadName { name: String::from(".0"), + scope: bytecode::NameScope::Local, }); } else { // Evaluate iterated item: @@ -1461,7 +1747,7 @@ impl Compiler { } => { self.compile_expression(value)?; self.emit(Instruction::FormatValue { - conversion: *conversion, + conversion: conversion.map(compile_conversion_flag), spec: spec.clone(), }); } @@ -1469,9 +1755,29 @@ impl Compiler { Ok(()) } + // Scope helpers: + fn enter_scope(&mut self) { + // println!("Enter scope {:?}", self.scope_stack); + // Enter first subscope! + let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0); + self.scope_stack.push(scope); + } + + fn leave_scope(&mut self) { + // println!("Leave scope {:?}", self.scope_stack); + let scope = self.scope_stack.pop().unwrap(); + assert!(scope.sub_scopes.is_empty()); + } + + fn lookup_name(&self, name: &str) -> &SymbolRole { + // println!("Looking up {:?}", name); + let scope = self.scope_stack.last().unwrap(); + scope.lookup(name).unwrap() + } + // Low level helper functions: fn emit(&mut self, instruction: Instruction) { - let location = self.current_source_location.clone(); + let location = compile_location(&self.current_source_location); let cur_code_obj = self.current_code_object(); cur_code_obj.instructions.push(instruction); cur_code_obj.locations.push(location); @@ -1501,7 +1807,15 @@ impl Compiler { } fn get_source_line_number(&mut self) -> usize { - self.current_source_location.get_row() + self.current_source_location.row() + } + + fn create_qualified_name(&self, name: &str, suffix: &str) -> String { + if let Some(ref qualified_path) = self.current_qualified_path { + format!("{}.{}{}", qualified_path, name, suffix) + } else { + format!("{}{}", name, suffix) + } } fn mark_generator(&mut self) { @@ -1509,12 +1823,48 @@ impl Compiler { } } +fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option) { + if let Some(val) = body.get(0) { + if let ast::Statement::Expression { ref expression } = val.node { + if let ast::Expression::String { ref value } = expression { + if let ast::StringGroup::Constant { ref value } = value { + if let Some((_, body_rest)) = body.split_first() { + return (body_rest, Some(value.to_string())); + } + } + } + } + } + (body, None) +} + +fn compile_location(location: &ast::Location) -> bytecode::Location { + bytecode::Location::new(location.row(), location.column()) +} + +fn compile_varargs(varargs: &ast::Varargs) -> bytecode::Varargs { + match varargs { + ast::Varargs::None => bytecode::Varargs::None, + ast::Varargs::Unnamed => bytecode::Varargs::Unnamed, + ast::Varargs::Named(param) => bytecode::Varargs::Named(param.arg.clone()), + } +} + +fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::ConversionFlag { + match conversion_flag { + ast::ConversionFlag::Ascii => bytecode::ConversionFlag::Ascii, + ast::ConversionFlag::Repr => bytecode::ConversionFlag::Repr, + ast::ConversionFlag::Str => bytecode::ConversionFlag::Str, + } +} + #[cfg(test)] mod tests { use super::Compiler; - use crate::bytecode::CodeObject; - use crate::bytecode::Constant::*; - use crate::bytecode::Instruction::*; + use crate::symboltable::make_symbol_table; + use rustpython_bytecode::bytecode::CodeObject; + use rustpython_bytecode::bytecode::Constant::*; + use rustpython_bytecode::bytecode::Instruction::*; use rustpython_parser::parser; fn compile_exec(source: &str) -> CodeObject { @@ -1522,7 +1872,8 @@ mod tests { compiler.source_path = Some("source_path".to_string()); compiler.push_new_code_object("".to_string()); let ast = parser::parse_program(&source.to_string()).unwrap(); - compiler.compile_program(&ast).unwrap(); + let symbol_scope = make_symbol_table(&ast).unwrap(); + compiler.compile_program(&ast, symbol_scope).unwrap(); compiler.pop_code_object() } diff --git a/compiler/src/error.rs b/compiler/src/error.rs new file mode 100644 index 0000000000..bd9950c8d3 --- /dev/null +++ b/compiler/src/error.rs @@ -0,0 +1,67 @@ +use rustpython_parser::error::ParseError; +use rustpython_parser::lexer::Location; + +use std::error::Error; +use std::fmt; + +#[derive(Debug)] +pub struct CompileError { + pub error: CompileErrorType, + pub location: Location, +} + +impl From for CompileError { + fn from(error: ParseError) -> Self { + CompileError { + error: CompileErrorType::Parse(error), + location: Default::default(), // TODO: extract location from parse error! + } + } +} + +#[derive(Debug)] +pub enum CompileErrorType { + /// Invalid assignment, cannot store value in target. + Assign(&'static str), + /// Invalid delete + Delete(&'static str), + /// Expected an expression got a statement + ExpectExpr, + /// Parser error + Parse(ParseError), + SyntaxError(String), + /// Multiple `*` detected + StarArgs, + /// Break statement outside of loop. + InvalidBreak, + /// Continue statement outside of loop. + InvalidContinue, + InvalidReturn, + InvalidYield, +} + +impl fmt::Display for CompileError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.error { + CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target), + CompileErrorType::Delete(target) => write!(f, "can't delete {}", target), + CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"), + CompileErrorType::Parse(err) => write!(f, "{}", err), + CompileErrorType::SyntaxError(err) => write!(f, "{}", err), + CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"), + CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"), + CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"), + CompileErrorType::InvalidReturn => write!(f, "'return' outside function"), + CompileErrorType::InvalidYield => write!(f, "'yield' outside function"), + }?; + + // Print line number: + write!(f, " at line {:?}", self.location.row()) + } +} + +impl Error for CompileError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs new file mode 100644 index 0000000000..e0e6fb2bf1 --- /dev/null +++ b/compiler/src/lib.rs @@ -0,0 +1,9 @@ +//! Compile a Python AST or source code into bytecode consumable by RustPython or +//! (eventually) CPython. + +#[macro_use] +extern crate log; + +pub mod compile; +pub mod error; +mod symboltable; diff --git a/compiler/src/symboltable.rs b/compiler/src/symboltable.rs new file mode 100644 index 0000000000..04e5d9ede5 --- /dev/null +++ b/compiler/src/symboltable.rs @@ -0,0 +1,596 @@ +/* Python code is pre-scanned for symbols in the ast. + +This ensures that global and nonlocal keywords are picked up. +Then the compiler can use the symbol table to generate proper +load and store instructions for names. + +Inspirational file: https://github.com/python/cpython/blob/master/Python/symtable.c +*/ + +use crate::error::{CompileError, CompileErrorType}; +use rustpython_parser::ast; +use rustpython_parser::lexer::Location; +use std::collections::HashMap; + +pub fn make_symbol_table(program: &ast::Program) -> Result { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_program(program)?; + assert_eq!(builder.scopes.len(), 1); + + let symbol_table = builder.scopes.pop().unwrap(); + analyze_symbol_table(&symbol_table, None)?; + Ok(symbol_table) +} + +pub fn statements_to_symbol_table( + statements: &[ast::LocatedStatement], +) -> Result { + let mut builder = SymbolTableBuilder::new(); + builder.enter_scope(); + builder.scan_statements(statements)?; + assert_eq!(builder.scopes.len(), 1); + + let symbol_table = builder.scopes.pop().unwrap(); + analyze_symbol_table(&symbol_table, None)?; + Ok(symbol_table) +} + +#[derive(Debug)] +pub enum SymbolRole { + Global, + Nonlocal, + Used, + Assigned, +} + +/// Captures all symbols in the current scope, and has a list of subscopes in this scope. +pub struct SymbolScope { + /// A set of symbols present on this scope level. + pub symbols: HashMap, + + /// A list of subscopes in the order as found in the + /// AST nodes. + pub sub_scopes: Vec, +} + +#[derive(Debug)] +pub struct SymbolTableError { + error: String, + location: Location, +} + +impl From for CompileError { + fn from(error: SymbolTableError) -> Self { + CompileError { + error: CompileErrorType::SyntaxError(error.error), + location: error.location, + } + } +} + +type SymbolTableResult = Result<(), SymbolTableError>; + +impl SymbolScope { + pub fn new() -> Self { + SymbolScope { + symbols: HashMap::new(), + sub_scopes: vec![], + } + } + + pub fn lookup(&self, name: &str) -> Option<&SymbolRole> { + self.symbols.get(name) + } +} + +impl std::fmt::Debug for SymbolScope { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "SymbolScope({:?} symbols, {:?} sub scopes)", + self.symbols.len(), + self.sub_scopes.len() + ) + } +} + +/* Perform some sort of analysis on nonlocals, globals etc.. + See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410 +*/ +fn analyze_symbol_table( + symbol_scope: &SymbolScope, + parent_symbol_scope: Option<&SymbolScope>, +) -> SymbolTableResult { + // Analyze sub scopes: + for sub_scope in &symbol_scope.sub_scopes { + analyze_symbol_table(&sub_scope, Some(symbol_scope))?; + } + + // Analyze symbols: + for (symbol_name, symbol_role) in &symbol_scope.symbols { + analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?; + } + + Ok(()) +} + +#[allow(clippy::single_match)] +fn analyze_symbol( + symbol_name: &str, + symbol_role: &SymbolRole, + parent_symbol_scope: Option<&SymbolScope>, +) -> SymbolTableResult { + match symbol_role { + SymbolRole::Nonlocal => { + // check if name is defined in parent scope! + if let Some(parent_symbol_scope) = parent_symbol_scope { + if !parent_symbol_scope.symbols.contains_key(symbol_name) { + return Err(SymbolTableError { + error: format!("no binding for nonlocal '{}' found", symbol_name), + location: Default::default(), + }); + } + } else { + return Err(SymbolTableError { + error: format!( + "nonlocal {} defined at place without an enclosing scope", + symbol_name + ), + location: Default::default(), + }); + } + } + // TODO: add more checks for globals + _ => {} + } + Ok(()) +} + +pub struct SymbolTableBuilder { + // Scope stack. + pub scopes: Vec, +} + +impl SymbolTableBuilder { + pub fn new() -> Self { + SymbolTableBuilder { scopes: vec![] } + } + + pub fn enter_scope(&mut self) { + let scope = SymbolScope::new(); + self.scopes.push(scope); + } + + fn leave_scope(&mut self) { + // Pop scope and add to subscopes of parent scope. + let scope = self.scopes.pop().unwrap(); + self.scopes.last_mut().unwrap().sub_scopes.push(scope); + } + + pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { + self.scan_statements(&program.statements)?; + Ok(()) + } + + pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult { + for statement in statements { + self.scan_statement(statement)?; + } + Ok(()) + } + + fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter(parameter)?; + } + Ok(()) + } + + fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + self.register_name(¶meter.arg, SymbolRole::Assigned) + } + + fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { + for parameter in parameters { + self.scan_parameter_annotation(parameter)?; + } + Ok(()) + } + + fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { + if let Some(annotation) = ¶meter.annotation { + self.scan_expression(&annotation)?; + } + Ok(()) + } + + fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult { + match &statement.node { + ast::Statement::Global { names } => { + for name in names { + self.register_name(name, SymbolRole::Global)?; + } + } + ast::Statement::Nonlocal { names } => { + for name in names { + self.register_name(name, SymbolRole::Nonlocal)?; + } + } + ast::Statement::FunctionDef { + name, + body, + args, + decorator_list, + returns, + } + | ast::Statement::AsyncFunctionDef { + name, + body, + args, + decorator_list, + returns, + } => { + self.scan_expressions(decorator_list)?; + self.register_name(name, SymbolRole::Assigned)?; + + self.enter_function(args)?; + + self.scan_statements(body)?; + if let Some(expression) = returns { + self.scan_expression(expression)?; + } + self.leave_scope(); + } + ast::Statement::ClassDef { + name, + body, + bases, + keywords, + decorator_list, + } => { + self.register_name(name, SymbolRole::Assigned)?; + self.enter_scope(); + self.scan_statements(body)?; + self.leave_scope(); + self.scan_expressions(bases)?; + for keyword in keywords { + self.scan_expression(&keyword.value)?; + } + self.scan_expressions(decorator_list)?; + } + ast::Statement::Expression { expression } => self.scan_expression(expression)?, + ast::Statement::If { test, body, orelse } => { + self.scan_expression(test)?; + self.scan_statements(body)?; + if let Some(code) = orelse { + self.scan_statements(code)?; + } + } + ast::Statement::For { + target, + iter, + body, + orelse, + } + | ast::Statement::AsyncFor { + target, + iter, + body, + orelse, + } => { + self.scan_expression(target)?; + self.scan_expression(iter)?; + self.scan_statements(body)?; + if let Some(code) = orelse { + self.scan_statements(code)?; + } + } + ast::Statement::While { test, body, orelse } => { + self.scan_expression(test)?; + self.scan_statements(body)?; + if let Some(code) = orelse { + self.scan_statements(code)?; + } + } + ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => { + // No symbols here. + } + ast::Statement::Import { import_parts } => { + for part in import_parts { + if let Some(alias) = &part.alias { + // `import mymodule as myalias` + self.register_name(alias, SymbolRole::Assigned)?; + } else { + if part.symbols.is_empty() { + // `import module` + self.register_name(&part.module, SymbolRole::Assigned)?; + } else { + // `from mymodule import myimport` + for symbol in &part.symbols { + if let Some(alias) = &symbol.alias { + // `from mymodule import myimportname as myalias` + self.register_name(alias, SymbolRole::Assigned)?; + } else { + self.register_name(&symbol.symbol, SymbolRole::Assigned)?; + } + } + } + } + } + } + ast::Statement::Return { value } => { + if let Some(expression) = value { + self.scan_expression(expression)?; + } + } + ast::Statement::Assert { test, msg } => { + self.scan_expression(test)?; + if let Some(expression) = msg { + self.scan_expression(expression)?; + } + } + ast::Statement::Delete { targets } => { + self.scan_expressions(targets)?; + } + ast::Statement::Assign { targets, value } => { + self.scan_expressions(targets)?; + self.scan_expression(value)?; + } + ast::Statement::AugAssign { target, value, .. } => { + self.scan_expression(target)?; + self.scan_expression(value)?; + } + ast::Statement::With { items, body } => { + for item in items { + self.scan_expression(&item.context_expr)?; + if let Some(expression) = &item.optional_vars { + self.scan_expression(expression)?; + } + } + self.scan_statements(body)?; + } + ast::Statement::Try { + body, + handlers, + orelse, + finalbody, + } => { + self.scan_statements(body)?; + for handler in handlers { + if let Some(expression) = &handler.typ { + self.scan_expression(expression)?; + } + if let Some(name) = &handler.name { + self.register_name(name, SymbolRole::Assigned)?; + } + self.scan_statements(&handler.body)?; + } + if let Some(code) = orelse { + self.scan_statements(code)?; + } + if let Some(code) = finalbody { + self.scan_statements(code)?; + } + } + ast::Statement::Raise { exception, cause } => { + if let Some(expression) = exception { + self.scan_expression(expression)?; + } + if let Some(expression) = cause { + self.scan_expression(expression)?; + } + } + } + Ok(()) + } + + fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult { + for expression in expressions { + self.scan_expression(expression)?; + } + Ok(()) + } + + fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult { + match expression { + ast::Expression::Binop { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; + } + ast::Expression::BoolOp { a, b, .. } => { + self.scan_expression(a)?; + self.scan_expression(b)?; + } + ast::Expression::Compare { vals, .. } => { + self.scan_expressions(vals)?; + } + ast::Expression::Subscript { a, b } => { + self.scan_expression(a)?; + self.scan_expression(b)?; + } + ast::Expression::Attribute { value, .. } => { + self.scan_expression(value)?; + } + ast::Expression::Dict { elements } => { + for (key, value) in elements { + if let Some(key) = key { + self.scan_expression(key)?; + } else { + // dict unpacking marker + } + self.scan_expression(value)?; + } + } + ast::Expression::Await { value } => { + self.scan_expression(value)?; + } + ast::Expression::Yield { value } => { + if let Some(expression) = value { + self.scan_expression(expression)?; + } + } + ast::Expression::YieldFrom { value } => { + self.scan_expression(value)?; + } + ast::Expression::Unop { a, .. } => { + self.scan_expression(a)?; + } + ast::Expression::True + | ast::Expression::False + | ast::Expression::None + | ast::Expression::Ellipsis => {} + ast::Expression::Number { .. } => {} + ast::Expression::Starred { value } => { + self.scan_expression(value)?; + } + ast::Expression::Bytes { .. } => {} + ast::Expression::Tuple { elements } + | ast::Expression::Set { elements } + | ast::Expression::List { elements } + | ast::Expression::Slice { elements } => { + self.scan_expressions(elements)?; + } + ast::Expression::Comprehension { kind, generators } => { + match **kind { + ast::ComprehensionKind::GeneratorExpression { ref element } + | ast::ComprehensionKind::List { ref element } + | ast::ComprehensionKind::Set { ref element } => { + self.scan_expression(element)?; + } + ast::ComprehensionKind::Dict { ref key, ref value } => { + self.scan_expression(&key)?; + self.scan_expression(&value)?; + } + } + + for generator in generators { + self.scan_expression(&generator.target)?; + self.scan_expression(&generator.iter)?; + for if_expr in &generator.ifs { + self.scan_expression(if_expr)?; + } + } + } + ast::Expression::Call { + function, + args, + keywords, + } => { + self.scan_expression(function)?; + self.scan_expressions(args)?; + for keyword in keywords { + self.scan_expression(&keyword.value)?; + } + } + ast::Expression::String { value } => { + self.scan_string_group(value)?; + } + ast::Expression::Identifier { name } => { + self.register_name(name, SymbolRole::Used)?; + } + ast::Expression::Lambda { args, body } => { + self.enter_function(args)?; + self.scan_expression(body)?; + self.leave_scope(); + } + ast::Expression::IfExpression { test, body, orelse } => { + self.scan_expression(test)?; + self.scan_expression(body)?; + self.scan_expression(orelse)?; + } + } + Ok(()) + } + + fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { + // Evaluate eventual default parameters: + self.scan_expressions(&args.defaults)?; + for kw_default in &args.kw_defaults { + if let Some(expression) = kw_default { + self.scan_expression(&expression)?; + } + } + + // Annotations are scanned in outer scope: + self.scan_parameters_annotations(&args.args)?; + self.scan_parameters_annotations(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter_annotation(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter_annotation(name)?; + } + + self.enter_scope(); + + // Fill scope with parameter names: + self.scan_parameters(&args.args)?; + self.scan_parameters(&args.kwonlyargs)?; + if let ast::Varargs::Named(name) = &args.vararg { + self.scan_parameter(name)?; + } + if let ast::Varargs::Named(name) = &args.kwarg { + self.scan_parameter(name)?; + } + Ok(()) + } + + fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult { + match group { + ast::StringGroup::Constant { .. } => {} + ast::StringGroup::FormattedValue { value, .. } => { + self.scan_expression(value)?; + } + ast::StringGroup::Joined { values } => { + for subgroup in values { + self.scan_string_group(subgroup)?; + } + } + } + Ok(()) + } + + #[allow(clippy::single_match)] + fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { + let scope_depth = self.scopes.len(); + let current_scope = self.scopes.last_mut().unwrap(); + let location = Default::default(); + if current_scope.symbols.contains_key(name) { + // Role already set.. + match role { + SymbolRole::Global => { + return Err(SymbolTableError { + error: format!("name '{}' is used prior to global declaration", name), + location, + }) + } + SymbolRole::Nonlocal => { + return Err(SymbolTableError { + error: format!("name '{}' is used prior to nonlocal declaration", name), + location, + }) + } + _ => { + // Ok? + } + } + } else { + match role { + SymbolRole::Nonlocal => { + if scope_depth < 2 { + return Err(SymbolTableError { + error: format!("cannot define nonlocal '{}' at top level.", name), + location, + }); + } + } + _ => { + // Ok! + } + } + current_scope.symbols.insert(name.to_string(), role); + } + Ok(()) + } +} diff --git a/derive/Cargo.toml b/derive/Cargo.toml index c15d47ad58..fa304d0bdd 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,13 +1,17 @@ [package] -name = "rustpython_derive" +name = "rustpython-derive" version = "0.1.0" -authors = ["Joey "] +authors = ["RustPython Team"] edition = "2018" [lib] proc-macro = true [dependencies] -syn = "0.15.29" +syn = { version = "0.15.29", features = ["full"] } quote = "0.6.11" proc-macro2 = "0.4.27" +rustpython-compiler = { path = "../compiler" } +rustpython-bytecode = { path = "../bytecode" } +bincode = "1.1" +proc-macro-hack = "0.5" diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs new file mode 100644 index 0000000000..8fa9f0c7f1 --- /dev/null +++ b/derive/src/compile_bytecode.rs @@ -0,0 +1,174 @@ +//! Parsing and processing for this form: +//! ```ignore +//! py_compile_input!( +//! // either: +//! source = "python_source_code", +//! // or +//! file = "file/path/relative/to/$CARGO_MANIFEST_DIR", +//! +//! // the mode to compile the code in +//! mode = "exec", // or "eval" or "single" +//! // the path put into the CodeObject, defaults to "frozen" +//! module_name = "frozen", +//! ) +//! ``` + +use crate::{extract_spans, Diagnostic}; +use bincode; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::quote; +use rustpython_bytecode::bytecode::CodeObject; +use rustpython_compiler::compile; +use std::env; +use std::fs; +use std::path::PathBuf; +use syn::parse::{Parse, ParseStream, Result as ParseResult}; +use syn::{self, parse2, Lit, LitByteStr, Meta, Token}; + +enum CompilationSourceKind { + File(PathBuf), + SourceCode(String), +} + +struct CompilationSource { + kind: CompilationSourceKind, + span: (Span, Span), +} + +impl CompilationSource { + fn compile(self, mode: &compile::Mode, module_name: String) -> Result { + let compile = |source| { + compile::compile(source, mode, module_name).map_err(|err| { + Diagnostic::spans_error(self.span, format!("Compile error: {}", err)) + }) + }; + + match &self.kind { + CompilationSourceKind::File(rel_path) => { + let mut path = PathBuf::from( + env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"), + ); + path.push(rel_path); + let source = fs::read_to_string(&path).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Error reading file {:?}: {}", path, err), + ) + })?; + compile(&source) + } + CompilationSourceKind::SourceCode(code) => compile(code), + } + } +} + +/// This is essentially just a comma-separated list of Meta nodes, aka the inside of a MetaList. +struct PyCompileInput { + span: Span, + metas: Vec, +} + +impl PyCompileInput { + fn compile(&self) -> Result { + let mut module_name = None; + let mut mode = None; + let mut source: Option = None; + + fn assert_source_empty(source: &Option) -> Result<(), Diagnostic> { + if let Some(source) = source { + Err(Diagnostic::spans_error( + source.span.clone(), + "Cannot have more than one source", + )) + } else { + Ok(()) + } + } + + for meta in &self.metas { + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "mode" { + mode = Some(match &name_value.lit { + Lit::Str(s) => match s.value().as_str() { + "exec" => compile::Mode::Exec, + "eval" => compile::Mode::Eval, + "single" => compile::Mode::Single, + _ => bail_span!(s, "mode must be exec, eval, or single"), + }, + _ => bail_span!(name_value.lit, "mode must be a string"), + }) + } else if name_value.ident == "module_name" { + module_name = Some(match &name_value.lit { + Lit::Str(s) => s.value(), + _ => bail_span!(name_value.lit, "module_name must be string"), + }) + } else if name_value.ident == "source" { + assert_source_empty(&source)?; + let code = match &name_value.lit { + Lit::Str(s) => s.value(), + _ => bail_span!(name_value.lit, "source must be a string"), + }; + source = Some(CompilationSource { + kind: CompilationSourceKind::SourceCode(code), + span: extract_spans(&name_value).unwrap(), + }); + } else if name_value.ident == "file" { + assert_source_empty(&source)?; + let path = match &name_value.lit { + Lit::Str(s) => PathBuf::from(s.value()), + _ => bail_span!(name_value.lit, "source must be a string"), + }; + source = Some(CompilationSource { + kind: CompilationSourceKind::File(path), + span: extract_spans(&name_value).unwrap(), + }); + } + } + _ => {} + } + } + + source + .ok_or_else(|| { + Diagnostic::span_error( + self.span.clone(), + "Must have either file or source in py_compile_bytecode!()", + ) + })? + .compile( + &mode.unwrap_or(compile::Mode::Exec), + module_name.unwrap_or_else(|| "frozen".to_string()), + ) + } +} + +impl Parse for PyCompileInput { + fn parse(input: ParseStream) -> ParseResult { + let span = input.cursor().span(); + let metas = input + .parse_terminated::(Meta::parse)? + .into_iter() + .collect(); + Ok(PyCompileInput { span, metas }) + } +} + +pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result { + let input: PyCompileInput = parse2(input)?; + + let code_obj = input.compile()?; + + let bytes = bincode::serialize(&code_obj).expect("Failed to serialize"); + let bytes = LitByteStr::new(&bytes, Span::call_site()); + + let output = quote! { + ({ + use ::rustpython_vm::__exports::bincode; + bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes) + .expect("Deserializing CodeObject failed") + }) + }; + + Ok(output) +} diff --git a/derive/src/error.rs b/derive/src/error.rs new file mode 100644 index 0000000000..b9f25b63ac --- /dev/null +++ b/derive/src/error.rs @@ -0,0 +1,167 @@ +// Taken from https://github.com/rustwasm/wasm-bindgen/blob/master/crates/backend/src/error.rs +// +// Copyright (c) 2014 Alex Crichton +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +#![allow(dead_code)] + +use proc_macro2::*; +use quote::{ToTokens, TokenStreamExt}; +use syn::parse::Error; + +macro_rules! err_span { + ($span:expr, $($msg:tt)*) => ( + $crate::Diagnostic::spanned_error(&$span, format!($($msg)*)) + ) +} + +macro_rules! bail_span { + ($($t:tt)*) => ( + return Err(err_span!($($t)*).into()) + ) +} + +macro_rules! push_err_span { + ($diagnostics:expr, $($t:tt)*) => { + $diagnostics.push(err_span!($($t)*)) + }; +} + +#[derive(Debug)] +pub struct Diagnostic { + inner: Repr, +} + +#[derive(Debug)] +enum Repr { + Single { + text: String, + span: Option<(Span, Span)>, + }, + SynError(Error), + Multi { + diagnostics: Vec, + }, +} + +impl Diagnostic { + pub fn error>(text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: None, + }, + } + } + + pub fn span_error>(span: Span, text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: Some((span, span)), + }, + } + } + + pub fn spans_error>(spans: (Span, Span), text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: Some(spans), + }, + } + } + + pub fn spanned_error>(node: &ToTokens, text: T) -> Diagnostic { + Diagnostic { + inner: Repr::Single { + text: text.into(), + span: extract_spans(node), + }, + } + } + + pub fn from_vec(diagnostics: Vec) -> Result<(), Diagnostic> { + if diagnostics.len() == 0 { + Ok(()) + } else { + Err(Diagnostic { + inner: Repr::Multi { diagnostics }, + }) + } + } + + #[allow(unconditional_recursion)] + pub fn panic(&self) -> ! { + match &self.inner { + Repr::Single { text, .. } => panic!("{}", text), + Repr::SynError(error) => panic!("{}", error), + Repr::Multi { diagnostics } => diagnostics[0].panic(), + } + } +} + +impl From for Diagnostic { + fn from(err: Error) -> Diagnostic { + Diagnostic { + inner: Repr::SynError(err), + } + } +} + +pub fn extract_spans(node: &dyn ToTokens) -> Option<(Span, Span)> { + let mut t = TokenStream::new(); + node.to_tokens(&mut t); + let mut tokens = t.into_iter(); + let start = tokens.next().map(|t| t.span()); + let end = tokens.last().map(|t| t.span()); + start.map(|start| (start, end.unwrap_or(start))) +} + +impl ToTokens for Diagnostic { + fn to_tokens(&self, dst: &mut TokenStream) { + match &self.inner { + Repr::Single { text, span } => { + let cs2 = (Span::call_site(), Span::call_site()); + let (start, end) = span.unwrap_or(cs2); + dst.append(Ident::new("compile_error", start)); + dst.append(Punct::new('!', Spacing::Alone)); + let mut message = TokenStream::new(); + message.append(Literal::string(text)); + let mut group = Group::new(Delimiter::Brace, message); + group.set_span(end); + dst.append(group); + } + Repr::Multi { diagnostics } => { + for diagnostic in diagnostics { + diagnostic.to_tokens(dst); + } + } + Repr::SynError(err) => { + err.to_compile_error().to_tokens(dst); + } + } + } +} diff --git a/derive/src/from_args.rs b/derive/src/from_args.rs new file mode 100644 index 0000000000..fe326fb555 --- /dev/null +++ b/derive/src/from_args.rs @@ -0,0 +1,221 @@ +use crate::Diagnostic; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{ + parse_quote, Attribute, Data, DeriveInput, Expr, Field, Fields, Ident, Lit, Meta, NestedMeta, +}; + +/// The kind of the python parameter, this corresponds to the value of Parameter.kind +/// (https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind) +enum ParameterKind { + PositionalOnly, + PositionalOrKeyword, + KeywordOnly, +} + +impl ParameterKind { + fn from_ident(ident: &Ident) -> Option { + if ident == "positional_only" { + Some(ParameterKind::PositionalOnly) + } else if ident == "positional_or_keyword" { + Some(ParameterKind::PositionalOrKeyword) + } else if ident == "keyword_only" { + Some(ParameterKind::KeywordOnly) + } else { + None + } + } +} + +struct ArgAttribute { + kind: ParameterKind, + default: Option, + optional: bool, +} + +impl ArgAttribute { + fn from_attribute(attr: &Attribute) -> Option> { + if !attr.path.is_ident("pyarg") { + return None; + } + let inner = move || match attr.parse_meta()? { + Meta::List(list) => { + let mut iter = list.nested.iter(); + let first_arg = iter.next().ok_or_else(|| { + err_span!(list, "There must be at least one argument to #[pyarg()]") + })?; + let kind = match first_arg { + NestedMeta::Meta(Meta::Word(ident)) => ParameterKind::from_ident(ident), + _ => None, + }; + let kind = kind.ok_or_else(|| { + err_span!( + first_arg, + "The first argument to #[pyarg()] must be the parameter type, either \ + 'positional_only', 'positional_or_keyword', or 'keyword_only'." + ) + })?; + + let mut attribute = ArgAttribute { + kind, + default: None, + optional: false, + }; + + while let Some(arg) = iter.next() { + attribute.parse_argument(arg)?; + } + + if attribute.default.is_some() && attribute.optional { + bail_span!(attr, "Can't set both a default value and optional"); + } + + Ok(attribute) + } + _ => bail_span!(attr, "pyarg must be a list, like #[pyarg(...)]"), + }; + Some(inner()) + } + + fn parse_argument(&mut self, arg: &NestedMeta) -> Result<(), Diagnostic> { + match arg { + NestedMeta::Meta(Meta::Word(ident)) => { + if ident == "default" { + if self.default.is_some() { + bail_span!(ident, "Default already set"); + } + let expr = parse_quote!(Default::default()); + self.default = Some(expr); + } else if ident == "optional" { + self.optional = true; + } else { + bail_span!(ident, "Unrecognised pyarg attribute"); + } + } + NestedMeta::Meta(Meta::NameValue(name_value)) => { + if name_value.ident == "default" { + if self.default.is_some() { + bail_span!(name_value, "Default already set"); + } + + match name_value.lit { + Lit::Str(ref val) => { + let expr = val.parse::().map_err(|_| { + err_span!(val, "Expected a valid expression for default argument") + })?; + self.default = Some(expr); + } + _ => bail_span!(name_value, "Expected string value for default argument"), + } + } else if name_value.ident == "optional" { + match name_value.lit { + Lit::Bool(ref val) => { + self.optional = val.value; + } + _ => bail_span!( + name_value.lit, + "Expected boolean value for optional argument" + ), + } + } else { + bail_span!(name_value, "Unrecognised pyarg attribute"); + } + } + _ => bail_span!(arg, "Unrecognised pyarg attribute"), + } + + Ok(()) + } +} + +fn generate_field(field: &Field) -> Result { + let mut pyarg_attrs = field + .attrs + .iter() + .filter_map(ArgAttribute::from_attribute) + .collect::, _>>()?; + let attr = if pyarg_attrs.is_empty() { + ArgAttribute { + kind: ParameterKind::PositionalOrKeyword, + default: None, + optional: false, + } + } else if pyarg_attrs.len() == 1 { + pyarg_attrs.remove(0) + } else { + bail_span!(field, "Multiple pyarg attributes on field"); + }; + + let name = &field.ident; + let middle = quote! { + .map(|x| ::rustpython_vm::pyobject::TryFromObject::try_from_object(vm, x)).transpose()? + }; + let ending = if let Some(default) = attr.default { + quote! { + .unwrap_or_else(|| #default) + } + } else if attr.optional { + quote! { + .map(::rustpython_vm::function::OptionalArg::Present) + .unwrap_or(::rustpython_vm::function::OptionalArg::Missing) + } + } else { + let err = match attr.kind { + ParameterKind::PositionalOnly | ParameterKind::PositionalOrKeyword => quote! { + ::rustpython_vm::function::ArgumentError::TooFewArgs + }, + ParameterKind::KeywordOnly => quote! { + ::rustpython_vm::function::ArgumentError::RequiredKeywordArgument(tringify!(#name)) + }, + }; + quote! { + .ok_or_else(|| #err)? + } + }; + + let file_output = match attr.kind { + ParameterKind::PositionalOnly => { + quote! { + #name: args.take_positional()#middle#ending, + } + } + ParameterKind::PositionalOrKeyword => { + quote! { + #name: args.take_positional_keyword(stringify!(#name))#middle#ending, + } + } + ParameterKind::KeywordOnly => { + quote! { + #name: args.take_keyword(stringify!(#name))#middle#ending, + } + } + }; + Ok(file_output) +} + +pub fn impl_from_args(input: DeriveInput) -> Result { + let fields = match input.data { + Data::Struct(syn::DataStruct { + fields: Fields::Named(fields), + .. + }) => fields + .named + .iter() + .map(generate_field) + .collect::>()?, + _ => bail_span!(input, "FromArgs input must be a struct with named fields"), + }; + + let name = input.ident; + let output = quote! { + impl ::rustpython_vm::function::FromArgs for #name { + fn from_args( + vm: &::rustpython_vm::VirtualMachine, + args: &mut ::rustpython_vm::function::PyFuncArgs + ) -> Result { + Ok(#name { #fields }) + } + } + }; + Ok(output) +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs index ad743e72fe..21ac982ccf 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,48 +1,55 @@ +#![recursion_limit = "128"] + extern crate proc_macro; +#[macro_use] +mod error; +mod compile_bytecode; +mod from_args; +mod pyclass; + +use error::{extract_spans, Diagnostic}; use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; -use quote::quote; -use syn::{Data, DeriveInput, Fields}; +use proc_macro_hack::proc_macro_hack; +use quote::ToTokens; +use syn::{parse_macro_input, AttributeArgs, DeriveInput, Item}; + +fn result_to_tokens(result: Result) -> TokenStream { + match result { + Ok(tokens) => tokens.into(), + Err(diagnostic) => diagnostic.into_token_stream().into(), + } +} -#[proc_macro_derive(FromArgs)] +#[proc_macro_derive(FromArgs, attributes(pyarg))] pub fn derive_from_args(input: TokenStream) -> TokenStream { - let ast: DeriveInput = syn::parse(input).unwrap(); + let input = parse_macro_input!(input as DeriveInput); + result_to_tokens(from_args::impl_from_args(input)) +} - let gen = impl_from_args(&ast); - gen.to_string().parse().unwrap() +#[proc_macro_attribute] +pub fn pyclass(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + result_to_tokens(pyclass::impl_pyclass(attr, item)) } -fn impl_from_args(input: &DeriveInput) -> TokenStream2 { - // FIXME: This references types using `crate` instead of `rustpython_vm` - // so that it can be used in the latter. How can we support both? - let fields = match input.data { - Data::Struct(ref data) => { - match data.fields { - Fields::Named(ref fields) => fields.named.iter().map(|field| { - let name = &field.ident; - quote! { - #name: crate::pyobject::TryFromObject::try_from_object( - vm, - args.take_keyword(stringify!(#name)).unwrap_or_else(|| vm.ctx.none()) - )?, - } - }), - Fields::Unnamed(_) | Fields::Unit => unimplemented!(), // TODO: better error message - } - } - Data::Enum(_) | Data::Union(_) => unimplemented!(), // TODO: better error message - }; - - let name = &input.ident; - quote! { - impl crate::function::FromArgs for #name { - fn from_args( - vm: &crate::vm::VirtualMachine, - args: &mut crate::function::PyFuncArgs - ) -> Result { - Ok(#name { #(#fields)* }) - } - } - } +#[proc_macro_attribute] +pub fn pyimpl(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + result_to_tokens(pyclass::impl_pyimpl(attr, item)) +} + +#[proc_macro_attribute] +pub fn pystruct_sequence(attr: TokenStream, item: TokenStream) -> TokenStream { + let attr = parse_macro_input!(attr as AttributeArgs); + let item = parse_macro_input!(item as Item); + result_to_tokens(pyclass::impl_pystruct_sequence(attr, item)) +} + +#[proc_macro_hack] +pub fn py_compile_bytecode(input: TokenStream) -> TokenStream { + result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into())) } diff --git a/derive/src/pyclass.rs b/derive/src/pyclass.rs new file mode 100644 index 0000000000..65c3c63134 --- /dev/null +++ b/derive/src/pyclass.rs @@ -0,0 +1,463 @@ +use super::Diagnostic; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use std::collections::HashMap; +use syn::{ + Attribute, AttributeArgs, Ident, ImplItem, Index, Item, Lit, Meta, MethodSig, NestedMeta, +}; + +enum ClassItem { + Method { + item_ident: Ident, + py_name: String, + }, + ClassMethod { + item_ident: Ident, + py_name: String, + }, + Property { + item_ident: Ident, + py_name: String, + setter: bool, + }, +} + +fn meta_to_vec(meta: Meta) -> Result, Meta> { + match meta { + Meta::Word(_) => Ok(Vec::new()), + Meta::List(list) => Ok(list.nested.into_iter().collect()), + Meta::NameValue(_) => Err(meta), + } +} + +impl ClassItem { + fn extract_from_syn( + attrs: &mut Vec, + sig: &MethodSig, + ) -> Result, Diagnostic> { + let mut item = None; + let mut attr_idx = None; + for (i, meta) in attrs + .iter() + .filter_map(|attr| attr.parse_meta().ok()) + .enumerate() + { + let name = meta.name(); + if name == "pymethod" { + if item.is_some() { + bail_span!( + sig.ident, + "You can only have one #[py*] attribute on an impl item" + ) + } + let nesteds = meta_to_vec(meta).map_err(|meta| { + err_span!( + meta, + "#[pymethod = \"...\"] cannot be a name/value, you probably meant \ + #[pymethod(name = \"...\")]", + ) + })?; + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => continue, + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + bail_span!( + &sig.ident, + "#[pymethod(name = ...)] must be a string" + ); + } + } + } + _ => {} + } + } + item = Some(ClassItem::Method { + item_ident: sig.ident.clone(), + py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), + }); + attr_idx = Some(i); + } else if name == "pyclassmethod" { + if item.is_some() { + bail_span!( + sig.ident, + "You can only have one #[py*] attribute on an impl item" + ) + } + let nesteds = meta_to_vec(meta).map_err(|meta| { + err_span!( + meta, + "#[pyclassmethod = \"...\"] cannot be a name/value, you probably meant \ + #[pyclassmethod(name = \"...\")]", + ) + })?; + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => continue, + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + bail_span!( + &sig.ident, + "#[pyclassmethod(name = ...)] must be a string" + ); + } + } + } + _ => {} + } + } + item = Some(ClassItem::ClassMethod { + item_ident: sig.ident.clone(), + py_name: py_name.unwrap_or_else(|| sig.ident.to_string()), + }); + attr_idx = Some(i); + } else if name == "pyproperty" { + if item.is_some() { + bail_span!( + sig.ident, + "You can only have one #[py*] attribute on an impl item" + ) + } + let nesteds = meta_to_vec(meta).map_err(|meta| { + err_span!( + meta, + "#[pyproperty = \"...\"] cannot be a name/value, you probably meant \ + #[pyproperty(name = \"...\")]" + ) + })?; + let mut setter = false; + let mut py_name = None; + for meta in nesteds { + let meta = match meta { + NestedMeta::Meta(meta) => meta, + NestedMeta::Literal(_) => continue, + }; + match meta { + Meta::NameValue(name_value) => { + if name_value.ident == "name" { + if let Lit::Str(s) = &name_value.lit { + py_name = Some(s.value()); + } else { + bail_span!( + &sig.ident, + "#[pyproperty(name = ...)] must be a string" + ); + } + } + } + Meta::Word(ident) => { + if ident == "setter" { + setter = true; + } + } + _ => {} + } + } + let py_name = match py_name { + Some(py_name) => py_name, + None => { + let item_ident = sig.ident.to_string(); + if setter { + if item_ident.starts_with("set_") { + let name = &item_ident["set_".len()..]; + if name.is_empty() { + bail_span!( + &sig.ident, + "A #[pyproperty(setter)] fn with a set_* name must \ + have something after \"set_\"" + ) + } else { + name.to_string() + } + } else { + bail_span!( + &sig.ident, + "A #[pyproperty(setter)] fn must either have a `name` \ + parameter or a fn name along the lines of \"set_*\"" + ) + } + } else { + item_ident + } + } + }; + item = Some(ClassItem::Property { + py_name, + item_ident: sig.ident.clone(), + setter, + }); + attr_idx = Some(i); + } + } + if let Some(attr_idx) = attr_idx { + attrs.remove(attr_idx); + } + Ok(item) + } +} + +pub fn impl_pyimpl(_attr: AttributeArgs, item: Item) -> Result { + let mut imp = if let Item::Impl(imp) = item { + imp + } else { + return Ok(quote!(#item)); + }; + + let mut diagnostics: Vec = Vec::new(); + + let items = imp + .items + .iter_mut() + .filter_map(|item| { + if let ImplItem::Method(meth) = item { + ClassItem::extract_from_syn(&mut meth.attrs, &meth.sig) + .map_err(|err| diagnostics.push(err)) + .unwrap_or_default() + } else { + None + } + }) + .collect::>(); + let ty = &imp.self_ty; + let mut properties: HashMap<&str, (Option<&Ident>, Option<&Ident>)> = HashMap::new(); + for item in items.iter() { + match item { + ClassItem::Property { + ref item_ident, + ref py_name, + setter, + } => { + let entry = properties.entry(py_name).or_default(); + let func = if *setter { &mut entry.1 } else { &mut entry.0 }; + if func.is_some() { + bail_span!( + item_ident, + "Multiple property accessors with name {:?}", + py_name + ) + } + *func = Some(item_ident); + } + _ => {} + } + } + let methods = items.iter().filter_map(|item| match item { + ClassItem::Method { + item_ident, + py_name, + } => Some(quote! { + class.set_str_attr(#py_name, ctx.new_rustfunc(Self::#item_ident)); + }), + ClassItem::ClassMethod { + item_ident, + py_name, + } => Some(quote! { + class.set_str_attr(#py_name, ctx.new_classmethod(Self::#item_ident)); + }), + _ => None, + }); + let properties = properties + .iter() + .map(|(name, prop)| { + let getter = match prop.0 { + Some(getter) => getter, + None => { + push_err_span!( + diagnostics, + prop.1.unwrap(), + "Property {:?} is missing a getter", + name + ); + return TokenStream2::new(); + } + }; + let add_setter = prop.1.map(|setter| quote!(.add_setter(Self::#setter))); + quote! { + class.set_str_attr( + #name, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(Self::#getter) + #add_setter + .create(), + ); + } + }) + .collect::>(); + + Diagnostic::from_vec(diagnostics)?; + + let ret = quote! { + #imp + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #(#methods)* + #(#properties)* + } + } + }; + Ok(ret) +} + +fn generate_class_def( + ident: &Ident, + attr_name: &'static str, + attr: AttributeArgs, + attrs: &Vec, +) -> Result { + let mut class_name = None; + for attr in attr { + if let NestedMeta::Meta(meta) = attr { + if let Meta::NameValue(name_value) = meta { + if name_value.ident == "name" { + if let Lit::Str(s) = name_value.lit { + class_name = Some(s.value()); + } else { + bail_span!( + name_value.lit, + "#[{}(name = ...)] must be a string", + attr_name + ); + } + } + } + } + } + let class_name = class_name.unwrap_or_else(|| ident.to_string()); + + let mut doc: Option> = None; + for attr in attrs.iter() { + if attr.path.is_ident("doc") { + let meta = attr.parse_meta().expect("expected doc attr to be a meta"); + if let Meta::NameValue(name_value) = meta { + if let Lit::Str(s) = name_value.lit { + let val = s.value().trim().to_string(); + match doc { + Some(ref mut doc) => doc.push(val), + None => doc = Some(vec![val]), + } + } + } + } + } + let doc = match doc { + Some(doc) => { + let doc = doc.join("\n"); + quote!(Some(#doc)) + } + None => quote!(None), + }; + + let ret = quote! { + impl ::rustpython_vm::pyobject::PyClassDef for #ident { + const NAME: &'static str = #class_name; + const DOC: Option<&'static str> = #doc; + } + }; + Ok(ret) +} + +pub fn impl_pyclass(attr: AttributeArgs, item: Item) -> Result { + let (item, ident, attrs) = match item { + Item::Struct(struc) => (quote!(#struc), struc.ident, struc.attrs), + Item::Enum(enu) => (quote!(#enu), enu.ident, enu.attrs), + other => bail_span!( + other, + "#[pyclass] can only be on a struct or enum declaration" + ), + }; + + let class_def = generate_class_def(&ident, "pyclass", attr, &attrs)?; + + let ret = quote! { + #item + #class_def + }; + Ok(ret) +} + +pub fn impl_pystruct_sequence(attr: AttributeArgs, item: Item) -> Result { + let struc = if let Item::Struct(struc) = item { + struc + } else { + bail_span!( + item, + "#[pystruct_sequence] can only be on a struct declaration" + ) + }; + let class_def = generate_class_def(&struc.ident, "pystruct_sequence", attr, &struc.attrs)?; + let mut properties = Vec::new(); + let mut field_names = Vec::new(); + for (i, field) in struc.fields.iter().enumerate() { + let idx = Index::from(i); + if let Some(ref field_name) = field.ident { + let field_name_str = field_name.to_string(); + // TODO add doc to the generated property + let property = quote! { + class.set_str_attr( + #field_name_str, + ::rustpython_vm::obj::objproperty::PropertyBuilder::new(ctx) + .add_getter(|zelf: &::rustpython_vm::obj::objtuple::PyTuple, + _vm: &::rustpython_vm::vm::VirtualMachine| + zelf.fast_getitem(#idx)) + .create(), + ); + }; + properties.push(property); + field_names.push(quote!(#field_name)); + } else { + field_names.push(quote!(#idx)); + } + } + + let ty = &struc.ident; + let ret = quote! { + #struc + #class_def + impl #ty { + fn into_struct_sequence(&self, + vm: &::rustpython_vm::vm::VirtualMachine, + cls: ::rustpython_vm::obj::objtype::PyClassRef, + ) -> ::rustpython_vm::pyobject::PyResult<::rustpython_vm::obj::objtuple::PyTupleRef> { + let tuple: ::rustpython_vm::obj::objtuple::PyTuple = + vec![#(::rustpython_vm::pyobject::IntoPyObject + ::into_pyobject(self.#field_names, vm)? + ),*].into(); + ::rustpython_vm::pyobject::PyValue::into_ref_with_type(tuple, vm, cls) + } + } + impl ::rustpython_vm::pyobject::PyClassImpl for #ty { + fn impl_extend_class( + ctx: &::rustpython_vm::pyobject::PyContext, + class: &::rustpython_vm::obj::objtype::PyClassRef, + ) { + #(#properties)* + } + + fn make_class( + ctx: &::rustpython_vm::pyobject::PyContext + ) -> ::rustpython_vm::obj::objtype::PyClassRef { + let py_class = ctx.new_class(::NAME, ctx.tuple_type()); + Self::extend_class(ctx, &py_class); + py_class + } + } + }; + Ok(ret) +} diff --git a/logo.png b/logo.png new file mode 100644 index 0000000000..7d85f5d81a Binary files /dev/null and b/logo.png differ diff --git a/parser/.gitignore b/parser/.gitignore deleted file mode 100644 index 401e8cc22b..0000000000 --- a/parser/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -src/python.rs -target/ -Cargo.lock - diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e57d421406..af0b19d39f 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -1,17 +1,20 @@ [package] -name = "rustpython_parser" +name = "rustpython-parser" version = "0.0.1" -authors = [ "Shing Lyu", "Windel Bouwman" ] +authors = [ "RustPython Team" ] build = "build.rs" edition = "2018" [build-dependencies] -lalrpop="0.15.1" +lalrpop="0.16.3" [dependencies] -lalrpop-util="0.15.1" +lalrpop-util="0.16.3" log="0.4.1" regex="0.2.2" num-bigint = "0.2" num-traits = "0.2" - +unicode-xid = "0.1.0" +unic-emoji-char = "0.9.0" +serde = { version = "1.0.66", features = ["derive"] } +wtf8 = "0.0.3" diff --git a/parser/build.rs b/parser/build.rs index d35ace0c07..19f1027217 100644 --- a/parser/build.rs +++ b/parser/build.rs @@ -1,5 +1,5 @@ use lalrpop; fn main() { - lalrpop::process_root().unwrap(); + lalrpop::process_root().unwrap() } diff --git a/parser/src/ast.rs b/parser/src/ast.rs index 257ceadf2a..0a32fe3bfa 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -4,6 +4,8 @@ pub use super::lexer::Location; use num_bigint::BigInt; +use serde::{Deserialize, Serialize}; + /* #[derive(Debug)] @@ -16,7 +18,7 @@ pub struct Node { #[derive(Debug, PartialEq)] pub enum Top { Program(Program), - Statement(LocatedStatement), + Statement(Vec), Expression(Expression), } @@ -25,12 +27,18 @@ pub struct Program { pub statements: Vec, } +#[derive(Debug, PartialEq)] +pub struct ImportSymbol { + pub symbol: String, + pub alias: Option, +} + #[derive(Debug, PartialEq)] pub struct SingleImport { pub module: String, - // (symbol name in module, name it should be assigned locally) - pub symbol: Option, pub alias: Option, + pub symbols: Vec, + pub level: usize, } #[derive(Debug, PartialEq)] @@ -42,12 +50,13 @@ pub struct Located { pub type LocatedStatement = Located; /// Abstract syntax tree nodes for python statements. +#[allow(clippy::large_enum_variant)] #[derive(Debug, PartialEq)] pub enum Statement { Break, Continue, Return { - value: Option>, + value: Option>, }, Import { import_parts: Vec, @@ -94,7 +103,13 @@ pub enum Statement { }, For { target: Expression, - iter: Vec, + iter: Expression, + body: Vec, + orelse: Option>, + }, + AsyncFor { + target: Expression, + iter: Expression, body: Vec, orelse: Option>, }, @@ -114,12 +129,17 @@ pub enum Statement { bases: Vec, keywords: Vec, decorator_list: Vec, - // TODO: docstring: String, }, FunctionDef { name: String, args: Parameters, - // docstring: String, + body: Vec, + decorator_list: Vec, + returns: Option, + }, + AsyncFunctionDef { + name: String, + args: Parameters, body: Vec, decorator_list: Vec, returns: Option, @@ -152,6 +172,9 @@ pub enum Expression { op: UnaryOperator, a: Box, }, + Await { + value: Box, + }, Yield { value: Option>, }, @@ -159,9 +182,8 @@ pub enum Expression { value: Box, }, Compare { - a: Box, - op: Comparison, - b: Box, + vals: Vec, + ops: Vec, }, Attribute { value: Box, @@ -182,7 +204,7 @@ pub enum Expression { elements: Vec, }, Dict { - elements: Vec<(Expression, Expression)>, + elements: Vec<(Option, Expression)>, }, Set { elements: Vec, @@ -230,6 +252,7 @@ impl Expression { match self { BoolOp { .. } | Binop { .. } | Unop { .. } => "operator", Subscript { .. } => "subscript", + Await { .. } => "await expression", Yield { .. } | YieldFrom { .. } => "yield expression", Compare { .. } => "comparison", Attribute { .. } => "attribute", @@ -367,7 +390,7 @@ pub enum Number { } /// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum ConversionFlag { /// Converts by calling `str()`. Str, diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index cad885531e..e1e758907e 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -5,7 +5,7 @@ use std::str; use lalrpop_util::ParseError as LalrpopError; use crate::ast::{ConversionFlag, StringGroup}; -use crate::lexer::{LexicalError, Location, Tok}; +use crate::lexer::{LexicalError, LexicalErrorType, Location, Tok}; use crate::parser::parse_expression; use self::FStringError::*; @@ -25,7 +25,10 @@ pub enum FStringError { impl From for LalrpopError { fn from(_err: FStringError) -> Self { lalrpop_util::ParseError::User { - error: LexicalError::StringError, + error: LexicalError { + error: LexicalErrorType::StringError, + location: Default::default(), + }, } } } diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 113f7afb1a..76bc421693 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1,12 +1,19 @@ //! This module takes care of lexing python source text. This means source //! code is translated into separate tokens. +extern crate unic_emoji_char; +extern crate unicode_xid; + pub use super::token::Tok; use num_bigint::BigInt; use num_traits::Num; +use serde::{Deserialize, Serialize}; use std::cmp::Ordering; use std::collections::HashMap; use std::str::FromStr; +use unic_emoji_char::is_emoji_presentation; +use unicode_xid::UnicodeXID; +use wtf8; #[derive(Clone, Copy, PartialEq, Debug)] struct IndentationLevel { @@ -45,20 +52,29 @@ pub struct Lexer> { at_begin_of_line: bool, nesting: usize, // Amount of parenthesis indentation_stack: Vec, - pending: Vec>, + pending: Vec, chr0: Option, chr1: Option, location: Location, + keywords: HashMap, +} + +#[derive(Debug)] +pub struct LexicalError { + pub error: LexicalErrorType, + pub location: Location, } #[derive(Debug)] -pub enum LexicalError { +pub enum LexicalErrorType { StringError, + UnicodeError, NestingError, UnrecognizedToken { tok: char }, + OtherError(String), } -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Location { row: usize, column: usize, @@ -69,11 +85,11 @@ impl Location { Location { row, column } } - pub fn get_row(&self) -> usize { + pub fn row(&self) -> usize { self.row } - pub fn get_column(&self) -> usize { + pub fn column(&self) -> usize { self.column } } @@ -90,6 +106,8 @@ pub fn get_keywords() -> HashMap { keywords.insert(String::from("and"), Tok::And); keywords.insert(String::from("as"), Tok::As); keywords.insert(String::from("assert"), Tok::Assert); + keywords.insert(String::from("async"), Tok::Async); + keywords.insert(String::from("await"), Tok::Await); keywords.insert(String::from("break"), Tok::Break); keywords.insert(String::from("class"), Tok::Class); keywords.insert(String::from("continue"), Tok::Continue); @@ -120,9 +138,10 @@ pub fn get_keywords() -> HashMap { keywords } -pub type Spanned = Result<(Location, Tok, Location), LexicalError>; +pub type Spanned = (Location, Tok, Location); +pub type LexResult = Result; -pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator> + 'a { +pub fn make_tokenizer<'a>(source: &'a str) -> impl Iterator + 'a { let nlh = NewlineHandler::new(source.chars()); let lch = LineContinationHandler::new(nlh); Lexer::new(lch) @@ -254,6 +273,7 @@ where chr0: None, location: Location::new(0, 0), chr1: None, + keywords: get_keywords(), }; lxr.next_char(); lxr.next_char(); @@ -264,7 +284,7 @@ where } // Lexer helper functions: - fn lex_identifier(&mut self) -> Spanned { + fn lex_identifier(&mut self) -> LexResult { let mut name = String::new(); let start_pos = self.get_pos(); @@ -300,21 +320,19 @@ where } } - while self.is_char() { + while self.is_identifier_continuation() { name.push(self.next_char().unwrap()); } let end_pos = self.get_pos(); - let mut keywords = get_keywords(); - - if keywords.contains_key(&name) { - Ok((start_pos, keywords.remove(&name).unwrap(), end_pos)) + if self.keywords.contains_key(&name) { + Ok((start_pos, self.keywords[&name].clone(), end_pos)) } else { Ok((start_pos, Tok::Name { name }, end_pos)) } } - fn lex_number(&mut self) -> Spanned { + fn lex_number(&mut self) -> LexResult { let start_pos = self.get_pos(); if self.chr0 == Some('0') { if self.chr1 == Some('x') || self.chr1 == Some('X') { @@ -340,12 +358,12 @@ where } } - fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> Spanned { + fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult { let mut value_text = String::new(); loop { - if self.is_number(radix) { - value_text.push(self.next_char().unwrap()); + if let Some(c) = self.take_number(radix) { + value_text.push(c); } else if self.chr0 == Some('_') { self.next_char(); } else { @@ -354,18 +372,21 @@ where } let end_pos = self.get_pos(); - let value = BigInt::from_str_radix(&value_text, radix).unwrap(); + let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError { + error: LexicalErrorType::OtherError(format!("{:?}", e)), + location: start_pos.clone(), + })?; Ok((start_pos, Tok::Int { value }, end_pos)) } - fn lex_normal_number(&mut self) -> Spanned { + fn lex_normal_number(&mut self) -> LexResult { let start_pos = self.get_pos(); let mut value_text = String::new(); // Normal number: - while self.is_number(10) { - value_text.push(self.next_char().unwrap()); + while let Some(c) = self.take_number(10) { + value_text.push(c); } // If float: @@ -373,8 +394,8 @@ where // Take '.': if self.chr0 == Some('.') { value_text.push(self.next_char().unwrap()); - while self.is_number(10) { - value_text.push(self.next_char().unwrap()); + while let Some(c) = self.take_number(10) { + value_text.push(c); } } @@ -387,8 +408,8 @@ where value_text.push(self.next_char().unwrap()); } - while self.is_number(10) { - value_text.push(self.next_char().unwrap()); + while let Some(c) = self.take_number(10) { + value_text.push(c); } } @@ -437,13 +458,34 @@ where } } + fn unicode_literal(&mut self, literal_number: usize) -> Result { + let mut p: u32 = 0u32; + let unicode_error = Err(LexicalError { + error: LexicalErrorType::UnicodeError, + location: self.get_pos(), + }); + for i in 1..=literal_number { + match self.next_char() { + Some(c) => match c.to_digit(16) { + Some(d) => p += d << (literal_number - i) * 4, + None => return unicode_error, + }, + None => return unicode_error, + } + } + match wtf8::CodePoint::from_u32(p) { + Some(cp) => return Ok(cp.to_char_lossy()), + None => return unicode_error, + } + } + fn lex_string( &mut self, is_bytes: bool, is_raw: bool, _is_unicode: bool, is_fstring: bool, - ) -> Spanned { + ) -> LexResult { let quote_char = self.next_char().unwrap(); let mut string_content = String::new(); let start_pos = self.get_pos(); @@ -461,8 +503,19 @@ where loop { match self.next_char() { Some('\\') => { - if is_raw { + if self.chr0 == Some(quote_char) { + string_content.push(quote_char); + self.next_char(); + } else if is_raw { string_content.push('\\'); + if let Some(c) = self.next_char() { + string_content.push(c) + } else { + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); + } } else { match self.next_char() { Some('\\') => { @@ -483,13 +536,19 @@ where Some('t') => { string_content.push('\t'); } + Some('u') => string_content.push(self.unicode_literal(4)?), + Some('U') => string_content.push(self.unicode_literal(8)?), + Some('x') if !is_bytes => string_content.push(self.unicode_literal(2)?), Some('v') => string_content.push('\x0b'), Some(c) => { string_content.push('\\'); string_content.push(c); } None => { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } } @@ -512,7 +571,10 @@ where } else { if c == '\n' { if !triple_quoted { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } self.new_line(); } @@ -520,15 +582,25 @@ where } } None => { - return Err(LexicalError::StringError); + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } } let end_pos = self.get_pos(); let tok = if is_bytes { - Tok::Bytes { - value: string_content.as_bytes().to_vec(), + if string_content.is_ascii() { + Tok::Bytes { + value: lex_byte(string_content)?, + } + } else { + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: self.get_pos(), + }); } } else { Tok::String { @@ -540,15 +612,26 @@ where Ok((start_pos, tok, end_pos)) } - fn is_char(&self) -> bool { - match self.chr0 { - Some('a'..='z') | Some('A'..='Z') | Some('_') | Some('0'..='9') => true, - _ => false, + fn is_identifier_start(&self, c: char) -> bool { + match c { + '_' => true, + c => UnicodeXID::is_xid_start(c), } } - fn is_number(&self, radix: u32) -> bool { - match radix { + fn is_identifier_continuation(&self) -> bool { + if let Some(c) = self.chr0 { + match c { + '_' | '0'..='9' => true, + c => UnicodeXID::is_xid_continue(c), + } + } else { + false + } + } + + fn take_number(&mut self, radix: u32) -> Option { + let take_char = match radix { 2 => match self.chr0 { Some('0'..='1') => true, _ => false, @@ -566,6 +649,12 @@ where _ => false, }, x => unimplemented!("Radix not implemented: {}", x), + }; + + if take_char { + Some(self.next_char().unwrap()) + } else { + None } } @@ -587,9 +676,67 @@ where self.location.column = 1; } - fn inner_next(&mut self) -> Option> { + /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character. + fn determine_indentation(&mut self) -> Result { + // Determine indentation: + let mut spaces: usize = 0; + let mut tabs: usize = 0; + loop { + match self.chr0 { + Some(' ') => { + /* + if tabs != 0 { + // Don't allow spaces after tabs as part of indentation. + // This is technically stricter than python3 but spaces after + // tabs is even more insane than mixing spaces and tabs. + return Some(Err(LexicalError { + error: LexicalErrorType::OtherError("Spaces not allowed as part of indentation after tabs".to_string()), + location: self.get_pos(), + })); + } + */ + self.next_char(); + spaces += 1; + } + Some('\t') => { + if spaces != 0 { + // Don't allow tabs after spaces as part of indentation. + // This is technically stricter than python3 but spaces before + // tabs is even more insane than mixing spaces and tabs. + return Err(LexicalError { + error: LexicalErrorType::OtherError( + "Tabs not allowed as part of indentation after spaces".to_string(), + ), + location: self.get_pos(), + }); + } + self.next_char(); + tabs += 1; + } + Some('#') => { + self.lex_comment(); + spaces = 0; + tabs = 0; + } + Some('\n') => { + // Empty line! + self.next_char(); + self.new_line(); + spaces = 0; + tabs = 0; + } + _ => { + break; + } + } + } + + Ok(IndentationLevel { spaces, tabs }) + } + + fn inner_next(&mut self) -> LexResult { if !self.pending.is_empty() { - return Some(self.pending.remove(0)); + return self.pending.remove(0); } 'top_loop: loop { @@ -597,44 +744,7 @@ where if self.at_begin_of_line { self.at_begin_of_line = false; - // Determine indentation: - let mut spaces: usize = 0; - let mut tabs: usize = 0; - loop { - match self.chr0 { - Some(' ') => { - self.next_char(); - spaces += 1; - } - Some('\t') => { - if spaces != 0 { - // Don't allow tabs after spaces as part of indentation. - // This is technically stricter than python3 but spaces before - // tabs is even more insane than mixing spaces and tabs. - panic!("Tabs not allowed as part of indentation after spaces"); - } - self.next_char(); - tabs += 1; - } - Some('#') => { - self.lex_comment(); - self.at_begin_of_line = true; - continue 'top_loop; - } - Some('\n') => { - // Empty line! - self.next_char(); - self.at_begin_of_line = true; - self.new_line(); - continue 'top_loop; - } - _ => { - break; - } - } - } - - let indentation_level = IndentationLevel { spaces, tabs }; + let indentation_level = self.determine_indentation()?; if self.nesting == 0 { // Determine indent or dedent: @@ -649,7 +759,7 @@ where self.indentation_stack.push(indentation_level); let tok_start = self.get_pos(); let tok_end = tok_start.clone(); - return Some(Ok((tok_start, Tok::Indent, tok_end))); + return Ok((tok_start, Tok::Indent, tok_end)); } Some(Ordering::Less) => { // One or more dedentations @@ -666,7 +776,10 @@ where self.pending.push(Ok((tok_start, Tok::Dedent, tok_end))); } None => { - panic!("inconsistent use of tabs and spaces in indentation") + return Err(LexicalError { + error: LexicalErrorType::OtherError("inconsistent use of tabs and spaces in indentation".to_string()), + location: self.get_pos(), + }); } _ => { break; @@ -676,362 +789,411 @@ where if indentation_level != *self.indentation_stack.last().unwrap() { // TODO: handle wrong indentations - panic!("Non matching indentation levels!"); + return Err(LexicalError { + error: LexicalErrorType::OtherError( + "Non matching indentation levels!".to_string(), + ), + location: self.get_pos(), + }); } - return Some(self.pending.remove(0)); + return self.pending.remove(0); + } + None => { + return Err(LexicalError { + error: LexicalErrorType::OtherError( + "inconsistent use of tabs and spaces in indentation" + .to_string(), + ), + location: self.get_pos(), + }); } - None => panic!("inconsistent use of tabs and spaces in indentation"), } } } - match self.chr0 { - Some('0'..='9') => return Some(self.lex_number()), - Some('_') | Some('a'..='z') | Some('A'..='Z') => { - return Some(self.lex_identifier()); - } - Some('#') => { - self.lex_comment(); - continue; - } - Some('"') => { - return Some(self.lex_string(false, false, false, false)); - } - Some('\'') => { - return Some(self.lex_string(false, false, false, false)); - } - Some('=') => { + // Check if we have some character: + if let Some(c) = self.chr0 { + // First check identifier: + if self.is_identifier_start(c) { + return self.lex_identifier(); + } else if is_emoji_presentation(c) { let tok_start = self.get_pos(); self.next_char(); - match self.chr0 { - Some('=') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::EqEqual, tok_end))); + let tok_end = self.get_pos(); + return Ok(( + tok_start, + Tok::Name { + name: c.to_string(), + }, + tok_end, + )); + } else { + match c { + '0'..='9' => return self.lex_number(), + '#' => { + self.lex_comment(); + continue; } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Equal, tok_end))); + '"' => { + return self.lex_string(false, false, false, false); } - } - } - Some('+') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::PlusEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Plus, tok_end))); - } - } - Some('*') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::StarEqual, tok_end))); + '\'' => { + return self.lex_string(false, false, false, false); } - Some('*') => { + '=' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleStarEqual, tok_end))); + return Ok((tok_start, Tok::EqEqual, tok_end)); } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleStar, tok_end))); + return Ok((tok_start, Tok::Equal, tok_end)); } } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Star, tok_end))); - } - } - } - Some('/') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { + '+' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::SlashEqual, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::PlusEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Plus, tok_end)); + } } - Some('/') => { + '*' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlashEqual, tok_end))); + return Ok((tok_start, Tok::StarEqual, tok_end)); + } + Some('*') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::DoubleStarEqual, tok_end)); + } + _ => { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::DoubleStar, tok_end)); + } + } } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::DoubleSlash, tok_end))); + return Ok((tok_start, Tok::Star, tok_end)); } } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Slash, tok_end))); + '/' => { + let tok_start = self.get_pos(); + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::SlashEqual, tok_end)); + } + Some('/') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::DoubleSlashEqual, tok_end)); + } + _ => { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::DoubleSlash, tok_end)); + } + } + } + _ => { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Slash, tok_end)); + } + } } - } - } - Some('%') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::PercentEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Percent, tok_end))); - } - } - Some('|') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::VbarEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Vbar, tok_end))); - } - } - Some('^') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::CircumflexEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::CircumFlex, tok_end))); - } - } - Some('&') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::AmperEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Amper, tok_end))); - } - } - Some('-') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { + '%' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::MinusEqual, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::PercentEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Percent, tok_end)); + } } - Some('>') => { + '|' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Rarrow, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::VbarEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Vbar, tok_end)); + } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Minus, tok_end))); + '^' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::CircumflexEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::CircumFlex, tok_end)); + } } - } - } - Some('@') => { - let tok_start = self.get_pos(); - self.next_char(); - if let Some('=') = self.chr0 { - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::AtEqual, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::At, tok_end))); - } - } - Some('!') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('=') => { + '&' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::NotEqual, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::AmperEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Amper, tok_end)); + } } - _ => panic!("Invalid token '!'"), - } - } - Some('~') => { - return Some(self.eat_single_char(Tok::Tilde)); - } - Some('(') => { - let result = self.eat_single_char(Tok::Lpar); - self.nesting += 1; - return Some(result); - } - Some(')') => { - let result = self.eat_single_char(Tok::Rpar); - if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); - } - self.nesting -= 1; - return Some(result); - } - Some('[') => { - let result = self.eat_single_char(Tok::Lsqb); - self.nesting += 1; - return Some(result); - } - Some(']') => { - let result = self.eat_single_char(Tok::Rsqb); - if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); - } - self.nesting -= 1; - return Some(result); - } - Some('{') => { - let result = self.eat_single_char(Tok::Lbrace); - self.nesting += 1; - return Some(result); - } - Some('}') => { - let result = self.eat_single_char(Tok::Rbrace); - if self.nesting == 0 { - return Some(Err(LexicalError::NestingError)); - } - self.nesting -= 1; - return Some(result); - } - Some(':') => { - return Some(self.eat_single_char(Tok::Colon)); - } - Some(';') => { - return Some(self.eat_single_char(Tok::Semi)); - } - Some('<') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('<') => { + '-' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::LeftShiftEqual, tok_end))); + return Ok((tok_start, Tok::MinusEqual, tok_end)); + } + Some('>') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Rarrow, tok_end)); } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::LeftShift, tok_end))); + return Ok((tok_start, Tok::Minus, tok_end)); } } } - Some('=') => { + '@' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::LessEqual, tok_end))); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::AtEqual, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::At, tok_end)); + } } - _ => { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Less, tok_end))); + '!' => { + let tok_start = self.get_pos(); + self.next_char(); + if let Some('=') = self.chr0 { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::NotEqual, tok_end)); + } else { + return Err(LexicalError { + error: LexicalErrorType::UnrecognizedToken { tok: '!' }, + location: tok_start, + }); + } } - } - } - Some('>') => { - let tok_start = self.get_pos(); - self.next_char(); - match self.chr0 { - Some('>') => { + '~' => { + return self.eat_single_char(Tok::Tilde); + } + '(' => { + let result = self.eat_single_char(Tok::Lpar); + self.nesting += 1; + return result; + } + ')' => { + let result = self.eat_single_char(Tok::Rpar); + if self.nesting == 0 { + return Err(LexicalError { + error: LexicalErrorType::NestingError, + location: self.get_pos(), + }); + } + self.nesting -= 1; + return result; + } + '[' => { + let result = self.eat_single_char(Tok::Lsqb); + self.nesting += 1; + return result; + } + ']' => { + let result = self.eat_single_char(Tok::Rsqb); + if self.nesting == 0 { + return Err(LexicalError { + error: LexicalErrorType::NestingError, + location: self.get_pos(), + }); + } + self.nesting -= 1; + return result; + } + '{' => { + let result = self.eat_single_char(Tok::Lbrace); + self.nesting += 1; + return result; + } + '}' => { + let result = self.eat_single_char(Tok::Rbrace); + if self.nesting == 0 { + return Err(LexicalError { + error: LexicalErrorType::NestingError, + location: self.get_pos(), + }); + } + self.nesting -= 1; + return result; + } + ':' => { + return self.eat_single_char(Tok::Colon); + } + ';' => { + return self.eat_single_char(Tok::Semi); + } + '<' => { + let tok_start = self.get_pos(); self.next_char(); match self.chr0 { + Some('<') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::LeftShiftEqual, tok_end)); + } + _ => { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::LeftShift, tok_end)); + } + } + } Some('=') => { self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::RightShiftEqual, tok_end))); + return Ok((tok_start, Tok::LessEqual, tok_end)); } _ => { let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::RightShift, tok_end))); + return Ok((tok_start, Tok::Less, tok_end)); } } } - Some('=') => { + '>' => { + let tok_start = self.get_pos(); self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::GreaterEqual, tok_end))); + match self.chr0 { + Some('>') => { + self.next_char(); + match self.chr0 { + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::RightShiftEqual, tok_end)); + } + _ => { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::RightShift, tok_end)); + } + } + } + Some('=') => { + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::GreaterEqual, tok_end)); + } + _ => { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Greater, tok_end)); + } + } } - _ => { + ',' => { + let tok_start = self.get_pos(); + self.next_char(); let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Greater, tok_end))); + return Ok((tok_start, Tok::Comma, tok_end)); } - } - } - Some(',') => { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Comma, tok_end))); - } - Some('.') => { - let tok_start = self.get_pos(); - self.next_char(); - if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { - self.next_char(); - self.next_char(); - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Ellipsis, tok_end))); - } else { - let tok_end = self.get_pos(); - return Some(Ok((tok_start, Tok::Dot, tok_end))); - } - } - Some('\n') => { - let tok_start = self.get_pos(); - self.next_char(); - let tok_end = self.get_pos(); - self.new_line(); + '.' => { + if let Some('0'..='9') = self.chr1 { + return self.lex_number(); + } else { + let tok_start = self.get_pos(); + self.next_char(); + if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) { + self.next_char(); + self.next_char(); + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Ellipsis, tok_end)); + } else { + let tok_end = self.get_pos(); + return Ok((tok_start, Tok::Dot, tok_end)); + } + } + } + '\n' => { + let tok_start = self.get_pos(); + self.next_char(); + let tok_end = self.get_pos(); + self.new_line(); - // Depending on the nesting level, we emit newline or not: - if self.nesting == 0 { - self.at_begin_of_line = true; - return Some(Ok((tok_start, Tok::Newline, tok_end))); - } else { - continue; + // Depending on the nesting level, we emit newline or not: + if self.nesting == 0 { + self.at_begin_of_line = true; + return Ok((tok_start, Tok::Newline, tok_end)); + } else { + continue; + } + } + ' ' => { + // Skip whitespaces + self.next_char(); + continue; + } + _ => { + let c = self.next_char(); + return Err(LexicalError { + error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, + location: self.get_pos(), + }); + } // Ignore all the rest.. } } - Some(' ') => { - // Skip whitespaces - self.next_char(); - continue; - } - None => return None, - _ => { - let c = self.next_char(); - return Some(Err(LexicalError::UnrecognizedToken { tok: c.unwrap() })); - } // Ignore all the rest.. + } else { + let tok_pos = self.get_pos(); + return Ok((tok_pos.clone(), Tok::EndOfFile, tok_pos)); } } } - fn eat_single_char(&mut self, ty: Tok) -> Spanned { + fn eat_single_char(&mut self, ty: Tok) -> LexResult { let tok_start = self.get_pos(); self.next_char(); let tok_end = self.get_pos(); @@ -1048,7 +1210,7 @@ impl Iterator for Lexer where T: Iterator, { - type Item = Spanned; + type Item = LexResult; fn next(&mut self) -> Option { // Idea: create some sort of hash map for single char tokens: @@ -1061,8 +1223,63 @@ where self.nesting, self.indentation_stack ); - token + + match token { + Ok((_, Tok::EndOfFile, _)) => None, + r => Some(r), + } + } +} + +fn lex_byte(s: String) -> Result, LexicalError> { + let mut res = vec![]; + let mut escape = false; //flag if previous was \ + let mut hex_on = false; // hex mode on or off + let mut hex_value = String::new(); + + for c in s.chars() { + if hex_on { + if c.is_ascii_hexdigit() { + if hex_value.is_empty() { + hex_value.push(c); + continue; + } else { + hex_value.push(c); + res.push(u8::from_str_radix(&hex_value, 16).unwrap()); + hex_on = false; + hex_value.clear(); + } + } else { + return Err(LexicalError { + error: LexicalErrorType::StringError, + location: Default::default(), + }); + } + } else { + match (c, escape) { + ('\\', true) => res.push(b'\\'), + ('\\', false) => { + escape = true; + continue; + } + ('x', true) => hex_on = true, + ('x', false) => res.push(b'x'), + ('t', true) => res.push(b'\t'), + ('t', false) => res.push(b't'), + ('n', true) => res.push(b'\n'), + ('n', false) => res.push(b'n'), + ('r', true) => res.push(b'\r'), + ('r', false) => res.push(b'r'), + (x, true) => { + res.push(b'\\'); + res.push(x as u8); + } + (x, false) => res.push(x as u8), + } + escape = false; + } } + Ok(res) } #[cfg(test)] @@ -1389,7 +1606,7 @@ mod tests { #[test] fn test_string() { - let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g'"#); + let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#); let tokens = lex_source(&source); assert_eq!( tokens, @@ -1418,6 +1635,10 @@ mod tests { value: String::from("\\g"), is_fstring: false, }, + Tok::String { + value: String::from("raw\'"), + is_fstring: false, + }, ] ); } @@ -1448,4 +1669,28 @@ mod tests { test_string_continuation_mac_eol: MAC_EOL, test_string_continuation_unix_eol: UNIX_EOL, } + + #[test] + fn test_byte() { + // single quote + let all = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##; + let source = String::from(all); + let tokens = lex_source(&source); + let res = (0..=255).collect::>(); + assert_eq!(tokens, vec![Tok::Bytes { value: res }]); + + // double quote + let all = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##; + let source = String::from(all); + let tokens = lex_source(&source); + let res = (0..=255).collect::>(); + assert_eq!(tokens, vec![Tok::Bytes { value: res }]); + + // backslash doesnt escape + let all = r##"b"omkmok\Xaa""##; + let source = String::from(all); + let tokens = lex_source(&source); + let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97]; + assert_eq!(tokens, vec![Tok::Bytes { value: res }]); + } } diff --git a/parser/src/lib.rs b/parser/src/lib.rs index b10c3d5514..5cc8ca3c44 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -1,11 +1,14 @@ #[macro_use] extern crate log; +use lalrpop_util::lalrpop_mod; pub mod ast; pub mod error; mod fstring; pub mod lexer; pub mod parser; -#[cfg_attr(rustfmt, rustfmt_skip)] -mod python; +lalrpop_mod!( + #[allow(clippy::all)] + python +); pub mod token; diff --git a/parser/src/parser.rs b/parser/src/parser.rs index a119740ec5..eee4193105 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -35,7 +35,7 @@ pub fn parse_program(source: &str) -> Result { do_lalr_parsing!(source, Program, StartProgram) } -pub fn parse_statement(source: &str) -> Result { +pub fn parse_statement(source: &str) -> Result, ParseError> { do_lalr_parsing!(source, Statement, StartStatement) } @@ -44,7 +44,6 @@ pub fn parse_statement(source: &str) -> Result => ast::Program { - statements: Vec::from_iter(lines.into_iter().filter_map(|e| e)) + statements: Vec::from_iter(lines.into_iter().flatten()) }, }; // A file line either has a declaration, or an empty newline: -FileLine: Option = { - => Some(s), - "\n" => None, +FileLine: Vec = { + Statement, + "\n" => vec![], }; Suite: Vec = { - => vec![s], - "\n" indent dedent => s, + SimpleStatement, + "\n" indent dedent => s.into_iter().flatten().collect(), }; -Statement: ast::LocatedStatement = { +Statement: Vec = { SimpleStatement, - CompoundStatement, + => vec![s], }; -SimpleStatement: ast::LocatedStatement = { - "\n" => s, - ";" => s, +SimpleStatement: Vec = { + ";"? "\n" => { + let mut statements = vec![s1]; + statements.extend(s2.into_iter().map(|e| e.1)); + statements + } }; SmallStatement: ast::LocatedStatement = { @@ -103,9 +105,7 @@ ExpressionStatement: ast::LocatedStatement = { } } }, - => { - // TODO: this works in most cases: - let rhs = e2.into_iter().next().unwrap(); + => { ast::LocatedStatement { location: loc, node: ast::Statement::AugAssign { @@ -118,31 +118,17 @@ ExpressionStatement: ast::LocatedStatement = { }; AssignSuffix: ast::Expression = { - "=" => { - if e.len() > 1 { - ast::Expression::Tuple { - elements: e - } - } else { - e.into_iter().next().unwrap() - } - }, + "=" => e, "=" => e, }; TestOrStarExprList: ast::Expression = { - => { - let mut res = vec![e]; - res.extend(e2.into_iter().map(|x| x.1)); - - // First build tuple from first item: - let expr = if (res.len() > 1) || comma.is_some() { - ast::Expression::Tuple { elements: res } + > => { + if elements.len() == 1 && comma.is_none() { + elements.into_iter().next().unwrap() } else { - res.into_iter().next().unwrap() - }; - - expr + ast::Expression::Tuple { elements } + } } }; @@ -183,7 +169,7 @@ FlowStatement: ast::LocatedStatement = { "return" => { ast::LocatedStatement { location: loc, - node: ast::Statement::Return { value: t }, + node: ast::Statement::Return { value: t.map(Box::new) }, } }, => { @@ -220,8 +206,9 @@ ImportStatement: ast::LocatedStatement = { .map(|(n, a)| ast::SingleImport { module: n.to_string(), - symbol: None, - alias: a.clone() + symbols: vec![], + alias: a.clone(), + level: 0, }) .collect() }, @@ -231,35 +218,30 @@ ImportStatement: ast::LocatedStatement = { ast::LocatedStatement { location: loc, node: ast::Statement::Import { - import_parts: i - .iter() - .map(|(i, a)| - ast::SingleImport { - module: n.to_string(), - symbol: Some(i.to_string()), - alias: a.clone() - }) - .collect() + import_parts: vec![ + ast::SingleImport { + module: n.0.to_string(), + symbols: i.iter() + .map(|(i, a)| + ast::ImportSymbol { + symbol: i.to_string(), + alias: a.clone(), + }) + .collect(), + alias: None, + level: n.1 + }] }, } }, }; -ImportFromLocation: String = { +ImportFromLocation: (String, usize) = { => { - let mut r = "".to_string(); - for _dot in dots { - r.push_str("."); - } - r.push_str(&name); - r + (name, dots.len()) }, => { - let mut r = "".to_string(); - for _dot in dots { - r.push_str("."); - } - r + ("".to_string(), dots.len()) }, }; @@ -310,15 +292,11 @@ NonlocalStatement: ast::LocatedStatement = { }; AssertStatement: ast::LocatedStatement = { - "assert" => { + "assert" => { ast::LocatedStatement { location: loc, node: ast::Statement::Assert { - test: t, - msg: match m { - Some(e) => Some(e.1), - None => None, - } + test, msg: msg.map(|e| e.1) } } }, @@ -335,12 +313,9 @@ CompoundStatement: ast::LocatedStatement = { }; IfStatement: ast::LocatedStatement = { - "if" ":" => { + "if" ":" => { // Determine last else: - let mut last = match s3 { - Some(s) => Some(s.2), - None => None, - }; + let mut last = s3.map(|s| s.2); // handle elif: for i in s2.into_iter().rev() { @@ -353,35 +328,30 @@ IfStatement: ast::LocatedStatement = { ast::LocatedStatement { location: loc, - node: ast::Statement::If { test: t, body: s1, orelse: last } + node: ast::Statement::If { test, body: s1, orelse: last } } }, }; WhileStatement: ast::LocatedStatement = { - "while" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + "while" ":" => { + let or_else = s2.map(|s| s.2); ast::LocatedStatement { location: loc, - node: ast::Statement::While { test: e, body: s, orelse: or_else }, + node: ast::Statement::While { test, body, orelse: or_else }, } }, }; ForStatement: ast::LocatedStatement = { - "for" "in" ":" => { - let or_else = match s2 { - Some(s) => Some(s.2), - None => None, - }; + "for" "in" ":" => { + let orelse = s2.map(|s| s.2); ast::LocatedStatement { location: loc, - node: ast::Statement::For { - target: e, - iter: t, body: s, orelse: or_else + node: if is_async.is_some() { + ast::Statement::AsyncFor { target, iter, body, orelse } + } else { + ast::Statement::For { target, iter, body, orelse } }, } }, @@ -389,14 +359,8 @@ ForStatement: ast::LocatedStatement = { TryStatement: ast::LocatedStatement = { "try" ":" => { - let or_else = match else_suite { - Some(s) => Some(s.2), - None => None, - }; - let finalbody = match finally { - Some(s) => Some(s.2), - None => None, - }; + let or_else = else_suite.map(|s| s.2); + let finalbody = finally.map(|s| s.2); ast::LocatedStatement { location: loc, node: ast::Statement::Try { @@ -437,51 +401,48 @@ WithStatement: ast::LocatedStatement = { WithItem: ast::WithItem = { => { - let optional_vars = match n { - Some(val) => Some(val.1), - None => None, - }; + let optional_vars = n.map(|val| val.1); ast::WithItem { context_expr: t, optional_vars } }, }; FuncDef: ast::LocatedStatement = { - "def" " Test)?> ":" => { + "def" " Test)?> ":" => { ast::LocatedStatement { location: loc, - node: ast::Statement::FunctionDef { - name: i, - args: a, - body: s, - decorator_list: d, - returns: r.map(|x| x.1), - } + node: if is_async.is_some() { + ast::Statement::AsyncFunctionDef { + name: i, + args: a, + body: s, + decorator_list: d, + returns: r.map(|x| x.1), + } + } else { + ast::Statement::FunctionDef { + name: i, + args: a, + body: s, + decorator_list: d, + returns: r.map(|x| x.1), + } + } } }, }; Parameters: ast::Parameters = { - "(" )?> ")" => { - match a { - Some(a) => a, - None => Default::default(), - } - }, + "(" )?> ")" => a.unwrap_or_else(Default::default), }; -// parameters are (String, None), kwargs are (String, Some(Test)) where Test is -// the default // Note that this is a macro which is used once for function defs, and // once for lambda defs. -TypedArgsList: ast::Parameters = { - > )?> => { +ParameterList: ast::Parameters = { + > )?> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: - let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 { - Some((_, x)) => x, - None => (None, vec![], vec![], None), - }; + let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); ast::Parameters { args: names, @@ -492,7 +453,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > )> => { + > )> ","? => { let (names, default_elements) = param1; // Now gather rest of parameters: @@ -510,7 +471,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { let (vararg, kwonlyargs, kw_defaults, kwarg) = params; ast::Parameters { args: vec![], @@ -521,7 +482,7 @@ TypedArgsList: ast::Parameters = { kw_defaults: kw_defaults, } }, - > => { + > ","? => { ast::Parameters { args: vec![], kwonlyargs: vec![], @@ -535,8 +496,8 @@ TypedArgsList: ast::Parameters = { // Use inline here to make sure the "," is not creating an ambiguity. #[inline] -TypedParameters: (Vec, Vec) = { - > )*> => { +ParameterDefs: (Vec, Vec) = { + > )*> => { // Combine first parameters: let mut args = vec![param1]; args.extend(param2.into_iter().map(|x| x.1)); @@ -564,7 +525,7 @@ TypedParameters: (Vec, Vec) = { } }; -TypedParameterDef: (ast::Parameter, Option) = { +ParameterDef: (ast::Parameter, Option) = { => (i, None), "=" => (i, Some(e)), }; @@ -580,8 +541,11 @@ TypedParameter: ast::Parameter = { }, }; +// Use inline here to make sure the "," is not creating an ambiguity. +// TODO: figure out another grammar that makes this inline no longer required. +#[inline] ParameterListStarArgs: (Option>, Vec, Vec>, Option>) = { - "*" )*> )?> => { + "*" )*> )?> => { // Extract keyword arguments: let mut kwonlyargs = vec![]; let mut kw_defaults = vec![]; @@ -590,10 +554,7 @@ ParameterListStarArgs: (Option>, Vec Some(name), - None => None, - }; + let kwarg = kwarg.map(|n| n.1); (Some(va), kwonlyargs, kw_defaults, kwarg) } @@ -649,66 +610,59 @@ Decorator: ast::Expression = { }; YieldExpr: ast::Expression = { - "yield" => { - ast::Expression::Yield { - value: ex.map(|expr| Box::new( - if expr.len() > 1 { - ast::Expression::Tuple { elements: expr } - } else { - expr.into_iter().next().unwrap() - }) - ) - } - }, - "yield" "from" => { - ast::Expression::YieldFrom { - value: Box::new(e), - } - }, + "yield" => ast::Expression::Yield { value: value.map(Box::new) }, + "yield" "from" => ast::Expression::YieldFrom { value: Box::new(e) }, }; Test: ast::Expression = { - => { - match c { - Some(c) => { - ast::Expression::IfExpression { - test: Box::new(c.1), - body: Box::new(e), - orelse: Box::new(c.3), - } - }, - None => e, + => { + if let Some(c) = condition { + ast::Expression::IfExpression { + test: Box::new(c.1), + body: Box::new(expr), + orelse: Box::new(c.3), + } + } else { + expr } }, - => e, + LambdaDef, }; LambdaDef: ast::Expression = { - "lambda" ?> ":" => + "lambda" ?> ":" => ast::Expression::Lambda { args: p.unwrap_or(Default::default()), - body:Box::new(b) + body: Box::new(body) } } OrTest: ast::Expression = { - => e, + AndTest, "or" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::Or, b: Box::new(e2) }, }; AndTest: ast::Expression = { - => e, + NotTest, "and" => ast::Expression::BoolOp { a: Box::new(e1), op: ast::BooleanOperator::And, b: Box::new(e2) }, }; NotTest: ast::Expression = { "not" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Not }, - => e, + Comparison, }; Comparison: ast::Expression = { - => ast::Expression::Compare { a: Box::new(e1), op: op, b: Box::new(e2) }, - => e, + => { + let mut vals = vec![e]; + let mut ops = vec![]; + for x in comparisons { + ops.push(x.0); + vals.push(x.1); + } + ast::Expression::Compare { vals, ops } + }, + Expression, }; CompOp: ast::Comparison = { @@ -726,7 +680,7 @@ CompOp: ast::Comparison = { Expression: ast::Expression = { "|" => ast::Expression::Binop { a: Box::new(e1), op: ast::Operator::BitOr, b: Box::new(e2) }, - => e, + XorExpression, }; XorExpression: ast::Expression = { @@ -776,7 +730,7 @@ Factor: ast::Expression = { "+" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Pos }, "-" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Neg }, "~" => ast::Expression::Unop { a: Box::new(e), op: ast::UnaryOperator::Inv }, - => e, + Power, }; Power: ast::Expression = { @@ -789,10 +743,20 @@ Power: ast::Expression = { }; AtomExpr: ast::Expression = { - => e, - "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, - "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, - "." => ast::Expression::Attribute { value: Box::new(e), name: n }, + => { + if is_await.is_some() { + ast::Expression::Await { value: Box::new(atom) } + } else { + atom + } + } +} + +AtomExpr2: ast::Expression = { + Atom, + "(" ")" => ast::Expression::Call { function: Box::new(f), args: a.0, keywords: a.1 }, + "[" "]" => ast::Expression::Subscript { a: Box::new(e), b: Box::new(s) }, + "." => ast::Expression::Attribute { value: Box::new(e), name: n }, }; SubscriptList: ast::Expression = { @@ -810,7 +774,7 @@ SubscriptList: ast::Expression = { }; Subscript: ast::Expression = { - => e, + Test, ":" => { let s1 = e1.unwrap_or(ast::Expression::None); let s2 = e2.unwrap_or(ast::Expression::None); @@ -824,30 +788,17 @@ SliceOp: ast::Expression = { } Atom: ast::Expression = { - => ast::Expression::String { value: s }, - => ast::Expression::Bytes { value: b }, - => ast::Expression::Number { value: n }, - => ast::Expression::Identifier { name: i }, + => ast::Expression::String { value }, + => ast::Expression::Bytes { value }, + => ast::Expression::Number { value }, + => ast::Expression::Identifier { name }, "[" "]" => { let elements = e.unwrap_or(Vec::new()); ast::Expression::List { elements } }, - "[" "]" => { - // List comprehension: - e - }, - "(" ")" => { - match e { - None => ast::Expression::Tuple { elements: Vec::new() }, - Some(elements) => { - if elements.len() == 1 && trailing_comma.is_none() { - // This is "(e)", which is equivalent to "e" - elements.into_iter().next().unwrap() - } else { - ast::Expression::Tuple { elements } - } - } - } + "[" "]" => e, + "(" ")" => { + elements.unwrap_or(ast::Expression::Tuple { elements: Vec::new() }) }, "(" ")" => { ast::Expression::Comprehension { @@ -866,9 +817,7 @@ Atom: ast::Expression = { }; TestListComp: Vec = { - > <_trailing_comma:","?> => { - e - }, + > <_trailing_comma:","?> => e, }; TestListComp2: ast::Expression = { @@ -880,10 +829,8 @@ TestListComp2: ast::Expression = { }, }; -TestDict: Vec<(ast::Expression, ast::Expression)> = { - > <_trailing_comma:","?> => { - e1 - } +TestDict: Vec<(Option, ast::Expression)> = { + > <_trailing_comma:","?> => elements, }; TestDictComp: ast::Expression = { @@ -899,10 +846,13 @@ DictEntry: (ast::Expression, ast::Expression) = { ":" => (e1, e2), }; +DictElement: (Option, ast::Expression) = { + => (Some(e.0), e.1), + "**" => (None, e), +}; + TestSet: Vec = { - > ","? => { - e1 - } + > ","? => e1 }; TestSetComp: ast::Expression = { @@ -914,31 +864,37 @@ TestSetComp: ast::Expression = { } }; +ExpressionOrStarExpression = { + Expression, + StarExpr +}; + ExpressionList: ast::Expression = { - => { - if e.len() == 1 { - e.into_iter().next().unwrap() + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() } else { - ast::Expression::Tuple { elements: e } + ast::Expression::Tuple { elements } } }, }; ExpressionList2: Vec = { - ","? => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - }, + > ","? => elements, }; -#[inline] -TestList: Vec = { - => { - let mut l = vec![e1]; - l.extend(e2.into_iter().map(|x| x.1)); - l - } +// A test list is one of: +// - a list of expressions +// - a single expression +// - a single expression followed by a trailing comma +TestList: ast::Expression = { + > => { + if elements.len() == 1 && trailing_comma.is_none() { + elements.into_iter().next().unwrap() + } else { + ast::Expression::Tuple { elements } + } + } }; // Test @@ -947,27 +903,16 @@ StarExpr: ast::Expression = { }; // Comprehensions: -CompFor: Vec = { - => c, -}; +CompFor: Vec = => c; SingleForComprehension: ast::Comprehension = { - "for" "in" => { - ast::Comprehension { - target: e, - iter: i, - ifs: c2, - } + "for" "in" => { + ast::Comprehension { target, iter, ifs: c2 } } }; -ExpressionNoCond: ast::Expression = { - OrTest, -}; - -ComprehensionIf: ast::Expression = { - "if" => c, -}; +ExpressionNoCond: ast::Expression = OrTest; +ComprehensionIf: ast::Expression = "if" => c; ArgumentList: (Vec, Vec) = { > => { @@ -979,8 +924,15 @@ ArgumentList: (Vec, Vec) = { keywords.push(ast::Keyword { name: n, value: value }); }, None => { - if keywords.len() > 0 { - panic!("positional argument follows keyword argument"); + // Allow starred args after keyword arguments. + let is_starred = if let ast::Expression::Starred { .. } = &value { + true + } else { + false + }; + + if keywords.len() > 0 && !is_starred { + panic!("positional argument follows keyword argument {:?}", keywords); }; args.push(value); }, @@ -1024,8 +976,8 @@ OneOrMore: Vec = { }; Number: ast::Number = { - => { ast::Number::Integer { value: s } }, - => { ast::Number::Float { value: s } }, + => { ast::Number::Integer { value } }, + => { ast::Number::Float { value } }, => { ast::Number::Complex { real: s.0, imag: s.1 } }, }; @@ -1115,6 +1067,8 @@ extern { "and" => lexer::Tok::And, "as" => lexer::Tok::As, "assert" => lexer::Tok::Assert, + "async" => lexer::Tok::Async, + "await" => lexer::Tok::Await, "break" => lexer::Tok::Break, "class" => lexer::Tok::Class, "continue" => lexer::Tok::Continue, diff --git a/parser/src/token.rs b/parser/src/token.rs index ebe5050fef..982d8de924 100644 --- a/parser/src/token.rs +++ b/parser/src/token.rs @@ -3,7 +3,7 @@ use num_bigint::BigInt; /// Python source code can be tokenized in a sequence of these tokens. -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Tok { Name { name: String }, Int { value: BigInt }, @@ -17,6 +17,7 @@ pub enum Tok { StartProgram, StartStatement, StartExpression, + EndOfFile, Lpar, Rpar, Lsqb, @@ -72,6 +73,8 @@ pub enum Tok { And, As, Assert, + Async, + Await, Break, Class, Continue, diff --git a/src/main.rs b/src/main.rs index a0683ec3ed..d1b33ab4e0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,18 +3,22 @@ extern crate clap; extern crate env_logger; #[macro_use] extern crate log; -extern crate rustpython_parser; -extern crate rustpython_vm; extern crate rustyline; use clap::{App, Arg}; +use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType}; use rustpython_parser::error::ParseError; use rustpython_vm::{ - compile, error::CompileError, frame::Scope, import, obj::objstr, print_exception, - pyobject::PyResult, util, VirtualMachine, + frame::Scope, + import, + obj::objstr, + print_exception, + pyobject::{ItemProtocol, PyResult}, + util, VirtualMachine, }; + use rustyline::{error::ReadlineError, Editor}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; fn main() { env_logger::init(); @@ -47,6 +51,9 @@ fn main() { // Construct vm: let vm = VirtualMachine::new(); + let res = import::init_importlib(&vm, true); + handle_exception(&vm, res); + // Figure out if a -c option was given: let result = if let Some(command) = matches.value_of("c") { run_command(&vm, command.to_string()) @@ -65,19 +72,13 @@ fn main() { } fn _run_string(vm: &VirtualMachine, source: &str, source_path: String) -> PyResult { - let code_obj = compile::compile( - source, - &compile::Mode::Exec, - source_path, - vm.ctx.code_type(), - ) - .map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - })?; + let code_obj = vm + .compile(source, &compile::Mode::Exec, source_path.clone()) + .map_err(|err| vm.new_syntax_error(&err))?; // trace!("Code object: {:?}", code_obj.borrow()); - let vars = vm.ctx.new_scope(); // Keep track of local variables - vm.run_code_obj(code_obj, vars) + let attrs = vm.ctx.new_dict(); + attrs.set_item("__file__", vm.new_str(source_path), vm)?; + vm.run_code_obj(code_obj, Scope::with_builtins(None, attrs, vm)) } fn handle_exception(vm: &VirtualMachine, result: PyResult) { @@ -97,41 +98,95 @@ fn run_command(vm: &VirtualMachine, mut source: String) -> PyResult { fn run_module(vm: &VirtualMachine, module: &str) -> PyResult { debug!("Running module {}", module); - let current_path = PathBuf::from("."); - import::import_module(vm, current_path, module) + vm.import(module, &vm.ctx.new_tuple(vec![]), 0) } fn run_script(vm: &VirtualMachine, script_file: &str) -> PyResult { debug!("Running file {}", script_file); // Parse an ast from it: - let file_path = Path::new(script_file); - match util::read_file(file_path) { + let file_path = PathBuf::from(script_file); + let file_path = if file_path.is_file() { + file_path + } else if file_path.is_dir() { + let main_file_path = file_path.join("__main__.py"); + if main_file_path.is_file() { + main_file_path + } else { + error!( + "can't find '__main__' module in '{}'", + file_path.to_str().unwrap() + ); + std::process::exit(1); + } + } else { + error!( + "can't open file '{}': No such file or directory", + file_path.to_str().unwrap() + ); + std::process::exit(1); + }; + + let dir = file_path.parent().unwrap().to_str().unwrap().to_string(); + let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap(); + vm.call_method(&sys_path, "insert", vec![vm.new_int(0), vm.new_str(dir)])?; + + match util::read_file(&file_path) { Ok(source) => _run_string(vm, &source, file_path.to_str().unwrap().to_string()), Err(err) => { - error!("Failed reading file: {:?}", err.kind()); + error!( + "Failed reading file '{}': {:?}", + file_path.to_str().unwrap(), + err.kind() + ); std::process::exit(1); } } } +#[test] +fn test_run_script() { + let vm = VirtualMachine::new(); + + // test file run + let r = run_script(&vm, "tests/snippets/dir_main/__main__.py"); + assert!(r.is_ok()); + + // test module run + let r = run_script(&vm, "tests/snippets/dir_main"); + assert!(r.is_ok()); +} + fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> Result<(), CompileError> { - match compile::compile( - source, - &compile::Mode::Single, - "".to_string(), - vm.ctx.code_type(), - ) { + match vm.compile(source, &compile::Mode::Single, "".to_string()) { Ok(code) => { - if let Err(err) = vm.run_code_obj(code, scope) { - print_exception(vm, &err); + match vm.run_code_obj(code, scope.clone()) { + Ok(value) => { + // Save non-None values as "_" + + use rustpython_vm::pyobject::{IdProtocol, IntoPyObject}; + + if !value.is(&vm.get_none()) { + let key = objstr::PyString::from("_").into_pyobject(vm); + scope.globals.set_item(key, value, vm).unwrap(); + } + } + + Err(err) => { + print_exception(vm, &err); + } } + Ok(()) } // Don't inject syntax errors for line continuation - Err(err @ CompileError::Parse(ParseError::EOF(_))) => Err(err), + Err( + err @ CompileError { + error: CompileErrorType::Parse(ParseError::EOF(_)), + .. + }, + ) => Err(err), Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - let exc = vm.new_exception(syntax_error, format!("{}", err)); + let exc = vm.new_syntax_error(&err); print_exception(vm, &exc); Err(err) } @@ -163,10 +218,10 @@ fn get_prompt(vm: &VirtualMachine, prompt_name: &str) -> String { fn run_shell(vm: &VirtualMachine) -> PyResult { println!( - "Welcome to the magnificent Rust Python {} interpreter", + "Welcome to the magnificent Rust Python {} interpreter \u{1f631} \u{1f596}", crate_version!() ); - let vars = vm.ctx.new_scope(); + let vars = vm.new_scope_with_builtins(); // Read a single line: let mut input = String::new(); @@ -202,7 +257,10 @@ fn run_shell(vm: &VirtualMachine) -> PyResult { } match shell_exec(vm, &input, vars.clone()) { - Err(CompileError::Parse(ParseError::EOF(_))) => { + Err(CompileError { + error: CompileErrorType::Parse(ParseError::EOF(_)), + .. + }) => { continuing = true; continue; } diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000..8829b75811 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +snippets/whats_left_*.py diff --git a/tests/.travis-runner.sh b/tests/.travis-runner.sh index 8a004d3b19..f555265898 100755 --- a/tests/.travis-runner.sh +++ b/tests/.travis-runner.sh @@ -13,6 +13,7 @@ pip install pipenv if [ $CODE_COVERAGE = "true" ] then find . -name '*.gcda' -delete + find . -name '*.gcno' -delete export CARGO_INCREMENTAL=0 export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads" @@ -31,7 +32,7 @@ then zip -0 ccov.zip `find . \( -name "rustpython*.gc*" \) -print` # Install grcov - curl -L https://github.com/mozilla/grcov/releases/download/v0.4.1/grcov-linux-x86_64.tar.bz2 | tar jxf - + curl -L https://github.com/mozilla/grcov/releases/download/v0.4.2/grcov-linux-x86_64.tar.bz2 | tar jxf - ./grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir "/*" -p "x" > lcov.info diff --git a/tests/Pipfile b/tests/Pipfile index 8ccbddc4da..4bbae44ef9 100644 --- a/tests/Pipfile +++ b/tests/Pipfile @@ -10,4 +10,4 @@ pytest = "*" [dev-packages] [requires] -python_version = "3" +python_version = "3.6" diff --git a/tests/Pipfile.lock b/tests/Pipfile.lock index 78d20f6053..02b33bbdba 100644 --- a/tests/Pipfile.lock +++ b/tests/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "b2d2d68e7d4330ff8d889816c56b9cee4bf54962c86b2c11382108176a201ec8" + "sha256": "ce98de5914393363a8cb86a4753b3964caa53a4659a403a3ef357e2086363ef7" }, "pipfile-spec": 6, "requires": { - "python_version": "3" + "python_version": "3.6" }, "sources": [ { @@ -16,74 +16,99 @@ ] }, "default": { - "aenum": { - "hashes": [ - "sha256:3df9b84cce5dc9ed77c337079f97b66c44c0053eb87d6f4d46b888dc45801e38", - "sha256:7a77c205c4bc9d7fe9bd73b3193002d724aebf5909fa0d297534208953891ec8", - "sha256:a3208e4b28db3a7b232ff69b934aef2ea1bf27286d9978e1e597d46f490e4687" - ], - "version": "==2.1.2" - }, "atomicwrites": { "hashes": [ - "sha256:240831ea22da9ab882b551b31d4225591e5e447a68c5e188db5b89ca1d487585", - "sha256:a24da68318b08ac9c9c45029f4a10371ab5b20e4226738e150e6e7c571630ae6" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.1.5" + "version": "==1.3.0" }, "attrs": { "hashes": [ - "sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265", - "sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.1.0" + "version": "==19.1.0" }, "bytecode": { "hashes": [ - "sha256:cc6931151c7f0a542f8cf7619fe1639af3b9529c4678860fa3239397cb0f7de0", - "sha256:e464004d4a9eeeca987cb4950dba11b827964b6c90cd331c1f20abd2dab3c962" + "sha256:68b1d591c7af0e5c5273e028d3cc0299fbe374dff0cf9149ec7e569be0c573e7", + "sha256:c43d5052cbff076bfdf5b0b93ff6c76e461aab628ce47d30637bb200b6b7bb2c" ], "index": "pypi", - "version": "==0.7.0" + "version": "==0.8.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", + "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + ], + "version": "==0.18" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" + ], + "markers": "python_version > '2.7'", + "version": "==7.0.0" + }, + "packaging": { + "hashes": [ + "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", + "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" ], - "version": "==4.3.0" + "version": "==19.0" }, "pluggy": { "hashes": [ - "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", - "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" ], - "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.2.*'", - "version": "==0.7.1" + "version": "==0.12.0" }, "py": { "hashes": [ - "sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7", - "sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "markers": "python_version != '3.3.*' and python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.2.*'", - "version": "==1.5.4" + "version": "==1.8.0" + }, + "pyparsing": { + "hashes": [ + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + ], + "version": "==2.4.0" }, "pytest": { "hashes": [ - "sha256:86a8dbf407e437351cef4dba46736e9c5a6e3c3ac71b2e942209748e76ff2086", - "sha256:e74466e97ac14582a8188ff4c53e6cc3810315f342f6096899332ae864c1d432" + "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45", + "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da" ], "index": "pypi", - "version": "==3.7.1" + "version": "==4.6.3" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "zipp": { + "hashes": [ + "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", + "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" ], - "version": "==1.11.0" + "version": "==0.5.1" } }, "develop": {} diff --git a/tests/README.md b/tests/README.md index a225ba3b68..ba24ab8bc2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,4 +1,3 @@ - # Test snippets This directory contains two sets of test snippets which can be run in @@ -6,6 +5,10 @@ Python. The `snippets/` directory contains functional tests, and the `benchmarks/` directory contains snippets for use in benchmarking RustPython's performance. +## Generates the test for not implemented methods + +run using cpython not_impl_gen.py it automatically generate a +test snippet to check not yet implemented methods ## Running with CPython + RustPython @@ -15,4 +18,3 @@ compilation to bytecode. When this is done, run the bytecode with rustpython. ## Running with RustPython The other option is to run all snippets with RustPython. - diff --git a/tests/generator/not_impl_methods_footer.txt b/tests/generator/not_impl_methods_footer.txt new file mode 100644 index 0000000000..5cd500999a --- /dev/null +++ b/tests/generator/not_impl_methods_footer.txt @@ -0,0 +1,12 @@ +not_implemented = [(name, method) + for name, (val, methods) in expected_methods.items() + for method in methods + if not hasattr(val, method)] + +for r in not_implemented: + print(r[0], ".", r[1], sep="") +if not not_implemented: + print("Not much \\o/") + +if platform.python_implementation() == "CPython": + assert len(not_implemented) == 0, "CPython should have all the methods" diff --git a/tests/generator/not_impl_methods_header.txt b/tests/generator/not_impl_methods_header.txt new file mode 100644 index 0000000000..4ceefff4d6 --- /dev/null +++ b/tests/generator/not_impl_methods_header.txt @@ -0,0 +1,7 @@ +# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE +# EDIT tests/not_impl_gen.py, NOT THIS FILE. +# RESULTS OF THIS TEST DEPEND ON THE CPYTHON +# VERSION USED TO RUN not_impl_gen.py + +import platform + diff --git a/tests/generator/not_impl_modules_footer.txt b/tests/generator/not_impl_modules_footer.txt new file mode 100644 index 0000000000..c475ca1782 --- /dev/null +++ b/tests/generator/not_impl_modules_footer.txt @@ -0,0 +1,32 @@ + + +rustpymods = list( + map( + lambda mod: mod[0], + filter( + lambda mod: (mod[1] == "" or mod[1] == ".py") and "LICENSE" not in mod[0], + map(os.path.splitext, os.listdir(libdir)), + ), + ) +) +rustpymods += list(sys.builtin_module_names) + +rustpymods = dict(map( + lambda mod: ( + mod, + set(dir(__import__(mod))) + if mod not in ("this", "antigravity") + else None, + ), + rustpymods +)) + +for modname, cpymod in cpymods.items(): + if modname in rustpymods: + rustpymod = rustpymods[modname] + if rustpymod: + for item in cpymod - rustpymod: + print(f"{modname}.{item}") + else: + print(f"{modname} (entire module)") + diff --git a/tests/generator/not_impl_modules_header.txt b/tests/generator/not_impl_modules_header.txt new file mode 100644 index 0000000000..8879230d40 --- /dev/null +++ b/tests/generator/not_impl_modules_header.txt @@ -0,0 +1,8 @@ +# WARNING: THIS IS AN AUTOMATICALLY GENERATED FILE +# EDIT tests/not_impl_mods_gen.sh, NOT THIS FILE. +# RESULTS OF THIS TEST DEPEND ON THE CPYTHON +# VERSION AND PYTHON ENVIRONMENT USED +# TO RUN not_impl_mods_gen.py + +import sys +import os diff --git a/tests/not_impl_gen.py b/tests/not_impl_gen.py new file mode 100644 index 0000000000..7e0c029e9f --- /dev/null +++ b/tests/not_impl_gen.py @@ -0,0 +1,105 @@ +# It's recommended to run this with `python3 -I not_impl_gen.py`, to make sure +# that nothing in your global Python environment interferes with what's being +# extracted here. + +import pkgutil +import os +import sys + +sys.path = list( + filter( + lambda path: "site-packages" not in path and "dist-packages" not in path, + sys.path, + ) +) + + +def attr_is_not_inherited(type_, attr): + """ + returns True if type_'s attr is not inherited from any of its base classes + """ + + bases = type_.__mro__[1:] + + return getattr(type_, attr) not in (getattr(base, attr, None) for base in bases) + + +def gen_methods(header, footer, output): + objects = [ + bool, + bytearray, + bytes, + complex, + dict, + float, + frozenset, + int, + list, + memoryview, + range, + set, + str, + tuple, + object, + ] + + output.write(header.read()) + output.write("expected_methods = {\n") + + for obj in objects: + output.write(f" '{obj.__name__}': ({obj.__name__}, [\n") + output.write( + "\n".join( + f" {attr!r}," + for attr in dir(obj) + if attr_is_not_inherited(obj, attr) + ) + ) + output.write("\n ])," + ("\n" if objects[-1] == obj else "\n\n")) + + output.write("}\n\n") + output.write(footer.read()) + +def get_module_methods(name): + try: + return set(dir(__import__(name))) if name not in ("this", "antigravity") else None + except ModuleNotFoundError: + return None + + +def gen_modules(header, footer, output): + output.write(header.read()) + + modules = dict( + map( + lambda mod: ( + mod.name, + # check name b/c modules listed have side effects on import, + # e.g. printing something or opening a webpage + get_module_methods(mod.name) + ), + pkgutil.iter_modules(), + ) + ) + + print( + f""" +cpymods = {modules!r} +libdir = {os.path.abspath("../Lib/")!r} +""", + file=output, + ) + + output.write(footer.read()) + + +gen_funcs = {"methods": gen_methods, "modules": gen_modules} + + +for name, gen_func in gen_funcs.items(): + gen_func( + header=open(f"generator/not_impl_{name}_header.txt"), + footer=open(f"generator/not_impl_{name}_footer.txt"), + output=open(f"snippets/whats_left_{name}.py", "w"), + ) + diff --git a/tests/snippets/append.py b/tests/snippets/append.py deleted file mode 100644 index a0490cb6d3..0000000000 --- a/tests/snippets/append.py +++ /dev/null @@ -1,2 +0,0 @@ -x = [] -x.append(1) diff --git a/tests/snippets/ast_snippet.py b/tests/snippets/ast_snippet.py index 43bf74756b..dad767b45b 100644 --- a/tests/snippets/ast_snippet.py +++ b/tests/snippets/ast_snippet.py @@ -21,3 +21,10 @@ def foo(): assert foo.body[0].value.func.id == 'print' assert foo.body[0].lineno == 3 assert foo.body[1].lineno == 4 + +n = ast.parse("3 < 4 > 5\n") +assert n.body[0].value.left.n == 3 +assert 'Lt' in str(n.body[0].value.ops[0]) +assert 'Gt' in str(n.body[0].value.ops[1]) +assert n.body[0].value.comparators[0].n == 4 +assert n.body[0].value.comparators[1].n == 5 diff --git a/tests/snippets/bools.py b/tests/snippets/bools.py index 0c277143bc..23f22dce79 100644 --- a/tests/snippets/bools.py +++ b/tests/snippets/bools.py @@ -51,3 +51,29 @@ def __bool__(self): assert int(True) == 1 assert True.conjugate() == 1 assert isinstance(True.conjugate(), int) + +# Boolean operations on pairs of Bools should return Bools, not ints +assert (False | True) is True +assert (False & True) is False +assert (False ^ True) is True +# But only if both are Bools +assert (False | 1) is not True +assert (0 | True) is not True +assert (False & 1) is not False +assert (0 & True) is not False +assert (False ^ 1) is not True +assert (0 ^ True) is not True + +# Check that the same works with __XXX__ methods +assert False.__or__(0) is not False +assert False.__or__(False) is False +assert False.__ror__(0) is not False +assert False.__ror__(False) is False +assert False.__and__(0) is not False +assert False.__and__(False) is False +assert False.__rand__(0) is not False +assert False.__rand__(False) is False +assert False.__xor__(0) is not False +assert False.__xor__(False) is False +assert False.__rxor__(0) is not False +assert False.__rxor__(False) is False diff --git a/tests/snippets/builtin_all.py b/tests/snippets/builtin_all.py new file mode 100644 index 0000000000..4195d5f2ed --- /dev/null +++ b/tests/snippets/builtin_all.py @@ -0,0 +1,10 @@ +from testutils import assert_raises +from testutils import TestFailingBool, TestFailingIter + +assert all([True]) +assert not all([False]) +assert all([]) +assert not all([False, TestFailingBool()]) + +assert_raises(RuntimeError, lambda: all(TestFailingIter())) +assert_raises(RuntimeError, lambda: all([TestFailingBool()])) diff --git a/tests/snippets/builtin_any.py b/tests/snippets/builtin_any.py index a8679d5fe3..59b4514f85 100644 --- a/tests/snippets/builtin_any.py +++ b/tests/snippets/builtin_any.py @@ -1,13 +1,10 @@ -assert any([1]); -assert not any([]); -assert not any([0,0,0,0]); -assert any([0,0,1,0,0]); -def anything(a): - return a +from testutils import assert_raises +from testutils import TestFailingBool, TestFailingIter -class Test: - def __iter__(self): - while True: - yield True +assert any([True]) +assert not any([False]) +assert not any([]) +assert any([True, TestFailingBool()]) -assert any(map(anything, Test())) +assert_raises(RuntimeError, lambda: any(TestFailingIter())) +assert_raises(RuntimeError, lambda: any([TestFailingBool()])) diff --git a/tests/snippets/builtin_callable.py b/tests/snippets/builtin_callable.py index c22db07797..db554df245 100644 --- a/tests/snippets/builtin_callable.py +++ b/tests/snippets/builtin_callable.py @@ -1,9 +1,8 @@ assert not callable(1) def f(): pass -# TODO uncomment when callable types get unified __call__ (or equivalent) -#assert callable(f) -#assert callable(len) -#assert callable(lambda: 1) +assert callable(f) +assert callable(len) +assert callable(lambda: 1) assert callable(int) class C: @@ -13,7 +12,7 @@ def __init__(self): def f(self): pass assert callable(C) assert not callable(C()) -#assert callable(C().f) +assert callable(C().f) class C: def __call__(self): pass diff --git a/tests/snippets/builtin_chr.py b/tests/snippets/builtin_chr.py new file mode 100644 index 0000000000..1ad642a586 --- /dev/null +++ b/tests/snippets/builtin_chr.py @@ -0,0 +1,8 @@ +from testutils import assert_raises + +assert "a" == chr(97) +assert "é" == chr(233) +assert "🤡" == chr(129313) + +assert_raises(TypeError, lambda: chr(), "chr() takes exactly one argument (0 given)") +assert_raises(ValueError, lambda: chr(0x110005), "ValueError: chr() arg not in range(0x110000)") diff --git a/tests/snippets/builtin_complex.py b/tests/snippets/builtin_complex.py index c073f74337..87e25b9b38 100644 --- a/tests/snippets/builtin_complex.py +++ b/tests/snippets/builtin_complex.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + # __abs__ assert abs(complex(3, 4)) == 5 @@ -21,26 +23,121 @@ assert complex(1, 2) != complex(1, 1) assert complex(1, 2) != 'foo' assert complex(1, 2).__eq__('foo') == NotImplemented +assert 1j != 10 ** 1000 + +# __mul__, __rmul__ + +assert complex(2, -3) * complex(-5, 7) == complex(11, 29) +assert complex(2, -3) * 5 == complex(10, -15) +assert 5 * complex(2, -3) == complex(2, -3) * 5 + +# __truediv__, __rtruediv__ + +assert complex(2, -3) / 2 == complex(1, -1.5) +assert 5 / complex(3, -4) == complex(0.6, 0.8) + +# __mod__, __rmod__ + +assert_raises( + TypeError, + lambda: complex(2, -3) % 2, + "can't mod complex numbers.") +assert_raises( + TypeError, + lambda: 2 % complex(2, -3), + "can't mod complex numbers.") + +# __floordiv__, __rfloordiv__ + +assert_raises( + TypeError, + lambda: complex(2, -3) // 2, + "can't take floor of complex number.") +assert_raises( + TypeError, + lambda: 2 // complex(2, -3), + "can't take floor of complex number.") + +# __divmod__, __rdivmod__ + +assert_raises( + TypeError, + lambda: divmod(complex(2, -3), 2), + "can't take floor or mod of complex number.") +assert_raises( + TypeError, + lambda: divmod(2, complex(2, -3)), + "can't take floor or mod of complex number.") + +# __pow__, __rpow__ + +# assert 1j ** 2 == -1 +assert complex(1) ** 2 == 1 +assert 2 ** complex(2) == 4 # __neg__ assert -complex(1, -1) == complex(-1, 1) assert -complex(0, 0) == complex(0, 0) -# real +# __bool__ + +assert bool(complex(0, 0)) is False +assert bool(complex(0, 1)) is True +assert bool(complex(1, 0)) is True + +# __hash__ + +assert hash(complex(1)) == hash(float(1)) == hash(int(1)) +assert hash(complex(-1)) == hash(float(-1)) == hash(int(-1)) +assert hash(complex(3.14)) == hash(float(3.14)) +assert hash(complex(-float('inf'))) == hash(-float('inf')) +assert hash(1j) != hash(1) + +# numbers.Complex a = complex(3, 4) b = 4j assert a.real == 3 assert b.real == 0 -# imag - assert a.imag == 4 assert b.imag == 4 +assert a.conjugate() == 3 - 4j +assert b.conjugate() == -4j + # int and complex addition assert 1 + 1j == complex(1, 1) assert 1j + 1 == complex(1, 1) assert (1j + 1) + 3 == complex(4, 1) assert 3 + (1j + 1) == complex(4, 1) + +# float and complex addition +assert 1.1 + 1.2j == complex(1.1, 1.2) +assert 1.3j + 1.4 == complex(1.4, 1.3) +assert (1.5j + 1.6) + 3 == complex(4.6, 1.5) +assert 3.5 + (1.1j + 1.2) == complex(4.7, 1.1) + +# subtraction +assert 1 - 1j == complex(1, -1) +assert 1j - 1 == complex(-1, 1) +assert 2j - 1j == complex(0, 1) + +# type error addition +assert_raises(TypeError, lambda: 1j + 'str') +assert_raises(TypeError, lambda: 1j - 'str') +assert_raises(TypeError, lambda: 'str' + 1j) +assert_raises(TypeError, lambda: 'str' - 1j) + +# overflow +msg = 'int too large to convert to float' +assert_raises(OverflowError, lambda: complex(10 ** 1000, 0), msg) +assert_raises(OverflowError, lambda: complex(0, 10 ** 1000), msg) +assert_raises(OverflowError, lambda: 0j + 10 ** 1000, msg) + +# str/repr +assert '(1+1j)' == str(1+1j) +assert '(1-1j)' == str(1-1j) +assert '(1+1j)' == repr(1+1j) +assert '(1-1j)' == repr(1-1j) diff --git a/tests/snippets/builtin_dir.py b/tests/snippets/builtin_dir.py index a121fb718c..3e808597c1 100644 --- a/tests/snippets/builtin_dir.py +++ b/tests/snippets/builtin_dir.py @@ -1,3 +1,5 @@ +assert isinstance(dir(), list) +assert '__builtins__' in dir() class A: def test(): @@ -8,6 +10,9 @@ def test(): assert "test" in dir(a), "test not in a" assert "test" in dir(A), "test not in A" +a.x = 3 +assert "x" in dir(a), "x not in a" + class B(A): def __dir__(self): return ('q', 'h') @@ -18,6 +23,16 @@ def __dir__(self): # This calls type.__dir__ so isn't changed (but inheritance works)! assert 'test' in dir(A) +# eval() takes any mapping-like type, so dir() must support them +# TODO: eval() should take any mapping as locals, not just dict-derived types +class A(dict): + def __getitem__(self, x): + return dir + def keys(self): + yield 6 + yield 5 +assert eval("dir()", {}, A()) == [5, 6] + import socket assert "AF_INET" in dir(socket) diff --git a/tests/snippets/builtin_file.py b/tests/snippets/builtin_file.py new file mode 100644 index 0000000000..2c80d15904 --- /dev/null +++ b/tests/snippets/builtin_file.py @@ -0,0 +1,7 @@ +import os + +from import_file import import_file + +import_file() + +assert os.path.basename(__file__) == "builtin_file.py" diff --git a/tests/snippets/builtin_format.py b/tests/snippets/builtin_format.py index 6c06cbd98c..55a6a3da12 100644 --- a/tests/snippets/builtin_format.py +++ b/tests/snippets/builtin_format.py @@ -7,3 +7,8 @@ assert format({}) == "{}" assert_raises(TypeError, lambda: format({}, 'b'), 'format_spec not empty for dict') + +class BadFormat: + def __format__(self, spec): + return 42 +assert_raises(TypeError, lambda: format(BadFormat())) diff --git a/tests/snippets/builtin_open.py b/tests/snippets/builtin_open.py index 41bcf2b800..d7c310a921 100644 --- a/tests/snippets/builtin_open.py +++ b/tests/snippets/builtin_open.py @@ -6,5 +6,14 @@ assert_raises(FileNotFoundError, lambda: open('DoesNotExist')) # Use open as a context manager -with open('README.md') as fp: - fp.read() +with open('README.md', 'rt') as fp: + contents = fp.read() + assert type(contents) == str, "type is " + str(type(contents)) + +with open('README.md', 'r') as fp: + contents = fp.read() + assert type(contents) == str, "type is " + str(type(contents)) + +with open('README.md', 'rb') as fp: + contents = fp.read() + assert type(contents) == bytes, "type is " + str(type(contents)) diff --git a/tests/snippets/builtin_ord.py b/tests/snippets/builtin_ord.py index 2549f28e7d..c9bc40f5f3 100644 --- a/tests/snippets/builtin_ord.py +++ b/tests/snippets/builtin_ord.py @@ -3,7 +3,11 @@ assert ord("a") == 97 assert ord("é") == 233 assert ord("🤡") == 129313 +assert ord(b'a') == 97 +assert ord(bytearray(b'a')) == 97 assert_raises(TypeError, lambda: ord(), "ord() is called with no argument") assert_raises(TypeError, lambda: ord(""), "ord() is called with an empty string") assert_raises(TypeError, lambda: ord("ab"), "ord() is called with more than one character") +assert_raises(TypeError, lambda: ord(b"ab"), "ord() expected a character, but string of length 2 found") +assert_raises(TypeError, lambda: ord(1), "ord() expected a string, bytes or bytearray, but found int") diff --git a/tests/snippets/builtin_range.py b/tests/snippets/builtin_range.py index 89014880ce..03d1cb2cf7 100644 --- a/tests/snippets/builtin_range.py +++ b/tests/snippets/builtin_range.py @@ -17,6 +17,7 @@ assert_raises(ValueError, lambda: range(10).index(10), 'out of bounds') assert_raises(ValueError, lambda: range(4, 10, 2).index(5), 'out of step') assert_raises(ValueError, lambda: range(10).index('foo'), 'not an int') +assert_raises(ValueError, lambda: range(1, 10, 0), 'step is zero') # count tests assert range(10).count(2) == 1 @@ -27,6 +28,11 @@ assert range(4, 10, 2).count(7) == 0 assert range(10).count("foo") == 0 +# __eq__ +assert range(1, 2, 3) == range(1, 2, 3) +assert range(1, 2, 1) == range(1, 2) +assert range(2) == range(0, 2) + # __bool__ assert bool(range(1)) assert bool(range(1, 2)) @@ -50,3 +56,13 @@ assert list(reversed(range(5))) == [4, 3, 2, 1, 0] assert list(reversed(range(5, 0, -1))) == [1, 2, 3, 4, 5] assert list(reversed(range(1,10,5))) == [6, 1] + +# range retains the original int refs +i = 2**64 +assert range(i).stop is i + +# negative index +assert range(10)[-1] == 9 +assert_raises(IndexError, lambda: range(10)[-11], 'out of bound') +assert range(10)[-2:4] == range(8, 4) +assert range(10)[-6:-2] == range(4, 8) diff --git a/tests/snippets/builtin_reversed.py b/tests/snippets/builtin_reversed.py index 2bbfcb98a2..261b5c3263 100644 --- a/tests/snippets/builtin_reversed.py +++ b/tests/snippets/builtin_reversed.py @@ -1 +1,4 @@ assert list(reversed(range(5))) == [4, 3, 2, 1, 0] + +l = [5,4,3,2,1] +assert list(reversed(l)) == [1,2,3,4,5] diff --git a/tests/snippets/builtin_slice.py b/tests/snippets/builtin_slice.py index 1d8a93b479..46592769ef 100644 --- a/tests/snippets/builtin_slice.py +++ b/tests/snippets/builtin_slice.py @@ -2,24 +2,25 @@ a = [] assert a[:] == [] -assert a[:2**100] == [] -assert a[-2**100:] == [] -assert a[::2**100] == [] +assert a[: 2 ** 100] == [] +assert a[-2 ** 100 :] == [] +assert a[:: 2 ** 100] == [] assert a[10:20] == [] assert a[-20:-10] == [] b = [1, 2] assert b[:] == [1, 2] -assert b[:2**100] == [1, 2] -assert b[-2**100:] == [1, 2] -assert b[2**100:] == [] -assert b[::2**100] == [1] +assert b[slice(None)] == [1, 2] +assert b[: 2 ** 100] == [1, 2] +assert b[-2 ** 100 :] == [1, 2] +assert b[2 ** 100 :] == [] +assert b[:: 2 ** 100] == [1] assert b[-10:1] == [1] assert b[0:0] == [] assert b[1:0] == [] -assert_raises(ValueError, lambda: b[::0], 'zero step slice') +assert_raises(ValueError, lambda: b[::0], "zero step slice") assert b[::-1] == [2, 1] assert b[1::-1] == [2, 1] @@ -33,7 +34,7 @@ assert c[9:6:-3] == [9] assert c[9::-3] == [9, 6, 3, 0] assert c[9::-4] == [9, 5, 1] -assert c[8::-2**100] == [8] +assert c[8 :: -2 ** 100] == [8] assert c[7:7:-2] == [] assert c[7:8:-2] == [] @@ -43,6 +44,7 @@ assert d[3::-1] == "4321" assert d[4::-3] == "52" +assert [1, 2, 3, 5, 6][-1:-5:-1] == [6, 5, 3, 2] # #746 slice_a = slice(5) assert slice_a.start is None @@ -59,6 +61,12 @@ assert slice_c.stop == 5 assert slice_c.step == 2 +a = object() +slice_d = slice(a, "v", 1.0) +assert slice_d.start is a +assert slice_d.stop == "v" +assert slice_d.step == 1.0 + class SubScript(object): def __getitem__(self, item): @@ -71,3 +79,53 @@ def __setitem__(self, key, value): ss = SubScript() _ = ss[:] ss[:1] = 1 + + +class CustomIndex: + def __init__(self, x): + self.x = x + + def __index__(self): + return self.x + + +assert c[CustomIndex(1):CustomIndex(3)] == [1, 2] +assert d[CustomIndex(1):CustomIndex(3)] == "23" + + +def test_all_slices(): + """ + test all possible slices except big number + """ + + mod = __import__('cpython_generated_slices') + + ll = mod.LL + start = mod.START + end = mod.END + step = mod.STEP + slices_res = mod.SLICES_RES + + count = 0 + failures = [] + for s in start: + for e in end: + for t in step: + lhs = ll[s:e:t] + try: + assert lhs == slices_res[count] + except AssertionError: + failures.append( + "start: {} ,stop: {}, step {}. Expected: {}, found: {}".format( + s, e, t, lhs, slices_res[count] + ) + ) + count += 1 + + if failures: + for f in failures: + print(f) + print(len(failures), "slices failed") + + +test_all_slices() diff --git a/tests/snippets/builtins_module.py b/tests/snippets/builtins_module.py new file mode 100644 index 0000000000..e3bdcdc16c --- /dev/null +++ b/tests/snippets/builtins_module.py @@ -0,0 +1,28 @@ +from testutils import assertRaises + +assert '__builtins__' in globals() +# assert type(__builtins__).__name__ == 'module' +with assertRaises(AttributeError): + __builtins__.__builtins__ + +__builtins__.x = 'new' +assert x == 'new' + +exec('assert "__builtins__" in globals()', dict()) +exec('assert __builtins__ == 7', {'__builtins__': 7}) +exec('assert not isinstance(__builtins__, dict)') +exec('assert isinstance(__builtins__, dict)', {}) + +namespace = {} +exec('', namespace) +assert namespace['__builtins__'] == __builtins__.__dict__ + +# with assertRaises(NameError): +# exec('print(__builtins__)', {'__builtins__': {}}) + +# __builtins__ is deletable but names are alive +del __builtins__ +with assertRaises(NameError): + __builtins__ + +assert print diff --git a/tests/snippets/bytearray.py b/tests/snippets/bytearray.py index 563da2c53b..f69edeac1b 100644 --- a/tests/snippets/bytearray.py +++ b/tests/snippets/bytearray.py @@ -1,53 +1,605 @@ -#__getitem__ not implemented yet -#a = bytearray(b'abc') -#assert a[0] == b'a' -#assert a[1] == b'b' +from testutils import assertRaises -assert len(bytearray([1,2,3])) == 3 +# new +assert bytearray([1, 2, 3]) +assert bytearray((1, 2, 3)) +assert bytearray(range(4)) +assert bytearray(3) +assert b"bla" +assert ( + bytearray("bla", "utf8") == bytearray("bla", encoding="utf-8") == bytearray(b"bla") +) +with assertRaises(TypeError): + bytearray("bla") +with assertRaises(TypeError): + bytearray("bla", encoding=b"jilj") -assert bytearray(b'1a23').isalnum() -assert not bytearray(b'1%a23').isalnum() +assert bytearray( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" +) == bytearray(range(0, 256)) +assert bytearray( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" +) == bytearray(range(0, 256)) +assert bytearray(b"omkmok\Xaa") == bytearray( + [111, 109, 107, 109, 111, 107, 92, 88, 97, 97] +) -assert bytearray(b'abc').isalpha() -assert not bytearray(b'abc1').isalpha() + +a = bytearray(b"abcd") +b = bytearray(b"ab") +c = bytearray(b"abcd") + + +# repr +assert repr(bytearray([0, 1, 2])) == repr(bytearray(b"\x00\x01\x02")) +assert ( + repr(bytearray([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255])) + == "bytearray(b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff')" +) +assert repr(bytearray(b"abcd")) == "bytearray(b'abcd')" + +# len +assert len(bytearray("abcdé", "utf8")) == 6 + +# comp +assert a == b"abcd" +assert a > b +assert a >= b +assert b < a +assert b <= a + +assert bytearray(b"foobar").__eq__(2) == NotImplemented +assert bytearray(b"foobar").__ne__(2) == NotImplemented +assert bytearray(b"foobar").__gt__(2) == NotImplemented +assert bytearray(b"foobar").__ge__(2) == NotImplemented +assert bytearray(b"foobar").__lt__(2) == NotImplemented +assert bytearray(b"foobar").__le__(2) == NotImplemented + +# # hash +with assertRaises(TypeError): + hash(bytearray(b"abcd")) # unashable + +# # iter +[i for i in bytearray(b"abcd")] == ["a", "b", "c", "d"] +assert list(bytearray(3)) == [0, 0, 0] + +# add +assert a + b == bytearray(b"abcdab") + +# contains +assert bytearray(b"ab") in bytearray(b"abcd") +assert bytearray(b"cd") in bytearray(b"abcd") +assert bytearray(b"abcd") in bytearray(b"abcd") +assert bytearray(b"a") in bytearray(b"abcd") +assert bytearray(b"d") in bytearray(b"abcd") +assert bytearray(b"dc") not in bytearray(b"abcd") +assert 97 in bytearray(b"abcd") +assert 150 not in bytearray(b"abcd") +with assertRaises(ValueError): + 350 in bytearray(b"abcd") + + +# getitem +d = bytearray(b"abcdefghij") + +assert d[1] == 98 +assert d[-1] == 106 +assert d[2:6] == bytearray(b"cdef") +assert d[-6:] == bytearray(b"efghij") +assert d[1:8:2] == bytearray(b"bdfh") +assert d[8:1:-2] == bytearray(b"igec") + + +# # is_xx methods + +assert bytearray(b"1a23").isalnum() +assert not bytearray(b"1%a23").isalnum() + +assert bytearray(b"abc").isalpha() +assert not bytearray(b"abc1").isalpha() # travis doesn't like this -#assert bytearray(b'xyz').isascii() -#assert not bytearray([128, 157, 32]).isascii() +# assert bytearray(b'xyz').isascii() +# assert not bytearray([128, 157, 32]).isascii() + +assert bytearray(b"1234567890").isdigit() +assert not bytearray(b"12ab").isdigit() -assert bytearray(b'1234567890').isdigit() -assert not bytearray(b'12ab').isdigit() +l = bytearray(b"lower") +b = bytearray(b"UPPER") -l = bytearray(b'lower') assert l.islower() assert not l.isupper() -assert l.upper().isupper() -assert not bytearray(b'Super Friends').islower() +assert b.isupper() +assert not bytearray(b"Super Friends").islower() -assert bytearray(b' \n\t').isspace() -assert not bytearray(b'\td\n').isspace() +assert bytearray(b" \n\t").isspace() +assert not bytearray(b"\td\n").isspace() -b = bytearray(b'UPPER') assert b.isupper() assert not b.islower() -assert b.lower().islower() -assert not bytearray(b'tuPpEr').isupper() +assert l.islower() +assert not bytearray(b"tuPpEr").isupper() -assert bytearray(b'Is Title Case').istitle() -assert not bytearray(b'is Not title casE').istitle() +assert bytearray(b"Is Title Case").istitle() +assert not bytearray(b"is Not title casE").istitle() -a = bytearray(b'abcd') -a.clear() -assert len(a) == 0 +# upper lower, capitalize, swapcase +l = bytearray(b"lower") +b = bytearray(b"UPPER") +assert l.lower().islower() +assert b.upper().isupper() +assert l.capitalize() == b"Lower" +assert b.capitalize() == b"Upper" +assert bytearray().capitalize() == bytearray() +assert b"AaBbCc123'@/".swapcase().swapcase() == b"AaBbCc123'@/" +assert b"AaBbCc123'@/".swapcase() == b"aAbBcC123'@/" +# # hex from hex +assert bytearray([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" + +bytearray.fromhex("62 6c7a 34350a ") == b"blz45\n" try: - bytearray([400]) -except ValueError: - pass -else: - assert False + bytearray.fromhex("62 a 21") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 4" +try: + bytearray.fromhex("6Z2") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 1" +with assertRaises(TypeError): + bytearray.fromhex(b"hhjjk") +# center +assert [bytearray(b"koki").center(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"|koki|", + b"||koki|", + b"||koki||", + b"|||koki||", +] + +assert [bytearray(b"kok").center(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"|kok|", + b"|kok||", + b"||kok||", + b"||kok|||", + b"|||kok|||", +] +bytearray(b"kok").center(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + bytearray(b"b").center(2, "a") +with assertRaises(TypeError): + bytearray(b"b").center(2, b"ba") +with assertRaises(TypeError): + bytearray(b"b").center(b"ba") +assert bytearray(b"kok").center(5, bytearray(b"x")) == b"xkokx" +bytearray(b"kok").center(-5) == b"kok" + + +# ljust +assert [bytearray(b"koki").ljust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"koki|", + b"koki||", + b"koki|||", + b"koki||||", + b"koki|||||", +] +assert [bytearray(b"kok").ljust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"kok||", + b"kok|||", + b"kok||||", + b"kok|||||", + b"kok||||||", +] + +bytearray(b"kok").ljust(4) == b"kok " # " test no arg" +with assertRaises(TypeError): + bytearray(b"b").ljust(2, "a") +with assertRaises(TypeError): + bytearray(b"b").ljust(2, b"ba") +with assertRaises(TypeError): + bytearray(b"b").ljust(b"ba") +assert bytearray(b"kok").ljust(5, bytearray(b"x")) == b"kokxx" +assert bytearray(b"kok").ljust(-5) == b"kok" + +# rjust +assert [bytearray(b"koki").rjust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"||koki", + b"|||koki", + b"||||koki", + b"|||||koki", +] +assert [bytearray(b"kok").rjust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"|kok", + b"||kok", + b"|||kok", + b"||||kok", + b"|||||kok", + b"||||||kok", +] + + +bytearray(b"kok").rjust(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + bytearray(b"b").rjust(2, "a") +with assertRaises(TypeError): + bytearray(b"b").rjust(2, b"ba") +with assertRaises(TypeError): + bytearray(b"b").rjust(b"ba") +assert bytearray(b"kok").rjust(5, bytearray(b"x")) == b"xxkok" +assert bytearray(b"kok").rjust(-5) == b"kok" + + +# count +assert bytearray(b"azeazerazeazopia").count(b"aze") == 3 +assert bytearray(b"azeazerazeazopia").count(b"az") == 4 +assert bytearray(b"azeazerazeazopia").count(b"a") == 5 +assert bytearray(b"123456789").count(b"") == 10 +assert bytearray(b"azeazerazeazopia").count(bytearray(b"aze")) == 3 +assert bytearray(b"azeazerazeazopia").count(memoryview(b"aze")) == 3 +assert bytearray(b"azeazerazeazopia").count(memoryview(b"aze"), 1, 9) == 1 +assert bytearray(b"azeazerazeazopia").count(b"aze", None, None) == 3 +assert bytearray(b"azeazerazeazopia").count(b"aze", 2, None) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", 2) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", None, 7) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", None, 7) == 2 +assert bytearray(b"azeazerazeazopia").count(b"aze", 2, 7) == 1 +assert bytearray(b"azeazerazeazopia").count(b"aze", -13, -10) == 1 +assert bytearray(b"azeazerazeazopia").count(b"aze", 1, 10000) == 2 +with assertRaises(ValueError): + bytearray(b"ilj").count(3550) +assert bytearray(b"azeazerazeazopia").count(97) == 5 + +# join +assert bytearray(b"").join( + (b"jiljl", bytearray(b"kmoomk"), memoryview(b"aaaa")) +) == bytearray(b"jiljlkmoomkaaaa") +with assertRaises(TypeError): + bytearray(b"").join((b"km", "kl")) + + +# endswith startswith +assert bytearray(b"abcde").endswith(b"de") +assert bytearray(b"abcde").endswith(b"") +assert not bytearray(b"abcde").endswith(b"zx") +assert bytearray(b"abcde").endswith(b"bc", 0, 3) +assert not bytearray(b"abcde").endswith(b"bc", 2, 3) +assert bytearray(b"abcde").endswith((b"c", bytearray(b"de"))) + +assert bytearray(b"abcde").startswith(b"ab") +assert bytearray(b"abcde").startswith(b"") +assert not bytearray(b"abcde").startswith(b"zx") +assert bytearray(b"abcde").startswith(b"cd", 2) +assert not bytearray(b"abcde").startswith(b"cd", 1, 4) +assert bytearray(b"abcde").startswith((b"a", bytearray(b"bc"))) + + +# index find +assert bytearray(b"abcd").index(b"cd") == 2 +assert bytearray(b"abcd").index(b"cd", 0) == 2 +assert bytearray(b"abcd").index(b"cd", 1) == 2 +assert bytearray(b"abcd").index(99) == 2 +with assertRaises(ValueError): + bytearray(b"abcde").index(b"c", 3, 1) +with assertRaises(ValueError): + bytearray(b"abcd").index(b"cdaaaaa") +with assertRaises(ValueError): + bytearray(b"abcd").index(b"b", 3, 4) +with assertRaises(ValueError): + bytearray(b"abcd").index(1) + + +assert bytearray(b"abcd").find(b"cd") == 2 +assert bytearray(b"abcd").find(b"cd", 0) == 2 +assert bytearray(b"abcd").find(b"cd", 1) == 2 +assert bytearray(b"abcde").find(b"c", 3, 1) == -1 +assert bytearray(b"abcd").find(b"cdaaaaa") == -1 +assert bytearray(b"abcd").find(b"b", 3, 4) == -1 +assert bytearray(b"abcd").find(1) == -1 +assert bytearray(b"abcd").find(99) == 2 + +assert bytearray(b"abcdabcda").find(b"a") == 0 +assert bytearray(b"abcdabcda").rfind(b"a") == 8 +assert bytearray(b"abcdabcda").rfind(b"a", 2, 6) == 4 +assert bytearray(b"abcdabcda").rfind(b"a", None, 6) == 4 +assert bytearray(b"abcdabcda").rfind(b"a", 2, None) == 8 +assert bytearray(b"abcdabcda").index(b"a") == 0 +assert bytearray(b"abcdabcda").rindex(b"a") == 8 -b = bytearray(b'test') + +# make trans +# fmt: off +assert ( + bytearray.maketrans(memoryview(b"abc"), bytearray(b"zzz")) + == bytes([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 122, 122, 122, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]) +) +# fmt: on + +# translate +assert bytearray(b"hjhtuyjyujuyj").translate( + bytearray.maketrans(b"hj", bytearray(b"ab")), bytearray(b"h") +) == bytearray(b"btuybyubuyb") +assert bytearray(b"hjhtuyjyujuyj").translate( + bytearray.maketrans(b"hj", bytearray(b"ab")), bytearray(b"a") +) == bytearray(b"abatuybyubuyb") +assert bytearray(b"hjhtuyjyujuyj").translate( + bytearray.maketrans(b"hj", bytearray(b"ab")) +) == bytearray(b"abatuybyubuyb") +assert bytearray(b"hjhtuyfjtyhuhjuyj").translate(None, bytearray(b"ht")) == bytearray( + b"juyfjyujuyj" +) +assert bytearray(b"hjhtuyfjtyhuhjuyj").translate(None, delete=b"ht") == bytearray( + b"juyfjyujuyj" +) + + +# strip lstrip rstrip +assert bytearray(b" spacious ").strip() == bytearray(b"spacious") +assert bytearray(b"www.example.com").strip(b"cmowz.") == bytearray(b"example") +assert bytearray(b" spacious ").lstrip() == bytearray(b"spacious ") +assert bytearray(b"www.example.com").lstrip(b"cmowz.") == bytearray(b"example.com") +assert bytearray(b" spacious ").rstrip() == bytearray(b" spacious") +assert bytearray(b"mississippi").rstrip(b"ipz") == bytearray(b"mississ") + + + +# split +assert bytearray(b"1,2,3").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"1,2,3").split(bytearray(b","), maxsplit=1) == [bytearray(b"1"), bytearray(b"2,3")] +assert bytearray(b"1,2,,3,").split(bytearray(b",")) == [bytearray(b"1"), bytearray(b"2"), bytearray(b""), bytearray(b"3"), bytearray(b"")] +assert bytearray(b"1 2 3").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"1 2 3").split(maxsplit=1) == [bytearray(b"1"), bytearray(b"2 3")] +assert bytearray(b" 1 2 3 ").split() == [bytearray(b"1"), bytearray(b"2"), bytearray(b"3")] +assert bytearray(b"k\ruh\nfz e f").split() == [bytearray(b"k"), bytearray(b"uh"), bytearray(b"fz"), bytearray(b"e"), bytearray(b"f")] +assert bytearray(b"Two lines\n").split(bytearray(b"\n")) == [bytearray(b"Two lines"), bytearray(b"")] +assert bytearray(b"").split() == [] +assert bytearray(b"").split(bytearray(b"\n")) == [bytearray(b"")] +assert bytearray(b"\n").split(bytearray(b"\n")) == [bytearray(b""), bytearray(b"")] + +SPLIT_FIXTURES = [ + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + -1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + -1, + ], + # maxsplit + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], []], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], [3]], + 1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 4, 5], [1, 2, 3], [1, 2, 3]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104, 10, 102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102]], + 0, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122], [101, 102]], + 1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102, 9, 9]], + [[13, 13, 13, 117, 104], [102, 122], [101, 102]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 10, 10, 102, 122, 32, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [[49, 44, 50, 44, 51], [44], [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 44, 50, 44, 51], [44], [[49], [50, 44, 51]], [[49, 44, 50], [51]], 1], + [ + [49, 44, 50, 44, 44, 51, 44], + [44], + [[49], [50], [], [51], []], + [[49], [50], [], [51], []], + -1, + ], + [[49, 32, 50, 32, 51], None, [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 32, 50, 32, 51], None, [[49], [50, 32, 51]], [[49, 32, 50], [51]], 1], + [ + [32, 32, 32, 49, 32, 32, 32, 50, 32, 32, 32, 51, 32, 32, 32], + None, + [[49], [50], [51]], + [[49], [50], [51]], + -1, + ], +] + + +# for i in SPLIT_FIXTURES: # for not yet implemented : TypeError: Unsupported method: __next__ +n_sp = 0 +while n_sp < len(SPLIT_FIXTURES): + i = SPLIT_FIXTURES[n_sp] + sep = None if i[1] == None else bytearray(i[1]) + try: + assert bytearray(i[0]).split(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[2]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected : ", [list(x) for x in bytearray(i[0]).split(sep=sep, maxsplit=i[4])] + ) + break + + try: + assert bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [bytearray(j) for j in i[3]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected Rev : ", + [list(x) for x in bytearray(i[0]).rsplit(sep=sep, maxsplit=i[4])], + ) + break + + n_sp += 1 + + +# expandtabs +a = bytearray(b"\x01\x03\r\x05\t8CYZ\t\x06CYZ\t\x17cba`\n\x12\x13\x14") +assert ( + a.expandtabs() == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") +) +assert a.expandtabs(5) == bytearray(b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14") +assert bytearray(b"01\t012\t0123\t01234").expandtabs() == bytearray(b"01 012 0123 01234") +assert bytearray(b"01\t012\t0123\t01234").expandtabs(4) == bytearray(b"01 012 0123 01234") +assert bytearray(b"123\t123").expandtabs(-5) == bytearray(b"123123") +assert bytearray(b"123\t123").expandtabs(0) == bytearray(b"123123") + + +# # partition +assert bytearray(b"123456789").partition(b"45") == ((b"123"), bytearray(b"45"), bytearray(b"6789")) +assert bytearray(b"14523456789").partition(b"45") == ((b"1"), bytearray(b"45"), bytearray(b"23456789")) +a = bytearray(b"14523456789").partition(b"45") +assert isinstance(a[1], bytearray) +a = bytearray(b"14523456789").partition(memoryview(b"45")) +assert isinstance(a[1], bytearray) + +# partition +assert bytearray(b"123456789").rpartition(bytearray(b"45")) == ((bytearray(b"123")), bytearray(b"45"), bytearray(b"6789")) +assert bytearray(b"14523456789").rpartition(bytearray(b"45")) == ((bytearray(b"14523")), bytearray(b"45"), bytearray(b"6789")) +a = bytearray(b"14523456789").rpartition(b"45") +assert isinstance(a[1], bytearray) +a = bytearray(b"14523456789").rpartition(memoryview(b"45")) +assert isinstance(a[1], bytearray) + +# splitlines +assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines() == [bytearray(b"ab c"), bytearray(b""), bytearray(b"de fg"), bytearray(b"kl")] +assert bytearray(b"ab c\n\nde fg\rkl\r\n").splitlines(keepends=True) == [ + bytearray(b"ab c\n"), + bytearray(b"\n"), + bytearray(b"de fg\r"), + bytearray(b"kl\r\n"), +] +assert bytearray(b"").splitlines() == [] +assert bytearray(b"One line\n").splitlines() == [b"One line"] + +# zfill + +assert bytearray(b"42").zfill(5) == bytearray(b"00042") +assert bytearray(b"-42").zfill(5) == bytearray(b"-0042") +assert bytearray(b"42").zfill(1) == bytearray(b"42") +assert bytearray(b"42").zfill(-1) == bytearray(b"42") + +# replace +assert bytearray(b"123456789123").replace(b"23",b"XX") ==bytearray(b'1XX4567891XX') +assert bytearray(b"123456789123").replace(b"23",b"XX", 1) ==bytearray(b'1XX456789123') +assert bytearray(b"123456789123").replace(b"23",b"XX", 0) == bytearray(b"123456789123") +assert bytearray(b"123456789123").replace(b"23",b"XX", -1) ==bytearray(b'1XX4567891XX') +assert bytearray(b"123456789123").replace(b"23", bytearray(b"")) == bytearray(b"14567891") + + +# clear +a = bytearray(b"abcd") +a.clear() +assert len(a) == 0 + +b = bytearray(b"test") assert len(b) == 4 b.pop() assert len(b) == 3 @@ -65,3 +617,92 @@ pass else: assert False + +a = bytearray(b"appen") +assert len(a) == 5 +a.append(100) +assert a == bytearray(b"append") +assert len(a) == 6 +assert a.pop() == 100 + +# title +assert bytearray(b"Hello world").title() == bytearray(b"Hello World") +assert ( + bytearray(b"they're bill's friends from the UK").title() + == bytearray(b"They'Re Bill'S Friends From The Uk") +) + + +# repeat by multiply +a = bytearray(b'abcd') +assert a * 0 == bytearray(b'') +assert a * -1 == bytearray(b'') +assert a * 1 == bytearray(b'abcd') +assert a * 3 == bytearray(b'abcdabcdabcd') +assert 3 * a == bytearray(b'abcdabcdabcd') + +a = bytearray(b'abcd') +a.__imul__(3) +assert a == bytearray(b'abcdabcdabcd') +a.__imul__(0) +assert a == bytearray(b'') + + +# copy +a = bytearray(b"my bytearray") +b = a.copy() +assert a == b +assert a is not b +b.append(100) +assert a != b + + +# extend +a = bytearray(b"hello,") +# any iterable of ints should work +a.extend([32, 119, 111, 114]) +a.extend(b"ld") +assert a == bytearray(b"hello, world") + + +# insert +a = bytearray(b"hello, world") +a.insert(0, 119) +assert a == bytearray(b"whello, world"), a +# -1 is not at the end, but one before +a.insert(-1, 119) +assert a == bytearray(b"whello, worlwd"), a +# inserting before the beginning just inserts at the beginning +a.insert(-1000, 111) +assert a == bytearray(b"owhello, worlwd"), a +# inserting after the end just inserts at the end +a.insert(1000, 111) +assert a == bytearray(b"owhello, worlwdo"), a + + +# remove +a = bytearray(b'abcdabcd') +a.remove(99) # the letter c +# Only the first is removed +assert a == bytearray(b'abdabcd') + + +# reverse +a = bytearray(b'hello, world') +a.reverse() +assert a == bytearray(b'dlrow ,olleh') + +# __setitem__ +a = bytearray(b'test') +a[0] = 1 +assert a == bytearray(b'\x01est') +with assertRaises(TypeError): + a[0] = b'a' +with assertRaises(TypeError): + a[0] = memoryview(b'a') +a[:2] = [0, 9] +assert a == bytearray(b'\x00\x09st') +a[1:3] = b'test' +assert a == bytearray(b'\x00testt') +a[:6] = memoryview(b'test') +assert a == bytearray(b'test') diff --git a/tests/snippets/bytes.py b/tests/snippets/bytes.py new file mode 100644 index 0000000000..1fcbae3abf --- /dev/null +++ b/tests/snippets/bytes.py @@ -0,0 +1,599 @@ +from testutils import assertRaises + +# new +assert bytes([1, 2, 3]) +assert bytes((1, 2, 3)) +assert bytes(range(4)) +assert bytes(3) +assert b"bla" +assert bytes("bla", "utf8") == bytes("bla", encoding="utf-8") == b"bla" +with assertRaises(TypeError): + bytes("bla") +with assertRaises(TypeError): + bytes("bla", encoding=b"jilj") + +assert ( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + == bytes(range(0, 256)) +) +assert ( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" + == bytes(range(0, 256)) +) +assert b"omkmok\Xaa" == bytes([111, 109, 107, 109, 111, 107, 92, 88, 97, 97]) + + +a = b"abcd" +b = b"ab" +c = b"abcd" + +# +# repr +assert repr(bytes([0, 1, 2])) == repr(b"\x00\x01\x02") +assert repr( + bytes([0, 1, 9, 10, 11, 13, 31, 32, 33, 89, 120, 255]) + == "b'\\x00\\x01\\t\\n\\x0b\\r\\x1f !Yx\\xff'" +) +assert repr(b"abcd") == "b'abcd'" + +# len +assert len(bytes("abcdé", "utf8")) == 6 + +# comp +assert a == b"abcd" +assert a > b +assert a >= b +assert b < a +assert b <= a + +assert b"foobar".__eq__(2) == NotImplemented +assert b"foobar".__ne__(2) == NotImplemented +assert b"foobar".__gt__(2) == NotImplemented +assert b"foobar".__ge__(2) == NotImplemented +assert b"foobar".__lt__(2) == NotImplemented +assert b"foobar".__le__(2) == NotImplemented + +# hash +hash(a) == hash(b"abcd") + +# iter +[i for i in b"abcd"] == ["a", "b", "c", "d"] +assert list(bytes(3)) == [0, 0, 0] + +# add +assert a + b == b"abcdab" + +# contains +assert b"ab" in b"abcd" +assert b"cd" in b"abcd" +assert b"abcd" in b"abcd" +assert b"a" in b"abcd" +assert b"d" in b"abcd" +assert b"dc" not in b"abcd" +assert 97 in b"abcd" +assert 150 not in b"abcd" +with assertRaises(ValueError): + 350 in b"abcd" + + +# getitem +d = b"abcdefghij" + +assert d[1] == 98 +assert d[-1] == 106 +assert d[2:6] == b"cdef" +assert d[-6:] == b"efghij" +assert d[1:8:2] == b"bdfh" +assert d[8:1:-2] == b"igec" + + +# is_xx methods + +assert bytes(b"1a23").isalnum() +assert not bytes(b"1%a23").isalnum() + +assert bytes(b"abc").isalpha() +assert not bytes(b"abc1").isalpha() + +# travis doesn't like this +# assert bytes(b'xyz').isascii() +# assert not bytes([128, 157, 32]).isascii() + +assert bytes(b"1234567890").isdigit() +assert not bytes(b"12ab").isdigit() + +l = bytes(b"lower") +b = bytes(b"UPPER") + +assert l.islower() +assert not l.isupper() +assert b.isupper() +assert not bytes(b"Super Friends").islower() + +assert bytes(b" \n\t").isspace() +assert not bytes(b"\td\n").isspace() + +assert b.isupper() +assert not b.islower() +assert l.islower() +assert not bytes(b"tuPpEr").isupper() + +assert bytes(b"Is Title Case").istitle() +assert not bytes(b"is Not title casE").istitle() + +# upper lower, capitalize, swapcase +l = bytes(b"lower") +b = bytes(b"UPPER") +assert l.lower().islower() +assert b.upper().isupper() +assert l.capitalize() == b"Lower" +assert b.capitalize() == b"Upper" +assert bytes().capitalize() == bytes() +assert b"AaBbCc123'@/".swapcase().swapcase() == b"AaBbCc123'@/" +assert b"AaBbCc123'@/".swapcase() == b"aAbBcC123'@/" + +# hex from hex +assert bytes([0, 1, 9, 23, 90, 234]).hex() == "000109175aea" + +bytes.fromhex("62 6c7a 34350a ") == b"blz45\n" +try: + bytes.fromhex("62 a 21") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 4" +try: + bytes.fromhex("6Z2") +except ValueError as e: + str(e) == "non-hexadecimal number found in fromhex() arg at position 1" +with assertRaises(TypeError): + bytes.fromhex(b"hhjjk") +# center +assert [b"koki".center(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"|koki|", + b"||koki|", + b"||koki||", + b"|||koki||", +] + +assert [b"kok".center(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"|kok|", + b"|kok||", + b"||kok||", + b"||kok|||", + b"|||kok|||", +] +b"kok".center(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + b"b".center(2, "a") +with assertRaises(TypeError): + b"b".center(2, b"ba") +with assertRaises(TypeError): + b"b".center(b"ba") +assert b"kok".center(5, bytearray(b"x")) == b"xkokx" +b"kok".center(-5) == b"kok" + + +# ljust +assert [b"koki".ljust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"koki|", + b"koki||", + b"koki|||", + b"koki||||", + b"koki|||||", +] +assert [b"kok".ljust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"kok|", + b"kok||", + b"kok|||", + b"kok||||", + b"kok|||||", + b"kok||||||", +] + +b"kok".ljust(4) == b"kok " # " test no arg" +with assertRaises(TypeError): + b"b".ljust(2, "a") +with assertRaises(TypeError): + b"b".ljust(2, b"ba") +with assertRaises(TypeError): + b"b".ljust(b"ba") +assert b"kok".ljust(5, bytearray(b"x")) == b"kokxx" +assert b"kok".ljust(-5) == b"kok" + +# rjust +assert [b"koki".rjust(i, b"|") for i in range(3, 10)] == [ + b"koki", + b"koki", + b"|koki", + b"||koki", + b"|||koki", + b"||||koki", + b"|||||koki", +] +assert [b"kok".rjust(i, b"|") for i in range(2, 10)] == [ + b"kok", + b"kok", + b"|kok", + b"||kok", + b"|||kok", + b"||||kok", + b"|||||kok", + b"||||||kok", +] + + +b"kok".rjust(4) == b" kok" # " test no arg" +with assertRaises(TypeError): + b"b".rjust(2, "a") +with assertRaises(TypeError): + b"b".rjust(2, b"ba") +with assertRaises(TypeError): + b"b".rjust(b"ba") +assert b"kok".rjust(5, bytearray(b"x")) == b"xxkok" +assert b"kok".rjust(-5) == b"kok" + + +# count +assert b"azeazerazeazopia".count(b"aze") == 3 +assert b"azeazerazeazopia".count(b"az") == 4 +assert b"azeazerazeazopia".count(b"a") == 5 +assert b"123456789".count(b"") == 10 +assert b"azeazerazeazopia".count(bytearray(b"aze")) == 3 +assert b"azeazerazeazopia".count(memoryview(b"aze")) == 3 +assert b"azeazerazeazopia".count(memoryview(b"aze"), 1, 9) == 1 +assert b"azeazerazeazopia".count(b"aze", None, None) == 3 +assert b"azeazerazeazopia".count(b"aze", 2, None) == 2 +assert b"azeazerazeazopia".count(b"aze", 2) == 2 +assert b"azeazerazeazopia".count(b"aze", None, 7) == 2 +assert b"azeazerazeazopia".count(b"aze", None, 7) == 2 +assert b"azeazerazeazopia".count(b"aze", 2, 7) == 1 +assert b"azeazerazeazopia".count(b"aze", -13, -10) == 1 +assert b"azeazerazeazopia".count(b"aze", 1, 10000) == 2 +with assertRaises(ValueError): + b"ilj".count(3550) +assert b"azeazerazeazopia".count(97) == 5 + +# join +assert ( + b"".join((b"jiljl", bytearray(b"kmoomk"), memoryview(b"aaaa"))) + == b"jiljlkmoomkaaaa" +) +with assertRaises(TypeError): + b"".join((b"km", "kl")) + + +# endswith startswith +assert b"abcde".endswith(b"de") +assert b"abcde".endswith(b"") +assert not b"abcde".endswith(b"zx") +assert b"abcde".endswith(b"bc", 0, 3) +assert not b"abcde".endswith(b"bc", 2, 3) +assert b"abcde".endswith((b"c", b"de")) + +assert b"abcde".startswith(b"ab") +assert b"abcde".startswith(b"") +assert not b"abcde".startswith(b"zx") +assert b"abcde".startswith(b"cd", 2) +assert not b"abcde".startswith(b"cd", 1, 4) +assert b"abcde".startswith((b"a", b"bc")) + + +# index find +assert b"abcd".index(b"cd") == 2 +assert b"abcd".index(b"cd", 0) == 2 +assert b"abcd".index(b"cd", 1) == 2 +assert b"abcd".index(99) == 2 +with assertRaises(ValueError): + b"abcde".index(b"c", 3, 1) +with assertRaises(ValueError): + b"abcd".index(b"cdaaaaa") +with assertRaises(ValueError): + b"abcd".index(b"b", 3, 4) +with assertRaises(ValueError): + b"abcd".index(1) + + +assert b"abcd".find(b"cd") == 2 +assert b"abcd".find(b"cd", 0) == 2 +assert b"abcd".find(b"cd", 1) == 2 +assert b"abcde".find(b"c", 3, 1) == -1 +assert b"abcd".find(b"cdaaaaa") == -1 +assert b"abcd".find(b"b", 3, 4) == -1 +assert b"abcd".find(1) == -1 +assert b"abcd".find(99) == 2 + +assert b"abcdabcda".find(b"a") == 0 +assert b"abcdabcda".rfind(b"a") == 8 +assert b"abcdabcda".rfind(b"a", 2, 6) == 4 +assert b"abcdabcda".rfind(b"a", None, 6) == 4 +assert b"abcdabcda".rfind(b"a", 2, None) == 8 +assert b"abcdabcda".index(b"a") == 0 +assert b"abcdabcda".rindex(b"a") == 8 + + +# make trans +# fmt: off +assert ( + bytes.maketrans(memoryview(b"abc"), bytearray(b"zzz")) + == bytes([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 122, 122, 122, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]) +) +# fmt: on + +# translate +assert b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab"), b"h") == b"btuybyubuyb" +assert ( + b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab"), b"a") == b"abatuybyubuyb" +) +assert b"hjhtuyjyujuyj".translate(bytes.maketrans(b"hj", b"ab")) == b"abatuybyubuyb" +assert b"hjhtuyfjtyhuhjuyj".translate(None, b"ht") == b"juyfjyujuyj" +assert b"hjhtuyfjtyhuhjuyj".translate(None, delete=b"ht") == b"juyfjyujuyj" + + +# strip lstrip rstrip +assert b" spacious ".strip() == b"spacious" +assert b"www.example.com".strip(b"cmowz.") == b"example" +assert b" spacious ".lstrip() == b"spacious " +assert b"www.example.com".lstrip(b"cmowz.") == b"example.com" +assert b" spacious ".rstrip() == b" spacious" +assert b"mississippi".rstrip(b"ipz") == b"mississ" + + +# split +assert b"1,2,3".split(b",") == [b"1", b"2", b"3"] +assert b"1,2,3".split(b",", maxsplit=1) == [b"1", b"2,3"] +assert b"1,2,,3,".split(b",") == [b"1", b"2", b"", b"3", b""] +assert b"1 2 3".split() == [b"1", b"2", b"3"] +assert b"1 2 3".split(maxsplit=1) == [b"1", b"2 3"] +assert b" 1 2 3 ".split() == [b"1", b"2", b"3"] +assert b"k\ruh\nfz e f".split() == [b"k", b"uh", b"fz", b"e", b"f"] +assert b"Two lines\n".split(b"\n") == [b"Two lines", b""] +assert b"".split() == [] +assert b"".split(b"\n") == [b""] +assert b"\n".split(b"\n") == [b"", b""] + +SPLIT_FIXTURES = [ + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], []], + -1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + [[1, 2, 3], [1, 2, 3], [1, 2, 3], [3]], + -1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + [[], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + [[1], [2, 3], [1, 2, 3], [1, 2, 3]], + -1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + [[1, 2, 3], [], [], [1, 2, 3], [1, 2, 3]], + -1, + ], + # maxsplit + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], []], + 1, + ], + [ + [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3], + [4, 5], + [[1, 2, 3], [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 3]], + [[1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], [3]], + 1, + ], + [ + [4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 4, 5, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1], [2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 4, 5, 2, 3, 4, 5, 1, 2, 3], [1, 2, 3]], + 1, + ], + [ + [1, 2, 3, 4, 5, 4, 5, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3], + [4, 5], + [[1, 2, 3], [], [4, 5, 1, 2, 3, 4, 5, 1, 2, 3]], + [[1, 2, 3, 4, 5, 4, 5], [1, 2, 3], [1, 2, 3]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104, 10, 102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102]], + 0, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122, 32, 101, 102, 9, 9]], + [[13, 13, 13, 117, 104, 10, 102, 122], [101, 102]], + 1, + ], + [ + [13, 13, 13, 117, 104, 10, 102, 122, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102, 9, 9]], + [[13, 13, 13, 117, 104], [102, 122], [101, 102]], + 2, + ], + [ + [13, 13, 13, 117, 104, 10, 10, 10, 102, 122, 32, 32, 101, 102, 9, 9], + None, + [[117, 104], [102, 122], [101, 102]], + [[117, 104], [102, 122], [101, 102]], + -1, + ], + [[49, 44, 50, 44, 51], [44], [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 44, 50, 44, 51], [44], [[49], [50, 44, 51]], [[49, 44, 50], [51]], 1], + [ + [49, 44, 50, 44, 44, 51, 44], + [44], + [[49], [50], [], [51], []], + [[49], [50], [], [51], []], + -1, + ], + [[49, 32, 50, 32, 51], None, [[49], [50], [51]], [[49], [50], [51]], -1], + [[49, 32, 50, 32, 51], None, [[49], [50, 32, 51]], [[49, 32, 50], [51]], 1], + [ + [32, 32, 32, 49, 32, 32, 32, 50, 32, 32, 32, 51, 32, 32, 32], + None, + [[49], [50], [51]], + [[49], [50], [51]], + -1, + ], +] + + +# for i in SPLIT_FIXTURES: # for not yet implemented : TypeError: Unsupported method: __next__ +n_sp = 0 +while n_sp < len(SPLIT_FIXTURES): + i = SPLIT_FIXTURES[n_sp] + sep = None if i[1] == None else bytes(i[1]) + try: + assert bytes(i[0]).split(sep=sep, maxsplit=i[4]) == [bytes(j) for j in i[2]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected : ", [list(x) for x in bytes(i[0]).split(sep=sep, maxsplit=i[4])] + ) + break + + try: + assert bytes(i[0]).rsplit(sep=sep, maxsplit=i[4]) == [bytes(j) for j in i[3]] + except AssertionError: + print(i[0], i[1], i[2]) + print( + "Expected Rev : ", + [list(x) for x in bytes(i[0]).rsplit(sep=sep, maxsplit=i[4])], + ) + break + + n_sp += 1 + + +# expandtabs +a = b"\x01\x03\r\x05\t8CYZ\t\x06CYZ\t\x17cba`\n\x12\x13\x14" +assert ( + a.expandtabs() == b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +) +assert a.expandtabs(5) == b"\x01\x03\r\x05 8CYZ \x06CYZ \x17cba`\n\x12\x13\x14" +assert b"01\t012\t0123\t01234".expandtabs() == b"01 012 0123 01234" +assert b"01\t012\t0123\t01234".expandtabs(4) == b"01 012 0123 01234" +assert b"123\t123".expandtabs(-5) == b"123123" +assert b"123\t123".expandtabs(0) == b"123123" + + +# partition +assert b"123456789".partition(b"45") == (b"123", b"45", b"6789") +assert b"14523456789".partition(b"45") == (b"1", b"45", b"23456789") +a = b"14523456789".partition(bytearray(b"45")) +assert isinstance(a[1], bytearray) +a = b"14523456789".partition(memoryview(b"45")) +assert isinstance(a[1], memoryview) + +# partition +assert b"123456789".rpartition(b"45") == (b"123", b"45", b"6789") +assert b"14523456789".rpartition(b"45") == (b"14523", b"45", b"6789") +a = b"14523456789".rpartition(bytearray(b"45")) +assert isinstance(a[1], bytearray) +a = b"14523456789".rpartition(memoryview(b"45")) +assert isinstance(a[1], memoryview) + +# splitlines +assert b"ab c\n\nde fg\rkl\r\n".splitlines() == [b"ab c", b"", b"de fg", b"kl"] +assert b"ab c\n\nde fg\rkl\r\n".splitlines(keepends=True) == [ + b"ab c\n", + b"\n", + b"de fg\r", + b"kl\r\n", +] +assert b"".splitlines() == [] +assert b"One line\n".splitlines() == [b"One line"] + +# zfill + +assert b"42".zfill(5) == b"00042" +assert b"-42".zfill(5) == b"-0042" +assert b"42".zfill(1) == b"42" +assert b"42".zfill(-1) == b"42" + +# replace +assert b"123456789123".replace(b"23", b"XX") == b"1XX4567891XX" +assert b"123456789123".replace(b"23", b"XX", 1) == b"1XX456789123" +assert b"123456789123".replace(b"23", b"XX", 0) == b"123456789123" +assert b"123456789123".replace(b"23", b"XX", -1) == b"1XX4567891XX" +assert b"123456789123".replace(b"23", b"") == b"14567891" + +# title +assert b"Hello world".title() == b"Hello World" +assert ( + b"they're bill's friends from the UK".title() + == b"They'Re Bill'S Friends From The Uk" +) + + +# repeat by multiply +a = b'abcd' +assert a * 0 == b'' +assert a * -1 == b'' +assert a * 1 == b'abcd' +assert a * 3 == b'abcdabcdabcd' +assert 3 * a == b'abcdabcdabcd' diff --git a/tests/snippets/bytes_io.py b/tests/snippets/bytes_io.py new file mode 100644 index 0000000000..f6f97e6841 --- /dev/null +++ b/tests/snippets/bytes_io.py @@ -0,0 +1,50 @@ + +from io import BytesIO + +def test_01(): + bytes_string = b'Test String 1' + + f = BytesIO() + f.write(bytes_string) + + assert f.getvalue() == bytes_string + +def test_02(): + bytes_string = b'Test String 2' + f = BytesIO(bytes_string) + + assert f.read() == bytes_string + assert f.read() == b'' + +def test_03(): + """ + Tests that the read method (integer arg) + returns the expected value + """ + string = b'Test String 3' + f = BytesIO(string) + + assert f.read(1) == b'T' + assert f.read(1) == b'e' + assert f.read(1) == b's' + assert f.read(1) == b't' + +def test_04(): + """ + Tests that the read method increments the + cursor position and the seek method moves + the cursor to the appropriate position + """ + string = b'Test String 4' + f = BytesIO(string) + + assert f.read(4) == b'Test' + assert f.seek(0) == 0 + assert f.read(4) == b'Test' + +if __name__ == "__main__": + test_01() + test_02() + test_03() + test_04() + diff --git a/tests/snippets/class.py b/tests/snippets/class.py index 35d043a6d4..389c227126 100644 --- a/tests/snippets/class.py +++ b/tests/snippets/class.py @@ -1,3 +1,6 @@ +__name__ = "class" + + class Foo: def __init__(self, x): assert x == 5 @@ -14,7 +17,12 @@ def square(self): assert foo.y == Foo.y assert foo.x == 5 assert foo.square() == 25 - +assert Foo.__name__ == "Foo" +assert Foo.__qualname__ == "Foo" +assert Foo.__module__ == "class" +assert Foo.square.__name__ == "square" +assert Foo.square.__qualname__ == "Foo.square" +assert Foo.square.__module__ == "class" class Bar: """ W00t """ @@ -37,8 +45,7 @@ def kungfu(x): assert x == 3 -# TODO: -# assert Bar.__doc__ == " W00t " +assert Bar.__doc__ == " W00t " bar = Bar(42) @@ -64,6 +71,10 @@ class B(): def test1(self): return 200 + @classmethod + def test3(cls): + return 300 + class C(A,B): def test(self): return super().test() @@ -71,9 +82,15 @@ def test(self): def test1(self): return super().test1() + @classmethod + def test3(cls): + return super().test3() + c = C() assert c.test() == 100 assert c.test1() == 200 +assert c.test3() == 300 +assert C.test3() == 300 class Me(): @@ -107,3 +124,34 @@ def f(self): assert type(a) is super assert a.conjugate() == 1 + +class T1: + "test1" + +assert T1.__doc__ == "test1" + +class T2: + '''test2''' + +assert T2.__doc__ == "test2" + +class T3: + """ + test3 + """ + +assert T3.__doc__ == "\n test3\n " + +class T4: + + """test4""" + + def t1(self): + """t1""" + pass + +assert T4.__doc__ == "test4" +assert T4.t1.__doc__ == "t1" + +cm = classmethod(lambda cls: cls) +assert cm.__func__(int) is int diff --git a/tests/snippets/comparisons.py b/tests/snippets/comparisons.py new file mode 100644 index 0000000000..d68dcaf3e5 --- /dev/null +++ b/tests/snippets/comparisons.py @@ -0,0 +1,23 @@ +from testutils import assert_raises + +assert 1 < 2 +assert 1 < 2 < 3 +assert 5 == 5 == 5 +assert (5 == 5) == True +assert 5 == 5 != 4 == 4 > 3 > 2 < 3 <= 3 != 0 == 0 + +assert not 1 > 2 +assert not 5 == 5 == True +assert not 5 == 5 != 5 == 5 +assert not 1 < 2 < 3 > 4 +assert not 1 < 2 > 3 < 4 +assert not 1 > 2 < 3 < 4 + +def test_type_error(x, y): + assert_raises(TypeError, lambda: x < y) + assert_raises(TypeError, lambda: x <= y) + assert_raises(TypeError, lambda: x > y) + assert_raises(TypeError, lambda: x >= y) + +test_type_error([], 0) +test_type_error((), 0) diff --git a/tests/snippets/comprehensions.py b/tests/snippets/comprehensions.py index 14c545de30..132f3cdcb9 100644 --- a/tests/snippets/comprehensions.py +++ b/tests/snippets/comprehensions.py @@ -22,3 +22,8 @@ y = [a+2 for a in x if a % 2] print(y) assert y == [3, 5] + +z = [(9,), (10,)] +w = [x for x, in z] +assert w == [9, 10] + diff --git a/tests/snippets/decorators.py b/tests/snippets/decorators.py index fcc196f936..2633308f18 100644 --- a/tests/snippets/decorators.py +++ b/tests/snippets/decorators.py @@ -15,6 +15,16 @@ def add(a, b): assert c == 14 +@logged +def add3(a, b, c=2): + return a + b + c + + +d = add3(12, 5) + +assert d == 20 + + def f(func): return lambda: 42 class A: pass a = A() diff --git a/tests/snippets/delete.py b/tests/snippets/delete.py index 078b6dcd9f..aa7ae0536c 100644 --- a/tests/snippets/delete.py +++ b/tests/snippets/delete.py @@ -1,3 +1,4 @@ +from testutils import assert_raises, assertRaises a = 1 del a @@ -10,3 +11,11 @@ class MyObject: pass assert not hasattr(foo, 'bar') +x = 1 +y = 2 +del (x, y) +assert_raises(NameError, lambda: x) +assert_raises(NameError, lambda: y) + +with assertRaises(NameError): + del y diff --git a/tests/snippets/dict.py b/tests/snippets/dict.py index 170a6492dc..c453cff2b4 100644 --- a/tests/snippets/dict.py +++ b/tests/snippets/dict.py @@ -1,19 +1,26 @@ -def dict_eq(d1, d2): - return (all(k in d2 and d1[k] == d2[k] for k in d1) - and all(k in d1 and d1[k] == d2[k] for k in d2)) +from testutils import assertRaises +assert dict(a=2, b=3) == {'a': 2, 'b': 3} +assert dict({'a': 2, 'b': 3}, b=4) == {'a': 2, 'b': 4} +assert dict([('a', 2), ('b', 3)]) == {'a': 2, 'b': 3} -assert dict_eq(dict(a=2, b=3), {'a': 2, 'b': 3}) -assert dict_eq(dict({'a': 2, 'b': 3}, b=4), {'a': 2, 'b': 4}) -assert dict_eq(dict([('a', 2), ('b', 3)]), {'a': 2, 'b': 3}) +assert {} == {} +assert not {'a': 2} == {} +assert not {} == {'a': 2} +assert not {'b': 2} == {'a': 2} +assert not {'a': 4} == {'a': 2} +assert {'a': 2} == {'a': 2} + +nan = float('nan') +assert {'a': nan} == {'a': nan} a = {'g': 5} b = {'a': a, 'd': 9} c = dict(b) c['d'] = 3 c['a']['g'] = 2 -assert dict_eq(a, {'g': 2}) -assert dict_eq(b, {'a': a, 'd': 9}) +assert a == {'g': 2} +assert b == {'a': a, 'd': 9} a.clear() assert len(a) == 0 @@ -21,16 +28,216 @@ def dict_eq(d1, d2): a = {'a': 5, 'b': 6} res = set() for value in a.values(): - res.add(value) + res.add(value) assert res == set([5,6]) count = 0 for (key, value) in a.items(): - assert a[key] == value - count += 1 + assert a[key] == value + count += 1 assert count == len(a) res = set() for key in a.keys(): - res.add(key) + res.add(key) assert res == set(['a','b']) + +# Deleted values are correctly skipped over: +x = {'a': 1, 'b': 2, 'c': 3, 'd': 3} +del x['c'] +it = iter(x.items()) +assert ('a', 1) == next(it) +assert ('b', 2) == next(it) +assert ('d', 3) == next(it) +with assertRaises(StopIteration): + next(it) + +with assertRaises(KeyError) as cm: + del x[10] +assert cm.exception.args[0] == 10 + +# Iterating a dictionary is just its keys: +assert ['a', 'b', 'd'] == list(x) + +# Iterating view captures dictionary when iterated. +data = {1: 2, 3: 4} +items = data.items() +assert list(items) == [(1, 2), (3, 4)] +data[5] = 6 +assert list(items) == [(1, 2), (3, 4), (5, 6)] + +# Values can be changed during iteration. +data = {1: 2, 3: 4} +items = iter(data.items()) +assert (1, 2) == next(items) +data[3] = "changed" +assert (3, "changed") == next(items) + +# But we can't add or delete items during iteration. +d = {} +a = iter(d.items()) +d['a'] = 2 +b = iter(d.items()) +assert ('a', 2) == next(b) +with assertRaises(RuntimeError): + next(a) +del d['a'] +with assertRaises(RuntimeError): + next(b) + +# View isn't itself an iterator. +with assertRaises(TypeError): + next(data.keys()) + +assert len(data.keys()) == 2 + +x = {} +x[1] = 1 +assert x[1] == 1 + +x[7] = 7 +x[2] = 2 +x[(5, 6)] = 5 + +with assertRaises(TypeError): + x[[]] # Unhashable type. + +x["here"] = "here" +assert x.get("not here", "default") == "default" +assert x.get("here", "default") == "here" +assert x.get("not here") == None + +class LengthDict(dict): + def __getitem__(self, k): + return len(k) + +x = LengthDict() +assert type(x) == LengthDict +assert x['word'] == 4 +assert x.get('word') is None + +assert 5 == eval("a + word", LengthDict()) + + +class Squares(dict): + def __missing__(self, k): + v = k * k + self[k] = v + return v + +x = Squares() +assert x[-5] == 25 + +# An object that hashes to the same value always, and compares equal if any its values match. +class Hashable(object): + def __init__(self, *args): + self.values = args + def __hash__(self): + return 1 + def __eq__(self, other): + for x in self.values: + for y in other.values: + if x == y: + return True + return False + +x = {} +x[Hashable(1,2)] = 8 + +assert x[Hashable(1,2)] == 8 +assert x[Hashable(3,1)] == 8 + +x[Hashable(8)] = 19 +x[Hashable(19,8)] = 1 +assert x[Hashable(8)] == 1 +assert len(x) == 2 + +assert list({'a': 2, 'b': 10}) == ['a', 'b'] +x = {} +x['a'] = 2 +x['b'] = 10 +assert list(x) == ['a', 'b'] + +y = x.copy() +x['c'] = 12 +assert y == {'a': 2, 'b': 10} + +y.update({'c': 19, "d": -1, 'b': 12}) +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} + +y.update(y) +assert y == {'a': 2, 'b': 12, 'c': 19, 'd': -1} # hasn't changed + +# KeyError has object that used as key as an .args[0] +with assertRaises(KeyError) as cm: + x['not here'] +assert cm.exception.args[0] == "not here" +with assertRaises(KeyError) as cm: + x.pop('not here') +assert cm.exception.args[0] == "not here" + +with assertRaises(KeyError) as cm: + x[10] +assert cm.exception.args[0] == 10 +with assertRaises(KeyError) as cm: + x.pop(10) +assert cm.exception.args[0] == 10 + +class MyClass: pass +obj = MyClass() + +with assertRaises(KeyError) as cm: + x[obj] +assert cm.exception.args[0] == obj +with assertRaises(KeyError) as cm: + x.pop(obj) +assert cm.exception.args[0] == obj + +x = {1: 'a', '1': None} +assert x.pop(1) == 'a' +assert x.pop('1') is None +assert x == {} + +x = {1: 'a'} +assert (1, 'a') == x.popitem() +assert x == {} +with assertRaises(KeyError) as cm: + x.popitem() +assert cm.exception.args == ('popitem(): dictionary is empty',) + +x = {'a': 4} +assert 4 == x.setdefault('a', 0) +assert x['a'] == 4 +assert 0 == x.setdefault('b', 0) +assert x['b'] == 0 +assert None == x.setdefault('c') +assert x['c'] is None + +assert {1: None, "b": None} == dict.fromkeys([1, "b"]) +assert {1: 0, "b": 0} == dict.fromkeys([1, "b"], 0) + +x = {'a': 1, 'b': 1, 'c': 1} +y = {'b': 2, 'c': 2, 'd': 2} +z = {'c': 3, 'd': 3, 'e': 3} + +w = {1: 1, **x, 2: 2, **y, 3: 3, **z, 4: 4} +assert w == {1: 1, 'a': 1, 'b': 2, 'c': 3, 2: 2, 'd': 3, 3: 3, 'e': 3, 4: 4} + +assert str({True: True, 1.0: 1.0}) == str({True: 1.0}) + +class A: + def __hash__(self): + return 1 + def __eq__(self, other): + return isinstance(other, A) +class B: + def __hash__(self): + return 1 + def __eq__(self, other): + return isinstance(other, B) + +s = {1: 0, A(): 1, B(): 2} +assert len(s) == 3 +assert s[1] == 0 +assert s[A()] == 1 +assert s[B()] == 2 diff --git a/tests/snippets/dir_main/__main__.py b/tests/snippets/dir_main/__main__.py new file mode 100644 index 0000000000..c324c2e6e5 --- /dev/null +++ b/tests/snippets/dir_main/__main__.py @@ -0,0 +1 @@ +print('Hello') diff --git a/tests/snippets/dir_module/__init__.py b/tests/snippets/dir_module/__init__.py new file mode 100644 index 0000000000..5d9faff33d --- /dev/null +++ b/tests/snippets/dir_module/__init__.py @@ -0,0 +1,2 @@ +from .relative import value +from .dir_module_inner import value2 diff --git a/tests/snippets/dir_module/dir_module_inner/__init__.py b/tests/snippets/dir_module/dir_module_inner/__init__.py new file mode 100644 index 0000000000..20e95590f1 --- /dev/null +++ b/tests/snippets/dir_module/dir_module_inner/__init__.py @@ -0,0 +1,3 @@ +from ..relative import value + +value2 = value + 2 diff --git a/tests/snippets/dir_module/relative.py b/tests/snippets/dir_module/relative.py new file mode 100644 index 0000000000..78ed77f40c --- /dev/null +++ b/tests/snippets/dir_module/relative.py @@ -0,0 +1 @@ +value = 5 diff --git a/tests/snippets/ellipsis.py b/tests/snippets/ellipsis.py index 88b1c965d8..20bb3a33c6 100644 --- a/tests/snippets/ellipsis.py +++ b/tests/snippets/ellipsis.py @@ -3,6 +3,14 @@ a = ... b = ... c = type(a)() # Test singleton behavior +d = Ellipsis +e = Ellipsis assert a is b assert b is c +assert b is d +assert d is e + +assert Ellipsis is ... +Ellipsis = 2 +assert Ellipsis is not ... diff --git a/tests/snippets/exceptions.py b/tests/snippets/exceptions.py new file mode 100644 index 0000000000..cb178fc066 --- /dev/null +++ b/tests/snippets/exceptions.py @@ -0,0 +1,45 @@ +# KeyError +empty_exc = KeyError() +assert str(empty_exc) == '' +assert repr(empty_exc) == 'KeyError()' +assert len(empty_exc.args) == 0 +assert type(empty_exc.args) == tuple + +exc = KeyError('message') +assert str(exc) == "'message'" +assert repr(exc) == "KeyError('message',)" + +exc = KeyError('message', 'another message') +assert str(exc) == "('message', 'another message')" +assert repr(exc) == "KeyError('message', 'another message')" +assert exc.args[0] == 'message' +assert exc.args[1] == 'another message' + +class A: + def __repr__(self): + return 'repr' + def __str__(self): + return 'str' + +exc = KeyError(A()) +assert str(exc) == 'repr' +assert repr(exc) == 'KeyError(repr,)' + +# ImportError / ModuleNotFoundError +exc = ImportError() +assert exc.name is None +assert exc.path is None +assert exc.msg is None +assert exc.args == () + +exc = ImportError('hello') +assert exc.name is None +assert exc.path is None +assert exc.msg == 'hello' +assert exc.args == ('hello',) + +exc = ImportError('hello', name='name', path='path') +assert exc.name == 'name' +assert exc.path == 'path' +assert exc.msg == 'hello' +assert exc.args == ('hello',) diff --git a/tests/snippets/floats.py b/tests/snippets/floats.py index 6675615c56..101d58cc92 100644 --- a/tests/snippets/floats.py +++ b/tests/snippets/floats.py @@ -7,10 +7,16 @@ a = 1.2 b = 1.3 c = 1.2 +z = 2 +ov = 10 ** 1000 + +assert -a == -1.2 + assert a < b assert not b < a assert a <= b assert a <= c +assert a < z assert b > a assert not a > b @@ -18,13 +24,39 @@ assert b >= a assert c >= a assert not a >= b +assert z > a assert a + b == 2.5 assert a - c == 0 assert a / c == 1 +assert a % c == 0 +assert a + z == 3.2 +assert z + a == 3.2 +assert a - z == -0.8 +assert z - a == 0.8 +assert a / z == 0.6 +assert 6 / a == 5.0 +assert 2.0 % z == 0.0 +assert z % 2.0 == 0.0 +assert_raises(OverflowError, lambda: a + ov) +assert_raises(OverflowError, lambda: a - ov) +assert_raises(OverflowError, lambda: a * ov) +assert_raises(OverflowError, lambda: a / ov) +assert_raises(OverflowError, lambda: a // ov) +assert_raises(OverflowError, lambda: a % ov) +assert_raises(OverflowError, lambda: a ** ov) +assert_raises(OverflowError, lambda: ov + a) +assert_raises(OverflowError, lambda: ov - a) +assert_raises(OverflowError, lambda: ov * a) +assert_raises(OverflowError, lambda: ov / a) +assert_raises(OverflowError, lambda: ov // a) +assert_raises(OverflowError, lambda: ov % a) +# assert_raises(OverflowError, lambda: ov ** a) assert a < 5 assert a <= 5 +assert a < 5.5 +assert a <= 5.5 try: assert a < 'a' except TypeError: @@ -66,6 +98,21 @@ assert_raises(ValueError, lambda: float('foo')) assert_raises(OverflowError, lambda: float(2**10000)) +# check eq and hash for small numbers + +assert 1.0 == 1 +assert 1.0 == True +assert 0.0 == 0 +assert 0.0 == False +assert hash(1.0) == hash(1) +assert hash(1.0) == hash(True) +assert hash(0.0) == hash(0) +assert hash(0.0) == hash(False) +assert hash(1.0) != hash(1.0000000001) + +assert 5.0 in {3, 4, 5} +assert {-1: 2}[-1.0] == 2 + # check that magic methods are implemented for ints and floats assert 1.0.__add__(1.0) == 2.0 @@ -74,6 +121,8 @@ assert 2.0.__rmul__(1.0) == 2.0 assert 1.0.__truediv__(2.0) == 0.5 assert 1.0.__rtruediv__(2.0) == 2.0 +assert 2.5.__divmod__(2.0) == (1.0, 0.5) +assert 2.0.__rdivmod__(2.5) == (1.0, 0.5) assert 1.0.__add__(1) == 2.0 assert 1.0.__radd__(1) == 2.0 @@ -83,8 +132,41 @@ assert 1.0.__rtruediv__(2) == 2.0 assert 2.0.__mul__(1) == 2.0 assert 2.0.__rsub__(1) == -1.0 +assert 2.0.__mod__(2) == 0.0 +assert 2.0.__rmod__(2) == 0.0 +assert_raises(ZeroDivisionError, lambda: 2.0 / 0) +assert_raises(ZeroDivisionError, lambda: 2.0 // 0) +assert_raises(ZeroDivisionError, lambda: 2.0 % 0) +assert_raises(ZeroDivisionError, lambda: divmod(2.0, 0)) +assert_raises(ZeroDivisionError, lambda: 2 / 0.0) +assert_raises(ZeroDivisionError, lambda: 2 // 0.0) +assert_raises(ZeroDivisionError, lambda: 2 % 0.0) +# assert_raises(ZeroDivisionError, lambda: divmod(2, 0.0)) + +assert 1.2.__int__() == 1 +assert 1.2.__float__() == 1.2 +assert 1.2.__trunc__() == 1 +assert int(1.2) == 1 +assert float(1.2) == 1.2 +assert math.trunc(1.2) == 1 +assert_raises(OverflowError, float('inf').__trunc__) +assert_raises(ValueError, float('nan').__trunc__) +assert 0.5.__round__() == 0.0 +assert 1.5.__round__() == 2.0 +assert 0.5.__round__(0) == 0.0 +assert 1.5.__round__(0) == 2.0 +assert 0.5.__round__(None) == 0.0 +assert 1.5.__round__(None) == 2.0 +assert_raises(OverflowError, float('inf').__round__) +assert_raises(ValueError, float('nan').__round__) + +assert 1.2 ** 2 == 1.44 +assert_raises(OverflowError, lambda: 1.2 ** (10 ** 1000)) +assert 3 ** 2.0 == 9.0 assert (1.7).real == 1.7 +assert (1.7).imag == 0.0 +assert (1.7).conjugate() == 1.7 assert (1.3).is_integer() == False assert (1.0).is_integer() == True @@ -106,3 +188,29 @@ assert_raises(OverflowError, float('-inf').as_integer_ratio) assert_raises(ValueError, float('nan').as_integer_ratio) +assert str(1.0) == '1.0' +assert str(0.0) == '0.0' +assert str(1.123456789) == '1.123456789' + +# Test special case for lexer, float starts with a dot: +a = .5 +assert a == 0.5 + +assert float.fromhex('0x0.0p+0') == 0.0 +assert float.fromhex('-0x0.0p+0') == -0.0 +assert float.fromhex('0x1.000000p+0') == 1.0 +assert float.fromhex('-0x1.800000p+0') == -1.5 +assert float.fromhex('inf') == float('inf') +assert math.isnan(float.fromhex('nan')) + +assert (0.0).hex() == '0x0.0p+0' +assert (-0.0).hex() == '-0x0.0p+0' +assert (1.0).hex() == '0x1.0000000000000p+0' +assert (-1.5).hex() == '-0x1.8000000000000p+0' +assert float('inf').hex() == 'inf' +assert float('-inf').hex() == '-inf' +assert float('nan').hex() == 'nan' + +#for _ in range(10000): +# f = random.random() * random.randint(0, 0x10000000000000000) +# assert f == float.fromhex(f.hex()) diff --git a/tests/snippets/for.py b/tests/snippets/for.py index 07e138e3c0..e913ac907f 100644 --- a/tests/snippets/for.py +++ b/tests/snippets/for.py @@ -10,3 +10,15 @@ x = 3 assert x == 3 + +y = [] +for x, in [(9,), [2]]: + y.append(x) + +assert y == [9, 2], str(y) + +y = [] +for x, *z in [(9,88,'b'), [2, 'bla'], [None]*4]: + y.append(z) + +assert y == [[88, 'b'], ['bla'], [None]*3], str(y) diff --git a/tests/snippets/frozen.py b/tests/snippets/frozen.py new file mode 100644 index 0000000000..d03658c191 --- /dev/null +++ b/tests/snippets/frozen.py @@ -0,0 +1,2 @@ +import __hello__ +assert __hello__.initialized == True diff --git a/tests/snippets/function.py b/tests/snippets/function.py index a69178bb78..b499c6776b 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -1,4 +1,74 @@ + +__name__ = "function" + + def foo(): + """test""" return 42 assert foo() == 42 +assert foo.__doc__ == "test" +assert foo.__name__ == "foo" +assert foo.__qualname__ == "foo" +assert foo.__module__ == "function" + +def my_func(a,): + return a+2 + +assert my_func(2) == 4 + +def fubar(): + return 42, + +assert fubar() == (42,) + +def f1(): + + """test1""" + pass + +assert f1.__doc__ == "test1" + +def f2(): + '''test2''' + pass + +assert f2.__doc__ == "test2" + +def f3(): + """ + test3 + """ + pass + +assert f3.__doc__ == "\n test3\n " + +def f4(): + "test4" + pass + +assert f4.__doc__ == "test4" + + +def revdocstr(f): + d = f.__doc__ + d = d + 'w00t' + f.__doc__ = d + return f + +@revdocstr +def f5(): + """abc""" + +assert f5.__doc__ == 'abcw00t', f5.__doc__ + + +def f6(): + def nested(): + pass + + assert nested.__name__ == "nested" + assert nested.__qualname__ == "f6..nested" + + +f6() diff --git a/tests/snippets/function_args.py b/tests/snippets/function_args.py index 745648f9d2..86c92617ac 100644 --- a/tests/snippets/function_args.py +++ b/tests/snippets/function_args.py @@ -1,21 +1,28 @@ +from testutils import assertRaises + + def sum(x, y): return x+y -# def total(a, b, c, d): -# return sum(sum(a,b), sum(c,d)) -# -# assert total(1,1,1,1) == 4 -# assert total(1,2,3,4) == 10 +def total(a, b, c, d): + return sum(sum(a,b), sum(c,d)) + + +assert total(1,1,1,1) == 4 +assert total(1,2,3,4) == 10 assert sum(1, 1) == 2 assert sum(1, 3) == 4 + def sum2y(x, y): return x+y*2 + assert sum2y(1, 1) == 3 assert sum2y(1, 3) == 7 + def va(a, b=2, *c, d, **e): assert a == 1 assert b == 22 @@ -23,22 +30,68 @@ def va(a, b=2, *c, d, **e): assert d == 1337 assert e['f'] == 42 + va(1, 22, 3, 4, d=1337, f=42) +assert va.__defaults__ == (2,) +assert va.__kwdefaults__ is None + + def va2(*args, **kwargs): assert args == (5, 4) assert len(kwargs) == 0 + va2(5, 4) x = (5, 4) va2(*x) va2(5, *x[1:]) -# def va3(x, *, b=2): -# pass -# va3(1, 2, 3, b=10) + +def va3(x, *, a, b=2, c=9): + return x + b + c + + +assert va3(1, a=1, b=10) == 20 + +with assertRaises(TypeError): + va3(1, 2, 3, a=1, b=10) + +with assertRaises(TypeError): + va3(1, b=10) + + +assert va3.__defaults__ is None +kw_defaults = va3.__kwdefaults__ +# assert va3.__kwdefaults__ == {'b': 2, 'c': 9} +assert set(kw_defaults) == {'b', 'c'} +assert kw_defaults['b'] == 2 +assert kw_defaults['c'] == 9 x = {'f': 42, 'e': 1337} y = {'d': 1337} va(1, 22, 3, 4, **x, **y) + +# star arg after keyword args: +def fubar(x, y, obj=None): + assert x == 4 + assert y == 5 + assert obj == 6 + +rest = [4, 5] +fubar(obj=6, *rest) + + +# https://www.python.org/dev/peps/pep-0468/ +def func(**kwargs): + return list(kwargs.items()) + +empty_kwargs = func() +assert empty_kwargs == [] + +kwargs = func(a=1, b=2) +assert kwargs == [('a', 1), ('b', 2)] + +kwargs = func(a=1, b=2, c=3) +assert kwargs == [('a', 1), ('b', 2), ('c', 3)] diff --git a/tests/snippets/generators.py b/tests/snippets/generators.py index 5f64c49050..3b7dc77c39 100644 --- a/tests/snippets/generators.py +++ b/tests/snippets/generators.py @@ -1,3 +1,5 @@ +from testutils import assertRaises + r = [] @@ -35,3 +37,30 @@ def g3(): # print(r) assert r == [23, 1, 2, 3, 44] +def g4(): + yield + yield 2, + +r = list(g4()) +assert r == [None, (2,)] + + +def catch_exception(): + try: + yield 1 + except ValueError: + yield 2 + yield 3 + + +g = catch_exception() +assert next(g) == 1 + +assert g.throw(ValueError, ValueError(), None) == 2 +assert next(g) == 3 + +g = catch_exception() +assert next(g) == 1 + +with assertRaises(KeyError): + assert g.throw(KeyError, KeyError(), None) == 2 diff --git a/tests/snippets/global_nonlocal.py b/tests/snippets/global_nonlocal.py new file mode 100644 index 0000000000..ddf5e1ed22 --- /dev/null +++ b/tests/snippets/global_nonlocal.py @@ -0,0 +1,60 @@ +from testutils import assertRaises + +# Test global and nonlocal funkyness + +a = 2 + +def b(): + global a + a = 4 + +assert a == 2 +b() +assert a == 4 + + +def x(): + def y(): + nonlocal b + b = 3 + b = 2 + y() + return b + +res = x() +assert res == 3, str(res) + +# Invalid syntax: +src = """ +b = 2 +global b +""" + +with assertRaises(SyntaxError): + exec(src) + +# Invalid syntax: +src = """ +nonlocal c +""" + +with assertRaises(SyntaxError): + exec(src) + + +# Invalid syntax: +src = """ +def f(): + def x(): + nonlocal c +c = 2 +""" + +with assertRaises(SyntaxError): + exec(src) + + +# class X: +# nonlocal c +# c = 2 + diff --git a/tests/snippets/hash.py b/tests/snippets/hash.py new file mode 100644 index 0000000000..b108db7055 --- /dev/null +++ b/tests/snippets/hash.py @@ -0,0 +1,23 @@ + +from testutils import assertRaises + + +class A: + pass + + +assert type(hash(None)) is int +assert type(hash(object())) is int +assert type(hash(A())) is int +assert type(hash(1)) is int +assert type(hash(1.1)) is int +assert type(hash("")) is int + +with assertRaises(TypeError): + hash({}) + +with assertRaises(TypeError): + hash(set()) + +with assertRaises(TypeError): + hash([]) diff --git a/tests/snippets/if_expressions.py b/tests/snippets/if_expressions.py new file mode 100644 index 0000000000..143cb64580 --- /dev/null +++ b/tests/snippets/if_expressions.py @@ -0,0 +1,26 @@ +def ret(expression): + return expression + + +assert ret("0" if True else "1") == "0" +assert ret("0" if False else "1") == "1" + +assert ret("0" if False else ("1" if True else "2")) == "1" +assert ret("0" if False else ("1" if False else "2")) == "2" + +assert ret(("0" if True else "1") if True else "2") == "0" +assert ret(("0" if False else "1") if True else "2") == "1" + +a = True +b = False +assert ret("0" if a or b else "1") == "0" +assert ret("0" if a and b else "1") == "1" + + +def func1(): + return 0 + +def func2(): + return 20 + +assert ret(func1() or func2()) == 20 diff --git a/tests/snippets/imp.py b/tests/snippets/imp.py new file mode 100644 index 0000000000..44e7e10423 --- /dev/null +++ b/tests/snippets/imp.py @@ -0,0 +1,28 @@ +import _imp +import time as import_time + +assert _imp.is_builtin("time") == True +assert _imp.is_builtin("os") == False +assert _imp.is_builtin("not existing module") == False + +assert _imp.is_frozen("__hello__") == True +assert _imp.is_frozen("os") == False + +class FakeSpec: + def __init__(self, name): + self.name = name + +A = FakeSpec("time") + +imp_time = _imp.create_builtin(A) +assert imp_time.sleep == import_time.sleep + +B = FakeSpec("not existing module") +assert _imp.create_builtin(B) == None + +_imp.exec_builtin(imp_time) == 0 + +_imp.get_frozen_object("__hello__") + +hello = _imp.init_frozen("__hello__") +assert hello.initialized == True diff --git a/tests/snippets/import.py b/tests/snippets/import.py index a481e3f1ae..7c730c2659 100644 --- a/tests/snippets/import.py +++ b/tests/snippets/import.py @@ -3,9 +3,12 @@ from import_target import func as aliased_func, other_func as aliased_other_func from import_star import * +import import_mutual1 assert import_target.X == import_target.func() assert import_target.X == func() +assert import_mutual1.__name__ == "import_mutual1" + assert import_target.Y == other_func() assert import_target.X == aliased.X @@ -22,6 +25,11 @@ except ImportError: pass +try: + import mymodule +except ModuleNotFoundError as exc: + assert exc.name == 'mymodule' + test = __import__("import_target") assert test.X == import_target.X diff --git a/tests/snippets/import_file.py b/tests/snippets/import_file.py new file mode 100644 index 0000000000..3f9eeed704 --- /dev/null +++ b/tests/snippets/import_file.py @@ -0,0 +1,4 @@ +import os + +def import_file(): + assert os.path.basename(__file__) == "import_file.py" diff --git a/tests/snippets/import_module.py b/tests/snippets/import_module.py new file mode 100644 index 0000000000..b82aad5091 --- /dev/null +++ b/tests/snippets/import_module.py @@ -0,0 +1,3 @@ +import dir_module +assert dir_module.value == 5 +assert dir_module.value2 == 7 diff --git a/tests/snippets/import_mutual1.py b/tests/snippets/import_mutual1.py new file mode 100644 index 0000000000..0dca4a34e0 --- /dev/null +++ b/tests/snippets/import_mutual1.py @@ -0,0 +1,4 @@ + +# Mutual recursive import: +import import_mutual2 + diff --git a/tests/snippets/import_mutual2.py b/tests/snippets/import_mutual2.py new file mode 100644 index 0000000000..388ce25217 --- /dev/null +++ b/tests/snippets/import_mutual2.py @@ -0,0 +1,3 @@ + +# Mutual recursive import: +import import_mutual1 diff --git a/tests/snippets/int_float_comparisons.py b/tests/snippets/int_float_comparisons.py new file mode 100644 index 0000000000..400d47d07f --- /dev/null +++ b/tests/snippets/int_float_comparisons.py @@ -0,0 +1,61 @@ +# 10**308 cannot be represented exactly in f64, thus it is not equal to 1e308 float +assert not (10**308 == 1e308) +# but the 1e308 float can be converted to big int and then it still should be equal to itself +assert int(1e308) == 1e308 + +# and the equalities should be the same when operands switch sides +assert not (1e308 == 10**308) +assert 1e308 == int(1e308) + +# floats that cannot be converted to big ints shouldn’t crash the vm +import math +assert not (10**500 == math.inf) +assert not (math.inf == 10**500) +assert not (10**500 == math.nan) +assert not (math.nan == 10**500) + +# comparisons +# floats with worse than integer precision +assert 2.**54 > 2**54 - 1 +assert 2.**54 < 2**54 + 1 +assert 2.**54 >= 2**54 - 1 +assert 2.**54 <= 2**54 + 1 +assert 2.**54 == 2**54 +assert not 2.**54 == 2**54 + 1 + +# inverse operands +assert 2**54 - 1 < 2.**54 +assert 2**54 + 1 > 2.**54 +assert 2**54 - 1 <= 2.**54 +assert 2**54 + 1 >= 2.**54 +assert 2**54 == 2.**54 +assert not 2**54 + 1 == 2.**54 + +assert not 2.**54 < 2**54 - 1 +assert not 2.**54 > 2**54 + 1 + +# sub-int numbers +assert 1.3 > 1 +assert 1.3 >= 1 +assert 2.5 > 2 +assert 2.5 >= 2 +assert -0.3 < 0 +assert -0.3 <= 0 + +# int out of float range comparisons +assert 10**500 > 2.**54 +assert -10**500 < -0.12 + +# infinity and NaN comparisons +assert math.inf > 10**500 +assert math.inf >= 10**500 +assert not math.inf < 10**500 + +assert -math.inf < -10*500 +assert -math.inf <= -10*500 +assert not -math.inf > -10*500 + +assert not math.nan > 123 +assert not math.nan < 123 +assert not math.nan >= 123 +assert not math.nan <= 123 diff --git a/tests/snippets/int_float_equality.py b/tests/snippets/int_float_equality.py deleted file mode 100644 index fb240f8d3c..0000000000 --- a/tests/snippets/int_float_equality.py +++ /dev/null @@ -1,15 +0,0 @@ -# 10**308 cannot be represented exactly in f64, thus it is not equal to 1e308 float -assert not (10**308 == 1e308) -# but the 1e308 float can be converted to big int and then it still should be equal to itself -assert int(1e308) == 1e308 - -# and the equalities should be the same when operands switch sides -assert not (1e308 == 10**308) -assert 1e308 == int(1e308) - -# floats that cannot be converted to big ints shouldn’t crash the vm -import math -assert not (10**500 == math.inf) -assert not (math.inf == 10**500) -assert not (10**500 == math.nan) -assert not (math.nan == 10**500) diff --git a/tests/snippets/ints.py b/tests/snippets/ints.py index 2d36afcb6e..40061e7e52 100644 --- a/tests/snippets/ints.py +++ b/tests/snippets/ints.py @@ -1,4 +1,4 @@ -from testutils import assert_raises +from testutils import assert_raises, assertRaises # int to int comparisons @@ -37,6 +37,17 @@ assert (2).__rmul__(1) == 2 assert (2).__truediv__(1) == 2.0 assert (2).__rtruediv__(1) == 0.5 +assert (2).__pow__(3) == 8 +assert (10).__pow__(-1) == 0.1 +assert (2).__rpow__(3) == 9 +assert (10).__mod__(5) == 0 +assert (10).__mod__(6) == 4 +with assertRaises(ZeroDivisionError): + (10).__mod__(0) +assert (5).__rmod__(10) == 0 +assert (6).__rmod__(10) == 4 +with assertRaises(ZeroDivisionError): + (0).__rmod__(10) # real/imag attributes assert (1).real == 1 @@ -58,3 +69,73 @@ assert (2).__rmul__(1.0) == NotImplemented assert (2).__truediv__(1.0) == NotImplemented assert (2).__rtruediv__(1.0) == NotImplemented +assert (2).__pow__(3.0) == NotImplemented +assert (2).__rpow__(3.0) == NotImplemented +assert (2).__mod__(3.0) == NotImplemented +assert (2).__rmod__(3.0) == NotImplemented + +assert 10 // 4 == 2 +assert -10 // 4 == -3 +assert 10 // -4 == -3 +assert -10 // -4 == 2 + +assert int() == 0 +assert int("101", 2) == 5 +assert int("101", base=2) == 5 +assert int(1) == 1 + +assert int.from_bytes(b'\x00\x10', 'big') == 16 +assert int.from_bytes(b'\x00\x10', 'little') == 4096 +assert int.from_bytes(b'\xfc\x00', 'big', signed=True) == -1024 +assert int.from_bytes(b'\xfc\x00', 'big', signed=False) == 64512 + +assert (1024).to_bytes(4, 'big') == b'\x00\x00\x04\x00' +assert (1024).to_bytes(2, 'little', signed=True) == b'\x00\x04' +assert (-1024).to_bytes(4, 'big', signed=True) == b'\xff\xff\xfc\x00' +assert (-1024).to_bytes(4, 'little', signed=True) == b'\x00\xfc\xff\xff' +assert (2147483647).to_bytes(8, 'big', signed=False) == b'\x00\x00\x00\x00\x7f\xff\xff\xff' +assert (-2147483648).to_bytes(8, 'little', signed=True) == b'\x00\x00\x00\x80\xff\xff\xff\xff' + +with assertRaises(TypeError): + int(base=2) + +with assertRaises(TypeError): + int(1, base=2) + +with assertRaises(TypeError): + # check that first parameter is truly positional only + int(val_options=1) + +class A(object): + def __int__(self): + return 10 + +assert int(A()) == 10 + +class B(object): + pass + +b = B() +b.__int__ = lambda: 20 + +with assertRaises(TypeError): + assert int(b) == 20 + +class C(object): + def __int__(self): + return 'str' + +with assertRaises(TypeError): + int(C()) + +class I(int): + def __int__(self): + return 3 + +assert int(I(1)) == 3 + +class F(float): + def __int__(self): + return 3 + +assert int(F(1.2)) == 3 diff --git a/tests/snippets/invalid_syntax.py b/tests/snippets/invalid_syntax.py new file mode 100644 index 0000000000..3c9c6d46c8 --- /dev/null +++ b/tests/snippets/invalid_syntax.py @@ -0,0 +1,18 @@ +from testutils import assertRaises + +src = """ +def valid_func(): + pass + +yield 2 +""" + +try: + compile(src, 'test.py', 'exec') +except SyntaxError as ex: + assert ex.lineno == 5 +else: + raise AssertionError("Must throw syntax error") + +with assertRaises(SyntaxError): + compile('0xX', 'test.py', 'exec') diff --git a/tests/snippets/iterable.py b/tests/snippets/iterable.py new file mode 100644 index 0000000000..45bb1cf4dc --- /dev/null +++ b/tests/snippets/iterable.py @@ -0,0 +1,32 @@ +from testutils import assert_raises + +def test_container(x): + assert 3 in x + assert 4 not in x + assert list(x) == list(iter(x)) + assert list(x) == [0, 1, 2, 3] + assert [*x] == [0, 1, 2, 3] + lst = [] + lst.extend(x) + assert lst == [0, 1, 2, 3] + +class C: + def __iter__(self): + return iter([0, 1, 2, 3]) +test_container(C()) + +class C: + def __getitem__(self, x): + return (0, 1, 2, 3)[x] # raises IndexError on x==4 +test_container(C()) + +class C: + def __getitem__(self, x): + if x > 3: + raise StopIteration + return x +test_container(C()) + +class C: pass +assert_raises(TypeError, lambda: 5 in C()) +assert_raises(TypeError, lambda: iter(C)) diff --git a/tests/snippets/json_snippet.py b/tests/snippets/json_snippet.py index 917556a478..a8c0ecc19e 100644 --- a/tests/snippets/json_snippet.py +++ b/tests/snippets/json_snippet.py @@ -1,3 +1,4 @@ +from testutils import assert_raises import json def round_trip_test(obj): @@ -25,8 +26,13 @@ def round_trip_test(obj): assert [1, "string", 1.0, True] == json.loads(json.dumps((1, "string", 1.0, True))) assert '{}' == json.dumps({}) -# TODO: uncomment once dict comparison is implemented -# round_trip_test({'a': 'b'}) +round_trip_test({'a': 'b'}) + +# should reject non-str keys in jsons +assert_raises(json.JSONDecodeError, lambda: json.loads('{3: "abc"}')) + +# should serialize non-str keys as strings +assert json.dumps({'3': 'abc'}) == json.dumps({3: 'abc'}) assert 1 == json.loads("1") assert -1 == json.loads("-1") @@ -45,12 +51,23 @@ class String(str): pass assert "string" == json.loads(String('"string"')) assert '"string"' == json.dumps(String("string")) -# TODO: Uncomment and test once int/float construction is supported -# class Int(int): pass -# class Float(float): pass +class Int(int): pass +class Float(float): pass + +assert '1' == json.dumps(Int(1)) +assert '0.5' == json.dumps(Float(0.5)) + +class List(list): pass +class Tuple(tuple): pass +class Dict(dict): pass + +assert '[1]' == json.dumps(List([1])) +assert json.dumps((1, "string", 1.0, True)) == json.dumps(Tuple((1, "string", 1.0, True))) +assert json.dumps({'a': 'b'}) == json.dumps(Dict({'a': 'b'})) -# TODO: Uncomment and test once sequence/dict subclasses are supported by -# json.dumps -# class List(list): pass -# class Tuple(tuple): pass -# class Dict(dict): pass +# big ints should not crash VM +# TODO: test for correct output when actual serialization implemented and doesn’t throw +try: + json.dumps(7*500) +except: + pass diff --git a/tests/snippets/list.py b/tests/snippets/list.py index 6b9839321d..e8e731de97 100644 --- a/tests/snippets/list.py +++ b/tests/snippets/list.py @@ -14,6 +14,10 @@ assert x * 0 == [], "list __mul__ by 0 failed" assert x * -1 == [], "list __mul__ by -1 failed" assert x * 2 == [1, 2, 3, 1, 2, 3], "list __mul__ by 2 failed" +y = x +x *= 2 +assert y is x +assert x == [1, 2, 3] * 2 # index() assert ['a', 'b', 'c'].index('b') == 1 @@ -170,3 +174,317 @@ def f(x): return x assert_raises(ValueError, lambda: lst.sort(key=f)) # "list modified during sort" assert lst == [1, 2, 3, 4, 5] + +# __delitem__ +x = ['a', 'b', 'c'] +del x[0] +assert x == ['b', 'c'] + +x = ['a', 'b', 'c'] +del x[-1] +assert x == ['a', 'b'] + +x = y = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[2:14:3] +assert x == y +assert x == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] +assert y == [1, 2, 4, 5, 7, 8, 11, 12, 14, 15] + +x = [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15] +del x[-5:] +assert x == [1, 2, 3, 4, 5, 6, 7, 8, 10] + +x = list(range(12)) +del x[10:2:-2] +assert x == [0,1,2,3,5,7,9,11] + +def bad_del_1(): + del ['a', 'b']['a'] +assert_raises(TypeError, bad_del_1) + +def bad_del_2(): + del ['a', 'b'][2] +assert_raises(IndexError, bad_del_2) + +# __setitem__ + +# simple index +x = [1, 2, 3, 4, 5] +x[0] = 'a' +assert x == ['a', 2, 3, 4, 5] +x[-1] = 'b' +assert x == ['a', 2, 3, 4, 'b'] +# make sure refrences are assigned correctly +y = [] +x[1] = y +y.append(100) +assert x[1] == y +assert x[1] == [100] + +#index bounds +def set_index_out_of_bounds_high(): + x = [0, 1, 2, 3, 4] + x[5] = 'a' + +def set_index_out_of_bounds_low(): + x = [0, 1, 2, 3, 4] + x[-6] = 'a' + +assert_raises(IndexError, set_index_out_of_bounds_high) +assert_raises(IndexError, set_index_out_of_bounds_low) + +# non stepped slice index +a = list(range(10)) +x = a[:] +y = a[:] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +# replace whole list +x[:] = ['a', 'b', 'c'] +y[::1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c'] +assert x == y +# splice list start +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[:1] = ['a', 'b', 'c'] +y[0:1] = ['a', 'b', 'c'] +z[:1:1] = ['a', 'b', 'c'] +zz[0:1:1] = ['a', 'b', 'c'] +assert x == ['a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# splice list end +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[5:] = ['a', 'b', 'c'] +y[5::1] = ['a', 'b', 'c'] +z[5:10] = ['a', 'b', 'c'] +zz[5:10:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 'a', 'b', 'c'] +assert x == y +assert x == z +assert x == zz +# insert sec +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[1:1] = ['a', 'b', 'c'] +y[1:0] = ['a', 'b', 'c'] +z[1:1:1] = ['a', 'b', 'c'] +zz[1:0:1] = ['a', 'b', 'c'] +assert x == [0, 'a', 'b', 'c', 1, 2, 3, 4, 5, 6, 7, 8, 9] +assert x == y +assert x == z +assert x == zz +# same but negative indexes? +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +x[-1:-1] = ['a', 'b', 'c'] +y[-1:9] = ['a', 'b', 'c'] +z[-1:-1:1] = ['a', 'b', 'c'] +zz[-1:9:1] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c', 9] +assert x == y +assert x == z +assert x == zz +# splice mid +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', 'e'] +y[3:5:1] = ['a', 'b', 'c', 'd', 'e'] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 5, 6, 7, 8, 9] +assert x == y +x = a[:] +x[3:5] = ['a'] +assert x == [0, 1, 2, 'a', 5, 6, 7, 8, 9] +# assign empty to non stepped empty slice does nothing +x = a[:] +y = a[:] +x[5:2] = [] +y[5:2:1] = [] +assert x == a +assert y == a +# assign empty to non stepped slice removes elems +x = a[:] +y = a[:] +x[2:8] = [] +y[2:8:1] = [] +assert x == [0, 1, 8, 9] +assert x == y +# make sure refrences are assigned correctly +yy = [] +x = a[:] +y = a[:] +x[3:5] = ['a', 'b', 'c', 'd', yy] +y[3:5:1] = ['a', 'b', 'c', 'd', yy] +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [], 5, 6, 7, 8, 9] +assert x == y +yy.append(100) +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', [100], 5, 6, 7, 8, 9] +assert x == y +assert x[7] == yy +assert x[7] == [100] +assert y[7] == yy +assert y[7] == [100] + +# no zero step +def no_zero_step_set(): + x = [1, 2, 3, 4, 5] + x[0:4:0] = [11, 12, 13, 14, 15] +assert_raises(ValueError, no_zero_step_set) + +# stepped slice index +# forward slice +x = a[:] +x[2:8:2] = ['a', 'b', 'c'] +assert x == [0, 1, 'a', 3, 'b', 5, 'c', 7, 8, 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e'] +x[::2] = c +y[-10::2] = c +z[0:10:2] = c +zz[-13:13:2] = c # slice indexes will be truncated to bounds +assert x == ['a', 1, 'b', 3, 'c', 5, 'd', 7, 'e', 9] +assert x == y +assert x == z +assert x == zz +# backward slice +x = a[:] +x[8:2:-2] = ['a', 'b', 'c'] +assert x == [0, 1, 2, 3, 'c', 5, 'b', 7, 'a', 9] +x = a[:] +y = a[:] +z = a[:] +zz = a[:] +c = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'] +x[::-1] = c +y[9:-11:-1] = c +z[9::-1] = c +zz[11:-13:-1] = c # slice indexes will be truncated to bounds +assert x == ['j', 'i', 'h', 'g', 'f', 'e', 'd', 'c', 'b', 'a'] +assert x == y +assert x == z +assert x == zz +# step size bigger than len +x = a[:] +x[::200] = ['a'] +assert x == ['a', 1, 2, 3, 4, 5, 6, 7, 8, 9] +x = a[:] +x[5::200] = ['a'] +assert x == [0, 1, 2, 3, 4, 'a', 6, 7, 8, 9] + +# bad stepped slices +def stepped_slice_assign_too_big(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b', 'c', 'd'] + +assert_raises(ValueError, stepped_slice_assign_too_big) + +def stepped_slice_assign_too_small(): + x = [0, 1, 2, 3, 4] + x[::2] = ['a', 'b'] + +assert_raises(ValueError, stepped_slice_assign_too_small) + +# must assign iter t0 slice +def must_assign_iter_to_slice(): + x = [0, 1, 2, 3, 4] + x[::2] = 42 + +assert_raises(TypeError, must_assign_iter_to_slice) + +# other iterables? +a = list(range(10)) + +# string +x = a[:] +x[3:8] = "abcdefghi" +assert x == [0, 1, 2, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 8, 9] + +# tuple +x = a[:] +x[3:8] = (11, 12, 13, 14, 15) +assert x == [0, 1, 2, 11, 12, 13, 14, 15, 8, 9] + +# class +# __next__ +class CIterNext: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + self.index = 0 + def __iter__(self): + return self + def __next__(self): + if self.index >= len(self.sec): + raise StopIteration + v = self.sec[self.index] + self.index += 1 + return v + +x = list(range(10)) +x[3:8] = CIterNext() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# __iter__ yield +class CIter: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __iter__(self): + for n in self.sec: + yield n + +x = list(range(10)) +x[3:8] = CIter() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# __getitem but no __iter__ sequence +class CGetItem: + def __init__(self, sec=(1, 2, 3)): + self.sec = sec + def __getitem__(self, sub): + return self.sec[sub] + +x = list(range(10)) +x[3:8] = CGetItem() +assert x == [0, 1, 2, 1, 2, 3, 8, 9] + +# iter raises error +class CIterError: + def __iter__(self): + for i in range(10): + if i > 5: + raise RuntimeError + yield i + +def bad_iter_assign(): + x = list(range(10)) + x[3:8] = CIterError() + +assert_raises(RuntimeError, bad_iter_assign) + +# slice assign when step or stop is -1 +a = list(range(10)) +x = a[:] +x[-1:-5:-1] = ['a', 'b', 'c', 'd'] +assert x == [0, 1, 2, 3, 4, 5, 'd', 'c', 'b', 'a'] +x = a[:] +x[-5:-1:-1] = [] +assert x == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +# is step != 1 and start or stop of slice == -1 +x = list(range(10)) +del x[-1:-5:-1] +assert x == [0, 1, 2, 3, 4, 5] +x = list(range(10)) +del x[-5:-1:-1] diff --git a/tests/snippets/literals.py b/tests/snippets/literals.py new file mode 100644 index 0000000000..f481b33f5d --- /dev/null +++ b/tests/snippets/literals.py @@ -0,0 +1,8 @@ +# Integer literals +assert 0b101010 == 42 +assert 0B101010 == 42 +assert 0o777 == 511 +assert 0O777 == 511 +assert 0xcafebabe == 3405691582 +assert 0Xcafebabe == 3405691582 +assert 0xCAFEBABE == 3405691582 diff --git a/tests/snippets/mappingproxy.py b/tests/snippets/mappingproxy.py new file mode 100644 index 0000000000..254f2a9251 --- /dev/null +++ b/tests/snippets/mappingproxy.py @@ -0,0 +1,20 @@ +from testutils import assertRaises + +class A(dict): + def a(): + pass + + def b(): + pass + + +assert A.__dict__['a'] == A.a +with assertRaises(KeyError) as cm: + A.__dict__['not here'] + +assert cm.exception.args[0] == "not here" + +assert 'b' in A.__dict__ +assert 'c' not in A.__dict__ + +assert '__dict__' in A.__dict__ diff --git a/tests/snippets/math_basics.py b/tests/snippets/math_basics.py index 09f3ed3b3a..f02ba3e60c 100644 --- a/tests/snippets/math_basics.py +++ b/tests/snippets/math_basics.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + a = 4 #print(a ** 3) @@ -12,18 +14,58 @@ assert a * 3 == 12 assert a / 2 == 2 assert 2 == a / 2 -# assert a % 3 == 1 +assert a % 3 == 1 assert a - 3 == 1 assert -a == -4 assert +a == 4 -# import math -# assert(math.exp(2) == math.exp(2.0)) -# assert(math.exp(True) == math.exp(1.0)) -# -# class Conversible(): -# def __float__(self): -# print("Converting to float now!") -# return 1.1111 -# -# assert math.log(1.1111) == math.log(Conversible()) +assert round(1.2) == 1 +assert round(1.8) == 2 +assert round(0.5) == 0 +assert round(1.5) == 2 +assert round(-0.5) == 0 +assert round(-1.5) == -2 + +assert_raises( + ValueError, + lambda: round(float('nan')), + 'ValueError: cannot convert float NaN to integer') +assert_raises( + OverflowError, + lambda: round(float('inf')), + 'OverflowError: cannot convert float infinity to integer') +assert_raises( + OverflowError, + lambda: round(-float('inf')), + 'OverflowError: cannot convert float infinity to integer') + +assert pow(0, 0) == 1 +assert pow(2, 2) == 4 +assert pow(1, 2.0) == 1.0 +assert pow(2.0, 1) == 2.0 +assert pow(0, 10**1000) == 0 +assert pow(1, 10**1000) == 1 +assert pow(-1, 10**1000+1) == -1 +assert pow(-1, 10**1000) == 1 + +assert pow(2, 4, 5) == 1 +assert_raises( + TypeError, + lambda: pow(2, 4, 5.0), + 'pow() 3rd argument not allowed unless all arguments are integers') +assert_raises( + TypeError, + lambda: pow(2, 4.0, 5), + 'pow() 3rd argument not allowed unless all arguments are integers') +assert_raises( + TypeError, + lambda: pow(2.0, 4, 5), + 'pow() 3rd argument not allowed unless all arguments are integers') +assert_raises( + ValueError, + lambda: pow(2, -1, 5), + 'pow() 2nd argument cannot be negative when 3rd argument specified') +assert_raises( + ValueError, + lambda: pow(2, 2, 0), + 'pow() 3rd argument cannot be 0') diff --git a/tests/snippets/math_module.py b/tests/snippets/math_module.py new file mode 100644 index 0000000000..dc7fc07062 --- /dev/null +++ b/tests/snippets/math_module.py @@ -0,0 +1,89 @@ +import math +from testutils import assertRaises, assert_raises + +# assert(math.exp(2) == math.exp(2.0)) +# assert(math.exp(True) == math.exp(1.0)) +# +# class Conversible(): +# def __float__(self): +# print("Converting to float now!") +# return 1.1111 +# +# assert math.log(1.1111) == math.log(Conversible()) + +# roundings +assert int.__trunc__ +assert int.__floor__ +assert int.__ceil__ + +# assert float.__trunc__ +with assertRaises(AttributeError): + assert float.__floor__ +with assertRaises(AttributeError): + assert float.__ceil__ + +assert math.trunc(2) == 2 +assert math.ceil(3) == 3 +assert math.floor(4) == 4 + +assert math.trunc(2.2) == 2 +assert math.ceil(3.3) == 4 +assert math.floor(4.4) == 4 + +class A(object): + def __trunc__(self): + return 2 + + def __ceil__(self): + return 3 + + def __floor__(self): + return 4 + +assert math.trunc(A()) == 2 +assert math.ceil(A()) == 3 +assert math.floor(A()) == 4 + +class A(object): + def __trunc__(self): + return 2.2 + + def __ceil__(self): + return 3.3 + + def __floor__(self): + return 4.4 + +assert math.trunc(A()) == 2.2 +assert math.ceil(A()) == 3.3 +assert math.floor(A()) == 4.4 + +class A(object): + def __trunc__(self): + return 'trunc' + + def __ceil__(self): + return 'ceil' + + def __floor__(self): + return 'floor' + +assert math.trunc(A()) == 'trunc' +assert math.ceil(A()) == 'ceil' +assert math.floor(A()) == 'floor' + +with assertRaises(TypeError): + math.trunc(object()) +with assertRaises(TypeError): + math.ceil(object()) +with assertRaises(TypeError): + math.floor(object()) + +assert str(math.frexp(0.0)) == str((+0.0, 0)) +assert str(math.frexp(-0.0)) == str((-0.0, 0)) +assert math.frexp(1) == (0.5, 1) +assert math.frexp(1.5) == (0.75, 1) + +assert math.frexp(float('inf')) == (float('inf'), 0) +assert str(math.frexp(float('nan'))) == str((float('nan'), 0)) +assert_raises(TypeError, lambda: math.frexp(None)) diff --git a/tests/snippets/membership.py b/tests/snippets/membership.py index a944c45398..2987c3c0fe 100644 --- a/tests/snippets/membership.py +++ b/tests/snippets/membership.py @@ -12,9 +12,8 @@ assert "whatever" not in "foobar" # test bytes -# TODO: uncomment this when bytes are implemented -# assert b"foo" in b"foobar" -# assert b"whatever" not in b"foobar" +assert b"foo" in b"foobar" +assert b"whatever" not in b"foobar" assert b"1" < b"2" assert b"1" <= b"2" assert b"5" <= b"5" @@ -32,18 +31,20 @@ assert 3 not in set([1, 2]) # test dicts -# TODO: test dicts when keys other than strings will be allowed assert "a" in {"a": 0, "b": 0} assert "c" not in {"a": 0, "b": 0} +assert 1 in {1: 5, 7: 12} +assert 5 not in {9: 10, 50: 100} +assert True in {True: 5} +assert False not in {True: 5} # test iter assert 3 in iter([1, 2, 3]) assert 3 not in iter([1, 2]) # test sequence -# TODO: uncomment this when ranges are usable -# assert 1 in range(0, 2) -# assert 3 not in range(0, 2) +assert 1 in range(0, 2) +assert 3 not in range(0, 2) # test __contains__ in user objects class MyNotContainingClass(): diff --git a/tests/snippets/memoryview.py b/tests/snippets/memoryview.py new file mode 100644 index 0000000000..ebf27a9a6f --- /dev/null +++ b/tests/snippets/memoryview.py @@ -0,0 +1,6 @@ + +obj = b"abcde" +a = memoryview(obj) +assert a.obj == obj + +assert a[2:3] == b"c" diff --git a/tests/snippets/printing.py b/tests/snippets/printing.py index e420c6c7fd..b2c9ca55df 100644 --- a/tests/snippets/printing.py +++ b/tests/snippets/printing.py @@ -1,4 +1,5 @@ from testutils import assert_raises +import io print(2 + 3) @@ -9,3 +10,7 @@ print('test', end=None, sep=None, flush=None) except: assert False, 'Expected None passed to end, sep, and flush to not raise errors' + +buf = io.StringIO() +print('hello, world', file=buf) +assert buf.getvalue() == 'hello, world\n', buf.getvalue() diff --git a/tests/snippets/property.py b/tests/snippets/property.py index c61f131899..da2c1ee632 100644 --- a/tests/snippets/property.py +++ b/tests/snippets/property.py @@ -59,6 +59,8 @@ def foo(self): with assertRaises(TypeError): property.__new__(object) +# assert p.__doc__ is None + p1 = property("a", "b", "c") @@ -75,3 +77,7 @@ def foo(self): assert p1.deleter(None).fdel == "c" assert p1.__get__(None, object) is p1 +# assert p1.__doc__ is 'a'.__doc__ + +p2 = property('a', doc='pdoc') +# assert p2.__doc__ == 'pdoc' diff --git a/tests/snippets/set.py b/tests/snippets/set.py index a7a20b0efb..50b8f97d84 100644 --- a/tests/snippets/set.py +++ b/tests/snippets/set.py @@ -27,6 +27,26 @@ assert not set([1,2]) < set([1,2]) assert not set([1,3]) < set([1,2]) +assert (set() == []) is False +assert set().__eq__([]) == NotImplemented +assert_raises(TypeError, lambda: set() < [], "'<' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() <= [], "'<=' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() > [], "'>' not supported between instances of 'set' and 'list'") +assert_raises(TypeError, lambda: set() >= [], "'>=' not supported between instances of 'set' and 'list'") +assert set().issuperset([]) +assert set().issubset([]) +assert not set().issuperset([1, 2, 3]) +assert set().issubset([1, 2]) + +assert (set() == 3) is False +assert set().__eq__(3) == NotImplemented +assert_raises(TypeError, lambda: set() < 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() <= 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() > 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set() >= 3, "'int' object is not iterable") +assert_raises(TypeError, lambda: set().issuperset(3), "'int' object is not iterable") +assert_raises(TypeError, lambda: set().issubset(3), "'int' object is not iterable") + class Hashable(object): def __init__(self, obj): self.obj = obj @@ -49,27 +69,45 @@ def __hash__(self): assert set([1,2,3]).union(set([4,5])) == set([1,2,3,4,5]) assert set([1,2,3]).union(set([1,2,3,4,5])) == set([1,2,3,4,5]) +assert set([1,2,3]).union([1,2,3,4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) | set([1,2,3,4,5]) == set([1,2,3,4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) | [1,2,3,4,5]) assert set([1,2,3]).intersection(set([1,2])) == set([1,2]) assert set([1,2,3]).intersection(set([5,6])) == set([]) +assert set([1,2,3]).intersection([1,2]) == set([1,2]) assert set([1,2,3]) & set([4,5]) == set([]) assert set([1,2,3]) & set([1,2,3,4,5]) == set([1,2,3]) +assert_raises(TypeError, lambda: set([1,2,3]) & [1,2,3,4,5]) assert set([1,2,3]).difference(set([1,2])) == set([3]) assert set([1,2,3]).difference(set([5,6])) == set([1,2,3]) +assert set([1,2,3]).difference([1,2]) == set([3]) assert set([1,2,3]) - set([4,5]) == set([1,2,3]) assert set([1,2,3]) - set([1,2,3,4,5]) == set([]) +assert_raises(TypeError, lambda: set([1,2,3]) - [1,2,3,4,5]) assert set([1,2,3]).symmetric_difference(set([1,2])) == set([3]) assert set([1,2,3]).symmetric_difference(set([5,6])) == set([1,2,3,5,6]) +assert set([1,2,3]).symmetric_difference([1,2]) == set([3]) assert set([1,2,3]) ^ set([4,5]) == set([1,2,3,4,5]) assert set([1,2,3]) ^ set([1,2,3,4,5]) == set([4,5]) +assert_raises(TypeError, lambda: set([1,2,3]) ^ [1,2,3,4,5]) + +assert set([1,2,3]).isdisjoint(set([5,6])) == True +assert set([1,2,3]).isdisjoint(set([2,5,6])) == False +assert set([1,2,3]).isdisjoint([5,6]) == True + +assert_raises(TypeError, lambda: set() & []) +assert_raises(TypeError, lambda: set() | []) +assert_raises(TypeError, lambda: set() ^ []) +assert_raises(TypeError, lambda: set() + []) +assert_raises(TypeError, lambda: set() - []) assert_raises(TypeError, lambda: set([[]])) assert_raises(TypeError, lambda: set().add([])) @@ -91,7 +129,7 @@ def __hash__(self): b = a.pop() assert b in [1,2] c = a.pop() -assert (c in [1,2] and c != b) +assert (c in [1,2] and c != b) assert_raises(KeyError, lambda: a.pop()) a = set([1,2,3]) @@ -111,6 +149,8 @@ def __hash__(self): assert a == set([1,2,3,4,5]) with assertRaises(TypeError): a |= 1 +with assertRaises(TypeError): + a |= [1,2,3] a = set([1,2,3]) a.intersection_update([2,3,4,5]) @@ -122,6 +162,8 @@ def __hash__(self): assert a == set([2,3]) with assertRaises(TypeError): a &= 1 +with assertRaises(TypeError): + a &= [1,2,3] a = set([1,2,3]) a.difference_update([3,4,5]) @@ -133,6 +175,8 @@ def __hash__(self): assert a == set([1,2]) with assertRaises(TypeError): a -= 1 +with assertRaises(TypeError): + a -= [1,2,3] a = set([1,2,3]) a.symmetric_difference_update([3,4,5]) @@ -144,3 +188,142 @@ def __hash__(self): assert a == set([1,2,4,5]) with assertRaises(TypeError): a ^= 1 +with assertRaises(TypeError): + a ^= [1,2,3] + +# frozen set + +assert frozenset([1,2]) == frozenset([1,2]) +assert not frozenset([1,2,3]) == frozenset([1,2]) + +assert frozenset([1,2,3]) >= frozenset([1,2]) +assert frozenset([1,2]) >= frozenset([1,2]) +assert not frozenset([1,3]) >= frozenset([1,2]) + +assert frozenset([1,2,3]).issuperset(frozenset([1,2])) +assert frozenset([1,2]).issuperset(frozenset([1,2])) +assert not frozenset([1,3]).issuperset(frozenset([1,2])) + +assert frozenset([1,2,3]) > frozenset([1,2]) +assert not frozenset([1,2]) > frozenset([1,2]) +assert not frozenset([1,3]) > frozenset([1,2]) + +assert frozenset([1,2]) <= frozenset([1,2,3]) +assert frozenset([1,2]) <= frozenset([1,2]) +assert not frozenset([1,3]) <= frozenset([1,2]) + +assert frozenset([1,2]).issubset(frozenset([1,2,3])) +assert frozenset([1,2]).issubset(frozenset([1,2])) +assert not frozenset([1,3]).issubset(frozenset([1,2])) + +assert frozenset([1,2]) < frozenset([1,2,3]) +assert not frozenset([1,2]) < frozenset([1,2]) +assert not frozenset([1,3]) < frozenset([1,2]) + +a = frozenset([1, 2, 3]) +assert len(a) == 3 +b = a.copy() +assert b == a + +assert frozenset([1,2,3]).union(frozenset([4,5])) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]).union(frozenset([1,2,3,4,5])) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]).union([1,2,3,4,5]) == frozenset([1,2,3,4,5]) + +assert frozenset([1,2,3]) | frozenset([4,5]) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]) | frozenset([1,2,3,4,5]) == frozenset([1,2,3,4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) | [1,2,3,4,5]) + +assert frozenset([1,2,3]).intersection(frozenset([1,2])) == frozenset([1,2]) +assert frozenset([1,2,3]).intersection(frozenset([5,6])) == frozenset([]) +assert frozenset([1,2,3]).intersection([1,2]) == frozenset([1,2]) + +assert frozenset([1,2,3]) & frozenset([4,5]) == frozenset([]) +assert frozenset([1,2,3]) & frozenset([1,2,3,4,5]) == frozenset([1,2,3]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) & [1,2,3,4,5]) + +assert frozenset([1,2,3]).difference(frozenset([1,2])) == frozenset([3]) +assert frozenset([1,2,3]).difference(frozenset([5,6])) == frozenset([1,2,3]) +assert frozenset([1,2,3]).difference([1,2]) == frozenset([3]) + +assert frozenset([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) +assert frozenset([1,2,3]) - frozenset([1,2,3,4,5]) == frozenset([]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) - [1,2,3,4,5]) + +assert frozenset([1,2,3]).symmetric_difference(frozenset([1,2])) == frozenset([3]) +assert frozenset([1,2,3]).symmetric_difference(frozenset([5,6])) == frozenset([1,2,3,5,6]) +assert frozenset([1,2,3]).symmetric_difference([1,2]) == frozenset([3]) + +assert frozenset([1,2,3]) ^ frozenset([4,5]) == frozenset([1,2,3,4,5]) +assert frozenset([1,2,3]) ^ frozenset([1,2,3,4,5]) == frozenset([4,5]) +assert_raises(TypeError, lambda: frozenset([1,2,3]) ^ [1,2,3,4,5]) + +assert frozenset([1,2,3]).isdisjoint(frozenset([5,6])) == True +assert frozenset([1,2,3]).isdisjoint(frozenset([2,5,6])) == False +assert frozenset([1,2,3]).isdisjoint([5,6]) == True + +assert_raises(TypeError, lambda: frozenset([[]])) + +a = frozenset([1,2,3]) +b = set() +for e in a: + assert e == 1 or e == 2 or e == 3 + b.add(e) +assert a == b + +# set and frozen set +assert frozenset([1,2,3]).union(set([4,5])) == frozenset([1,2,3,4,5]) +assert set([1,2,3]).union(frozenset([4,5])) == set([1,2,3,4,5]) + +assert frozenset([1,2,3]) | set([4,5]) == frozenset([1,2,3,4,5]) +assert set([1,2,3]) | frozenset([4,5]) == set([1,2,3,4,5]) + +assert frozenset([1,2,3]).intersection(set([5,6])) == frozenset([]) +assert set([1,2,3]).intersection(frozenset([5,6])) == set([]) + +assert frozenset([1,2,3]) & set([1,2,3,4,5]) == frozenset([1,2,3]) +assert set([1,2,3]) & frozenset([1,2,3,4,5]) == set([1,2,3]) + +assert frozenset([1,2,3]).difference(set([5,6])) == frozenset([1,2,3]) +assert set([1,2,3]).difference(frozenset([5,6])) == set([1,2,3]) + +assert frozenset([1,2,3]) - set([4,5]) == frozenset([1,2,3]) +assert set([1,2,3]) - frozenset([4,5]) == frozenset([1,2,3]) + +assert frozenset([1,2,3]).symmetric_difference(set([1,2])) == frozenset([3]) +assert set([1,2,3]).symmetric_difference(frozenset([1,2])) == set([3]) + +assert frozenset([1,2,3]) ^ set([4,5]) == frozenset([1,2,3,4,5]) +assert set([1,2,3]) ^ frozenset([4,5]) == set([1,2,3,4,5]) + +class A: + def __hash__(self): + return 1 +class B: + def __hash__(self): + return 1 + +s = {1, A(), B()} +assert len(s) == 3 + +s = {True} +s.add(1.0) +assert str(s) == '{True}' + +class EqObject: + def __init__(self, eq): + self.eq = eq + def __eq__(self, other): + return self.eq + def __hash__(self): + return bool(self.eq) + +assert 'x' == (EqObject('x') == EqObject('x')) +s = {EqObject('x')} +assert EqObject('x') in s +assert '[]' == (EqObject('[]') == EqObject('[]')) +s = {EqObject([])} +assert EqObject([]) not in s +x = object() +assert x == (EqObject(x) == EqObject(x)) +s = {EqObject(x)} +assert EqObject(x) in s diff --git a/tests/snippets/statements.py b/tests/snippets/statements.py new file mode 100644 index 0000000000..1921363f56 --- /dev/null +++ b/tests/snippets/statements.py @@ -0,0 +1,9 @@ + +# Test several statement types + +# small ones, seperated by ';': + +if True: + 5;4; + b=4 + b; diff --git a/tests/snippets/stdlib_binascii.py b/tests/snippets/stdlib_binascii.py new file mode 100644 index 0000000000..e56b3e47d8 --- /dev/null +++ b/tests/snippets/stdlib_binascii.py @@ -0,0 +1,39 @@ +import binascii +from testutils import assertRaises + + +# hexlify tests +h = binascii.hexlify + +assert h(b"abc") == b"616263" +assert h(1000 * b"x") == 1000 * b"78" +# bytearray not supported yet +# assert h(bytearray(b"a")) = b"61" +assert binascii.b2a_hex(b"aa") == b"6161" + +with assertRaises(TypeError): + h("a") + + +# unhexlify tests +uh = binascii.unhexlify + +assert uh(b"616263") == b"abc" +assert uh(1000 * b"78") == 1000 * b"x" +x = 1000 * b"1234" +assert uh(h(x)) == x +assert uh(b"ABCDEF") == b"\xab\xcd\xef" +assert binascii.a2b_hex(b"6161") == b"aa" + +# unhexlify on strings not supported yet +# assert uh("abcd") == b"\xab\xcd" + +with assertRaises(ValueError): + uh(b"a") # Odd-length string + +with assertRaises(ValueError): + uh(b"nn") # Non-hexadecimal digit found + +assert binascii.crc32(b"hello world") == 222957957 +assert binascii.crc32(b"hello world", 555555) == 1216827162 +assert binascii.crc32(b"goodbye interesting world",777777) == 1885538403 diff --git a/tests/snippets/stdlib_io.py b/tests/snippets/stdlib_io.py index 503759df7a..08786c2198 100644 --- a/tests/snippets/stdlib_io.py +++ b/tests/snippets/stdlib_io.py @@ -1,4 +1,5 @@ from io import BufferedReader, FileIO +import os fi = FileIO('README.md') bb = BufferedReader(fi) @@ -8,3 +9,15 @@ assert len(result) <= 8*1024 assert len(result) >= 0 assert isinstance(result, bytes) + +with FileIO('README.md') as fio: + res = fio.read() + assert len(result) <= 8*1024 + assert len(result) >= 0 + assert isinstance(result, bytes) + +fd = os.open('README.md', os.O_RDONLY) + +with FileIO(fd) as fio: + res2 = fio.read() + assert res == res2 diff --git a/tests/snippets/stdlib_itertools.py b/tests/snippets/stdlib_itertools.py new file mode 100644 index 0000000000..e8285fac95 --- /dev/null +++ b/tests/snippets/stdlib_itertools.py @@ -0,0 +1,185 @@ +import itertools + +from testutils import assertRaises + + +# itertools.chain tests +chain = itertools.chain + +# empty +assert list(chain()) == [] +assert list(chain([], "", b"", ())) == [] + +assert list(chain([1, 2, 3, 4])) == [1, 2, 3, 4] +assert list(chain("ab", "cd", (), 'e')) == ['a', 'b', 'c', 'd', 'e'] +with assertRaises(TypeError): + list(chain(1)) + +x = chain("ab", 1) +assert next(x) == 'a' +assert next(x) == 'b' +with assertRaises(TypeError): + next(x) + +# itertools.count tests + +# default arguments +c = itertools.count() +assert next(c) == 0 +assert next(c) == 1 +assert next(c) == 2 + +# positional +c = itertools.count(2, 3) +assert next(c) == 2 +assert next(c) == 5 +assert next(c) == 8 + +# backwards +c = itertools.count(1, -10) +assert next(c) == 1 +assert next(c) == -9 +assert next(c) == -19 + +# step = 0 +c = itertools.count(5, 0) +assert next(c) == 5 +assert next(c) == 5 + +# itertools.count TODOs: kwargs and floats + +# step kwarg +# c = itertools.count(step=5) +# assert next(c) == 0 +# assert next(c) == 5 + +# start kwarg +# c = itertools.count(start=10) +# assert next(c) == 10 + +# float start +# c = itertools.count(0.5) +# assert next(c) == 0.5 +# assert next(c) == 1.5 +# assert next(c) == 2.5 + +# float step +# c = itertools.count(1, 0.5) +# assert next(c) == 1 +# assert next(c) == 1.5 +# assert next(c) == 2 + +# float start + step +# c = itertools.count(0.5, 0.5) +# assert next(c) == 0.5 +# assert next(c) == 1 +# assert next(c) == 1.5 + + +# itertools.repeat tests + +# no times +r = itertools.repeat(5) +assert next(r) == 5 +assert next(r) == 5 +assert next(r) == 5 + +# times +r = itertools.repeat(1, 2) +assert next(r) == 1 +assert next(r) == 1 +with assertRaises(StopIteration): + next(r) + +# timees = 0 +r = itertools.repeat(1, 0) +with assertRaises(StopIteration): + next(r) + +# negative times +r = itertools.repeat(1, -1) +with assertRaises(StopIteration): + next(r) + + +# itertools.starmap tests +starmap = itertools.starmap + +assert list(starmap(pow, zip(range(3), range(1,7)))) == [0**1, 1**2, 2**3] +assert list(starmap(pow, [])) == [] +assert list(starmap(pow, [iter([4,5])])) == [4**5] +with assertRaises(TypeError): + starmap(pow) + + +# itertools.takewhile tests + +from itertools import takewhile as tw + +t = tw(lambda n: n < 5, [1, 2, 5, 1, 3]) +assert next(t) == 1 +assert next(t) == 2 +with assertRaises(StopIteration): + next(t) + +# not iterable +with assertRaises(TypeError): + tw(lambda n: n < 1, 1) + +# not callable +t = tw(5, [1, 2]) +with assertRaises(TypeError): + next(t) + +# non-bool predicate +t = tw(lambda n: n, [1, 2, 0]) +assert next(t) == 1 +assert next(t) == 2 +with assertRaises(StopIteration): + next(t) + +# bad predicate prototype +t = tw(lambda: True, [1]) +with assertRaises(TypeError): + next(t) + +# StopIteration before attempting to call (bad) predicate +t = tw(lambda: True, []) +with assertRaises(StopIteration): + next(t) + +# doesn't try again after the first predicate failure +t = tw(lambda n: n < 1, [1, 0]) +with assertRaises(StopIteration): + next(t) +with assertRaises(StopIteration): + next(t) + + +# itertools.islice tests + +def assert_matches_seq(it, seq): + assert list(it) == list(seq) + +i = itertools.islice + +it = i([1, 2, 3, 4, 5], 3) +assert_matches_seq(it, [1, 2, 3]) + +it = i([0.5, 1, 1.5, 2, 2.5, 3, 4, 5], 1, 6, 2) +assert_matches_seq(it, [1, 2, 3]) + +it = i([1, 2], None) +assert_matches_seq(it, [1, 2]) + +it = i([1, 2, 3], None, None, None) +assert_matches_seq(it, [1, 2, 3]) + +it = i([1, 2, 3], 1, None, None) +assert_matches_seq(it, [2, 3]) + +it = i([1, 2, 3], None, 2, None) +assert_matches_seq(it, [1, 2]) + +it = i([1, 2, 3], None, None, 3) +assert_matches_seq(it, [1]) diff --git a/tests/snippets/stdlib_marshal.py b/tests/snippets/stdlib_marshal.py new file mode 100644 index 0000000000..118bb1bddf --- /dev/null +++ b/tests/snippets/stdlib_marshal.py @@ -0,0 +1,7 @@ +import marshal +orig = compile("1 + 1", "", 'eval') + +dumped = marshal.dumps(orig) +loaded = marshal.loads(dumped) + +assert eval(loaded) == eval(orig) diff --git a/tests/snippets/stdlib_os.py b/tests/snippets/stdlib_os.py index ed9e91c708..d2e39c1ce5 100644 --- a/tests/snippets/stdlib_os.py +++ b/tests/snippets/stdlib_os.py @@ -1,13 +1,253 @@ -import os +import os +import time +import stat from testutils import assert_raises -assert os.open('README.md', 0) > 0 +fd = os.open('README.md', os.O_RDONLY) +assert fd > 0 +os.close(fd) +assert_raises(OSError, lambda: os.read(fd, 10)) +assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', os.O_RDONLY)) +assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', os.O_WRONLY)) +assert_raises(FileNotFoundError, lambda: os.rename('DOES_NOT_EXIST', 'DOES_NOT_EXIST 2')) + +try: + os.open('DOES_NOT_EXIST', 0) +except OSError as err: + assert err.errno == 2 -assert_raises(FileNotFoundError, lambda: os.open('DOES_NOT_EXIST', 0)) assert os.O_RDONLY == 0 assert os.O_WRONLY == 1 assert os.O_RDWR == 2 + +ENV_KEY = "TEST_ENV_VAR" +ENV_VALUE = "value" + +assert os.getenv(ENV_KEY) == None +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY, 5) == 5 +os.environ[ENV_KEY] = ENV_VALUE +assert ENV_KEY in os.environ +assert os.getenv(ENV_KEY) == ENV_VALUE +del os.environ[ENV_KEY] +assert ENV_KEY not in os.environ +assert os.getenv(ENV_KEY) == None + +if os.name == "posix": + os.putenv(ENV_KEY, ENV_VALUE) + os.unsetenv(ENV_KEY) + assert os.getenv(ENV_KEY) == None + +assert os.curdir == "." +assert os.pardir == ".." +assert os.extsep == "." + +if os.name == "nt": + assert os.sep == "\\" + assert os.linesep == "\r\n" + assert os.altsep == "/" + assert os.pathsep == ";" +else: + assert os.sep == "/" + assert os.linesep == "\n" + assert os.altsep == None + assert os.pathsep == ":" + +assert os.fspath("Testing") == "Testing" +assert os.fspath(b"Testing") == b"Testing" +assert_raises(TypeError, lambda: os.fspath([1,2,3])) + +class TestWithTempDir(): + def __enter__(self): + if os.name == "nt": + base_folder = os.environ["TEMP"] + else: + base_folder = "/tmp" + name = os.path.join(base_folder, "rustpython_test_os_" + str(int(time.time()))) + os.mkdir(name) + self.name = name + return name + + def __exit__(self, exc_type, exc_val, exc_tb): + # TODO: Delete temp dir + pass + + +class TestWithTempCurrentDir(): + def __enter__(self): + self.prev_cwd = os.getcwd() + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.prev_cwd) + + +FILE_NAME = "test1" +FILE_NAME2 = "test2" +FILE_NAME3 = "test3" +SYMLINK_FILE = "symlink" +SYMLINK_FOLDER = "symlink1" +FOLDER = "dir1" +CONTENT = b"testing" +CONTENT2 = b"rustpython" +CONTENT3 = b"BOYA" + +with TestWithTempDir() as tmpdir: + fname = os.path.join(tmpdir, FILE_NAME) + fd = os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + assert os.write(fd, CONTENT2) == len(CONTENT2) + os.close(fd) + + fd = os.open(fname, os.O_WRONLY | os.O_APPEND) + assert os.write(fd, CONTENT3) == len(CONTENT3) + os.close(fd) + + assert_raises(FileExistsError, lambda: os.open(fname, os.O_WRONLY | os.O_CREAT | os.O_EXCL)) + + fd = os.open(fname, os.O_RDONLY) + assert os.read(fd, len(CONTENT2)) == CONTENT2 + assert os.read(fd, len(CONTENT3)) == CONTENT3 + os.close(fd) + + fname3 = os.path.join(tmpdir, FILE_NAME3) + os.rename(fname, fname3) + assert os.path.exists(fname) == False + assert os.path.exists(fname3) == True + + fd = os.open(fname3, 0) + assert os.read(fd, len(CONTENT2) + len(CONTENT3)) == CONTENT2 + CONTENT3 + os.close(fd) + + os.rename(fname3, fname) + assert os.path.exists(fname3) == False + assert os.path.exists(fname) == True + + # wait a little bit to ensure that the file times aren't the same + time.sleep(0.1) + + fname2 = os.path.join(tmpdir, FILE_NAME2) + with open(fname2, "wb"): + pass + folder = os.path.join(tmpdir, FOLDER) + os.mkdir(folder) + + symlink_file = os.path.join(tmpdir, SYMLINK_FILE) + os.symlink(fname, symlink_file) + symlink_folder = os.path.join(tmpdir, SYMLINK_FOLDER) + os.symlink(folder, symlink_folder) + + names = set() + paths = set() + dirs = set() + dirs_no_symlink = set() + files = set() + files_no_symlink = set() + symlinks = set() + for dir_entry in os.scandir(tmpdir): + names.add(dir_entry.name) + paths.add(dir_entry.path) + if dir_entry.is_dir(): + assert stat.S_ISDIR(dir_entry.stat().st_mode) == True + dirs.add(dir_entry.name) + if dir_entry.is_dir(follow_symlinks=False): + assert stat.S_ISDIR(dir_entry.stat().st_mode) == True + dirs_no_symlink.add(dir_entry.name) + if dir_entry.is_file(): + files.add(dir_entry.name) + assert stat.S_ISREG(dir_entry.stat().st_mode) == True + if dir_entry.is_file(follow_symlinks=False): + files_no_symlink.add(dir_entry.name) + assert stat.S_ISREG(dir_entry.stat().st_mode) == True + if dir_entry.is_symlink(): + symlinks.add(dir_entry.name) + + assert names == set([FILE_NAME, FILE_NAME2, FOLDER, SYMLINK_FILE, SYMLINK_FOLDER]) + assert paths == set([fname, fname2, folder, symlink_file, symlink_folder]) + assert dirs == set([FOLDER, SYMLINK_FOLDER]) + assert dirs_no_symlink == set([FOLDER]) + assert files == set([FILE_NAME, FILE_NAME2, SYMLINK_FILE]) + assert files_no_symlink == set([FILE_NAME, FILE_NAME2]) + assert symlinks == set([SYMLINK_FILE, SYMLINK_FOLDER]) + + # Stat + stat_res = os.stat(fname) + print(stat_res.st_mode) + assert stat.S_ISREG(stat_res.st_mode) == True + print(stat_res.st_ino) + print(stat_res.st_dev) + print(stat_res.st_nlink) + print(stat_res.st_uid) + print(stat_res.st_gid) + print(stat_res.st_size) + assert stat_res.st_size == len(CONTENT2) + len(CONTENT3) + print(stat_res.st_atime) + print(stat_res.st_ctime) + print(stat_res.st_mtime) + # test that it all of these times are greater than the 10 May 2019, when this test was written + assert stat_res.st_atime > 1557500000 + assert stat_res.st_ctime > 1557500000 + assert stat_res.st_mtime > 1557500000 + + stat_file2 = os.stat(fname2) + print(stat_file2.st_ctime) + assert stat_file2.st_ctime > stat_res.st_ctime + + # wait a little bit to ensures that the access/modify time will change + time.sleep(0.1) + + old_atime = stat_res.st_atime + old_mtime = stat_res.st_mtime + + fd = os.open(fname, os.O_RDWR) + os.write(fd, CONTENT) + os.fsync(fd) + + # wait a little bit to ensures that the access/modify time is different + time.sleep(0.1) + + os.read(fd, 1) + os.fsync(fd) + os.close(fd) + + # retrieve update file stats + stat_res = os.stat(fname) + print(stat_res.st_atime) + print(stat_res.st_ctime) + print(stat_res.st_mtime) + if os.name != "nt": + # access time on windows has a resolution ranging from 1 hour to 1 day + # https://docs.microsoft.com/en-gb/windows/desktop/api/minwinbase/ns-minwinbase-filetime + assert stat_res.st_atime > old_atime, "Access time should be update" + assert stat_res.st_atime > stat_res.st_mtime + assert stat_res.st_mtime > old_mtime, "Modified time should be update" + + # stat default is follow_symlink=True + os.stat(fname).st_ino == os.stat(symlink_file).st_ino + os.stat(fname).st_mode == os.stat(symlink_file).st_mode + + os.stat(fname, follow_symlinks=False).st_ino == os.stat(symlink_file, follow_symlinks=False).st_ino + os.stat(fname, follow_symlinks=False).st_mode == os.stat(symlink_file, follow_symlinks=False).st_mode + + # os.path + assert os.path.exists(fname) == True + assert os.path.exists("NO_SUCH_FILE") == False + assert os.path.isfile(fname) == True + assert os.path.isdir(folder) == True + assert os.path.isfile(folder) == False + assert os.path.isdir(fname) == False + + assert os.path.basename(fname) == FILE_NAME + assert os.path.dirname(fname) == tmpdir + + with TestWithTempCurrentDir(): + os.chdir(tmpdir) + assert os.getcwd() == os.path.realpath(tmpdir) + os.path.exists(FILE_NAME) + +# supports +assert isinstance(os.supports_fd, set) +assert isinstance(os.supports_dir_fd, set) +assert isinstance(os.supports_follow_symlinks, set) diff --git a/tests/snippets/stdlib_socket.py b/tests/snippets/stdlib_socket.py index 79efaa3639..d7afe1bd35 100644 --- a/tests/snippets/stdlib_socket.py +++ b/tests/snippets/stdlib_socket.py @@ -1,4 +1,5 @@ import socket +import os from testutils import assertRaises MESSAGE_A = b'aaaa' @@ -21,6 +22,18 @@ recv_b = connector.recv(len(MESSAGE_B)) assert recv_a == MESSAGE_A assert recv_b == MESSAGE_B + +# fileno +if os.name == "posix": + connector_fd = connector.fileno() + connection_fd = connection.fileno() + os.write(connector_fd, MESSAGE_A) + connection.send(MESSAGE_B) + recv_a = connection.recv(len(MESSAGE_A)) + recv_b = os.read(connector_fd, (len(MESSAGE_B))) + assert recv_a == MESSAGE_A + assert recv_b == MESSAGE_B + connection.close() connector.close() listener.close() diff --git a/tests/snippets/stdlib_types.py b/tests/snippets/stdlib_types.py new file mode 100644 index 0000000000..b8011bf40b --- /dev/null +++ b/tests/snippets/stdlib_types.py @@ -0,0 +1,10 @@ +import types + +from testutils import assertRaises + +ns = types.SimpleNamespace(a=2, b='Rust') + +assert ns.a == 2 +assert ns.b == "Rust" +with assertRaises(AttributeError): + _ = ns.c diff --git a/tests/snippets/string_io.py b/tests/snippets/string_io.py new file mode 100644 index 0000000000..0b4182527d --- /dev/null +++ b/tests/snippets/string_io.py @@ -0,0 +1,56 @@ + +from io import StringIO + +def test_01(): + """ + Test that the constructor and getvalue + method return expected values + """ + string = 'Test String 1' + f = StringIO() + f.write(string) + + assert f.getvalue() == string + +def test_02(): + """ + Test that the read method (no arg) + results the expected value + """ + string = 'Test String 2' + f = StringIO(string) + + assert f.read() == string + assert f.read() == '' + +def test_03(): + """ + Tests that the read method (integer arg) + returns the expected value + """ + string = 'Test String 3' + f = StringIO(string) + + assert f.read(1) == 'T' + assert f.read(1) == 'e' + assert f.read(1) == 's' + assert f.read(1) == 't' + +def test_04(): + """ + Tests that the read method increments the + cursor position and the seek method moves + the cursor to the appropriate position + """ + string = 'Test String 4' + f = StringIO(string) + + assert f.read(4) == 'Test' + assert f.seek(0) == 0 + assert f.read(4) == 'Test' + +if __name__ == "__main__": + test_01() + test_02() + test_03() + test_04() diff --git a/tests/snippets/strings.py b/tests/snippets/strings.py index 4483b37f62..e073544099 100644 --- a/tests/snippets/strings.py +++ b/tests/snippets/strings.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + assert "a" == 'a' assert """a""" == "a" assert len(""" " "" " "" """) == 11 @@ -27,14 +29,29 @@ assert str(["a", "b", "can't"]) == "['a', 'b', \"can't\"]" +assert "xy" * 3 == "xyxyxy" +assert "x" * 0 == "" +assert "x" * -1 == "" + +assert 3 * "xy" == "xyxyxy" +assert 0 * "x" == "" +assert -1 * "x" == "" + +assert_raises(OverflowError, lambda: 'xy' * 234234234234234234234234234234) + a = 'Hallo' assert a.lower() == 'hallo' assert a.upper() == 'HALLO' -assert a.split('al') == ['H', 'lo'] assert a.startswith('H') +assert a.startswith(('H', 1)) +assert a.startswith(('A', 'H')) assert not a.startswith('f') +assert not a.startswith(('A', 'f')) assert a.endswith('llo') +assert a.endswith(('lo', 1)) +assert a.endswith(('A', 'lo')) assert not a.endswith('on') +assert not a.endswith(('A', 'll')) assert a.zfill(8) == '000Hallo' assert a.isalnum() assert not a.isdigit() @@ -43,16 +60,58 @@ assert a.istitle() assert a.isalpha() - +s = '1 2 3' +assert s.split(' ', 1) == ['1', '2 3'] +assert s.rsplit(' ', 1) == ['1 2', '3'] b = ' hallo ' assert b.strip() == 'hallo' assert b.lstrip() == 'hallo ' assert b.rstrip() == ' hallo' +s = '^*RustPython*^' +assert s.strip('^*') == 'RustPython' +assert s.lstrip('^*') == 'RustPython*^' +assert s.rstrip('^*') == '^*RustPython' + +s = 'RustPython' +assert s.ljust(8) == 'RustPython' +assert s.rjust(8) == 'RustPython' +assert s.ljust(12) == 'RustPython ' +assert s.rjust(12) == ' RustPython' +assert s.ljust(12, '_') == 'RustPython__' +assert s.rjust(12, '_') == '__RustPython' +# The fill character must be exactly one character long +assert_raises(TypeError, lambda: s.ljust(12, '__')) +assert_raises(TypeError, lambda: s.rjust(12, '__')) + c = 'hallo' assert c.capitalize() == 'Hallo' assert c.center(11, '-') == '---hallo---' +assert ["koki".center(i, "|") for i in range(3, 10)] == [ + "koki", + "koki", + "|koki", + "|koki|", + "||koki|", + "||koki||", + "|||koki||", +] + + +assert ["kok".center(i, "|") for i in range(2, 10)] == [ + "kok", + "kok", + "kok|", + "|kok|", + "|kok||", + "||kok||", + "||kok|||", + "|||kok|||", +] + + +# requires CPython 3.7, and the CI currently runs with 3.6 # assert c.isascii() assert c.index('a') == 1 assert c.rindex('l') == 3 @@ -94,6 +153,7 @@ assert '___a__'.find('a', 4, 3) == -1 assert 'abcd'.startswith('b', 1) +assert 'abcd'.startswith(('b', 'z'), 1) assert not 'abcd'.startswith('b', -4) assert 'abcd'.startswith('b', -3) @@ -107,16 +167,56 @@ assert '-'.join(['1', '2', '3']) == '1-2-3' assert 'HALLO'.isupper() assert "hello, my name is".partition("my ") == ('hello, ', 'my ', 'name is') +assert "hello".partition("is") == ('hello', '', '') assert "hello, my name is".rpartition("is") == ('hello, my name ', 'is', '') +assert "hello".rpartition("is") == ('', '', 'hello') assert not ''.isdecimal() assert '123'.isdecimal() assert not '\u00B2'.isdecimal() +assert not ''.isidentifier() +assert 'python'.isidentifier() +assert '_'.isidentifier() +assert '유니코드'.isidentifier() +assert not '😂'.isidentifier() +assert not '123'.isidentifier() + # String Formatting -assert "{} {}".format(1,2) == "1 2" -assert "{0} {1}".format(2,3) == "2 3" +assert "{} {}".format(1, 2) == "1 2" +assert "{0} {1}".format(2, 3) == "2 3" assert "--{:s>4}--".format(1) == "--sss1--" assert "{keyword} {0}".format(1, keyword=2) == "2 1" +assert "repr() shows quotes: {!r}; str() doesn't: {!s}".format( + 'test1', 'test2' +) == "repr() shows quotes: 'test1'; str() doesn't: test2", 'Output: {!r}, {!s}'.format('test1', 'test2') + + +class Foo: + def __str__(self): + return 'str(Foo)' + + def __repr__(self): + return 'repr(Foo)' + + +f = Foo() +assert "{} {!s} {!r} {!a}".format(f, f, f, f) == 'str(Foo) str(Foo) repr(Foo) repr(Foo)' +assert "{foo} {foo!s} {foo!r} {foo!a}".format(foo=f) == 'str(Foo) str(Foo) repr(Foo) repr(Foo)' +# assert '{} {!r} {:10} {!r:10} {foo!r:10} {foo!r} {foo}'.format('txt1', 'txt2', 'txt3', 'txt4', 'txt5', foo='bar') + + +# Printf-style String formatting +assert "%d %d" % (1, 2) == "1 2" +assert "%*c " % (3, '❤') == " ❤ " +assert "%(first)s %(second)s" % {'second': 'World!', 'first': "Hello,"} == "Hello, World!" +assert "%(key())s" % {'key()': 'aaa'} +assert "%s %a %r" % (f, f, f) == "str(Foo) repr(Foo) repr(Foo)" +assert "repr() shows quotes: %r; str() doesn't: %s" % ("test1", "test2") == "repr() shows quotes: 'test1'; str() doesn't: test2" + +assert_raises(TypeError, lambda: "My name is %s and I'm %(age)d years old" % ("Foo", 25), msg="format requires a mapping") +assert_raises(TypeError, lambda: "My name is %(name)s" % "Foo", msg="format requires a mapping") +assert_raises(ValueError, lambda: "This %(food}s is great!" % {"food": "cookie"}, msg="incomplete format key") +assert_raises(ValueError, lambda: "My name is %" % "Foo", msg="incomplete format") assert 'a' < 'b' assert 'a' <= 'b' @@ -124,3 +224,42 @@ assert 'z' > 'b' assert 'z' >= 'b' assert 'a' >= 'a' + +# str.translate +assert "abc".translate({97: '🎅', 98: None, 99: "xd"}) == "🎅xd" + +# str.maketrans +assert str.maketrans({"a": "abc", "b": None, "c": 33}) == {97: "abc", 98: None, 99: 33} +assert str.maketrans("hello", "world", "rust") == {104: 119, 101: 111, 108: 108, 111: 100, 114: None, 117: None, 115: None, 116: None} + +def try_mutate_str(): + word = "word" + word[0] = 'x' + +assert_raises(TypeError, try_mutate_str) + +ss = ['Hello', '안녕', '👋'] +bs = [b'Hello', b'\xec\x95\x88\xeb\x85\x95', b'\xf0\x9f\x91\x8b'] + +for s, b in zip(ss, bs): + assert s.encode() == b + +for s, b, e in zip(ss, bs, ['u8', 'U8', 'utf-8', 'UTF-8', 'utf_8']): + assert s.encode(e) == b + # assert s.encode(encoding=e) == b + +# str.isisprintable +assert "".isprintable() +assert " ".isprintable() +assert "abcdefg".isprintable() +assert not "abcdefg\n".isprintable() +assert "ʹ".isprintable() + +# test unicode iterals +assert "\xac" == "¬" +assert "\u0037" == "7" +assert "\u0040" == "@" +assert "\u0041" == "A" +assert "\u00BE" == "¾" +assert "\u9487" == "钇" +assert "\U0001F609" == "😉" diff --git a/tests/snippets/sysmod.py b/tests/snippets/sysmod.py index bfcf17201e..adabb57d2b 100644 --- a/tests/snippets/sysmod.py +++ b/tests/snippets/sysmod.py @@ -2,3 +2,21 @@ print(sys.argv) assert sys.argv[0].endswith('.py') + +assert sys.platform == "linux" or sys.platform == "darwin" or sys.platform == "win32" or sys.platform == "unknown" + +assert isinstance(sys.builtin_module_names, tuple) +assert 'sys' in sys.builtin_module_names + +assert isinstance(sys.implementation.name, str) +assert isinstance(sys.implementation.cache_tag, str) + +assert sys.getfilesystemencoding() == 'utf-8' +assert sys.getfilesystemencodeerrors().startswith('surrogate') + +assert sys.byteorder == "little" or sys.byteorder == "big" + +assert isinstance(sys.flags, tuple) +assert type(sys.flags).__name__ == "flags" +assert type(sys.flags.optimize) is int +assert sys.flags[3] == sys.flags.optimize diff --git a/tests/snippets/test_abc.py b/tests/snippets/test_abc.py new file mode 100644 index 0000000000..391932c36a --- /dev/null +++ b/tests/snippets/test_abc.py @@ -0,0 +1,39 @@ +import abc + +from testutils import assertRaises + + +class CustomInterface(abc.ABC): + @abc.abstractmethod + def a(self): + pass + + @classmethod + def __subclasshook__(cls, subclass): + return NotImplemented + + +# TODO raise an error if there are in any abstract methods not fulfilled +# with assertRaises(TypeError): +# CustomInterface() + + +class Concrete: + def a(self): + pass + + +CustomInterface.register(Concrete) + + +class SubConcrete(Concrete): + pass + + +assert issubclass(Concrete, CustomInterface) +assert issubclass(SubConcrete, CustomInterface) +assert not issubclass(tuple, CustomInterface) + +assert isinstance(Concrete(), CustomInterface) +assert isinstance(SubConcrete(), CustomInterface) +assert not isinstance((), CustomInterface) diff --git a/tests/snippets/test_async_stuff.py b/tests/snippets/test_async_stuff.py new file mode 100644 index 0000000000..a585692962 --- /dev/null +++ b/tests/snippets/test_async_stuff.py @@ -0,0 +1,13 @@ + +import sys +import ast + +src = """ +async def x(): + async for x in [1,2,3]: + await y() +""" + +mod = ast.parse(src) +# print(mod) +# print(ast.dump(mod)) diff --git a/tests/snippets/test_exec.py b/tests/snippets/test_exec.py index 61932c24cb..506882e165 100644 --- a/tests/snippets/test_exec.py +++ b/tests/snippets/test_exec.py @@ -44,3 +44,23 @@ def f(): exec("del x") assert 'x' not in g + +assert 'g' in globals() +assert 'g' in locals() +exec("assert 'g' in globals()") +exec("assert 'g' in locals()") +exec("assert 'g' not in globals()", {}) +exec("assert 'g' not in locals()", {}) + +del g + +def f(): + g = 1 + assert 'g' not in globals() + assert 'g' in locals() + exec("assert 'g' not in globals()") + exec("assert 'g' in locals()") + exec("assert 'g' not in globals()", {}) + exec("assert 'g' not in locals()", {}) + +f() diff --git a/tests/snippets/test_hashlib.py b/tests/snippets/test_hashlib.py new file mode 100644 index 0000000000..796f6aefd2 --- /dev/null +++ b/tests/snippets/test_hashlib.py @@ -0,0 +1,33 @@ + +import hashlib + +# print(hashlib.md5) +h = hashlib.md5() +h.update(b'a') +assert h.name == 'md5' +print(h.hexdigest()) + +assert h.hexdigest() == '0cc175b9c0f1b6a831c399e269772661' +assert h.digest_size == 16 + +h = hashlib.sha256() +h.update(b'a') +assert h.name == 'sha256' +assert h.digest_size == 32 +print(h.hexdigest()) + +assert h.hexdigest() == 'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb' + +h = hashlib.sha512() +assert h.name == 'sha512' +h.update(b'a') +print(h.hexdigest()) + +assert h.hexdigest() == '1f40fc92da241694750979ee6cf582f2d5d7d28e18335de05abc54d0560e0f5302860c652bf08d560252aa5e74210546f369fbbbce8c12cfc7957b2652fe9a75' + +h = hashlib.new("blake2s", b"fubar") +print(h.hexdigest()) +assert h.hexdigest() == 'a0e1ad0c123c9c65e8ef850db2ce4b5cef2c35b06527c615b0154353574d0415' +h.update(b'bla') +print(h.hexdigest()) +assert h.hexdigest() == '25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81' diff --git a/tests/snippets/test_io.py b/tests/snippets/test_io.py deleted file mode 100644 index bc49b72870..0000000000 --- a/tests/snippets/test_io.py +++ /dev/null @@ -1,14 +0,0 @@ - - -import io - -f = io.StringIO() -f.write('bladibla') -assert f.getvalue() == 'bladibla' - -# TODO: -# print('fubar', file=f, end='') -print(f.getvalue()) - -# TODO: -# assert f.getvalue() == 'bladiblafubar' diff --git a/tests/snippets/test_re.py b/tests/snippets/test_re.py index 2463f380b5..2bc7308e6a 100644 --- a/tests/snippets/test_re.py +++ b/tests/snippets/test_re.py @@ -11,3 +11,5 @@ # assert isinstance(mo, re.Match) assert mo.start() == 1 assert mo.end() == 5 + +assert re.escape('python.exe') == 'python\\.exe' diff --git a/tests/snippets/test_struct.py b/tests/snippets/test_struct.py index c13e713a16..8f535f63cc 100644 --- a/tests/snippets/test_struct.py +++ b/tests/snippets/test_struct.py @@ -8,3 +8,17 @@ assert v1 == 14 assert v2 == 12 +data = struct.pack('IH', 14, 12) +assert data == bytes([0, 0, 0, 14, 0, 12]) + +v1, v2 = struct.unpack('>IH', data) +assert v1 == 14 +assert v2 == 12 + diff --git a/tests/snippets/test_xdrlib.py b/tests/snippets/test_xdrlib.py new file mode 100644 index 0000000000..989a603c7d --- /dev/null +++ b/tests/snippets/test_xdrlib.py @@ -0,0 +1,13 @@ + +# This probably will be superceeded by the python unittests when that works. + +import xdrlib + +p = xdrlib.Packer() +p.pack_int(1337) + +d = p.get_buffer() + +print(d) + +assert d == b'\x00\x00\x059' diff --git a/tests/snippets/testutils.py b/tests/snippets/testutils.py index 7dda57b17a..21662c4614 100644 --- a/tests/snippets/testutils.py +++ b/tests/snippets/testutils.py @@ -14,23 +14,35 @@ def assert_raises(exc_type, expr, msg=None): except exc_type: pass else: - failmsg = '{!s} was not raised'.format(exc_type.__name__) + failmsg = '{} was not raised'.format(exc_type.__name__) if msg is not None: - failmsg += ': {!s}'.format(msg) + failmsg += ': {}'.format(msg) assert False, failmsg class assertRaises: def __init__(self, expected): self.expected = expected + self.exception = None def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: - failmsg = '{!s} was not raised'.format(self.expected.__name__) - assert False, failmsg + failmsg = '{} was not raised'.format(self.expected.__name__) + assert False, failmsg if not issubclass(exc_type, self.expected): return False + + self.exception = exc_val return True + + +class TestFailingBool: + def __bool__(self): + raise RuntimeError + +class TestFailingIter: + def __iter__(self): + raise RuntimeError diff --git a/tests/snippets/try_exceptions.py b/tests/snippets/try_exceptions.py index 4e3b7269aa..f37101b611 100644 --- a/tests/snippets/try_exceptions.py +++ b/tests/snippets/try_exceptions.py @@ -1,4 +1,4 @@ - +from testutils import assertRaises try: raise BaseException() @@ -95,3 +95,128 @@ def __init__(self): l.append(3) print('boom', type(ex)) assert l == [1, 3] + +cause = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__cause__ == None + cause = ex + raise NameError from ex +except NameError as ex2: + assert ex2.__cause__ == cause + assert ex2.__context__ == cause + +try: + raise ZeroDivisionError from None +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__cause__ == None + +with assertRaises(TypeError): + raise ZeroDivisionError from 5 + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + +with assertRaises(NameError): + try: + raise NameError + except: + raise + +with assertRaises(RuntimeError): + raise + +context = None +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + assert ex.__context__ == None + context = ex + raise NameError +except NameError as ex2: + assert ex2.__context__ == context + assert type(ex2.__context__) == ZeroDivisionError + +try: + raise ZeroDivisionError +except ZeroDivisionError as ex: + assert ex.__context__ == None + +try: + raise ZeroDivisionError from NameError +except ZeroDivisionError as ex: + assert type(ex.__cause__) == NameError + assert ex.__context__ == None + +try: + try: + raise ZeroDivisionError + except ZeroDivisionError as ex: + pass + finally: + raise NameError +except NameError as ex2: + assert ex2.__context__ == None + +def f(): + raise + +with assertRaises(ZeroDivisionError): + try: + 1/0 + except: + f() + +with assertRaises(ZeroDivisionError): + try: + 1/0 + except ZeroDivisionError: + try: + raise + except NameError: + pass + raise + +# Regression https://github.com/RustPython/RustPython/issues/867 +for _ in [1, 2]: + try: + raise ArithmeticError() + except ArithmeticError as e: + continue + + +def g(): + try: + 1/0 + except ArithmeticError: + return 5 + +try: + g() + raise NameError +except NameError as ex: + assert ex.__context__ == None + + +def y(): + try: + 1/0 + except ArithmeticError: + yield 5 + + +try: + y() + raise NameError +except NameError as ex: + assert ex.__context__ == None diff --git a/tests/snippets/tuple.py b/tests/snippets/tuple.py index d50f552dc0..86f12fe943 100644 --- a/tests/snippets/tuple.py +++ b/tests/snippets/tuple.py @@ -9,7 +9,7 @@ assert x + y == (1, 2, 1) assert x * 3 == (1, 2, 1, 2, 1, 2) -# assert 3 * x == (1, 2, 1, 2, 1, 2) +assert 3 * x == (1, 2, 1, 2, 1, 2) assert x * 0 == () assert x * -1 == () # integers less than zero treated as 0 @@ -34,3 +34,7 @@ def __eq__(self, x): foo = Foo() assert (foo,) == (foo,) + +a = (1, 2, 3) +a += 1, +assert a == (1, 2, 3, 1) diff --git a/tests/snippets/types_snippet.py b/tests/snippets/types_snippet.py index ac715751ef..13395cdbee 100644 --- a/tests/snippets/types_snippet.py +++ b/tests/snippets/types_snippet.py @@ -26,3 +26,33 @@ assert isinstance(type, type) assert issubclass(type, type) + +assert not isinstance(type, (int, float)) +assert isinstance(type, (int, object)) + +assert not issubclass(type, (int, float)) +assert issubclass(type, (int, type)) + +class A: pass +class B(A): pass +class C(A): pass +class D(B, C): pass + +assert A.__subclasses__() == [B, C] +assert B.__subclasses__() == [D] +assert C.__subclasses__() == [D] +assert D.__subclasses__() == [] + +del D + +try: # gc sweep is needed here for CPython... + import gc; gc.collect() +except: # ...while RustPython doesn't have `gc` yet. + pass + +assert B.__subclasses__() == [] +assert C.__subclasses__() == [] + +assert type in object.__subclasses__() + +assert cls.__name__ == 'Cls' diff --git a/tests/snippets/unicode_fu.py b/tests/snippets/unicode_fu.py new file mode 100644 index 0000000000..bae97a3720 --- /dev/null +++ b/tests/snippets/unicode_fu.py @@ -0,0 +1,21 @@ + +# Test the unicode support! 👋 + + +ᚴ=2 + +assert ᚴ*8 == 16 + +ᚴ="👋" + +c = ᚴ*3 + +assert c == '👋👋👋' + +import unicodedata +assert unicodedata.category('a') == 'Ll' +assert unicodedata.category('A') == 'Lu' +assert unicodedata.name('a') == 'LATIN SMALL LETTER A' +assert unicodedata.lookup('LATIN SMALL LETTER A') == 'a' +assert unicodedata.bidirectional('a') == 'L' +assert unicodedata.normalize('NFC', 'bla') == 'bla' diff --git a/tests/snippets/warnings.py b/tests/snippets/warnings.py new file mode 100644 index 0000000000..95092e5066 --- /dev/null +++ b/tests/snippets/warnings.py @@ -0,0 +1,3 @@ +import _warnings + +_warnings.warn("Test") diff --git a/tests/snippets/weakrefs.py b/tests/snippets/weakrefs.py index ffc61f4271..059ddd0000 100644 --- a/tests/snippets/weakrefs.py +++ b/tests/snippets/weakrefs.py @@ -1,4 +1,5 @@ -from _weakref import ref +from _weakref import ref, proxy +from testutils import assert_raises class X: @@ -11,3 +12,17 @@ class X: assert callable(b) assert b() is a + +class G: + def __init__(self, h): + self.h = h + + +g = G(5) +p = proxy(g) + +assert p.h == 5 + +del g + +assert_raises(ReferenceError, lambda: p.h) diff --git a/tests/snippets/whats_left_to_implement.py b/tests/snippets/whats_left_to_implement.py deleted file mode 100644 index b6c8cbe42b..0000000000 --- a/tests/snippets/whats_left_to_implement.py +++ /dev/null @@ -1,878 +0,0 @@ -expected_methods = [] - -# TODO: using tuple could have been better -expected_methods.append({'name': 'bool', 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -], 'type': bool}) -expected_methods.append({'name': 'bytearray', 'type': bytearray, 'methods': [ - '__add__', - '__alloc__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'capitalize', - 'center', - 'clear', - 'copy', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'extend', - 'find', - 'fromhex', - 'hex', - 'index', - 'insert', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'pop', - 'remove', - 'replace', - 'reverse', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'bytes', 'type': bytes, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'center', - 'count', - 'decode', - 'endswith', - 'expandtabs', - 'find', - 'fromhex', - 'hex', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdigit', - 'islower', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill', -]}) -expected_methods.append({'name': 'complex', 'type': complex, 'methods': [ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - 'conjugate', - 'imag', - 'real', -]}) -expected_methods.append({'name': 'dict', 'type': dict, 'methods': [ - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'clear', - 'copy', - 'fromkeys', - 'get', - 'items', - 'keys', - 'pop', - 'popitem', - 'setdefault', - 'update', - 'values', -]}) -expected_methods.append({'name': 'float','type':float,'methods':[ - '__abs__', - '__add__', - '__bool__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getformat__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__int__', - '__le__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__pos__', - '__pow__', - '__radd__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rmod__', - '__rmul__', - '__round__', - '__rpow__', - '__rsub__', - '__rtruediv__', - '__set_format__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - 'as_integer_ratio', - 'conjugate', - 'fromhex', - 'hex', - 'imag', - 'is_integer', - 'real', -]}) -expected_methods.append({'name': 'frozenset','type':frozenset, 'methods': [ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'copy', - 'difference', - 'intersection', - 'isdisjoint', - 'issubset', - 'issuperset', - 'symmetric_difference', - 'union', -]}) -expected_methods.append({'name': 'int', 'type':int, 'methods': [ - '__abs__', - '__add__', - '__and__', - '__bool__', - '__ceil__', - '__class__', - '__delattr__', - '__dir__', - '__divmod__', - '__doc__', - '__eq__', - '__float__', - '__floor__', - '__floordiv__', - '__format__', - '__ge__', - '__getattribute__', - '__getnewargs__', - '__gt__', - '__hash__', - '__index__', - '__init__', - '__init_subclass__', - '__int__', - '__invert__', - '__le__', - '__lshift__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__neg__', - '__new__', - '__or__', - '__pos__', - '__pow__', - '__radd__', - '__rand__', - '__rdivmod__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rfloordiv__', - '__rlshift__', - '__rmod__', - '__rmul__', - '__ror__', - '__round__', - '__rpow__', - '__rrshift__', - '__rshift__', - '__rsub__', - '__rtruediv__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__truediv__', - '__trunc__', - '__xor__', - 'bit_length', - 'conjugate', - 'denominator', - 'from_bytes', - 'imag', - 'numerator', - 'real', - 'to_bytes', -]}) -expected_methods.append({'name': 'iter','type':iter,'methods':[ - '__class__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__length_hint__', - '__lt__', - '__ne__', - '__new__', - '__next__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setstate__', - '__sizeof__', - '__str__', - '__subclasshook__', -]}) -expected_methods.append({'name': 'list','type':list,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', -]}) -expected_methods.append({'name': 'memoryview','type':memoryview,'methods':[ - '__class__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__enter__', - '__eq__', - '__exit__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'c_contiguous', - 'cast', - 'contiguous', - 'f_contiguous', - 'format', - 'hex', - 'itemsize', - 'nbytes', - 'ndim', - 'obj', - 'readonly', - 'release', - 'shape', - 'strides', - 'suboffsets', - 'tobytes', - 'tolist', -]}) -expected_methods.append({'name': 'range','type':range,'methods':[ - '__bool__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', - 'start', - 'step', - 'stop', -]}) -expected_methods.append({'name': 'set','type':set,'methods':[ - '__and__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__gt__', - '__hash__', - '__iand__', - '__init__', - '__init_subclass__', - '__ior__', - '__isub__', - '__iter__', - '__ixor__', - '__le__', - '__len__', - '__lt__', - '__ne__', - '__new__', - '__or__', - '__rand__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__ror__', - '__rsub__', - '__rxor__', - '__setattr__', - '__sizeof__', - '__str__', - '__sub__', - '__subclasshook__', - '__xor__', - 'add', - 'clear', - 'copy', - 'difference', - 'difference_update', - 'discard', - 'intersection', - 'intersection_update', - 'isdisjoint', - 'issubset', - 'issuperset', - 'pop', - 'remove', - 'symmetric_difference', - 'symmetric_difference_update', - 'union', - 'update', -]}) -expected_methods.append({'name': 'string','type':str,'methods':[ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mod__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmod__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'capitalize', - 'casefold', - 'center', - 'count', - 'encode', - 'endswith', - 'expandtabs', - 'find', - 'format', - 'format_map', - 'index', - 'isalnum', - 'isalpha', - 'isascii', - 'isdecimal', - 'isdigit', - 'isidentifier', - 'islower', - 'isnumeric', - 'isprintable', - 'isspace', - 'istitle', - 'isupper', - 'join', - 'ljust', - 'lower', - 'lstrip', - 'maketrans', - 'partition', - 'replace', - 'rfind', - 'rindex', - 'rjust', - 'rpartition', - 'rsplit', - 'rstrip', - 'split', - 'splitlines', - 'startswith', - 'strip', - 'swapcase', - 'title', - 'translate', - 'upper', - 'zfill' -]}) -expected_methods.append({'name': 'tuple','type':tuple, 'methods': [ - '__add__', - '__class__', - '__contains__', - '__delattr__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getnewargs__', - '__gt__', - '__hash__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__rmul__', - '__setattr__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'count', - 'index', -]}) -expected_methods.append({'name': 'object', 'type':object, 'methods':[ - '__repr__', - '__hash__', - '__str__', - '__getattribute__', - '__setattr__', - '__delattr__', - '__lt__', - '__le__', - '__eq__', - '__ne__', - '__gt__', - '__ge__', - '__init__', - '__new__', - '__reduce_ex__', - '__reduce__', - '__subclasshook__', - '__init_subclass__', - '__format__', - '__sizeof__', - '__dir__', - '__class__', - '__doc__' -]}) - -not_implemented = [] - -for item in expected_methods: - for method in item['methods']: - try: - if not hasattr(item['type'], method): - not_implemented.append((item['name'], method)) - except NameError: - not_implemented.append((item['name'], method)) - -for r in not_implemented: - print(r[0], ".", r[1]) -else: - print("Not much \\o/") diff --git a/tests/snippets/xfail_3.1.2.17.py b/tests/snippets/xfail_3.1.2.17.py deleted file mode 100644 index fc956bef43..0000000000 --- a/tests/snippets/xfail_3.1.2.17.py +++ /dev/null @@ -1,7 +0,0 @@ -word = "Python" - -word[0] = "J" # Should raise a error, immutable -word[2:] = "Jy" # Should raise a error, immutable - - - diff --git a/tests/test_snippets.py b/tests/test_snippets.py index 4be73b7bde..17bcf0dcad 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -1,4 +1,3 @@ - # This is a python unittest class automatically populating with all tests # in the tests folder. @@ -11,25 +10,23 @@ import subprocess import contextlib import enum +from pathlib import Path +import shutil import compile_code class _TestType(enum.Enum): functional = 1 - benchmark = 2 -logger = logging.getLogger('tests') -ROOT_DIR = '..' -TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, 'tests')) -TEST_DIRS = { - _TestType.functional: os.path.join(TEST_ROOT, 'snippets'), - _TestType.benchmark: os.path.join(TEST_ROOT, 'benchmarks'), -} -CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, 'py_code_object')) +logger = logging.getLogger("tests") +ROOT_DIR = ".." +TEST_ROOT = os.path.abspath(os.path.join(ROOT_DIR, "tests")) +TEST_DIRS = {_TestType.functional: os.path.join(TEST_ROOT, "snippets")} +CPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR, "py_code_object")) RUSTPYTHON_RUNNER_DIR = os.path.abspath(os.path.join(ROOT_DIR)) - +RUSTPYTHON_LIB_DIR = os.path.abspath(os.path.join(ROOT_DIR, "Lib")) @contextlib.contextmanager def pushd(path): @@ -40,12 +37,12 @@ def pushd(path): def perform_test(filename, method, test_type): - logger.info('Running %s via %s', filename, method) - if method == 'cpython': + logger.info("Running %s via %s", filename, method) + if method == "cpython": run_via_cpython(filename) - elif method == 'cpython_bytecode': + elif method == "cpython_bytecode": run_via_cpython_bytecode(filename, test_type) - elif method == 'rustpython': + elif method == "rustpython": run_via_rustpython(filename, test_type) else: raise NotImplementedError(method) @@ -59,44 +56,44 @@ def run_via_cpython(filename): def run_via_cpython_bytecode(filename, test_type): # Step1: Create bytecode file: - bytecode_filename = filename + '.bytecode' - with open(bytecode_filename, 'w') as f: + bytecode_filename = filename + ".bytecode" + with open(bytecode_filename, "w") as f: compile_code.compile_to_bytecode(filename, out_file=f) # Step2: run cpython bytecode: env = os.environ.copy() - log_level = 'info' if test_type == _TestType.benchmark else 'debug' - env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level) - env['RUST_BACKTRACE'] = '1' + env["RUST_LOG"] = "info,cargo=error,jobserver=error" + env["RUST_BACKTRACE"] = "1" with pushd(CPYTHON_RUNNER_DIR): - subprocess.check_call(['cargo', 'run', bytecode_filename], env=env) + subprocess.check_call(["cargo", "run", bytecode_filename], env=env) def run_via_rustpython(filename, test_type): env = os.environ.copy() - log_level = 'info' if test_type == _TestType.benchmark else 'trace' - env['RUST_LOG'] = '{},cargo=error,jobserver=error'.format(log_level) + env['RUST_LOG'] = 'info,cargo=error,jobserver=error' env['RUST_BACKTRACE'] = '1' - if env.get('CODE_COVERAGE', 'false') == 'true': - subprocess.check_call( - ['cargo', 'run', filename], env=env) - else: - subprocess.check_call( - ['cargo', 'run', '--release', filename], env=env) + env['PYTHONPATH'] = RUSTPYTHON_LIB_DIR + + target = "release" + if env.get("CODE_COVERAGE", "false") == "true": + target = "debug" + binary = os.path.abspath(os.path.join(ROOT_DIR, "target", target, "rustpython")) + + subprocess.check_call([binary, filename], env=env) def create_test_function(cls, filename, method, test_type): """ Create a test function for a single snippet """ core_test_directory, snippet_filename = os.path.split(filename) - test_function_name = 'test_{}_'.format(method) \ - + os.path.splitext(snippet_filename)[0] \ - .replace('.', '_').replace('-', '_') + test_function_name = "test_{}_".format(method) + os.path.splitext(snippet_filename)[ + 0 + ].replace(".", "_").replace("-", "_") def test_function(self): perform_test(filename, method, test_type) if hasattr(cls, test_function_name): - raise ValueError('Duplicate test case {}'.format(test_function_name)) + raise ValueError("Duplicate test case {}".format(test_function_name)) setattr(cls, test_function_name, test_function) @@ -106,22 +103,61 @@ def wrapper(cls): for test_type, filename in get_test_files(): create_test_function(cls, filename, method, test_type) return cls + return wrapper def get_test_files(): """ Retrieve test files """ for test_type, test_dir in TEST_DIRS.items(): - for filepath in sorted(glob.iglob(os.path.join(test_dir, '*.py'))): + for filepath in sorted(glob.iglob(os.path.join(test_dir, "*.py"))): filename = os.path.split(filepath)[1] - if filename.startswith('xfail_'): + if filename.startswith("xfail_"): continue yield test_type, os.path.abspath(filepath) -@populate('cpython') +def generate_slices(path): + # loop used to build slices_res.py with cpython + ll = [0, 1, 2, 3] + start = list(range(-7, 7)) + end = list(range(-7, 7)) + step = list(range(-5, 5)) + step.pop(step.index(0)) + for i in [start, end, step]: + i.append(None) + + slices_res = [] + for s in start: + for e in end: + for t in step: + slices_res.append(ll[s:e:t]) + + path.write_text( + "SLICES_RES={}\nSTART= {}\nEND= {}\nSTEP= {}\nLL={}\n".format( + slices_res, start, end, step, ll + ) + ) + + +@populate("cpython") # @populate('cpython_bytecode') -@populate('rustpython') +@populate("rustpython") class SampleTestCase(unittest.TestCase): - pass + @classmethod + def setUpClass(cls): + # Here add resource files + cls.slices_resource_path = Path(TEST_DIRS[_TestType.functional]) / "cpython_generated_slices.py" + if cls.slices_resource_path.exists(): + cls.slices_resource_path.unlink() + + generate_slices(cls.slices_resource_path) + + # cargo stuff + subprocess.check_call(["cargo", "build"]) + subprocess.check_call(["cargo", "build", "--release"]) + + @classmethod + def tearDownClass(cls): + cls.slices_resource_path.unlink() diff --git a/vm/Cargo.toml b/vm/Cargo.toml index e1b1cc9ffa..6c2533cb3f 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -1,22 +1,33 @@ [package] -name = "rustpython_vm" +name = "rustpython-vm" version = "0.1.0" -authors = ["Shing Lyu "] +authors = ["RustPython Team"] edition = "2018" +[features] +default = ["rustpython-parser", "rustpython-compiler"] + [dependencies] -bitflags = "1.0.4" -num-complex = "0.2" -num-bigint = "0.2.1" +# Crypto: +digest = "0.8.1" +md-5 = "0.8" +sha-1 = "0.8" +sha2 = "0.8" +sha3 = "0.8" +blake2 = "0.8" + +num-complex = { version = "0.2", features = ["serde"] } +num-bigint = { version = "0.2.1", features = ["serde"] } num-traits = "0.2" num-integer = "0.1.39" num-rational = "0.2.1" rand = "0.5" log = "0.3" -rustpython_derive = {path = "../derive"} -rustpython_parser = {path = "../parser"} -serde = "1.0.66" -serde_derive = "1.0.66" +rustpython-derive = {path = "../derive"} +rustpython-parser = {path = "../parser", optional = true} +rustpython-compiler = {path = "../compiler", optional = true} +rustpython-bytecode = { path = "../bytecode" } +serde = { version = "1.0.66", features = ["derive"] } serde_json = "1.0.26" byteorder = "1.2.6" regex = "1" @@ -24,5 +35,27 @@ rustc_version_runtime = "0.1.*" statrs = "0.10.0" caseless = "0.2.1" unicode-segmentation = "1.2.1" +unicode-xid = "0.1.0" lazy_static = "^1.0.1" lexical = "2.0.0" +itertools = "^0.8.0" +hex = "0.3.2" +hexf = "0.1.0" +indexmap = "1.0.2" +crc = "^1.0.0" +bincode = "1.1.4" +unicode_categories = "0.1.1" +unicode_names2 = "0.2.2" +unic = "0.9.0" +maplit = "1.0" +proc-macro-hack = "0.5" +bitflags = "1.1" + + +# TODO: release and publish to crates.io +[dependencies.unicode-casing] +git = "https://github.com/OddCoincidence/unicode-casing" +rev = "90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb" + +[target.'cfg(all(unix, not(target_os = "android")))'.dependencies] +pwd = "1" diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 68e933db3e..fa05febf68 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -2,132 +2,116 @@ //! //! Implements functions listed here: https://docs.python.org/3/library/builtins.html +use std::cell::Cell; use std::char; use std::io::{self, Write}; -use std::path::PathBuf; +use std::str; -use num_traits::{Signed, ToPrimitive}; +use num_bigint::Sign; +use num_traits::{Signed, ToPrimitive, Zero}; -use crate::compile; -use crate::import::import_module; use crate::obj::objbool; -use crate::obj::objdict; -use crate::obj::objint; +use crate::obj::objbytes::PyBytesRef; +use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::PyDictRef; +use crate::obj::objint::{self, PyIntRef}; use crate::obj::objiter; -use crate::obj::objstr::{self, PyStringRef}; -use crate::obj::objtype; +use crate::obj::objstr::{self, PyString, PyStringRef}; +use crate::obj::objtype::{self, PyClassRef}; +#[cfg(feature = "rustpython-compiler")] +use rustpython_compiler::compile; use crate::frame::Scope; -use crate::function::{Args, OptionalArg, PyFuncArgs}; +use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; use crate::pyobject::{ - AttributeProtocol, DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol, + Either, IdProtocol, IntoPyObject, ItemProtocol, PyIterable, PyObjectRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use crate::obj::objbyteinner::PyByteInner; #[cfg(not(target_arch = "wasm32"))] use crate::stdlib::io::io_open; -fn get_locals(vm: &VirtualMachine) -> PyObjectRef { - let d = vm.new_dict(); - // TODO: implement dict_iter_items? - let locals = vm.get_locals(); - let key_value_pairs = objdict::get_key_value_pairs(&locals); - for (key, value) in key_value_pairs { - objdict::set_item(&d, vm, &key, &value); - } - d -} - -fn dir_locals(vm: &VirtualMachine) -> PyObjectRef { - get_locals(vm) +fn builtin_abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let method = vm.get_method_or_type_error(x.clone(), "__abs__", || { + format!("bad operand type for abs(): '{}'", x.class().name) + })?; + vm.invoke(method, PyFuncArgs::new(vec![], vec![])) } -fn builtin_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(x, None)]); - match vm.get_method(x.clone(), "__abs__") { - Ok(attrib) => vm.invoke(attrib, PyFuncArgs::new(vec![], vec![])), - Err(..) => Err(vm.new_type_error("bad operand for abs".to_string())), - } -} - -fn builtin_all(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iterable, None)]); - let items = vm.extract_elements(iterable)?; - for item in items { - let result = objbool::boolval(vm, item)?; - if !result { - return Ok(vm.new_bool(false)); +fn builtin_all(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + if !item? { + return Ok(false); } } - Ok(vm.new_bool(true)) + Ok(true) } -fn builtin_any(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iterable, None)]); - let iterator = objiter::get_iter(vm, iterable)?; - - while let Some(item) = objiter::get_next_object(vm, &iterator)? { - let result = objbool::boolval(vm, item)?; - if result { - return Ok(vm.new_bool(true)); +fn builtin_any(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in iterable.iter(vm)? { + if item? { + return Ok(true); } } - - Ok(vm.new_bool(false)) + Ok(false) } // builtin_ascii -fn builtin_bin(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(number, Some(vm.ctx.int_type()))]); - - let n = objint::get_value(number); - let s = if n.is_negative() { - format!("-0b{:b}", n.abs()) +fn builtin_bin(x: PyIntRef, _vm: &VirtualMachine) -> String { + let x = x.as_bigint(); + if x.is_negative() { + format!("-0b{:b}", x.abs()) } else { - format!("0b{:b}", n) - }; - - Ok(vm.new_str(s)) + format!("0b{:b}", x) + } } // builtin_breakpoint -fn builtin_callable(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, None)]); - let is_callable = obj.typ().has_attr("__call__"); - Ok(vm.new_bool(is_callable)) +fn builtin_callable(obj: PyObjectRef, vm: &VirtualMachine) -> bool { + vm.is_callable(&obj) } -fn builtin_chr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.int_type()))]); - - let code_point = objint::get_value(i).to_u32().unwrap(); +fn builtin_chr(i: u32, vm: &VirtualMachine) -> PyResult { + match char::from_u32(i) { + Some(value) => Ok(value.to_string()), + None => Err(vm.new_value_error("chr() arg not in range(0x110000)".to_string())), + } +} - let txt = match char::from_u32(code_point) { - Some(value) => value.to_string(), - None => '_'.to_string(), +#[derive(FromArgs)] +#[allow(dead_code)] +struct CompileArgs { + #[pyarg(positional_only, optional = false)] + source: Either, + #[pyarg(positional_only, optional = false)] + filename: PyStringRef, + #[pyarg(positional_only, optional = false)] + mode: PyStringRef, + #[pyarg(positional_or_keyword, optional = true)] + flags: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + dont_inherit: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + optimize: OptionalArg, +} + +#[cfg(feature = "rustpython-compiler")] +fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult { + // TODO: compile::compile should probably get bytes + let source = match args.source { + Either::A(string) => string.value.to_string(), + Either::B(bytes) => str::from_utf8(&bytes).unwrap().to_string(), }; - Ok(vm.new_str(txt)) -} - -fn builtin_compile(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (source, None), - (filename, Some(vm.ctx.str_type())), - (mode, Some(vm.ctx.str_type())) - ] - ); - let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); let mode = { - let mode = objstr::get_value(mode); + let mode = &args.mode.value; if mode == "exec" { compile::Mode::Exec } else if mode == "eval" { @@ -141,45 +125,39 @@ fn builtin_compile(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } }; - let filename = objstr::get_value(filename); - - compile::compile(&source, &mode, filename, vm.ctx.code_type()).map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - }) + vm.compile(&source, &mode, args.filename.value.to_string()) + .map_err(|err| vm.new_syntax_error(&err)) } -fn builtin_delattr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(obj, None), (attr, Some(vm.ctx.str_type()))] - ); - vm.del_attr(obj, attr.clone()) +fn builtin_delattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + vm.del_attr(&obj, attr.into_object()) } -fn builtin_dir(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - Ok(dir_locals(vm)) - } else { - let obj = args.args.into_iter().next().unwrap(); - let seq = vm.call_method(&obj, "__dir__", vec![])?; - let sorted = builtin_sorted(vm, PyFuncArgs::new(vec![seq], vec![]))?; - Ok(sorted) - } +fn builtin_dir(obj: OptionalArg, vm: &VirtualMachine) -> PyResult { + let seq = match obj { + OptionalArg::Present(obj) => vm.call_method(&obj, "__dir__", vec![])?, + OptionalArg::Missing => vm.call_method(&vm.get_locals().into_object(), "keys", vec![])?, + }; + let sorted = builtin_sorted(vm, PyFuncArgs::new(vec![seq], vec![]))?; + Ok(sorted) } fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(x, None), (y, None)]); - match vm.get_method(x.clone(), "__divmod__") { - Ok(attrib) => vm.invoke(attrib, vec![y.clone()]), - Err(..) => Err(vm.new_type_error("unsupported operand type(s) for divmod".to_string())), - } + arg_check!(vm, args, required = [(a, None), (b, None)]); + vm.call_or_reflection( + a.clone(), + b.clone(), + "__divmod__", + "__rdivmod__", + |vm, a, b| Err(vm.new_unsupported_operand_error(a, b, "divmod")), + ) } /// Implements `eval`. /// See also: https://docs.python.org/3/library/functions.html#eval +#[cfg(feature = "rustpython-compiler")] fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: support any mapping for `locals` arg_check!( vm, args, @@ -190,35 +168,32 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let scope = make_scope(vm, globals, locals)?; // Determine code object: - let code_obj = if objtype::isinstance(source, &vm.ctx.code_type()) { - source.clone() + let code_obj = if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) { + code_obj } else if objtype::isinstance(source, &vm.ctx.str_type()) { let mode = compile::Mode::Eval; let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(&source, &mode, "".to_string(), vm.ctx.code_type()).map_err( - |err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - }, - )? + vm.compile(&source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? } else { return Err(vm.new_type_error("code argument must be str or code object".to_string())); }; // Run the source: - vm.run_code_obj(code_obj.clone(), scope) + vm.run_code_obj(code_obj, scope) } /// Implements `exec` /// https://docs.python.org/3/library/functions.html#exec +#[cfg(feature = "rustpython-compiler")] fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, required = [(source, None)], - optional = [(globals, None), (locals, Some(vm.ctx.dict_type()))] + optional = [(globals, None), (locals, None)] ); let scope = make_scope(vm, globals, locals)?; @@ -229,14 +204,10 @@ fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let source = objstr::get_value(source); // TODO: fix this newline bug: let source = format!("{}\n", source); - compile::compile(&source, &mode, "".to_string(), vm.ctx.code_type()).map_err( - |err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.to_string()) - }, - )? - } else if objtype::isinstance(source, &vm.ctx.code_type()) { - source.clone() + vm.compile(&source, &mode, "".to_string()) + .map_err(|err| vm.new_syntax_error(&err))? + } else if let Ok(code_obj) = PyCodeRef::try_from_object(vm, source.clone()) { + code_obj } else { return Err(vm.new_type_error("source argument must be str or code object".to_string())); }; @@ -258,7 +229,7 @@ fn make_scope( } else if vm.isinstance(arg, &dict_type)? { Some(arg) } else { - let arg_typ = arg.typ(); + let arg_typ = arg.class(); let actual_type = vm.to_pystr(&arg_typ)?; let expected_type_name = vm.to_pystr(&dict_type)?; return Err(vm.new_type_error(format!( @@ -269,31 +240,54 @@ fn make_scope( } None => None, }; - let current_scope = vm.current_scope(); + let locals = match locals { + Some(dict) => dict.clone().downcast().ok(), + None => { + if globals.is_some() { + None + } else { + current_scope.get_only_locals() + } + } + }; let globals = match globals { - Some(dict) => dict.clone(), + Some(dict) => { + let dict: PyDictRef = dict.clone().downcast().unwrap(); + if !dict.contains_key("__builtins__", vm) { + let builtins_dict = vm.builtins.dict.as_ref().unwrap().as_object(); + dict.set_item("__builtins__", builtins_dict.clone(), vm) + .unwrap(); + } + dict + } None => current_scope.globals.clone(), }; - let locals = match locals { - Some(dict) => Some(dict.clone()), - None => current_scope.get_only_locals(), - }; - Ok(Scope::new(locals, globals)) + let scope = Scope::with_builtins(locals, globals, vm); + Ok(scope) } -fn builtin_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(obj, None)], - optional = [(format_spec, Some(vm.ctx.str_type()))] - ); - let format_spec = format_spec - .cloned() - .unwrap_or_else(|| vm.new_str("".to_string())); - vm.call_method(obj, "__format__", vec![format_spec]) +fn builtin_format( + value: PyObjectRef, + format_spec: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let format_spec = format_spec.into_option().unwrap_or_else(|| { + PyString { + value: "".to_string(), + } + .into_ref(vm) + }); + + vm.call_method(&value, "__format__", vec![format_spec.into_object()])? + .downcast() + .map_err(|obj| { + vm.new_type_error(format!( + "__format__ must return a str, not {}", + obj.class().name + )) + }) } fn catch_attr_exception(ex: PyObjectRef, default: T, vm: &VirtualMachine) -> PyResult { @@ -318,7 +312,7 @@ fn builtin_getattr( } } -fn builtin_globals(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { +fn builtin_globals(vm: &VirtualMachine) -> PyResult { Ok(vm.current_scope().globals.clone()) } @@ -332,8 +326,7 @@ fn builtin_hasattr(obj: PyObjectRef, attr: PyStringRef, vm: &VirtualMachine) -> fn builtin_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, None)]); - - vm.call_method(obj, "__hash__", vec![]) + vm._hash(obj).and_then(|v| Ok(vm.new_int(v))) } // builtin_help @@ -359,26 +352,36 @@ fn builtin_id(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // builtin_input -fn builtin_isinstance(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( +fn builtin_isinstance(obj: PyObjectRef, typ: PyObjectRef, vm: &VirtualMachine) -> PyResult { + single_or_tuple_any( + typ, + |cls: PyClassRef| vm.isinstance(&obj, &cls), + |o| { + format!( + "isinstance() arg 2 must be a type or tuple of types, not {}", + o.class() + ) + }, vm, - args, - required = [(obj, None), (typ, Some(vm.get_type()))] - ); - - let isinstance = vm.isinstance(obj, typ)?; - Ok(vm.new_bool(isinstance)) + ) } -fn builtin_issubclass(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( +fn builtin_issubclass( + subclass: PyClassRef, + typ: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + single_or_tuple_any( + typ, + |cls: PyClassRef| vm.issubclass(&subclass, &cls), + |o| { + format!( + "issubclass() arg 2 must be a class or tuple of classes, not {}", + o.class() + ) + }, vm, - args, - required = [(subclass, Some(vm.get_type())), (cls, Some(vm.get_type()))] - ); - - let issubclass = vm.issubclass(subclass, cls)?; - Ok(vm.context().new_bool(issubclass)) + ) } fn builtin_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -388,20 +391,14 @@ fn builtin_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { fn builtin_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, None)]); - let len_method_name = "__len__"; - match vm.get_method(obj.clone(), len_method_name) { - Ok(value) => vm.invoke(value, PyFuncArgs::default()), - Err(..) => Err(vm.new_type_error(format!( - "object of type '{}' has no method {:?}", - objtype::get_type_name(&obj.typ()), - len_method_name - ))), - } + let method = vm.get_method_or_type_error(obj.clone(), "__len__", || { + format!("object of type '{}' has no len()", obj.class().name) + })?; + vm.invoke(method, PyFuncArgs::default()) } -fn builtin_locals(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args); - Ok(vm.get_locals()) +fn builtin_locals(vm: &VirtualMachine) -> PyDictRef { + vm.get_locals() } fn builtin_max(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -538,20 +535,39 @@ fn builtin_oct(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } fn builtin_ord(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(string, Some(vm.ctx.str_type()))]); - let string = objstr::get_value(string); - let string_len = string.chars().count(); - if string_len > 1 { - return Err(vm.new_type_error(format!( - "ord() expected a character, but string of length {} found", - string_len - ))); - } - match string.chars().next() { - Some(character) => Ok(vm.context().new_int(character as i32)), - None => Err(vm.new_type_error( - "ord() could not guess the integer representing this character".to_string(), - )), + arg_check!(vm, args, required = [(string, None)]); + if objtype::isinstance(string, &vm.ctx.str_type()) { + let string = objstr::borrow_value(string); + let string_len = string.chars().count(); + if string_len != 1 { + return Err(vm.new_type_error(format!( + "ord() expected a character, but string of length {} found", + string_len + ))); + } + match string.chars().next() { + Some(character) => Ok(vm.context().new_int(character as i32)), + None => Err(vm.new_type_error( + "ord() could not guess the integer representing this character".to_string(), + )), + } + } else if objtype::isinstance(string, &vm.ctx.bytearray_type()) + || objtype::isinstance(string, &vm.ctx.bytes_type()) + { + let inner = PyByteInner::try_from_object(vm, string.clone()).unwrap(); + let bytes_len = inner.elements.len(); + if bytes_len != 1 { + return Err(vm.new_type_error(format!( + "ord() expected a character, but string of length {} found", + bytes_len + ))); + } + Ok(vm.context().new_int(inner.elements[0])) + } else { + Err(vm.new_type_error(format!( + "ord() expected a string, bytes or bytearray, but found {}", + string.class().name + ))) } } @@ -562,59 +578,125 @@ fn builtin_pow(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(x, None), (y, None)], optional = [(mod_value, Some(vm.ctx.int_type()))] ); - let pow_method_name = "__pow__"; - let result = match vm.get_method(x.clone(), pow_method_name) { - Ok(attrib) => vm.invoke(attrib, vec![y.clone()]), - Err(..) => Err(vm.new_type_error("unsupported operand type(s) for pow".to_string())), - }; - //Check if the 3rd argument is defined and perform modulus on the result - //this should be optimized in the future to perform a "power-mod" algorithm in - //order to improve performance + match mod_value { - Some(mod_value) => { - let mod_method_name = "__mod__"; - match vm.get_method(result.expect("result not defined").clone(), mod_method_name) { - Ok(value) => vm.invoke(value, vec![mod_value.clone()]), - Err(..) => { - Err(vm.new_type_error("unsupported operand type(s) for mod".to_string())) - } + None => vm.call_or_reflection(x.clone(), y.clone(), "__pow__", "__rpow__", |vm, x, y| { + Err(vm.new_unsupported_operand_error(x, y, "pow")) + }), + Some(m) => { + // Check if the 3rd argument is defined and perform modulus on the result + if !(objtype::isinstance(x, &vm.ctx.int_type()) + && objtype::isinstance(y, &vm.ctx.int_type())) + { + return Err(vm.new_type_error( + "pow() 3rd argument not allowed unless all arguments are integers".to_string(), + )); } + let y = objint::get_value(y); + if y.sign() == Sign::Minus { + return Err(vm.new_value_error( + "pow() 2nd argument cannot be negative when 3rd argument specified".to_string(), + )); + } + let m = objint::get_value(m); + if m.is_zero() { + return Err(vm.new_value_error("pow() 3rd argument cannot be 0".to_string())); + } + let x = objint::get_value(x); + Ok(vm.new_int(x.modpow(&y, &m))) } - None => result, } } #[derive(Debug, FromArgs)] pub struct PrintOptions { + #[pyarg(keyword_only, default = "None")] sep: Option, + #[pyarg(keyword_only, default = "None")] end: Option, + #[pyarg(keyword_only, default = "false")] flush: bool, + #[pyarg(keyword_only, default = "None")] + file: Option, +} + +trait Printer { + fn write(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()>; + fn flush(&mut self, vm: &VirtualMachine) -> PyResult<()>; +} + +impl Printer for &'_ PyObjectRef { + fn write(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()> { + vm.call_method(self, "write", vec![obj])?; + Ok(()) + } + + fn flush(&mut self, vm: &VirtualMachine) -> PyResult<()> { + vm.call_method(self, "flush", vec![])?; + Ok(()) + } +} + +impl Printer for std::io::StdoutLock<'_> { + fn write(&mut self, vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<()> { + let s = &vm.to_str(&obj)?.value; + write!(self, "{}", s).unwrap(); + Ok(()) + } + + fn flush(&mut self, _vm: &VirtualMachine) -> PyResult<()> { + ::flush(self).unwrap(); + Ok(()) + } +} + +pub fn builtin_exit(exit_code_arg: OptionalArg, vm: &VirtualMachine) -> PyResult<()> { + if let OptionalArg::Present(exit_code_obj) = exit_code_arg { + match i32::try_from_object(&vm, exit_code_obj.clone()) { + Ok(code) => std::process::exit(code), + _ => println!("{}", vm.to_str(&exit_code_obj)?.as_str()), + } + } + std::process::exit(0); } pub fn builtin_print(objects: Args, options: PrintOptions, vm: &VirtualMachine) -> PyResult<()> { let stdout = io::stdout(); - let mut stdout_lock = stdout.lock(); + + let mut printer: Box = if let Some(file) = &options.file { + Box::new(file) + } else { + Box::new(stdout.lock()) + }; + + let sep = options + .sep + .as_ref() + .map_or(" ", |sep| &sep.value) + .into_pyobject(vm) + .unwrap(); + let mut first = true; for object in objects { if first { first = false; - } else if let Some(ref sep) = options.sep { - write!(stdout_lock, "{}", sep.value).unwrap(); } else { - write!(stdout_lock, " ").unwrap(); + printer.write(vm, sep.clone())?; } - let s = &vm.to_str(&object)?.value; - write!(stdout_lock, "{}", s).unwrap(); - } - if let Some(end) = options.end { - write!(stdout_lock, "{}", end.value).unwrap(); - } else { - writeln!(stdout_lock).unwrap(); + printer.write(vm, object)?; } + let end = options + .end + .as_ref() + .map_or("\n", |end| &end.value) + .into_pyobject(vm) + .unwrap(); + printer.write(vm, end)?; + if options.flush { - stdout_lock.flush().unwrap(); + printer.flush(vm)?; } Ok(()) @@ -627,16 +709,21 @@ fn builtin_repr(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult fn builtin_reversed(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(obj, None)]); - match vm.get_method(obj.clone(), "__reversed__") { - Ok(value) => vm.invoke(value, PyFuncArgs::default()), - // TODO: fallback to using __len__ and __getitem__, if object supports sequence protocol - Err(..) => Err(vm.new_type_error(format!( - "'{}' object is not reversible", - objtype::get_type_name(&obj.typ()), - ))), + if let Some(reversed_method) = vm.get_method(obj.clone(), "__reversed__") { + vm.invoke(reversed_method?, PyFuncArgs::default()) + } else { + vm.get_method_or_type_error(obj.clone(), "__getitem__", || { + format!("argument to reversed() must be a sequence") + })?; + let len = vm.call_method(&obj.clone(), "__len__", PyFuncArgs::default())?; + let obj_iterator = objiter::PySequenceIterator { + position: Cell::new(objint::get_value(&len).to_isize().unwrap() - 1), + obj: obj.clone(), + reversed: true, + }; + Ok(obj_iterator.into_ref(vm).into_object()) } } -// builtin_reversed fn builtin_round(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( @@ -678,42 +765,40 @@ fn builtin_sorted(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { Ok(lst) } -fn builtin_sum(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iterable, None)]); - let items = vm.extract_elements(iterable)?; - +fn builtin_sum(iterable: PyIterable, start: OptionalArg, vm: &VirtualMachine) -> PyResult { // Start with zero and add at will: - let mut sum = vm.ctx.new_int(0); - for item in items { - sum = vm._add(sum, item)?; + let mut sum = start.into_option().unwrap_or_else(|| vm.ctx.new_int(0)); + for item in iterable.iter(vm)? { + sum = vm._add(sum, item?)?; } Ok(sum) } // Should be renamed to builtin___import__? fn builtin_import(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(name, Some(vm.ctx.str_type()))], - optional = [ - (_globals, Some(vm.ctx.dict_type())), - (_locals, Some(vm.ctx.dict_type())) - ] - ); - let current_path = { - let mut source_pathbuf = PathBuf::from(&vm.current_frame().code.source_path); - source_pathbuf.pop(); - source_pathbuf - }; - - import_module(vm, current_path, &objstr::get_value(name)) + vm.invoke(vm.import_func.borrow().clone(), args) } // builtin_vars -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "__builtins__", { +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { + let ctx = &vm.ctx; + + #[cfg(target_arch = "wasm32")] + let open = vm.ctx.none(); + #[cfg(not(target_arch = "wasm32"))] + let open = vm.ctx.new_rustfunc(io_open); + + #[cfg(feature = "rustpython-compiler")] + { + extend_module!(vm, module, { + "compile" => ctx.new_rustfunc(builtin_compile), + "eval" => ctx.new_rustfunc(builtin_eval), + "exec" => ctx.new_rustfunc(builtin_exec), + }); + } + + extend_module!(vm, module, { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -727,15 +812,12 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "callable" => ctx.new_rustfunc(builtin_callable), "chr" => ctx.new_rustfunc(builtin_chr), "classmethod" => ctx.classmethod_type(), - "compile" => ctx.new_rustfunc(builtin_compile), "complex" => ctx.complex_type(), "delattr" => ctx.new_rustfunc(builtin_delattr), "dict" => ctx.dict_type(), "divmod" => ctx.new_rustfunc(builtin_divmod), "dir" => ctx.new_rustfunc(builtin_dir), "enumerate" => ctx.enumerate_type(), - "eval" => ctx.new_rustfunc(builtin_eval), - "exec" => ctx.new_rustfunc(builtin_exec), "float" => ctx.float_type(), "frozenset" => ctx.frozenset_type(), "filter" => ctx.filter_type(), @@ -759,6 +841,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "min" => ctx.new_rustfunc(builtin_min), "object" => ctx.object(), "oct" => ctx.new_rustfunc(builtin_oct), + "open" => open, "ord" => ctx.new_rustfunc(builtin_ord), "next" => ctx.new_rustfunc(builtin_next), "pow" => ctx.new_rustfunc(builtin_pow), @@ -779,10 +862,13 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "tuple" => ctx.tuple_type(), "type" => ctx.type_type(), "zip" => ctx.zip_type(), + "exit" => ctx.new_rustfunc(builtin_exit), + "quit" => ctx.new_rustfunc(builtin_exit), "__import__" => ctx.new_rustfunc(builtin_import), // Constants - "NotImplemented" => ctx.not_implemented.clone(), + "NotImplemented" => ctx.not_implemented(), + "Ellipsis" => vm.ctx.ellipsis.clone(), // Exceptions: "BaseException" => ctx.exceptions.base_exception_type.clone(), @@ -793,48 +879,81 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "NameError" => ctx.exceptions.name_error.clone(), "OverflowError" => ctx.exceptions.overflow_error.clone(), "RuntimeError" => ctx.exceptions.runtime_error.clone(), + "ReferenceError" => ctx.exceptions.reference_error.clone(), + "SyntaxError" => ctx.exceptions.syntax_error.clone(), "NotImplementedError" => ctx.exceptions.not_implemented_error.clone(), "TypeError" => ctx.exceptions.type_error.clone(), "ValueError" => ctx.exceptions.value_error.clone(), "IndexError" => ctx.exceptions.index_error.clone(), "ImportError" => ctx.exceptions.import_error.clone(), "FileNotFoundError" => ctx.exceptions.file_not_found_error.clone(), + "FileExistsError" => ctx.exceptions.file_exists_error.clone(), "StopIteration" => ctx.exceptions.stop_iteration.clone(), "ZeroDivisionError" => ctx.exceptions.zero_division_error.clone(), "KeyError" => ctx.exceptions.key_error.clone(), "OSError" => ctx.exceptions.os_error.clone(), + "ModuleNotFoundError" => ctx.exceptions.module_not_found_error.clone(), + "EOFError" => ctx.exceptions.eof_error.clone(), + + // Warnings + "Warning" => ctx.exceptions.warning.clone(), + "BytesWarning" => ctx.exceptions.bytes_warning.clone(), + "UnicodeWarning" => ctx.exceptions.unicode_warning.clone(), + "DeprecationWarning" => ctx.exceptions.deprecation_warning.clone(), + "PendingDeprecationWarning" => ctx.exceptions.pending_deprecation_warning.clone(), + "FutureWarning" => ctx.exceptions.future_warning.clone(), + "ImportWarning" => ctx.exceptions.import_warning.clone(), + "SyntaxWarning" => ctx.exceptions.syntax_warning.clone(), + "ResourceWarning" => ctx.exceptions.resource_warning.clone(), + "RuntimeWarning" => ctx.exceptions.runtime_warning.clone(), + "UserWarning" => ctx.exceptions.user_warning.clone(), }); - - #[cfg(not(target_arch = "wasm32"))] - ctx.set_attr(&py_mod, "open", ctx.new_rustfunc(io_open)); - - py_mod } -pub fn builtin_build_class_(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { - let function = args.shift(); - let name_arg = args.shift(); - let bases = args.args.clone(); - let mut metaclass = args.get_kwarg("metaclass", vm.get_type()); +pub fn builtin_build_class_( + function: PyObjectRef, + qualified_name: PyStringRef, + bases: Args, + mut kwargs: KwArgs, + vm: &VirtualMachine, +) -> PyResult { + let name = qualified_name.value.split('.').next_back().unwrap(); + let name_obj = vm.new_str(name.to_string()); + + let mut metaclass = if let Some(metaclass) = kwargs.pop_kwarg("metaclass") { + PyClassRef::try_from_object(vm, metaclass)? + } else { + vm.get_type() + }; for base in bases.clone() { - if objtype::issubclass(&base.typ(), &metaclass) { - metaclass = base.typ(); - } else if !objtype::issubclass(&metaclass, &base.typ()) { + if objtype::issubclass(&base.class(), &metaclass) { + metaclass = base.class(); + } else if !objtype::issubclass(&metaclass, &base.class()) { return Err(vm.new_type_error("metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases".to_string())); } } - let bases = vm.context().new_tuple(bases); + let bases = bases.into_tuple(vm); // Prepare uses full __getattribute__ resolution chain. - let prepare = vm.get_attribute(metaclass.clone(), "__prepare__")?; - let namespace = vm.invoke(prepare, vec![name_arg.clone(), bases.clone()])?; + let prepare = vm.get_attribute(metaclass.clone().into_object(), "__prepare__")?; + let namespace = vm.invoke(prepare, vec![name_obj.clone(), bases.clone()])?; + + let namespace: PyDictRef = TryFromObject::try_from_object(vm, namespace)?; - let cells = vm.new_dict(); + let cells = vm.ctx.new_dict(); vm.invoke_with_locals(function, cells.clone(), namespace.clone())?; - let class = vm.call_method(&metaclass, "__call__", vec![name_arg, bases, namespace])?; - cells.set_item(&vm.ctx, "__class__", class.clone()); + + namespace.set_item("__name__", name_obj.clone(), vm)?; + namespace.set_item("__qualname__", qualified_name.into_object(), vm)?; + + let class = vm.call_method( + metaclass.as_object(), + "__call__", + vec![name_obj, bases, namespace.into_object()], + )?; + cells.set_item("__class__", class.clone(), vm)?; Ok(class) } diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs new file mode 100644 index 0000000000..f8f98210b3 --- /dev/null +++ b/vm/src/cformat.rs @@ -0,0 +1,807 @@ +use crate::format::get_num_digits; +/// Implementation of Printf-Style string formatting +/// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting] +use num_bigint::{BigInt, Sign}; +use num_traits::Signed; +use std::cmp; +use std::fmt; +use std::str::FromStr; + +#[derive(Debug, PartialEq)] +pub enum CFormatErrorType { + UnmatchedKeyParentheses, + MissingModuloSign, + UnescapedModuloSignInLiteral, + UnsupportedFormatChar(char), + IncompleteFormat, + Unimplemented, +} + +// also contains how many chars the parsing function consumed +type ParsingError = (CFormatErrorType, usize); + +#[derive(Debug, PartialEq)] +pub struct CFormatError { + pub typ: CFormatErrorType, + pub index: usize, +} + +impl fmt::Display for CFormatError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use CFormatErrorType::*; + match self.typ { + UnmatchedKeyParentheses => write!(f, "incomplete format key"), + CFormatErrorType::IncompleteFormat => write!(f, "incomplete format"), + UnsupportedFormatChar(c) => write!( + f, + "unsupported format character '{}' ({:#x}) at index {}", + c, c as u32, self.index + ), + _ => write!(f, "unexpected error parsing format string"), + } + } +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPreconversor { + Repr, + Str, + Ascii, +} + +#[derive(Debug, PartialEq)] +pub enum CFormatCase { + Lowercase, + Uppercase, +} + +#[derive(Debug, PartialEq)] +pub enum CNumberType { + Decimal, + Octal, + Hex(CFormatCase), +} + +#[derive(Debug, PartialEq)] +pub enum CFloatType { + Exponent(CFormatCase), + PointDecimal, + General(CFormatCase), +} + +#[derive(Debug, PartialEq)] +pub enum CFormatType { + Number(CNumberType), + Float(CFloatType), + Character, + String(CFormatPreconversor), +} + +bitflags! { + pub struct CConversionFlags: u32 { + const ALTERNATE_FORM = 0b0000_0001; + const ZERO_PAD = 0b0000_0010; + const LEFT_ADJUST = 0b0000_0100; + const BLANK_SIGN = 0b0000_1000; + const SIGN_CHAR = 0b0001_0000; + } +} + +#[derive(Debug, PartialEq)] +pub enum CFormatQuantity { + Amount(usize), + FromValuesTuple, +} + +#[derive(Debug, PartialEq)] +pub struct CFormatSpec { + pub mapping_key: Option, + pub flags: CConversionFlags, + pub min_field_width: Option, + pub precision: Option, + pub format_type: CFormatType, + pub format_char: char, + chars_consumed: usize, +} + +impl CFormatSpec { + fn compute_fill_string(fill_char: char, fill_chars_needed: usize) -> String { + (0..fill_chars_needed) + .map(|_| fill_char) + .collect::() + } + + pub fn fill_string( + &self, + string: String, + fill_char: char, + num_prefix_chars: Option, + ) -> String { + let mut num_chars = string.chars().count(); + if let Some(num_prefix_chars) = num_prefix_chars { + num_chars = num_chars + num_prefix_chars; + } + let num_chars = num_chars; + + let width = match self.min_field_width { + Some(CFormatQuantity::Amount(width)) => cmp::max(width, num_chars), + _ => num_chars, + }; + let fill_chars_needed = width - num_chars; + let fill_string = CFormatSpec::compute_fill_string(fill_char, fill_chars_needed); + + if !fill_string.is_empty() { + if self.flags.contains(CConversionFlags::LEFT_ADJUST) { + format!("{}{}", string, fill_string) + } else { + format!("{}{}", fill_string, string) + } + } else { + string + } + } + + pub fn format_string(&self, string: String) -> String { + let mut string = string; + // truncate if needed + if let Some(CFormatQuantity::Amount(precision)) = self.precision { + if string.chars().count() > precision { + string = string.chars().take(precision).collect::(); + } + } + self.fill_string(string, ' ', None) + } + + pub fn format_number(&self, num: &BigInt) -> String { + use CFormatCase::{Lowercase, Uppercase}; + use CNumberType::*; + let magnitude = num.abs(); + let prefix = if self.flags.contains(CConversionFlags::ALTERNATE_FORM) { + match self.format_type { + CFormatType::Number(Octal) => "0o", + CFormatType::Number(Hex(Lowercase)) => "0x", + CFormatType::Number(Hex(Uppercase)) => "0X", + _ => "", + } + } else { + "" + }; + + let magnitude_string: String = match self.format_type { + CFormatType::Number(Decimal) => magnitude.to_str_radix(10), + CFormatType::Number(Octal) => magnitude.to_str_radix(8), + CFormatType::Number(Hex(Lowercase)) => magnitude.to_str_radix(16), + CFormatType::Number(Hex(Uppercase)) => { + let mut result = magnitude.to_str_radix(16); + result.make_ascii_uppercase(); + result + } + _ => unreachable!(), // Should not happen because caller has to make sure that this is a number + }; + + let sign_string = match num.sign() { + Sign::Minus => "-", + _ => { + if self.flags.contains(CConversionFlags::SIGN_CHAR) { + "+" + } else if self.flags.contains(CConversionFlags::BLANK_SIGN) { + " " + } else { + "" + } + } + }; + + let prefix = format!("{}{}", sign_string, prefix); + + if self.flags.contains(CConversionFlags::ZERO_PAD) { + let fill_char = if !self.flags.contains(CConversionFlags::LEFT_ADJUST) { + '0' + } else { + ' ' // '-' overrides the '0' conversion if both are given + }; + format!( + "{}{}", + prefix, + self.fill_string(magnitude_string, fill_char, Some(prefix.chars().count())) + ) + } else { + self.fill_string(format!("{}{}", prefix, magnitude_string), ' ', None) + } + } +} + +#[derive(Debug, PartialEq)] +pub enum CFormatPart { + Literal(String), + Spec(CFormatSpec), +} + +impl CFormatPart { + pub fn is_specifier(&self) -> bool { + match self { + CFormatPart::Spec(_) => true, + _ => false, + } + } + + pub fn has_key(&self) -> bool { + match self { + CFormatPart::Spec(s) => s.mapping_key.is_some(), + _ => false, + } + } +} + +#[derive(Debug, PartialEq)] +pub struct CFormatString { + pub format_parts: Vec<(usize, CFormatPart)>, +} + +impl FromStr for CFormatString { + type Err = CFormatError; + + fn from_str(text: &str) -> Result { + let mut cur_text: &str = text; + let mut index = 0; + let mut parts: Vec<(usize, CFormatPart)> = Vec::new(); + while !cur_text.is_empty() { + cur_text = parse_literal(cur_text) + .or_else(|_| parse_specifier(cur_text)) + .map(|(format_part, new_text, consumed)| { + parts.push((index, format_part)); + index = index + consumed; + new_text + }) + .map_err(|(e, consumed)| CFormatError { + typ: e, + index: index + consumed, + })?; + } + + Ok(CFormatString { + format_parts: parts, + }) + } +} + +fn parse_quantity(text: &str) -> (Option, &str) { + let num_digits: usize = get_num_digits(text); + if num_digits == 0 { + let mut chars = text.chars(); + return match chars.next() { + Some('*') => (Some(CFormatQuantity::FromValuesTuple), chars.as_str()), + _ => (None, text), + }; + } + // This should never fail + ( + Some(CFormatQuantity::Amount( + text[..num_digits].parse::().unwrap(), + )), + &text[num_digits..], + ) +} + +fn parse_precision(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('.') => parse_quantity(&chars.as_str()), + _ => (None, text), + } +} + +fn parse_literal_single(text: &str) -> Result<(char, &str), CFormatErrorType> { + let mut chars = text.chars(); + // TODO get rid of the unwrap + let first_char = chars.next().unwrap(); + if first_char == '%' { + // if we see a %, it has to be escaped + match chars.next() { + Some(next_char) => { + if next_char != first_char { + Err(CFormatErrorType::UnescapedModuloSignInLiteral) + } else { + Ok((first_char, chars.as_str())) + } + } + None => Err(CFormatErrorType::IncompleteFormat), + } + } else { + Ok((first_char, chars.as_str())) + } +} + +fn parse_literal(text: &str) -> Result<(CFormatPart, &str, usize), ParsingError> { + let mut cur_text = text; + let mut result_string = String::new(); + let mut consumed = 0; + while !cur_text.is_empty() { + match parse_literal_single(cur_text) { + Ok((next_char, remaining)) => { + result_string.push(next_char); + consumed = consumed + 1; + cur_text = remaining; + } + Err(err) => { + if !result_string.is_empty() { + return Ok(( + CFormatPart::Literal(result_string.to_string()), + cur_text, + consumed, + )); + } else { + return Err((err, consumed)); + } + } + } + } + Ok(( + CFormatPart::Literal(result_string.to_string()), + "", + text.chars().count(), + )) +} + +fn parse_text_inside_parentheses(text: &str) -> Option<(String, &str)> { + let mut counter = 1; + let mut chars = text.chars(); + let mut contained_text = String::new(); + while counter > 0 { + let c = chars.next(); + + match c { + Some('(') => { + counter += 1; + } + Some(')') => { + counter -= 1; + } + None => { + return None; + } + _ => (), + } + + if counter > 0 { + contained_text.push(c.unwrap()); + } + } + + Some((contained_text, chars.as_str())) +} + +fn parse_spec_mapping_key(text: &str) -> Result<(Option, &str), CFormatErrorType> { + let mut chars = text.chars(); + + let next_char = chars.next(); + if next_char == Some('(') { + match parse_text_inside_parentheses(chars.as_str()) { + Some((key, remaining_text)) => Ok((Some(key), remaining_text)), + None => Err(CFormatErrorType::UnmatchedKeyParentheses), + } + } else { + Ok((None, text)) + } +} + +fn parse_flag_single(text: &str) -> (Option, &str) { + let mut chars = text.chars(); + match chars.next() { + Some('#') => (Some(CConversionFlags::ALTERNATE_FORM), chars.as_str()), + Some('0') => (Some(CConversionFlags::ZERO_PAD), chars.as_str()), + Some('-') => (Some(CConversionFlags::LEFT_ADJUST), chars.as_str()), + Some(' ') => (Some(CConversionFlags::BLANK_SIGN), chars.as_str()), + Some('+') => (Some(CConversionFlags::SIGN_CHAR), chars.as_str()), + _ => (None, text), + } +} + +fn parse_flags(text: &str) -> (CConversionFlags, &str) { + let mut flags = CConversionFlags::empty(); + let mut cur_text = text; + while !cur_text.is_empty() { + match parse_flag_single(cur_text) { + (Some(flag), text) => { + flags |= flag; + cur_text = text; + } + + (None, text) => { + return (flags, text); + } + } + } + + (flags, "") +} + +fn consume_length(text: &str) -> &str { + let mut chars = text.chars(); + match chars.next() { + Some('h') | Some('l') | Some('L') => chars.as_str(), + _ => text, + } +} + +fn parse_format_type(text: &str) -> Result<(CFormatType, &str, char), CFormatErrorType> { + use CFloatType::*; + use CFormatCase::{Lowercase, Uppercase}; + use CNumberType::*; + let mut chars = text.chars(); + let next_char = chars.next(); + match next_char { + Some('d') | Some('i') | Some('u') => Ok(( + CFormatType::Number(Decimal), + chars.as_str(), + next_char.unwrap(), + )), + Some('o') => Ok(( + CFormatType::Number(Octal), + chars.as_str(), + next_char.unwrap(), + )), + Some('x') => Ok(( + CFormatType::Number(Hex(Lowercase)), + chars.as_str(), + next_char.unwrap(), + )), + Some('X') => Ok(( + CFormatType::Number(Hex(Uppercase)), + chars.as_str(), + next_char.unwrap(), + )), + Some('e') => Ok(( + CFormatType::Float(Exponent(Lowercase)), + chars.as_str(), + next_char.unwrap(), + )), + Some('E') => Ok(( + CFormatType::Float(Exponent(Uppercase)), + chars.as_str(), + next_char.unwrap(), + )), + Some('f') => Ok(( + CFormatType::Float(PointDecimal), + chars.as_str(), + next_char.unwrap(), + )), + Some('F') => Ok(( + CFormatType::Float(PointDecimal), + chars.as_str(), + next_char.unwrap(), + )), + Some('g') => Ok(( + CFormatType::Float(General(Lowercase)), + text, + next_char.unwrap(), + )), + Some('G') => Ok(( + CFormatType::Float(General(Uppercase)), + text, + next_char.unwrap(), + )), + Some('c') => Ok((CFormatType::Character, chars.as_str(), next_char.unwrap())), + Some('r') => Ok(( + CFormatType::String(CFormatPreconversor::Repr), + chars.as_str(), + next_char.unwrap(), + )), + Some('s') => Ok(( + CFormatType::String(CFormatPreconversor::Str), + chars.as_str(), + next_char.unwrap(), + )), + Some('a') => Ok(( + CFormatType::String(CFormatPreconversor::Ascii), + chars.as_str(), + next_char.unwrap(), + )), + Some(c) => Err(CFormatErrorType::UnsupportedFormatChar(c)), + None => Err(CFormatErrorType::IncompleteFormat), // should not happen because it is handled earlier in the parsing + } +} + +fn calc_consumed(a: &str, b: &str) -> usize { + a.chars().count() - b.chars().count() +} + +impl FromStr for CFormatSpec { + type Err = ParsingError; + + fn from_str(text: &str) -> Result { + let mut chars = text.chars(); + if chars.next() != Some('%') { + return Err((CFormatErrorType::MissingModuloSign, 1)); + } + + let after_modulo_sign = chars.as_str(); + let (mapping_key, after_mapping_key) = parse_spec_mapping_key(after_modulo_sign) + .map_err(|err| (err, calc_consumed(text, after_modulo_sign)))?; + let (flags, after_flags) = parse_flags(after_mapping_key); + let (width, after_width) = parse_quantity(after_flags); + let (precision, after_precision) = parse_precision(after_width); + // A length modifier (h, l, or L) may be present, + // but is ignored as it is not necessary for Python – so e.g. %ld is identical to %d. + let after_length = consume_length(after_precision); + let (format_type, remaining_text, format_char) = parse_format_type(after_length) + .map_err(|err| (err, calc_consumed(text, after_length)))?; + + // apply default precision for float types + let precision = match precision { + Some(precision) => Some(precision), + None => match format_type { + CFormatType::Float(_) => Some(CFormatQuantity::Amount(6)), + _ => None, + }, + }; + + Ok(CFormatSpec { + mapping_key: mapping_key, + flags: flags, + min_field_width: width, + precision: precision, + format_type: format_type, + format_char: format_char, + chars_consumed: calc_consumed(text, remaining_text), + }) + } +} + +fn parse_specifier(text: &str) -> Result<(CFormatPart, &str, usize), ParsingError> { + let spec = text.parse::()?; + let chars_consumed = spec.chars_consumed; + Ok(( + CFormatPart::Spec(spec), + &text[chars_consumed..], + chars_consumed, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_fill_and_align() { + assert_eq!( + "%10s" + .parse::() + .unwrap() + .format_string("test".to_string()), + " test".to_string() + ); + assert_eq!( + "%-10s" + .parse::() + .unwrap() + .format_string("test".to_string()), + "test ".to_string() + ); + assert_eq!( + "%#10x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + " 0x1337".to_string() + ); + assert_eq!( + "%-#10x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_string() + ); + } + + #[test] + fn test_parse_key() { + let expected = Ok(CFormatSpec { + mapping_key: Some("amount".to_string()), + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + chars_consumed: 10, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }); + assert_eq!("%(amount)d".parse::(), expected); + + let expected = Ok(CFormatSpec { + mapping_key: Some("m((u(((l((((ti))))p)))l))e".to_string()), + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + chars_consumed: 30, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }); + assert_eq!( + "%(m((u(((l((((ti))))p)))l))e)d".parse::(), + expected + ); + } + + #[test] + fn test_format_parse_key_fail() { + assert_eq!( + "%(aged".parse::(), + Err(CFormatError { + typ: CFormatErrorType::UnmatchedKeyParentheses, + index: 1 + }) + ); + } + + #[test] + fn test_format_parse_type_fail() { + assert_eq!( + "Hello %n".parse::(), + Err(CFormatError { + typ: CFormatErrorType::UnsupportedFormatChar('n'), + index: 7 + }) + ); + } + + #[test] + fn test_incomplete_format_fail() { + assert_eq!( + "Hello %".parse::(), + Err(CFormatError { + typ: CFormatErrorType::IncompleteFormat, + index: 7 + }) + ); + } + + #[test] + fn test_parse_flags() { + let expected = Ok(CFormatSpec { + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + chars_consumed: 17, + min_field_width: Some(CFormatQuantity::Amount(10)), + precision: None, + mapping_key: None, + flags: CConversionFlags::all(), + }); + let parsed = "% 0 -+++###10d".parse::(); + assert_eq!(parsed, expected); + assert_eq!( + parsed.unwrap().format_number(&BigInt::from(12)), + "+12 ".to_string() + ); + } + + #[test] + fn test_parse_and_format_string() { + assert_eq!( + "%5.4s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_string()), + " Hell".to_string() + ); + assert_eq!( + "%-5.4s" + .parse::() + .unwrap() + .format_string("Hello, World!".to_string()), + "Hell ".to_string() + ); + } + + #[test] + fn test_parse_and_format_unicode_string() { + assert_eq!( + "%.2s" + .parse::() + .unwrap() + .format_string("❤❤❤❤❤❤❤❤".to_string()), + "❤❤".to_string() + ); + } + + #[test] + fn test_parse_and_format_number() { + assert_eq!( + "%05d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "00027".to_string() + ); + assert_eq!( + "%+05d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + "+0027".to_string() + ); + assert_eq!( + "%-d" + .parse::() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_string() + ); + assert_eq!( + "% d" + .parse::() + .unwrap() + .format_number(&BigInt::from(27)), + " 27".to_string() + ); + assert_eq!( + "% d" + .parse::() + .unwrap() + .format_number(&BigInt::from(-27)), + "-27".to_string() + ); + assert_eq!( + "%08x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "00001337".to_string() + ); + assert_eq!( + "%#010x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x00001337".to_string() + ); + assert_eq!( + "%-#010x" + .parse::() + .unwrap() + .format_number(&BigInt::from(0x1337)), + "0x1337 ".to_string() + ); + } + + #[test] + fn test_format_parse() { + let fmt = "Hello, my name is %s and I'm %d years old"; + let expected = Ok(CFormatString { + format_parts: vec![ + (0, CFormatPart::Literal("Hello, my name is ".to_string())), + ( + 18, + CFormatPart::Spec(CFormatSpec { + format_type: CFormatType::String(CFormatPreconversor::Str), + format_char: 's', + chars_consumed: 2, + mapping_key: None, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }), + ), + (20, CFormatPart::Literal(" and I'm ".to_string())), + ( + 29, + CFormatPart::Spec(CFormatSpec { + format_type: CFormatType::Number(CNumberType::Decimal), + format_char: 'd', + chars_consumed: 2, + mapping_key: None, + min_field_width: None, + precision: None, + flags: CConversionFlags::empty(), + }), + ), + (31, CFormatPart::Literal(" years old".to_string())), + ], + }); + let result = fmt.parse::(); + assert_eq!( + result, expected, + "left = {:#?} \n\n\n right = {:#?}", + result, expected + ); + } +} diff --git a/vm/src/dictdatatype.rs b/vm/src/dictdatatype.rs index 436923ea95..e52e83b8c5 100644 --- a/vm/src/dictdatatype.rs +++ b/vm/src/dictdatatype.rs @@ -1,42 +1,77 @@ use crate::obj::objbool; -use crate::obj::objint; +use crate::pyhash; use crate::pyobject::{IdProtocol, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -use num_traits::ToPrimitive; /// Ordered dictionary implementation. /// Inspired by: https://morepypy.blogspot.com/2015/01/faster-more-memory-efficient-and-more.html /// And: https://www.youtube.com/watch?v=p33CVV29OG8 /// And: http://code.activestate.com/recipes/578375/ -use std::collections::HashMap; +use std::collections::{hash_map::DefaultHasher, HashMap}; +use std::hash::{Hash, Hasher}; -pub struct Dict { - size: usize, - indices: HashMap, - entries: Vec>, -} +/// hash value of an object returned by __hash__ +type HashValue = pyhash::PyHash; +/// index calculated by resolving collision +type HashIndex = pyhash::PyHash; +/// entry index mapped in indices +type EntryIndex = usize; -struct DictEntry { - hash: usize, - key: PyObjectRef, - value: PyObjectRef, +#[derive(Clone)] +pub struct Dict { + size: usize, + indices: HashMap, + entries: Vec>>, } -impl Dict { - pub fn new() -> Self { +impl Default for Dict { + fn default() -> Self { Dict { size: 0, indices: HashMap::new(), entries: Vec::new(), } } +} - /// Store a key - pub fn insert( +#[derive(Clone)] +struct DictEntry { + hash: HashValue, + key: PyObjectRef, + value: T, +} + +#[derive(Debug)] +pub struct DictSize { + size: usize, + entries_size: usize, +} + +impl Dict { + fn unchecked_push( &mut self, - vm: &VirtualMachine, + hash_index: HashIndex, + hash_value: HashValue, key: &PyObjectRef, - value: PyObjectRef, - ) -> PyResult<()> { + value: T, + ) { + let entry = DictEntry { + hash: hash_value, + key: key.clone(), + value, + }; + let entry_index = self.entries.len(); + self.entries.push(Some(entry)); + self.indices.insert(hash_index, entry_index); + self.size += 1; + } + + fn unchecked_delete(&mut self, entry_index: EntryIndex) { + self.entries[entry_index] = None; + self.size -= 1; + } + + /// Store a key + pub fn insert(&mut self, vm: &VirtualMachine, key: &PyObjectRef, value: T) -> PyResult<()> { match self.lookup(vm, key)? { LookupResult::Existing(index) => { // Update existing key @@ -52,54 +87,77 @@ impl Dict { hash_value, } => { // New key: - let entry = DictEntry { - hash: hash_value, - key: key.clone(), - value, - }; - let index = self.entries.len(); - self.entries.push(Some(entry)); - self.indices.insert(hash_index, index); - self.size += 1; + self.unchecked_push(hash_index, hash_value, key, value); Ok(()) } } } pub fn contains(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - if let LookupResult::Existing(_index) = self.lookup(vm, key)? { + if let LookupResult::Existing(_) = self.lookup(vm, key)? { Ok(true) } else { Ok(false) } } + fn unchecked_get(&self, index: EntryIndex) -> T { + if let Some(entry) = &self.entries[index] { + entry.value.clone() + } else { + panic!("Lookup returned invalid index into entries!"); + } + } + /// Retrieve a key - pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + pub fn get(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult> { if let LookupResult::Existing(index) = self.lookup(vm, key)? { - if let Some(entry) = &self.entries[index] { - Ok(entry.value.clone()) - } else { - panic!("Lookup returned invalid index into entries!"); - } + Ok(Some(self.unchecked_get(index))) } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Ok(None) } } + pub fn clear(&mut self) { + self.entries.clear(); + self.indices.clear(); + self.size = 0 + } + /// Delete a key pub fn delete(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult<()> { - if let LookupResult::Existing(index) = self.lookup(vm, key)? { - self.entries[index] = None; - self.size -= 1; + if self.delete_if_exists(vm, key)? { Ok(()) } else { - let key_repr = vm.to_pystr(key)?; - Err(vm.new_value_error(format!("Key not found: {}", key_repr))) + Err(vm.new_key_error(key.clone())) } } + pub fn delete_if_exists(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + if let LookupResult::Existing(entry_index) = self.lookup(vm, key)? { + self.unchecked_delete(entry_index); + Ok(true) + } else { + Ok(false) + } + } + + pub fn delete_or_insert( + &mut self, + vm: &VirtualMachine, + key: &PyObjectRef, + value: T, + ) -> PyResult<()> { + match self.lookup(vm, &key)? { + LookupResult::Existing(entry_index) => self.unchecked_delete(entry_index), + LookupResult::NewIndex { + hash_value, + hash_index, + } => self.unchecked_push(hash_index, hash_value, &key, value), + }; + Ok(()) + } + pub fn len(&self) -> usize { self.size } @@ -108,20 +166,41 @@ impl Dict { self.len() == 0 } - pub fn get_items(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - self.entries - .iter() - .filter(|e| e.is_some()) - .map(|e| e.as_ref().unwrap()) - .map(|e| (e.key.clone(), e.value.clone())) - .collect() + pub fn size(&self) -> DictSize { + DictSize { + size: self.size, + entries_size: self.entries.len(), + } + } + + pub fn next_entry(&self, position: &mut EntryIndex) -> Option<(&PyObjectRef, &T)> { + while *position < self.entries.len() { + if let Some(DictEntry { key, value, .. }) = &self.entries[*position] { + *position += 1; + return Some((key, value)); + } + *position += 1; + } + None + } + + pub fn has_changed_size(&self, position: &DictSize) -> bool { + position.size != self.size || self.entries.len() != position.entries_size + } + + pub fn keys<'a>(&'a self) -> Box + 'a> { + Box::new( + self.entries + .iter() + .filter_map(|v| v.as_ref().and_then(|v| Some(v.key.clone()))), + ) } /// Lookup the index for the given key. fn lookup(&self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - let hash_value = calc_hash(vm, key)?; + let hash_value = collection_hash(vm, key)?; let perturb = hash_value; - let mut hash_index: usize = hash_value; + let mut hash_index: HashIndex = hash_value; loop { if self.indices.contains_key(&hash_index) { // Now we have an index, lets check the key. @@ -159,29 +238,50 @@ impl Dict { // warn!("Perturb value: {}", i); } } + + /// Retrieve and delete a key + pub fn pop(&mut self, vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { + if let LookupResult::Existing(index) = self.lookup(vm, key)? { + let value = self.unchecked_get(index); + self.unchecked_delete(index); + Ok(value) + } else { + Err(vm.new_key_error(key.clone())) + } + } + + pub fn pop_front(&mut self) -> Option<(PyObjectRef, T)> { + let mut entry_index = 0; + match self.next_entry(&mut entry_index) { + Some((key, value)) => { + let item = (key.clone(), value.clone()); + self.unchecked_delete(entry_index - 1); + Some(item) + } + None => None, + } + } } enum LookupResult { NewIndex { - hash_value: usize, - hash_index: usize, + hash_value: HashValue, + hash_index: HashIndex, }, // return not found, index into indices - Existing(usize), // Existing record, index into entries + Existing(EntryIndex), // Existing record, index into entries } -fn calc_hash(vm: &VirtualMachine, key: &PyObjectRef) -> PyResult { - let hash = vm.call_method(key, "__hash__", vec![])?; - Ok(objint::get_value(&hash).to_usize().unwrap()) +fn collection_hash(vm: &VirtualMachine, object: &PyObjectRef) -> PyResult { + let raw_hash = vm._hash(object)?; + let mut hasher = DefaultHasher::new(); + raw_hash.hash(&mut hasher); + Ok(hasher.finish() as HashValue) } /// Invoke __eq__ on two keys -fn do_eq( - vm: &VirtualMachine, - key1: &PyObjectRef, - key2: &PyObjectRef, -) -> Result { - let result = vm._eq(key1, key2.clone())?; - Ok(objbool::get_value(&result)) +fn do_eq(vm: &VirtualMachine, key1: &PyObjectRef, key2: &PyObjectRef) -> Result { + let result = vm._eq(key1.clone(), key2.clone())?; + objbool::boolval(vm, result) } #[cfg(test)] @@ -191,7 +291,7 @@ mod tests { #[test] fn test_insert() { let mut vm = VirtualMachine::new(); - let mut dict = Dict::new(); + let mut dict = Dict::default(); assert_eq!(0, dict.len()); let key1 = vm.new_bool(true); diff --git a/vm/src/error.rs b/vm/src/error.rs deleted file mode 100644 index 06eb387e2e..0000000000 --- a/vm/src/error.rs +++ /dev/null @@ -1,46 +0,0 @@ -use rustpython_parser::error::ParseError; - -use std::error::Error; -use std::fmt; - -#[derive(Debug)] -pub enum CompileError { - /// Invalid assignment, cannot store value in target. - Assign(&'static str), - /// Invalid delete - Delete(&'static str), - /// Expected an expression got a statement - ExpectExpr, - /// Parser error - Parse(ParseError), - /// Multiple `*` detected - StarArgs, - /// Break statement outside of loop. - InvalidBreak, - /// Continue statement outside of loop. - InvalidContinue, - InvalidReturn, - InvalidYield, -} - -impl fmt::Display for CompileError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - CompileError::Assign(target) => write!(f, "can't assign to {}", target), - CompileError::Delete(target) => write!(f, "can't delete {}", target), - CompileError::ExpectExpr => write!(f, "Expecting expression, got statement"), - CompileError::Parse(err) => write!(f, "{}", err), - CompileError::StarArgs => write!(f, "Two starred expressions in assignment"), - CompileError::InvalidBreak => write!(f, "'break' outside loop"), - CompileError::InvalidContinue => write!(f, "'continue' outside loop"), - CompileError::InvalidReturn => write!(f, "'return' outside function"), - CompileError::InvalidYield => write!(f, "'yield' outside function"), - } - } -} - -impl Error for CompileError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - None - } -} diff --git a/vm/src/eval.rs b/vm/src/eval.rs index 202286eb13..f8ca7bb7b3 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,27 +1,15 @@ -extern crate rustpython_parser; - -use std::error::Error; - -use crate::compile; use crate::frame::Scope; use crate::pyobject::PyResult; use crate::vm::VirtualMachine; +use rustpython_compiler::compile; pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult { - match compile::compile( - source, - &compile::Mode::Eval, - source_path.to_string(), - vm.ctx.code_type(), - ) { + match vm.compile(source, &compile::Mode::Eval, source_path.to_string()) { Ok(bytecode) => { debug!("Code object: {:?}", bytecode); vm.run_code_obj(bytecode, scope) } - Err(err) => { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - Err(vm.new_exception(syntax_error, err.description().to_string())) - } + Err(err) => Err(vm.new_syntax_error(&err)), } } @@ -34,7 +22,7 @@ mod tests { fn test_print_42() { let source = String::from("print('Hello world')\n"); let mut vm = VirtualMachine::new(); - let vars = vm.ctx.new_scope(); + let vars = vm.new_scope_with_builtins(); let _result = eval(&mut vm, &source, vars, ""); // TODO: check result? diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index d37f174717..cfd611c3ff 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -1,63 +1,151 @@ use crate::function::PyFuncArgs; use crate::obj::objsequence; +use crate::obj::objtuple::{PyTuple, PyTupleRef}; use crate::obj::objtype; -use crate::pyobject::{create_type, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{create_type, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use std::fs::File; +use std::io::{BufRead, BufReader}; fn exception_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let zelf = args.args[0].clone(); - let msg = if args.args.len() > 1 { - args.args[1].clone() - } else { - vm.new_str("No msg".to_string()) - }; + let exc_self = args.args[0].clone(); + let exc_args = vm.ctx.new_tuple(args.args[1..].to_vec()); + vm.set_attr(&exc_self, "args", exc_args)?; + let traceback = vm.ctx.new_list(Vec::new()); - vm.ctx.set_attr(&zelf, "msg", msg); - vm.ctx.set_attr(&zelf, "__traceback__", traceback); + vm.set_attr(&exc_self, "__traceback__", traceback)?; Ok(vm.get_none()) } -// Print exception including traceback: +/// Print exception chain pub fn print_exception(vm: &VirtualMachine, exc: &PyObjectRef) { + let mut had_cause = false; + if let Ok(cause) = vm.get_attribute(exc.clone(), "__cause__") { + if !vm.get_none().is(&cause) { + had_cause = true; + print_exception(vm, &cause); + println!("\nThe above exception was the direct cause of the following exception:\n"); + } + } + if !had_cause { + if let Ok(context) = vm.get_attribute(exc.clone(), "__context__") { + if !vm.get_none().is(&context) { + print_exception(vm, &context); + println!("\nDuring handling of the above exception, another exception occurred:\n"); + } + } + } + print_exception_inner(vm, exc) +} + +fn print_source_line(filename: String, lineno: usize) { + // TODO: use io.open() method instead, when available, according to https://github.com/python/cpython/blob/master/Python/traceback.c#L393 + // TODO: support different encodings + let file = match File::open(filename) { + Ok(file) => file, + Err(_) => { + return; + } + }; + let file = BufReader::new(file); + + for (i, line) in file.lines().enumerate() { + if i + 1 == lineno { + if let Ok(line) = line { + // Indented with 4 spaces + println!(" {}", line.trim_start()); + } + return; + } + } +} + +/// Print exception occurrence location from traceback element +fn print_traceback_entry(vm: &VirtualMachine, tb_entry: &PyObjectRef) { + if objtype::isinstance(&tb_entry, &vm.ctx.tuple_type()) { + let location_attrs = objsequence::get_elements_tuple(&tb_entry); + let filename = if let Ok(x) = vm.to_str(&location_attrs[0]) { + x.value.clone() + } else { + "".to_string() + }; + + let lineno = if let Ok(x) = vm.to_str(&location_attrs[1]) { + x.value.clone() + } else { + "".to_string() + }; + + let obj_name = if let Ok(x) = vm.to_str(&location_attrs[2]) { + x.value.clone() + } else { + "".to_string() + }; + + println!( + r##" File "{}", line {}, in {}"##, + filename, lineno, obj_name + ); + print_source_line(filename, lineno.parse().unwrap()); + } else { + println!(" File ??"); + return; + } +} + +/// Print exception with traceback +pub fn print_exception_inner(vm: &VirtualMachine, exc: &PyObjectRef) { if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { println!("Traceback (most recent call last):"); if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let mut elements = objsequence::get_elements(&tb).to_vec(); - elements.reverse(); - for element in elements.iter() { - if objtype::isinstance(&element, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements(&element); - let filename = if let Ok(x) = vm.to_str(&element[0]) { - x.value.clone() - } else { - "".to_string() - }; - - let lineno = if let Ok(x) = vm.to_str(&element[1]) { - x.value.clone() - } else { - "".to_string() - }; - - let obj_name = if let Ok(x) = vm.to_str(&element[2]) { - x.value.clone() - } else { - "".to_string() - }; - - println!(" File {}, line {}, in {}", filename, lineno, obj_name); - } else { - println!(" File ??"); - } + let mut tb_entries = objsequence::get_elements_list(&tb).to_vec(); + tb_entries.reverse(); + + for exc_location in tb_entries.iter() { + print_traceback_entry(vm, exc_location); } } } else { println!("No traceback set on exception"); } - match vm.to_str(exc) { - Ok(txt) => println!("{}", txt.value), - Err(err) => println!("Error during error {:?}", err), + let varargs = vm + .get_attribute(exc.clone(), "args") + .unwrap() + .downcast::() + .expect("'args' must be a tuple"); + let args_repr = exception_args_as_string(vm, varargs); + + let exc_name = exc.class().name.clone(); + match args_repr.len() { + 0 => println!("{}", exc_name), + 1 => println!("{}: {}", exc_name, args_repr[0]), + _ => println!("{}: ({})", exc_name, args_repr.join(", ")), + } +} + +fn exception_args_as_string(vm: &VirtualMachine, varargs: PyTupleRef) -> Vec { + match varargs.elements.len() { + 0 => vec![], + 1 => { + let args0_repr = match vm.to_repr(&varargs.elements[0]) { + Ok(args0_repr) => args0_repr.value.clone(), + Err(_) => "".to_string(), + }; + vec![args0_repr] + } + _ => { + let mut args_vec = Vec::with_capacity(varargs.elements.len()); + for vararg in &varargs.elements { + let arg_repr = match vm.to_repr(vararg) { + Ok(arg_repr) => arg_repr.value.clone(), + Err(_) => "".to_string(), + }; + args_vec.push(arg_repr); + } + args_vec + } } } @@ -67,46 +155,84 @@ fn exception_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { args, required = [(exc, Some(vm.ctx.exceptions.exception_type.clone()))] ); - let type_name = objtype::get_type_name(&exc.typ()); - let msg = if let Ok(m) = vm.get_attribute(exc.clone(), "msg") { - match vm.to_pystr(&m) { - Ok(msg) => msg, - _ => "".to_string(), - } - } else { - panic!("Error message must be set"); + let args = vm + .get_attribute(exc.clone(), "args") + .unwrap() + .downcast::() + .expect("'args' must be a tuple"); + let args_str = exception_args_as_string(vm, args); + let joined_str = match args_str.len() { + 0 => "".to_string(), + 1 => args_str[0].to_string(), + _ => format!("({})", args_str.join(", ")), }; - let s = format!("{}: {}", type_name, msg); - Ok(vm.new_str(s)) + Ok(vm.new_str(joined_str)) +} + +fn exception_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(exc, Some(vm.ctx.exceptions.exception_type.clone()))] + ); + let args = vm + .get_attribute(exc.clone(), "args") + .unwrap() + .downcast::() + .expect("'args' must be a tuple"); + let args_repr = exception_args_as_string(vm, args); + + let exc_name = exc.class().name.clone(); + let joined_str = match args_repr.len() { + 0 => format!("{}()", exc_name), + 1 => format!("{}({},)", exc_name, args_repr[0]), + _ => format!("{}({})", exc_name, args_repr.join(", ")), + }; + Ok(vm.new_str(joined_str)) } #[derive(Debug)] pub struct ExceptionZoo { - pub arithmetic_error: PyObjectRef, - pub assertion_error: PyObjectRef, - pub attribute_error: PyObjectRef, - pub base_exception_type: PyObjectRef, - pub exception_type: PyObjectRef, - pub file_not_found_error: PyObjectRef, - pub import_error: PyObjectRef, - pub index_error: PyObjectRef, - pub key_error: PyObjectRef, - pub module_not_found_error: PyObjectRef, - pub name_error: PyObjectRef, - pub not_implemented_error: PyObjectRef, - pub os_error: PyObjectRef, - pub overflow_error: PyObjectRef, - pub permission_error: PyObjectRef, - pub runtime_error: PyObjectRef, - pub stop_iteration: PyObjectRef, - pub syntax_error: PyObjectRef, - pub type_error: PyObjectRef, - pub value_error: PyObjectRef, - pub zero_division_error: PyObjectRef, + pub arithmetic_error: PyClassRef, + pub assertion_error: PyClassRef, + pub attribute_error: PyClassRef, + pub base_exception_type: PyClassRef, + pub exception_type: PyClassRef, + pub file_not_found_error: PyClassRef, + pub file_exists_error: PyClassRef, + pub import_error: PyClassRef, + pub index_error: PyClassRef, + pub key_error: PyClassRef, + pub module_not_found_error: PyClassRef, + pub name_error: PyClassRef, + pub not_implemented_error: PyClassRef, + pub os_error: PyClassRef, + pub overflow_error: PyClassRef, + pub permission_error: PyClassRef, + pub reference_error: PyClassRef, + pub runtime_error: PyClassRef, + pub stop_iteration: PyClassRef, + pub syntax_error: PyClassRef, + pub type_error: PyClassRef, + pub value_error: PyClassRef, + pub zero_division_error: PyClassRef, + pub eof_error: PyClassRef, + + pub warning: PyClassRef, + pub bytes_warning: PyClassRef, + pub unicode_warning: PyClassRef, + pub deprecation_warning: PyClassRef, + pub pending_deprecation_warning: PyClassRef, + pub future_warning: PyClassRef, + pub import_warning: PyClassRef, + pub syntax_warning: PyClassRef, + pub resource_warning: PyClassRef, + pub runtime_warning: PyClassRef, + pub user_warning: PyClassRef, } impl ExceptionZoo { - pub fn new(type_type: &PyObjectRef, object_type: &PyObjectRef) -> Self { + pub fn new(type_type: &PyClassRef, object_type: &PyClassRef) -> Self { // Sorted By Hierarchy then alphabetized. let base_exception_type = create_type("BaseException", &type_type, &object_type); let exception_type = create_type("Exception", &type_type, &base_exception_type); @@ -119,6 +245,7 @@ impl ExceptionZoo { let name_error = create_type("NameError", &type_type, &exception_type); let os_error = create_type("OSError", &type_type, &exception_type); let runtime_error = create_type("RuntimeError", &type_type, &exception_type); + let reference_error = create_type("ReferenceError", &type_type, &exception_type); let stop_iteration = create_type("StopIteration", &type_type, &exception_type); let syntax_error = create_type("SyntaxError", &type_type, &exception_type); let type_error = create_type("TypeError", &type_type, &exception_type); @@ -129,6 +256,21 @@ impl ExceptionZoo { let not_implemented_error = create_type("NotImplementedError", &type_type, &runtime_error); let file_not_found_error = create_type("FileNotFoundError", &type_type, &os_error); let permission_error = create_type("PermissionError", &type_type, &os_error); + let file_exists_error = create_type("FileExistsError", &type_type, &os_error); + let eof_error = create_type("EOFError", &type_type, &exception_type); + + let warning = create_type("Warning", &type_type, &exception_type); + let bytes_warning = create_type("BytesWarning", &type_type, &warning); + let unicode_warning = create_type("UnicodeWarning", &type_type, &warning); + let deprecation_warning = create_type("DeprecationWarning", &type_type, &warning); + let pending_deprecation_warning = + create_type("PendingDeprecationWarning", &type_type, &warning); + let future_warning = create_type("FutureWarning", &type_type, &warning); + let import_warning = create_type("ImportWarning", &type_type, &warning); + let syntax_warning = create_type("SyntaxWarning", &type_type, &warning); + let resource_warning = create_type("ResourceWarning", &type_type, &warning); + let runtime_warning = create_type("RuntimeWarning", &type_type, &warning); + let user_warning = create_type("UserWarning", &type_type, &warning); ExceptionZoo { arithmetic_error, @@ -137,6 +279,7 @@ impl ExceptionZoo { base_exception_type, exception_type, file_not_found_error, + file_exists_error, import_error, index_error, key_error, @@ -152,21 +295,66 @@ impl ExceptionZoo { type_error, value_error, zero_division_error, + eof_error, + warning, + bytes_warning, + unicode_warning, + deprecation_warning, + pending_deprecation_warning, + future_warning, + import_warning, + syntax_warning, + resource_warning, + runtime_warning, + reference_error, + user_warning, } } } +fn import_error_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: call super().__init__(*args) instead + exception_init(vm, args.clone())?; + + let exc_self = args.args[0].clone(); + vm.set_attr( + &exc_self, + "name", + args.kwargs + .get("name") + .cloned() + .unwrap_or_else(|| vm.get_none()), + )?; + vm.set_attr( + &exc_self, + "path", + args.kwargs + .get("path") + .cloned() + .unwrap_or_else(|| vm.get_none()), + )?; + vm.set_attr( + &exc_self, + "msg", + args.args.get(1).cloned().unwrap_or_else(|| vm.get_none()), + )?; + Ok(vm.get_none()) +} + pub fn init(context: &PyContext) { let base_exception_type = &context.exceptions.base_exception_type; - context.set_attr( - &base_exception_type, - "__init__", - context.new_rustfunc(exception_init), - ); + extend_class!(context, base_exception_type, { + "__init__" => context.new_rustfunc(exception_init) + }); + let exception_type = &context.exceptions.exception_type; - context.set_attr( - &exception_type, - "__str__", - context.new_rustfunc(exception_str), - ); + extend_class!(context, exception_type, { + "__str__" => context.new_rustfunc(exception_str), + "__repr__" => context.new_rustfunc(exception_repr), + }); + + let import_error_type = &context.exceptions.import_error; + extend_class!(context, import_error_type, { + "__init__" => context.new_rustfunc(import_error_init) + }); } diff --git a/vm/src/format.rs b/vm/src/format.rs index e89e4f731d..727cbb05b7 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -3,6 +3,49 @@ use num_traits::Signed; use std::cmp; use std::str::FromStr; +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum FormatPreconversor { + Str, + Repr, + Ascii, +} + +impl FormatPreconversor { + pub fn from_char(c: char) -> Option { + match c { + 's' => Some(FormatPreconversor::Str), + 'r' => Some(FormatPreconversor::Repr), + 'a' => Some(FormatPreconversor::Ascii), + _ => None, + } + } + + pub fn from_str(text: &str) -> Option { + let mut chars = text.chars(); + if chars.next() != Some('!') { + return None; + } + + match chars.next() { + None => None, // Should fail instead? + Some(c) => FormatPreconversor::from_char(c), + } + } + + pub fn parse_and_consume(text: &str) -> (Option, &str) { + let preconversor = FormatPreconversor::from_str(text); + match preconversor { + None => (None, text), + Some(_) => { + let mut chars = text.chars(); + chars.next(); // Consume the bang + chars.next(); // Consume one r,s,a char + (preconversor, chars.as_str()) + } + } + } +} + #[derive(Debug, Copy, Clone, PartialEq)] pub enum FormatAlign { Left, @@ -56,6 +99,7 @@ pub enum FormatType { #[derive(Debug, PartialEq)] pub struct FormatSpec { + preconversor: Option, fill: Option, align: Option, sign: Option, @@ -66,7 +110,7 @@ pub struct FormatSpec { format_type: Option, } -fn get_num_digits(text: &str) -> usize { +pub fn get_num_digits(text: &str) -> usize { for (index, character) in text.char_indices() { if !character.is_digit(10) { return index; @@ -75,6 +119,10 @@ fn get_num_digits(text: &str) -> usize { text.len() } +fn parse_preconversor(text: &str) -> (Option, &str) { + FormatPreconversor::parse_and_consume(text) +} + fn parse_align(text: &str) -> (Option, &str) { let mut chars = text.chars(); let maybe_align = chars.next().and_then(FormatAlign::from_char); @@ -186,7 +234,8 @@ fn parse_format_type(text: &str) -> (Option, &str) { } fn parse_format_spec(text: &str) -> FormatSpec { - let (fill, align, after_align) = parse_fill_and_align(text); + let (preconversor, after_preconversor) = parse_preconversor(text); + let (fill, align, after_align) = parse_fill_and_align(after_preconversor); let (sign, after_sign) = parse_sign(after_align); let (alternate_form, after_alternate_form) = parse_alternate_form(after_sign); let after_zero = parse_zero(after_alternate_form); @@ -196,6 +245,7 @@ fn parse_format_spec(text: &str) -> FormatSpec { let (format_type, _) = parse_format_type(after_precision); FormatSpec { + preconversor, fill, align, sign, @@ -467,6 +517,18 @@ impl FormatString { String::new() }; + // On parts[0] can still be the preconversor (!r, !s, !a) + let parts: Vec<&str> = arg_part.splitn(2, '!').collect(); + // before the bang is a keyword or arg index, after the comma is maybe a conversor spec. + let arg_part = parts[0]; + + let preconversor_spec = if parts.len() > 1 { + "!".to_string() + parts[1] + } else { + String::new() + }; + let format_spec = preconversor_spec + &format_spec; + if arg_part.is_empty() { return Ok(FormatPart::AutoSpec(format_spec)); } @@ -551,6 +613,7 @@ mod tests { #[test] fn test_width_only() { let expected = FormatSpec { + preconversor: None, fill: None, align: None, sign: None, @@ -566,6 +629,7 @@ mod tests { #[test] fn test_fill_and_width() { let expected = FormatSpec { + preconversor: None, fill: Some('<'), align: Some(FormatAlign::Right), sign: None, @@ -581,6 +645,7 @@ mod tests { #[test] fn test_all() { let expected = FormatSpec { + preconversor: None, fill: Some('<'), align: Some(FormatAlign::Right), sign: Some(FormatSign::Minus), diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 0015557fec..61ea0a78a9 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -2,29 +2,27 @@ use std::cell::RefCell; use std::fmt; use std::rc::Rc; -use num_bigint::BigInt; - -use rustpython_parser::ast; - use crate::builtins; use crate::bytecode; use crate::function::PyFuncArgs; use crate::obj::objbool; -use crate::obj::objbuiltinfunc::PyBuiltinFunction; -use crate::obj::objcode; -use crate::obj::objdict; -use crate::obj::objdict::PyDict; -use crate::obj::objint::PyInt; +use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::{PyDict, PyDictRef}; use crate::obj::objiter; use crate::obj::objlist; use crate::obj::objslice::PySlice; use crate::obj::objstr; +use crate::obj::objstr::PyString; +use crate::obj::objtuple::PyTuple; use crate::obj::objtype; +use crate::obj::objtype::PyClassRef; use crate::pyobject::{ - DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; +use indexmap::IndexMap; +use itertools::Itertools; /* * So a scope is a linked list of scopes. @@ -79,8 +77,8 @@ impl<'a, T> Iterator for Iter<'a, T> { #[derive(Clone)] pub struct Scope { - locals: RcList, - pub globals: PyObjectRef, + locals: RcList, + pub globals: PyDictRef, } impl fmt::Debug for Scope { @@ -91,7 +89,7 @@ impl fmt::Debug for Scope { } impl Scope { - pub fn new(locals: Option, globals: PyObjectRef) -> Scope { + pub fn new(locals: Option, globals: PyDictRef) -> Scope { let locals = match locals { Some(dict) => RcList::new().insert(dict), None => RcList::new(), @@ -99,66 +97,100 @@ impl Scope { Scope { locals, globals } } - pub fn get_locals(&self) -> PyObjectRef { + pub fn with_builtins( + locals: Option, + globals: PyDictRef, + vm: &VirtualMachine, + ) -> Scope { + if !globals.contains_key("__builtins__", vm) { + globals + .clone() + .set_item("__builtins__", vm.builtins.clone(), vm) + .unwrap(); + } + Scope::new(locals, globals) + } + + pub fn get_locals(&self) -> PyDictRef { match self.locals.iter().next() { Some(dict) => dict.clone(), None => self.globals.clone(), } } - pub fn get_only_locals(&self) -> Option { + pub fn get_only_locals(&self) -> Option { self.locals.iter().next().cloned() } - pub fn child_scope_with_locals(&self, locals: PyObjectRef) -> Scope { + pub fn new_child_scope_with_locals(&self, locals: PyDictRef) -> Scope { Scope { locals: self.locals.clone().insert(locals), globals: self.globals.clone(), } } - pub fn child_scope(&self, ctx: &PyContext) -> Scope { - self.child_scope_with_locals(ctx.new_dict()) + pub fn new_child_scope(&self, ctx: &PyContext) -> Scope { + self.new_child_scope_with_locals(ctx.new_dict()) } } pub trait NameProtocol { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option; fn store_name(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); - fn delete_name(&self, vm: &VirtualMachine, name: &str); + fn delete_name(&self, vm: &VirtualMachine, name: &str) -> PyResult; fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option; + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef); } impl NameProtocol for Scope { fn load_name(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter() { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } - if let Some(value) = self.globals.get_item(name) { + if let Some(value) = self.globals.get_item_option(name, vm).unwrap() { return Some(value); } - vm.builtins.get_item(name) + vm.get_attribute(vm.builtins.clone(), name).ok() } - fn load_cell(&self, _vm: &VirtualMachine, name: &str) -> Option { + fn load_cell(&self, vm: &VirtualMachine, name: &str) -> Option { for dict in self.locals.iter().skip(1) { - if let Some(value) = dict.get_item(name) { + if let Some(value) = dict.get_item_option(name, vm).unwrap() { return Some(value); } } None } + fn store_cell(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.locals + .iter() + .nth(1) + .expect("no outer scope for non-local") + .set_item(name, value, vm) + .unwrap(); + } + fn store_name(&self, vm: &VirtualMachine, key: &str, value: PyObjectRef) { - self.get_locals().set_item(&vm.ctx, key, value) + self.get_locals().set_item(key, value, vm).unwrap(); + } + + fn delete_name(&self, vm: &VirtualMachine, key: &str) -> PyResult { + self.get_locals().del_item(key, vm) } - fn delete_name(&self, _vm: &VirtualMachine, key: &str) { - self.get_locals().del_item(key) + fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option { + self.globals.get_item_option(name, vm).unwrap() + } + + fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) { + self.globals.set_item(name, value, vm).unwrap(); } } @@ -183,8 +215,11 @@ enum BlockType { end: bytecode::Label, context_manager: PyObjectRef, }, + ExceptHandler, } +pub type FrameRef = PyRef; + pub struct Frame { pub code: bytecode::CodeObject, // We need 1 stack per frame @@ -195,7 +230,7 @@ pub struct Frame { } impl PyValue for Frame { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.frame_type() } } @@ -206,11 +241,11 @@ pub enum ExecutionResult { Yield(PyObjectRef), } -// A valid execution result, or an exception +/// A valid execution result, or an exception pub type FrameResult = Result, PyObjectRef>; impl Frame { - pub fn new(code: PyObjectRef, scope: Scope) -> Frame { + pub fn new(code: PyCodeRef, scope: Scope) -> Frame { //populate the globals and locals //TODO: This is wrong, check https://github.com/nedbat/byterun/blob/31e6c4a8212c35b5157919abff43a7daa0f377c6/byterun/pyvm2.py#L95 /* @@ -223,7 +258,7 @@ impl Frame { // locals.extend(callargs); Frame { - code: objcode::get_value(&code), + code: code.code.clone(), stack: RefCell::new(vec![]), blocks: RefCell::new(vec![]), // save the callargs as locals @@ -248,9 +283,11 @@ impl Frame { Ok(Some(value)) => { break Ok(value); } + // Instruction raised an exception Err(exception) => { - // unwind block stack on exception and find any handlers. - // Add an entry in the traceback: + // 1. Extract traceback from exception's '__traceback__' attr. + // 2. Add new entry with current execution position (filename, lineno, code_object) to traceback. + // 3. Unwind block stack till appropriate handler is found. assert!(objtype::isinstance( &exception, &vm.ctx.exceptions.base_exception_type @@ -259,13 +296,12 @@ impl Frame { .get_attribute(exception.clone(), "__traceback__") .unwrap(); trace!("Adding to traceback: {:?} {:?}", traceback, lineno); - let pos = vm.ctx.new_tuple(vec![ + let raise_location = vm.ctx.new_tuple(vec![ vm.ctx.new_str(filename.clone()), - vm.ctx.new_int(lineno.get_row()), + vm.ctx.new_int(lineno.row()), vm.ctx.new_str(run_obj_name.clone()), ]); - objlist::PyListRef::try_from_object(vm, traceback)?.append(pos, vm); - // exception.__trace + objlist::PyListRef::try_from_object(vm, traceback)?.append(raise_location, vm); match self.unwind_exception(vm, exception) { None => {} Some(exception) => { @@ -279,13 +315,24 @@ impl Frame { } } + pub fn throw( + &self, + vm: &VirtualMachine, + exception: PyObjectRef, + ) -> Result { + match self.unwind_exception(vm, exception) { + None => self.run(vm), + Some(exception) => Err(exception), + } + } + pub fn fetch_instruction(&self) -> &bytecode::Instruction { let ins2 = &self.code.instructions[*self.lasti.borrow()]; *self.lasti.borrow_mut() += 1; ins2 } - // Execute a single instruction: + /// Execute a single instruction. fn execute_instruction(&self, vm: &VirtualMachine) -> FrameResult { let instruction = self.fetch_instruction(); { @@ -308,11 +355,21 @@ impl Frame { } bytecode::Instruction::Import { ref name, - ref symbol, - } => self.import(vm, name, symbol), - bytecode::Instruction::ImportStar { ref name } => self.import_star(vm, name), - bytecode::Instruction::LoadName { ref name } => self.load_name(vm, name), - bytecode::Instruction::StoreName { ref name } => self.store_name(vm, name), + ref symbols, + ref level, + } => self.import(vm, name, symbols, level), + bytecode::Instruction::ImportStar { + ref name, + ref level, + } => self.import_star(vm, name, level), + bytecode::Instruction::LoadName { + ref name, + ref scope, + } => self.load_name(vm, name, scope), + bytecode::Instruction::StoreName { + ref name, + ref scope, + } => self.store_name(vm, name, scope), bytecode::Instruction::DeleteName { ref name } => self.delete_name(vm, name), bytecode::Instruction::StoreSubscript => self.execute_store_subscript(vm), bytecode::Instruction::DeleteSubscript => self.execute_delete_subscript(vm), @@ -384,44 +441,41 @@ impl Frame { } bytecode::Instruction::BuildMap { size, unpack } => { let map_obj = vm.ctx.new_dict(); - for _x in 0..*size { - let obj = self.pop_value(); - if *unpack { + if *unpack { + for obj in self.pop_multiple(*size) { // Take all key-value pairs from the dict: - let dict_elements = objdict::get_key_value_pairs(&obj); - for (key, value) in dict_elements.iter() { - objdict::set_item(&map_obj, vm, key, value); + let dict: PyDictRef = + obj.downcast().expect("Need a dictionary to build a map."); + for (key, value) in dict { + map_obj.set_item(key, value, vm).unwrap(); } - } else { - let key = self.pop_value(); - objdict::set_item(&map_obj, vm, &key, &obj); + } + } else { + for (key, value) in self.pop_multiple(2 * size).into_iter().tuples() { + map_obj.set_item(key, value, vm).unwrap(); } } - self.push_value(map_obj); + + self.push_value(map_obj.into_object()); Ok(None) } bytecode::Instruction::BuildSlice { size } => { assert!(*size == 2 || *size == 3); - let elements = self.pop_multiple(*size); - - let mut out: Vec> = elements - .into_iter() - .map(|x| { - if x.is(&vm.ctx.none()) { - None - } else if let Some(i) = x.payload::() { - Some(i.value.clone()) - } else { - panic!("Expect Int or None as BUILD_SLICE arguments") - } - }) - .collect(); - let start = out[0].take(); - let stop = out[1].take(); - let step = if out.len() == 3 { out[2].take() } else { None }; + let step = if *size == 3 { + Some(self.pop_value()) + } else { + None + }; + let stop = self.pop_value(); + let start = self.pop_value(); - let obj = PySlice { start, stop, step }.into_ref(vm); + let obj = PySlice { + start: Some(start), + stop, + step, + } + .into_ref(vm); self.push_value(obj.into_object()); Ok(None) } @@ -514,9 +568,7 @@ impl Frame { } = &block.typ { debug_assert!(end1 == end2); - - // call exit now with no exception: - self.with_exit(vm, &context_manager, None)?; + self.call_context_manager_exit_no_exception(vm, &context_manager)?; } else { unreachable!("Block stack is incorrect, expected a with block"); } @@ -560,29 +612,61 @@ impl Frame { } } bytecode::Instruction::MakeFunction { flags } => { - let _qualified_name = self.pop_value(); - let code_obj = self.pop_value(); + let qualified_name = self + .pop_value() + .downcast::() + .expect("qualified name to be a string"); + let code_obj = self + .pop_value() + .downcast() + .expect("Second to top value on the stack must be a code object"); let annotations = if flags.contains(bytecode::FunctionOpArg::HAS_ANNOTATIONS) { self.pop_value() } else { - vm.new_dict() + vm.ctx.new_dict().into_object() }; + let kw_only_defaults = + if flags.contains(bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS) { + Some( + self.pop_value().downcast::().expect( + "Stack value for keyword only defaults expected to be a dict", + ), + ) + } else { + None + }; + let defaults = if flags.contains(bytecode::FunctionOpArg::HAS_DEFAULTS) { - self.pop_value() + Some( + self.pop_value() + .downcast::() + .expect("Stack value for defaults expected to be a tuple"), + ) } else { - vm.get_none() + None }; // pop argc arguments // argument: name, args, globals let scope = self.scope.clone(); - let obj = vm.ctx.new_function(code_obj, scope, defaults); - - vm.ctx.set_attr(&obj, "__annotations__", annotations); - - self.push_value(obj); + let func_obj = vm + .ctx + .new_function(code_obj, scope, defaults, kw_only_defaults); + + let name = qualified_name.value.split('.').next_back().unwrap(); + vm.set_attr(&func_obj, "__name__", vm.new_str(name.to_string()))?; + vm.set_attr(&func_obj, "__qualname__", qualified_name)?; + let module = self + .scope + .globals + .get_item_option("__name__", vm)? + .unwrap_or_else(|| vm.get_none()); + vm.set_attr(&func_obj, "__module__", module)?; + vm.set_attr(&func_obj, "__annotations__", annotations)?; + + self.push_value(func_obj); Ok(None) } bytecode::Instruction::CallFunction { typ } => { @@ -591,7 +675,7 @@ impl Frame { let args: Vec = self.pop_multiple(*count); PyFuncArgs { args, - kwargs: vec![], + kwargs: IndexMap::new(), } } bytecode::CallType::Keyword(count) => { @@ -607,14 +691,14 @@ impl Frame { } bytecode::CallType::Ex(has_kwargs) => { let kwargs = if *has_kwargs { - let kw_dict = self.pop_value(); - let dict_elements = objdict::get_elements(&kw_dict).clone(); - dict_elements + let kw_dict: PyDictRef = + self.pop_value().downcast().expect("Kwargs must be a dict."); + kw_dict .into_iter() - .map(|elem| (elem.0, (elem.1).1)) + .map(|elem| (objstr::get_value(&elem.0), elem.1)) .collect() } else { - vec![] + IndexMap::new() }; let args = self.pop_value(); let args = vm.extract_elements(&args)?; @@ -651,29 +735,38 @@ impl Frame { } bytecode::Instruction::Raise { argc } => { + let cause = match argc { + 2 => self.get_exception(vm, true)?, + _ => vm.get_none(), + }; let exception = match argc { - 1 => self.pop_value(), - 0 | 2 | 3 => panic!("Not implemented!"), + 0 => match vm.pop_exception() { + Some(exc) => exc, + None => { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "No active exception to reraise".to_string(), + )); + } + }, + 1 | 2 => self.get_exception(vm, false)?, + 3 => panic!("Not implemented!"), _ => panic!("Invalid parameter for RAISE_VARARGS, must be between 0 to 3"), }; - if objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) { - info!("Exception raised: {:?}", exception); - Err(exception) - } else if objtype::isinstance(&exception, &vm.ctx.type_type()) - && objtype::issubclass(&exception, &vm.ctx.exceptions.base_exception_type) - { - let exception = vm.new_empty_exception(exception)?; - info!("Exception raised: {:?}", exception); - Err(exception) - } else { - let msg = format!( - "Can only raise BaseException derived types, not {}", - exception - ); - let type_error_type = vm.ctx.exceptions.type_error.clone(); - let type_error = vm.new_exception(type_error_type, msg); - Err(type_error) - } + let context = match argc { + 0 => vm.get_none(), // We have already got the exception, + _ => match vm.pop_exception() { + Some(exc) => exc, + None => vm.get_none(), + }, + }; + info!( + "Exception raised: {:?} with cause: {:?} and context: {:?}", + exception, cause, context + ); + vm.set_attr(&exception, vm.new_str("__cause__".to_string()), cause)?; + vm.set_attr(&exception, vm.new_str("__context__".to_string()), context)?; + Err(exception) } bytecode::Instruction::Break => { @@ -704,16 +797,14 @@ impl Frame { if !expr.is(&vm.get_none()) { let repr = vm.to_repr(&expr)?; // TODO: implement sys.displayhook - if let Some(print) = vm.ctx.get_attr(&vm.builtins, "print") { + if let Ok(print) = vm.get_attribute(vm.builtins.clone(), "print") { vm.invoke(print, vec![repr.into_object()])?; } } Ok(None) } bytecode::Instruction::LoadBuildClass => { - let rustfunc = - PyBuiltinFunction::new(Box::new(builtins::builtin_build_class_)).into_ref(vm); - self.push_value(rustfunc.into_object()); + self.push_value(vm.ctx.new_rustfunc(builtins::builtin_build_class_)); Ok(None) } bytecode::Instruction::UnpackSequence { size } => { @@ -772,7 +863,7 @@ impl Frame { Ok(None) } bytecode::Instruction::FormatValue { conversion, spec } => { - use ast::ConversionFlag::*; + use bytecode::ConversionFlag::*; let value = match conversion { Some(Str) => vm.to_str(&self.pop_value())?.into_object(), Some(Repr) => vm.to_repr(&self.pop_value())?.into_object(), @@ -785,6 +876,15 @@ impl Frame { self.push_value(formatted); Ok(None) } + bytecode::Instruction::PopException {} => { + let block = self.pop_block().unwrap(); // this asserts that the block is_some. + if let BlockType::ExceptHandler = block.typ { + assert!(vm.pop_exception().is_some()); + Ok(None) + } else { + panic!("Block type must be ExceptHandler here.") + } + } } } @@ -809,30 +909,40 @@ impl Frame { } } - fn import(&self, vm: &VirtualMachine, module: &str, symbol: &Option) -> FrameResult { - let module = vm.import(module)?; - - // If we're importing a symbol, look it up and use it, otherwise construct a module and return - // that - let obj = match symbol { - Some(symbol) => vm.get_attribute(module, symbol.as_str()).map_err(|_| { - let import_error = vm.context().exceptions.import_error.clone(); - vm.new_exception(import_error, format!("cannot import name '{}'", symbol)) - }), - None => Ok(module), - }; + fn import( + &self, + vm: &VirtualMachine, + module: &str, + symbols: &Vec, + level: &usize, + ) -> FrameResult { + let from_list = symbols + .iter() + .map(|symbol| vm.ctx.new_str(symbol.to_string())) + .collect(); + let module = vm.import(module, &vm.ctx.new_tuple(from_list), *level)?; - // Push module on stack: - self.push_value(obj?); + if symbols.is_empty() { + self.push_value(module); + } else { + for symbol in symbols { + let obj = vm + .get_attribute(module.clone(), symbol.as_str()) + .map_err(|_| vm.new_import_error(format!("cannot import name '{}'", symbol))); + self.push_value(obj?); + } + } Ok(None) } - fn import_star(&self, vm: &VirtualMachine, module: &str) -> FrameResult { - let module = vm.import(module)?; + fn import_star(&self, vm: &VirtualMachine, module: &str, level: &usize) -> FrameResult { + let module = vm.import(module, &vm.ctx.new_tuple(vec![]), *level)?; // Grab all the names from the module and put them in the context - for (k, v) in module.get_key_value_pairs().iter() { - self.scope.store_name(&vm, &objstr::get_value(k), v.clone()); + if let Some(dict) = &module.dict { + for (k, v) in dict { + self.scope.store_name(&vm, &objstr::get_value(&k), v); + } } Ok(None) } @@ -848,7 +958,7 @@ impl Frame { BlockType::With { context_manager, .. } => { - match self.with_exit(vm, &context_manager, None) { + match self.call_context_manager_exit_no_exception(vm, &context_manager) { Ok(..) => {} Err(exc) => { // __exit__ went wrong, @@ -856,6 +966,9 @@ impl Frame { } } } + BlockType::ExceptHandler => { + vm.pop_exception(); + } } } @@ -872,12 +985,15 @@ impl Frame { } BlockType::With { context_manager, .. - } => match self.with_exit(vm, &context_manager, None) { + } => match self.call_context_manager_exit_no_exception(vm, &context_manager) { Ok(..) => {} Err(exc) => { panic!("Exception in with __exit__ {:?}", exc); } }, + BlockType::ExceptHandler => { + vm.pop_exception(); + } } self.pop_block(); @@ -889,7 +1005,9 @@ impl Frame { while let Some(block) = self.pop_block() { match block.typ { BlockType::TryExcept { handler } => { - self.push_value(exc); + self.push_block(BlockType::ExceptHandler {}); + self.push_value(exc.clone()); + vm.push_exception(exc); self.jump(handler); return None; } @@ -897,12 +1015,12 @@ impl Frame { end, context_manager, } => { - match self.with_exit(vm, &context_manager, Some(exc.clone())) { - Ok(exit_action) => { - match objbool::boolval(vm, exit_action) { - Ok(handle_exception) => { - if handle_exception { - // We handle the exception, so return! + match self.call_context_manager_exit(vm, &context_manager, exc.clone()) { + Ok(exit_result_obj) => { + match objbool::boolval(vm, exit_result_obj) { + // If __exit__ method returned True, suppress the exception and continue execution. + Ok(suppress_exception) => { + if suppress_exception { self.jump(end); return None; } else { @@ -913,7 +1031,6 @@ impl Frame { return Some(exit_exc); } } - // if objtype::isinstance } Err(exit_exc) => { // TODO: what about original exception? @@ -922,84 +1039,110 @@ impl Frame { } } BlockType::Loop { .. } => {} + // Exception was already popped on Raised. + BlockType::ExceptHandler => {} } } Some(exc) } - fn with_exit( + fn call_context_manager_exit_no_exception( &self, vm: &VirtualMachine, context_manager: &PyObjectRef, - exc: Option, ) -> PyResult { - // Assume top of stack is __exit__ method: // TODO: do we want to put the exit call on the stack? - // let exit_method = self.pop_value(); - // let args = PyFuncArgs::default(); - // TODO: what happens when we got an error during handling exception? - let args = if let Some(exc) = exc { - let exc_type = exc.typ(); - let exc_val = exc.clone(); - let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? - vec![exc_type, exc_val, exc_tb] - } else { - let exc_type = vm.ctx.none(); - let exc_val = vm.ctx.none(); - let exc_tb = vm.ctx.none(); - vec![exc_type, exc_val, exc_tb] - }; - vm.call_method(context_manager, "__exit__", args) + // TODO: what happens when we got an error during execution of __exit__? + vm.call_method( + context_manager, + "__exit__", + vec![vm.ctx.none(), vm.ctx.none(), vm.ctx.none()], + ) } - fn store_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { + fn call_context_manager_exit( + &self, + vm: &VirtualMachine, + context_manager: &PyObjectRef, + exc: PyObjectRef, + ) -> PyResult { + // TODO: do we want to put the exit call on the stack? + // TODO: what happens when we got an error during execution of __exit__? + let exc_type = exc.class().into_object(); + let exc_val = exc.clone(); + let exc_tb = vm.ctx.none(); // TODO: retrieve traceback? + vm.call_method(context_manager, "__exit__", vec![exc_type, exc_val, exc_tb]) + } + + fn store_name( + &self, + vm: &VirtualMachine, + name: &str, + name_scope: &bytecode::NameScope, + ) -> FrameResult { let obj = self.pop_value(); - self.scope.store_name(&vm, name, obj); + match name_scope { + bytecode::NameScope::Global => { + self.scope.store_global(vm, name, obj); + } + bytecode::NameScope::NonLocal => { + self.scope.store_cell(vm, name, obj); + } + bytecode::NameScope::Local => { + self.scope.store_name(vm, name, obj); + } + } Ok(None) } fn delete_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { - self.scope.delete_name(vm, name); - Ok(None) + match self.scope.delete_name(vm, name) { + Ok(_) => Ok(None), + Err(_) => Err(vm.new_name_error(format!("name '{}' is not defined", name))), + } } - fn load_name(&self, vm: &VirtualMachine, name: &str) -> FrameResult { - match self.scope.load_name(&vm, name) { - Some(value) => { - self.push_value(value); - Ok(None) - } + fn load_name( + &self, + vm: &VirtualMachine, + name: &str, + name_scope: &bytecode::NameScope, + ) -> FrameResult { + let optional_value = match name_scope { + bytecode::NameScope::Global => self.scope.load_global(vm, name), + bytecode::NameScope::NonLocal => self.scope.load_cell(vm, name), + bytecode::NameScope::Local => self.scope.load_name(&vm, name), + }; + + let value = match optional_value { + Some(value) => value, None => { - let name_error_type = vm.ctx.exceptions.name_error.clone(); - let msg = format!("name '{}' is not defined", name); - let name_error = vm.new_exception(name_error_type, msg); - Err(name_error) + return Err(vm.new_name_error(format!("name '{}' is not defined", name))); } - } - } + }; - fn subscript(&self, vm: &VirtualMachine, a: PyObjectRef, b: PyObjectRef) -> PyResult { - vm.call_method(&a, "__getitem__", vec![b]) + self.push_value(value); + Ok(None) } fn execute_store_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); let value = self.pop_value(); - vm.call_method(&obj, "__setitem__", vec![idx, value])?; + obj.set_item(idx, value, vm)?; Ok(None) } fn execute_delete_subscript(&self, vm: &VirtualMachine) -> FrameResult { let idx = self.pop_value(); let obj = self.pop_value(); - vm.call_method(&obj, "__delitem__", vec![idx])?; + obj.del_item(idx, vm)?; Ok(None) } fn jump(&self, label: bytecode::Label) { let target_pc = self.code.label_map[&label]; - trace!("program counter from {:?} to {:?}", self.lasti, target_pc); + trace!("jump from {:?} to {:?}", self.lasti, target_pc); *self.lasti.borrow_mut() = target_pc; } @@ -1028,7 +1171,7 @@ impl Frame { bytecode::BinaryOperator::FloorDivide => vm._floordiv(a_ref, b_ref), // TODO: Subscript should probably have its own op bytecode::BinaryOperator::Subscript if inplace => unreachable!(), - bytecode::BinaryOperator::Subscript => self.subscript(vm, a_ref, b_ref), + bytecode::BinaryOperator::Subscript => a_ref.get_item(b_ref, vm), bytecode::BinaryOperator::Modulo if inplace => vm._imod(a_ref, b_ref), bytecode::BinaryOperator::Modulo => vm._mod(a_ref, b_ref), bytecode::BinaryOperator::Lshift if inplace => vm._ilshift(a_ref, b_ref), @@ -1066,36 +1209,14 @@ impl Frame { a.get_id() } - // https://docs.python.org/3/reference/expressions.html#membership-test-operations - fn _membership( - &self, - vm: &VirtualMachine, - needle: PyObjectRef, - haystack: &PyObjectRef, - ) -> PyResult { - vm.call_method(&haystack, "__contains__", vec![needle]) - // TODO: implement __iter__ and __getitem__ cases when __contains__ is - // not implemented. - } - fn _in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(found), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - objtype::get_type_name(&haystack.typ()) - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(objbool::boolval(vm, found)?)) } fn _not_in(&self, vm: &VirtualMachine, needle: PyObjectRef, haystack: PyObjectRef) -> PyResult { - match self._membership(vm, needle, &haystack) { - Ok(found) => Ok(vm.ctx.new_bool(!objbool::get_value(&found))), - Err(_) => Err(vm.new_type_error(format!( - "{} has no __contains__ method", - objtype::get_type_name(&haystack.typ()) - ))), - } + let found = vm._membership(haystack.clone(), needle)?; + Ok(vm.ctx.new_bool(!objbool::boolval(vm, found)?)) } fn _is(&self, a: PyObjectRef, b: PyObjectRef) -> bool { @@ -1154,7 +1275,7 @@ impl Frame { Ok(None) } - pub fn get_lineno(&self) -> ast::Location { + pub fn get_lineno(&self) -> bytecode::Location { self.code.locations[*self.lasti.borrow()].clone() } @@ -1200,6 +1321,28 @@ impl Frame { let stack = self.stack.borrow_mut(); stack[stack.len() - depth - 1].clone() } + + fn get_exception(&self, vm: &VirtualMachine, none_allowed: bool) -> PyResult { + let exception = self.pop_value(); + if none_allowed && vm.get_none().is(&exception) + || objtype::isinstance(&exception, &vm.ctx.exceptions.base_exception_type) + { + Ok(exception) + } else if let Ok(exc_type) = PyClassRef::try_from_object(vm, exception) { + if objtype::issubclass(&exc_type, &vm.ctx.exceptions.base_exception_type) { + let exception = vm.new_empty_exception(exc_type)?; + Ok(exception) + } else { + let msg = format!( + "Can only raise BaseException derived types, not {}", + exc_type + ); + Err(vm.new_type_error(msg)) + } + } else { + Err(vm.new_type_error("exceptions must derive from BaseException".to_string())) + } + } } impl fmt::Debug for Frame { @@ -1222,13 +1365,11 @@ impl fmt::Debug for Frame { .iter() .map(|elem| format!("\n > {:?}", elem)) .collect::(); - let local_str = match self.scope.get_locals().payload::() { - Some(dict) => objdict::get_key_value_pairs_from_content(&dict.entries.borrow()) - .iter() - .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) - .collect::(), - None => panic!("locals unexpectedly not wrapping a dict!",), - }; + let dict = self.scope.get_locals(); + let local_str = dict + .into_iter() + .map(|elem| format!("\n {:?} = {:?}", elem.0, elem.1)) + .collect::(); write!( f, "Frame Object {{ \n Stack:{}\n Blocks:{}\n Locals:{}\n}}", diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs new file mode 100644 index 0000000000..4f29088abf --- /dev/null +++ b/vm/src/frozen.rs @@ -0,0 +1,19 @@ +use crate::bytecode::CodeObject; +use std::collections::HashMap; + +pub fn get_module_inits() -> HashMap { + hashmap! { + "__hello__".into() => py_compile_bytecode!( + source = "initialized = True; print(\"Hello world!\")\n", + module_name = "__hello__", + ), + "_frozen_importlib".into() => py_compile_bytecode!( + file = "../Lib/importlib/_bootstrap.py", + module_name = "_frozen_importlib", + ), + "_frozen_importlib_external".into() => py_compile_bytecode!( + file = "../Lib/importlib/_bootstrap_external.py", + module_name = "_frozen_importlib_external", + ), + } +} diff --git a/vm/src/function.rs b/vm/src/function.rs index c5ec06cb1c..22802ac1fa 100644 --- a/vm/src/function.rs +++ b/vm/src/function.rs @@ -1,8 +1,14 @@ use std::collections::HashMap; +use std::mem; use std::ops::RangeInclusive; -use crate::obj::objtype; -use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; +use indexmap::IndexMap; + +use crate::obj::objtuple::PyTuple; +use crate::obj::objtype::{isinstance, PyClassRef}; +use crate::pyobject::{ + IntoPyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, +}; use crate::vm::VirtualMachine; use self::OptionalArg::*; @@ -13,7 +19,8 @@ use self::OptionalArg::*; #[derive(Debug, Default, Clone)] pub struct PyFuncArgs { pub args: Vec, - pub kwargs: Vec<(String, PyObjectRef)>, + // sorted map, according to https://www.python.org/dev/peps/pep-0468/ + pub kwargs: IndexMap, } /// Conversion from vector of python objects to function arguments. @@ -21,7 +28,7 @@ impl From> for PyFuncArgs { fn from(args: Vec) -> Self { PyFuncArgs { args, - kwargs: vec![], + kwargs: IndexMap::new(), } } } @@ -30,16 +37,36 @@ impl From for PyFuncArgs { fn from(arg: PyObjectRef) -> Self { PyFuncArgs { args: vec![arg], - kwargs: vec![], + kwargs: IndexMap::new(), } } } +impl From<(&Args, &KwArgs)> for PyFuncArgs { + fn from(arg: (&Args, &KwArgs)) -> Self { + let Args(args) = arg.0; + let KwArgs(kwargs) = arg.1; + PyFuncArgs { + args: args.clone(), + kwargs: kwargs.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + } + } +} + +impl FromArgs for PyFuncArgs { + fn from_args(_vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { + Ok(mem::replace(args, Default::default())) + } +} + impl PyFuncArgs { pub fn new(mut args: Vec, kwarg_names: Vec) -> PyFuncArgs { - let mut kwargs = vec![]; - for name in kwarg_names.iter().rev() { - kwargs.push((name.clone(), args.pop().unwrap())); + // last `kwarg_names.len()` elements of args in order of appearance in the call signature + let kwarg_values = args.drain((args.len() - kwarg_names.len())..); + + let mut kwargs = IndexMap::new(); + for (name, value) in kwarg_names.iter().zip(kwarg_values) { + kwargs.insert(name.clone(), value); } PyFuncArgs { args, kwargs } } @@ -58,36 +85,29 @@ impl PyFuncArgs { } pub fn get_kwarg(&self, key: &str, default: PyObjectRef) -> PyObjectRef { - for (arg_name, arg_value) in self.kwargs.iter() { - if arg_name == key { - return arg_value.clone(); - } - } - default.clone() + self.kwargs + .get(key) + .cloned() + .unwrap_or_else(|| default.clone()) } pub fn get_optional_kwarg(&self, key: &str) -> Option { - for (arg_name, arg_value) in self.kwargs.iter() { - if arg_name == key { - return Some(arg_value.clone()); - } - } - None + self.kwargs.get(key).cloned() } pub fn get_optional_kwarg_with_type( &self, key: &str, - ty: PyObjectRef, + ty: PyClassRef, vm: &VirtualMachine, ) -> Result, PyObjectRef> { match self.get_optional_kwarg(key) { Some(kwarg) => { - if objtype::isinstance(&kwarg, &ty) { + if isinstance(&kwarg, &ty) { Ok(Some(kwarg)) } else { let expected_ty_name = vm.to_pystr(&ty)?; - let actual_ty_name = vm.to_pystr(&kwarg.typ())?; + let actual_ty_name = vm.to_pystr(&kwarg.class())?; Err(vm.new_type_error(format!( "argument of type {} is required for named parameter `{}` (got: {})", expected_ty_name, key, actual_ty_name @@ -98,7 +118,7 @@ impl PyFuncArgs { } } - pub fn next_positional(&mut self) -> Option { + pub fn take_positional(&mut self) -> Option { if self.args.is_empty() { None } else { @@ -106,20 +126,17 @@ impl PyFuncArgs { } } + pub fn take_positional_keyword(&mut self, name: &str) -> Option { + self.take_positional().or_else(|| self.take_keyword(name)) + } + pub fn take_keyword(&mut self, name: &str) -> Option { - // TODO: change kwarg representation so this scan isn't necessary - if let Some(index) = self - .kwargs - .iter() - .position(|(arg_name, _)| arg_name == name) - { - Some(self.kwargs.remove(index).1) - } else { - None - } + self.kwargs.remove(name) } - pub fn remaining_keyword<'a>(&'a mut self) -> impl Iterator + 'a { + pub fn remaining_keywords<'a>( + &'a mut self, + ) -> impl Iterator + 'a { self.kwargs.drain(..) } @@ -131,7 +148,7 @@ impl PyFuncArgs { /// /// If the given `FromArgs` includes any conversions, exceptions raised /// during the conversion will halt the binding and return the error. - fn bind(mut self, vm: &VirtualMachine) -> PyResult { + pub fn bind(mut self, vm: &VirtualMachine) -> PyResult { let given_args = self.args.len(); let bound = match T::from_args(vm, &mut self) { Ok(args) => args, @@ -152,6 +169,9 @@ impl PyFuncArgs { Err(ArgumentError::InvalidKeywordArgument(name)) => { return Err(vm.new_type_error(format!("{} is an invalid keyword argument", name))); } + Err(ArgumentError::RequiredKeywordArgument(name)) => { + return Err(vm.new_type_error(format!("Required keyqord only argument {}", name))); + } Err(ArgumentError::Exception(ex)) => { return Err(ex); } @@ -164,7 +184,10 @@ impl PyFuncArgs { given_args, ))) } else if !self.kwargs.is_empty() { - Err(vm.new_type_error(format!("Unexpected keyword argument {}", self.kwargs[0].0))) + Err(vm.new_type_error(format!( + "Unexpected keyword argument {}", + self.kwargs.keys().next().unwrap() + ))) } else { Ok(bound) } @@ -180,6 +203,8 @@ pub enum ArgumentError { TooManyArgs, /// The function doesn't accept a keyword argument with the given name. InvalidKeywordArgument(String), + /// The function require a keyword argument with the given name, but one wasn't provided + RequiredKeywordArgument(String), /// An exception was raised while binding arguments to the function /// parameters. Exception(PyObjectRef), @@ -222,19 +247,34 @@ pub trait FromArgs: Sized { /// an appropriate FromArgs implementation must be created. pub struct KwArgs(HashMap); +impl KwArgs { + pub fn pop_kwarg(&mut self, name: &str) -> Option { + self.0.remove(name) + } +} + impl FromArgs for KwArgs where T: TryFromObject, { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { let mut kwargs = HashMap::new(); - for (name, value) in args.remaining_keyword() { + for (name, value) in args.remaining_keywords() { kwargs.insert(name, T::try_from_object(vm, value)?); } Ok(KwArgs(kwargs)) } } +impl IntoIterator for KwArgs { + type Item = (String, T); + type IntoIter = std::collections::hash_map::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + /// A list of positional argument values. /// /// A built-in function with a `Args` parameter is analagous to a Python @@ -243,15 +283,23 @@ where /// /// `Args` optionally accepts a generic type parameter to allow type checks /// or conversions of each argument. +#[derive(Clone)] pub struct Args(Vec); +impl Args> { + pub fn into_tuple(self, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx + .new_tuple(self.0.into_iter().map(PyRef::into_object).collect()) + } +} + impl FromArgs for Args where T: TryFromObject, { fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { let mut varargs = Vec::new(); - while let Some(value) = args.next_positional() { + while let Some(value) = args.take_positional() { varargs.push(T::try_from_object(vm, value)?); } Ok(Args(varargs)) @@ -276,7 +324,7 @@ where } fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { - if let Some(value) = args.next_positional() { + if let Some(value) = args.take_positional() { Ok(T::try_from_object(vm, value)?) } else { Err(ArgumentError::TooFewArgs) @@ -287,18 +335,56 @@ where /// An argument that may or may not be provided by the caller. /// /// This style of argument is not possible in pure Python. -pub enum OptionalArg { +#[derive(Debug)] +pub enum OptionalArg { Present(T), Missing, } impl OptionalArg { + #[inline] + pub fn is_present(&self) -> bool { + match self { + Present(_) => true, + Missing => false, + } + } + + #[inline] pub fn into_option(self) -> Option { match self { Present(value) => Some(value), Missing => None, } } + + pub fn unwrap_or(self, default: T) -> T { + match self { + Present(value) => value, + Missing => default, + } + } + + pub fn unwrap_or_else(self, f: F) -> T + where + F: FnOnce() -> T, + { + match self { + Present(value) => value, + Missing => f(), + } + } + + pub fn map_or_else(self, default: D, f: F) -> U + where + D: FnOnce() -> U, + F: FnOnce(T) -> U, + { + match self { + Present(value) => f(value), + Missing => default(), + } + } } impl FromArgs for OptionalArg @@ -310,7 +396,7 @@ where } fn from_args(vm: &VirtualMachine, args: &mut PyFuncArgs) -> Result { - if let Some(value) = args.next_positional() { + if let Some(value) = args.take_positional() { Ok(Present(T::try_from_object(vm, value)?)) } else { Ok(Missing) @@ -364,6 +450,7 @@ tuple_from_py_func_args!(A, B); tuple_from_py_func_args!(A, B, C); tuple_from_py_func_args!(A, B, C, D); tuple_from_py_func_args!(A, B, C, D, E); +tuple_from_py_func_args!(A, B, C, D, E, F); /// A built-in Python function. pub type PyNativeFunc = Box PyResult + 'static>; @@ -395,11 +482,8 @@ where } } -impl IntoPyNativeFunc for PyNativeFunc { - fn into_func(self) -> PyNativeFunc { - self - } -} +pub struct OwnedParam(std::marker::PhantomData); +pub struct RefParam(std::marker::PhantomData); // This is the "magic" that allows rust functions of varying signatures to // generate native python functions. @@ -407,11 +491,10 @@ impl IntoPyNativeFunc for PyNativeFunc { // Note that this could be done without a macro - it is simply to avoid repetition. macro_rules! into_py_native_func_tuple { ($(($n:tt, $T:ident)),*) => { - impl IntoPyNativeFunc<($($T,)*), R> for F + impl IntoPyNativeFunc<($(OwnedParam<$T>,)*), R> for F where F: Fn($($T,)* &VirtualMachine) -> R + 'static, $($T: FromArgs,)* - ($($T,)*): FromArgs, R: IntoPyObject, { fn into_func(self) -> PyNativeFunc { @@ -422,6 +505,22 @@ macro_rules! into_py_native_func_tuple { }) } } + + impl IntoPyNativeFunc<(RefParam, $(OwnedParam<$T>,)*), R> for F + where + F: Fn(&S, $($T,)* &VirtualMachine) -> R + 'static, + S: PyValue, + $($T: FromArgs,)* + R: IntoPyObject, + { + fn into_func(self) -> PyNativeFunc { + Box::new(move |vm, args| { + let (zelf, $($n,)*) = args.bind::<(PyRef, $($T,)*)>(vm)?; + + (self)(&zelf, $($n,)* vm).into_pyobject(vm) + }) + } + } }; } @@ -431,3 +530,28 @@ into_py_native_func_tuple!((a, A), (b, B)); into_py_native_func_tuple!((a, A), (b, B), (c, C)); into_py_native_func_tuple!((a, A), (b, B), (c, C), (d, D)); into_py_native_func_tuple!((a, A), (b, B), (c, C), (d, D), (e, E)); + +/// Tests that the predicate is True on a single value, or if the value is a tuple a tuple, then +/// test that any of the values contained within the tuples satisfies the predicate. Type parameter +/// T specifies the type that is expected, if the input value is not of that type or a tuple of +/// values of that type, then a TypeError is raised. +pub fn single_or_tuple_any) -> PyResult>( + obj: PyObjectRef, + predicate: F, + message: fn(&PyObjectRef) -> String, + vm: &VirtualMachine, +) -> PyResult { + match_class!(obj, + obj @ T => predicate(obj), + tuple @ PyTuple => { + for obj in tuple.elements.iter() { + let inner_val = PyRef::::try_from_object(vm, obj.clone())?; + if predicate(inner_val)? { + return Ok(true); + } + } + Ok(false) + }, + obj => Err(vm.new_type_error(message(&obj))) + ) +} diff --git a/vm/src/import.rs b/vm/src/import.rs index ed5a6d094b..2c2ac87289 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -2,80 +2,121 @@ * Import mechanics */ -use std::error::Error; -use std::path::PathBuf; - -use crate::compile; +use crate::bytecode::CodeObject; use crate::frame::Scope; -use crate::obj::{objsequence, objstr}; -use crate::pyobject::{DictProtocol, PyResult}; -use crate::util; +use crate::obj::{objcode, objsequence, objstr, objtype}; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[cfg(feature = "rustpython-compiler")] +use rustpython_compiler::compile; -fn import_uncached_module(vm: &VirtualMachine, current_path: PathBuf, module: &str) -> PyResult { - // Check for Rust-native modules - if let Some(module) = vm.stdlib_inits.borrow().get(module) { - return Ok(module(&vm.ctx).clone()); +pub fn init_importlib(vm: &VirtualMachine, external: bool) -> PyResult { + let importlib = import_frozen(vm, "_frozen_importlib")?; + let impmod = import_builtin(vm, "_imp")?; + let install = vm.get_attribute(importlib.clone(), "_install")?; + vm.invoke(install, vec![vm.sys_module.clone(), impmod])?; + vm.import_func + .replace(vm.get_attribute(importlib.clone(), "__import__")?); + if external && cfg!(feature = "rustpython-compiler") { + let install_external = + vm.get_attribute(importlib.clone(), "_install_external_importers")?; + vm.invoke(install_external, vec![])?; } + Ok(vm.get_none()) +} - let notfound_error = vm.context().exceptions.module_not_found_error.clone(); - let import_error = vm.context().exceptions.import_error.clone(); +pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { + vm.frozen + .borrow() + .get(module_name) + .ok_or_else(|| vm.new_import_error(format!("Cannot import frozen module {}", module_name))) + .and_then(|frozen| import_codeobj(vm, module_name, frozen.clone(), false)) +} - // Time to search for module in any place: - let file_path = find_source(vm, current_path, module) - .map_err(|e| vm.new_exception(notfound_error.clone(), e))?; - let source = util::read_file(file_path.as_path()) - .map_err(|e| vm.new_exception(import_error.clone(), e.description().to_string()))?; - let code_obj = compile::compile( - &source, - &compile::Mode::Exec, - file_path.to_str().unwrap().to_string(), - vm.ctx.code_type(), - ) - .map_err(|err| { - let syntax_error = vm.context().exceptions.syntax_error.clone(); - vm.new_exception(syntax_error, err.description().to_string()) - })?; - // trace!("Code object: {:?}", code_obj); +pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { + vm.stdlib_inits + .borrow() + .get(module_name) + .ok_or_else(|| vm.new_import_error(format!("Cannot import bultin module {}", module_name))) + .and_then(|make_module_func| { + let module = make_module_func(vm); + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; + sys_modules.set_item(module_name, module.clone(), vm)?; + Ok(module) + }) +} - let attrs = vm.ctx.new_dict(); - attrs.set_item(&vm.ctx, "__name__", vm.new_str(module.to_string())); - vm.run_code_obj(code_obj, Scope::new(None, attrs.clone()))?; - Ok(vm.ctx.new_module(module, attrs)) +#[cfg(feature = "rustpython-compiler")] +pub fn import_file( + vm: &VirtualMachine, + module_name: &str, + file_path: String, + content: String, +) -> PyResult { + let code_obj = compile::compile(&content, &compile::Mode::Exec, file_path) + .map_err(|err| vm.new_syntax_error(&err))?; + import_codeobj(vm, module_name, code_obj, true) } -pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &str) -> PyResult { - // First, see if we already loaded the module: - let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; - if let Some(module) = sys_modules.get_item(module_name) { - return Ok(module); +pub fn import_codeobj( + vm: &VirtualMachine, + module_name: &str, + code_obj: CodeObject, + set_file_attr: bool, +) -> PyResult { + let attrs = vm.ctx.new_dict(); + attrs.set_item("__name__", vm.new_str(module_name.to_string()), vm)?; + if set_file_attr { + attrs.set_item("__file__", vm.new_str(code_obj.source_path.to_owned()), vm)?; } - let module = import_uncached_module(vm, current_path, module_name)?; - sys_modules.set_item(&vm.ctx, module_name, module.clone()); + let module = vm.ctx.new_module(module_name, attrs.clone()); + + // Store module in cache to prevent infinite loop with mutual importing libs: + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules")?; + sys_modules.set_item(module_name, module.clone(), vm)?; + + // Execute main code in module: + vm.run_code_obj( + objcode::PyCode::new(code_obj).into_ref(vm), + Scope::with_builtins(None, attrs, vm), + )?; Ok(module) } -fn find_source(vm: &VirtualMachine, current_path: PathBuf, name: &str) -> Result { - let sys_path = vm.get_attribute(vm.sys_module.clone(), "path").unwrap(); - let mut paths: Vec = objsequence::get_elements(&sys_path) - .iter() - .map(|item| PathBuf::from(objstr::get_value(item))) - .collect(); +// TODO: This function should do nothing on verbose mode. +pub fn remove_importlib_frames(vm: &VirtualMachine, exc: &PyObjectRef) -> PyObjectRef { + let always_trim = objtype::isinstance(exc, &vm.ctx.exceptions.import_error); - paths.insert(0, current_path); - - let suffixes = [".py", "/__init__.py"]; - let mut file_paths = vec![]; - for path in paths { - for suffix in suffixes.iter() { - let mut file_path = path.clone(); - file_path.push(format!("{}{}", name, suffix)); - file_paths.push(file_path); + if let Ok(tb) = vm.get_attribute(exc.clone(), "__traceback__") { + if objtype::isinstance(&tb, &vm.ctx.list_type()) { + let tb_entries = objsequence::get_elements_list(&tb).to_vec(); + let mut in_importlib = false; + let new_tb = tb_entries + .iter() + .filter(|tb_entry| { + let location_attrs = objsequence::get_elements_tuple(&tb_entry); + let file_name = objstr::get_value(&location_attrs[0]); + if file_name == "_frozen_importlib" || file_name == "_frozen_importlib_external" + { + let run_obj_name = objstr::get_value(&location_attrs[2]); + if run_obj_name == "_call_with_frames_removed" { + in_importlib = true; + } + if always_trim || in_importlib { + false + } else { + true + } + } else { + in_importlib = false; + true + } + }) + .map(|x| x.clone()) + .collect(); + vm.set_attr(exc, "__traceback__", vm.ctx.new_list(new_tb)) + .unwrap(); } } - - match file_paths.iter().find(|p| p.exists()) { - Some(path) => Ok(path.to_path_buf()), - None => Err(format!("No module named '{}'", name)), - } + exc.clone() } diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 3892fe2a45..5754d98249 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -19,19 +19,21 @@ extern crate lazy_static; extern crate lexical; #[macro_use] extern crate log; +#[macro_use] +extern crate maplit; // extern crate env_logger; -extern crate num_bigint; -extern crate num_complex; -extern crate num_integer; -extern crate num_traits; -extern crate serde; -extern crate serde_json; -extern crate statrs; - -extern crate rustpython_parser; + #[macro_use] extern crate rustpython_derive; +extern crate self as rustpython_vm; + +pub use rustpython_derive::*; + +use proc_macro_hack::proc_macro_hack; +#[proc_macro_hack] +pub use rustpython_derive::py_compile_bytecode; + //extern crate eval; use eval::eval::*; // use py_code_object::{Function, NativeType, PyCodeObject}; @@ -40,16 +42,19 @@ extern crate rustpython_derive; pub mod macros; mod builtins; -pub mod bytecode; -pub mod compile; -pub mod error; +pub mod cformat; +mod dictdatatype; +#[cfg(feature = "rustpython-compiler")] pub mod eval; mod exceptions; pub mod format; pub mod frame; +mod frozen; pub mod function; pub mod import; pub mod obj; +pub mod py_serde; +mod pyhash; pub mod pyobject; pub mod stdlib; mod sysmodule; @@ -60,3 +65,9 @@ mod vm; // pub use self::pyobject::Executor; pub use self::exceptions::print_exception; pub use self::vm::VirtualMachine; +pub use rustpython_bytecode::*; + +#[doc(hidden)] +pub mod __exports { + pub use bincode; +} diff --git a/vm/src/macros.rs b/vm/src/macros.rs index 4d4e6ebe2e..3e5d3262f0 100644 --- a/vm/src/macros.rs +++ b/vm/src/macros.rs @@ -9,7 +9,7 @@ macro_rules! replace_expr { #[macro_export] macro_rules! count_tts { - ($($tts:tt)*) => {0usize $(+ replace_expr!($tts 1usize))*}; + ($($tts:tt)*) => {0usize $(+ $crate::replace_expr!($tts 1usize))*}; } #[macro_export] @@ -20,7 +20,9 @@ macro_rules! type_check { let arg = &$args.args[$arg_count]; if !$crate::obj::objtype::isinstance(arg, &expected_type) { - let arg_typ = arg.typ(); + use $crate::pyobject::TypeProtocol; + + let arg_typ = arg.class(); let expected_type_name = $vm.to_pystr(&expected_type)?; let actual_type = $vm.to_pystr(&arg_typ)?; return Err($vm.new_type_error(format!( @@ -45,14 +47,14 @@ macro_rules! arg_check { } }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*] ) => { - arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); + $crate::arg_check!($vm, $args, required=[$( ($arg_name, $arg_type) ),*], optional=[]); }; ( $vm: ident, $args:ident, required=[$( ($arg_name:ident, $arg_type:expr) ),*], optional=[$( ($optional_arg_name:ident, $optional_arg_type:expr) ),*] ) => { let mut arg_count = 0; // use macro magic to compile-time count number of required and optional arguments - let minimum_arg_count = count_tts!($($arg_name)*); - let maximum_arg_count = minimum_arg_count + count_tts!($($optional_arg_name)*); + let minimum_arg_count = $crate::count_tts!($($arg_name)*); + let maximum_arg_count = minimum_arg_count + $crate::count_tts!($($optional_arg_name)*); // verify that the number of given arguments is right if $args.args.len() < minimum_arg_count || $args.args.len() > maximum_arg_count { @@ -72,7 +74,7 @@ macro_rules! arg_check { // check if the type matches. If not, return with error // assign the arg to a variable $( - type_check!($vm, $args, arg_count, $arg_name, $arg_type); + $crate::type_check!($vm, $args, arg_count, $arg_name, $arg_type); let $arg_name = &$args.args[arg_count]; #[allow(unused_assignments)] { @@ -85,7 +87,7 @@ macro_rules! arg_check { // assign the arg to a variable $( let $optional_arg_name = if arg_count < $args.args.len() { - type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); + $crate::type_check!($vm, $args, arg_count, $optional_arg_name, $optional_arg_type); let ret = Some(&$args.args[arg_count]); #[allow(unused_assignments)] { @@ -114,14 +116,22 @@ macro_rules! no_kwargs { #[macro_export] macro_rules! py_module { - ( $ctx:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => { - { - let py_mod = $ctx.new_module($module_name, $ctx.new_dict()); - $( - $ctx.set_attr(&py_mod, $name, $value); - )* - py_mod - } + ( $vm:expr, $module_name:expr, { $($name:expr => $value:expr),* $(,)* }) => {{ + let module = $vm.ctx.new_module($module_name, $vm.ctx.new_dict()); + $vm.set_attr(&module, "__name__", $vm.ctx.new_str($module_name.to_string())).unwrap(); + $( + $vm.set_attr(&module, $name, $value).unwrap(); + )* + module + }}; +} + +#[macro_export] +macro_rules! extend_module { + ( $vm:expr, $module:expr, { $($name:expr => $value:expr),* $(,)* }) => { + $( + $vm.set_attr(&$module, $name, $value).unwrap(); + )* } } @@ -131,7 +141,7 @@ macro_rules! py_class { { let py_class = $ctx.new_class($class_name, $class_base); $( - $ctx.set_attr(&py_class, $name, $value); + py_class.set_str_attr($name, $value); )* py_class } @@ -143,7 +153,106 @@ macro_rules! extend_class { ( $ctx:expr, $class:expr, { $($name:expr => $value:expr),* $(,)* }) => { let class = $class; $( - $ctx.set_attr(&class, $name, $value); + class.set_str_attr($name, $value); )* } } + +#[macro_export] +macro_rules! py_namespace { + ( $vm:expr, { $($name:expr => $value:expr),* $(,)* }) => { + { + let namespace = $vm.ctx.new_namespace(); + $( + $vm.set_attr(&namespace, $name, $value).unwrap(); + )* + namespace + } + } +} + +/// Macro to match on the built-in class of a Python object. +/// +/// Like `match`, `match_class!` must be exhaustive, so a default arm with +/// the uncasted object is required. +/// +/// # Examples +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// assert_eq!( +/// "int", +/// match_class!(obj.clone(), +/// PyInt => "int", +/// PyFloat => "float", +/// _ => "neither", +/// ) +/// ); +/// +/// ``` +/// +/// With a binding to the downcasted type: +/// +/// ``` +/// use num_bigint::ToBigInt; +/// use num_traits::Zero; +/// +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::match_class; +/// use rustpython_vm::obj::objfloat::PyFloat; +/// use rustpython_vm::obj::objint::PyInt; +/// use rustpython_vm::pyobject::PyValue; +/// +/// let vm = VirtualMachine::new(); +/// let obj = PyInt::new(0).into_ref(&vm).into_object(); +/// +/// let int_value = match_class!(obj, +/// i @ PyInt => i.as_bigint().clone(), +/// f @ PyFloat => f.to_f64().to_bigint().unwrap(), +/// obj => panic!("non-numeric object {}", obj), +/// ); +/// +/// assert!(int_value.is_zero()); +/// ``` +#[macro_export] +macro_rules! match_class { + // The default arm. + ($obj:expr, _ => $default:expr $(,)?) => { + $default + }; + + // The default arm, binding the original object to the specified identifier. + ($obj:expr, $binding:ident => $default:expr $(,)?) => {{ + let $binding = $obj; + $default + }}; + + // An arm taken when the object is an instance of the specified built-in + // class and binding the downcasted object to the specified identifier. + ($obj:expr, $binding:ident @ $class:ty => $expr:expr, $($rest:tt)*) => { + match $obj.downcast::<$class>() { + Ok($binding) => $expr, + Err(_obj) => $crate::match_class!(_obj, $($rest)*), + } + }; + + // An arm taken when the object is an instance of the specified built-in + // class. + ($obj:expr, $class:ty => $expr:expr, $($rest:tt)*) => { + if $obj.payload_is::<$class>() { + $expr + } else { + $crate::match_class!($obj, $($rest)*) + } + }; +} diff --git a/vm/src/obj/mod.rs b/vm/src/obj/mod.rs index 0419d827f0..ad7af74d5d 100644 --- a/vm/src/obj/mod.rs +++ b/vm/src/obj/mod.rs @@ -3,6 +3,7 @@ pub mod objbool; pub mod objbuiltinfunc; pub mod objbytearray; +pub mod objbyteinner; pub mod objbytes; pub mod objclassmethod; pub mod objcode; @@ -19,8 +20,10 @@ pub mod objint; pub mod objiter; pub mod objlist; pub mod objmap; +pub mod objmappingproxy; pub mod objmemory; pub mod objmodule; +pub mod objnamespace; pub mod objnone; pub mod objobject; pub mod objproperty; @@ -33,5 +36,6 @@ pub mod objstr; pub mod objsuper; pub mod objtuple; pub mod objtype; +pub mod objweakproxy; pub mod objweakref; pub mod objzip; diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index 4feb1300bd..f294434b30 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -1,17 +1,10 @@ use num_traits::Zero; use crate::function::PyFuncArgs; -use crate::pyobject::{ - IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject, TypeProtocol, -}; +use crate::pyobject::{IntoPyObject, PyContext, PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; -use super::objdict::PyDict; -use super::objfloat::PyFloat; use super::objint::PyInt; -use super::objlist::PyList; -use super::objstr::PyString; -use super::objtuple::PyTuple; use super::objtype; impl IntoPyObject for bool { @@ -26,35 +19,21 @@ impl TryFromObject for bool { } } +/// Convert Python bool into Rust bool. pub fn boolval(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - if let Some(s) = obj.payload::() { - return Ok(!s.value.is_empty()); - } - if let Some(value) = obj.payload::() { - return Ok(*value != PyFloat::from(0.0)); - } - if let Some(dict) = obj.payload::() { - return Ok(!dict.entries.borrow().is_empty()); - } - if let Some(i) = obj.payload::() { - return Ok(!i.value.is_zero()); - } - if let Some(list) = obj.payload::() { - return Ok(!list.elements.borrow().is_empty()); - } - if let Some(tuple) = obj.payload::() { - return Ok(!tuple.elements.borrow().is_empty()); - } - - Ok(if let Ok(f) = vm.get_method(obj.clone(), "__bool__") { - let bool_res = vm.invoke(f, PyFuncArgs::default())?; - match bool_res.payload::() { - Some(i) => !i.value.is_zero(), - None => return Err(vm.new_type_error(String::from("TypeError"))), + let rs_bool = match vm.get_method(obj.clone(), "__bool__") { + Some(method_or_err) => { + // If descriptor returns Error, propagate it further + let method = method_or_err?; + let bool_obj = vm.invoke(method, PyFuncArgs::default())?; + match bool_obj.payload::() { + Some(int_obj) => !int_obj.as_bigint().is_zero(), + None => return Err(vm.new_type_error(String::from(""))), + } } - } else { - true - }) + None => true, + }; + Ok(rs_bool) } pub fn init(context: &PyContext) { @@ -65,9 +44,17 @@ The builtins True and False are the only two instances of the class bool. The class bool is a subclass of the class int, and cannot be subclassed."; let bool_type = &context.bool_type; - context.set_attr(&bool_type, "__new__", context.new_rustfunc(bool_new)); - context.set_attr(&bool_type, "__repr__", context.new_rustfunc(bool_repr)); - context.set_attr(&bool_type, "__doc__", context.new_str(bool_doc.to_string())); + extend_class!(context, bool_type, { + "__new__" => context.new_rustfunc(bool_new), + "__repr__" => context.new_rustfunc(bool_repr), + "__or__" => context.new_rustfunc(bool_or), + "__ror__" => context.new_rustfunc(bool_ror), + "__and__" => context.new_rustfunc(bool_and), + "__rand__" => context.new_rustfunc(bool_rand), + "__xor__" => context.new_rustfunc(bool_xor), + "__rxor__" => context.new_rustfunc(bool_rxor), + "__doc__" => context.new_str(bool_doc.to_string()) + }); } pub fn not(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { @@ -81,7 +68,7 @@ pub fn not(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { // Retrieve inner int value: pub fn get_value(obj: &PyObjectRef) -> bool { - !obj.payload::().unwrap().value.is_zero() + !obj.payload::().unwrap().as_bigint().is_zero() } fn bool_repr(vm: &VirtualMachine, args: PyFuncArgs) -> Result { @@ -95,6 +82,72 @@ fn bool_repr(vm: &VirtualMachine, args: PyFuncArgs) -> Result PyResult { + if objtype::isinstance(lhs, &vm.ctx.bool_type()) + && objtype::isinstance(rhs, &vm.ctx.bool_type()) + { + let lhs = get_value(lhs); + let rhs = get_value(rhs); + (lhs || rhs).into_pyobject(vm) + } else { + Ok(lhs.payload::().unwrap().or(rhs.clone(), vm)) + } +} + +fn bool_or(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + do_bool_or(vm, lhs, rhs) +} + +fn bool_ror(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(rhs, None), (lhs, None)]); + do_bool_or(vm, lhs, rhs) +} + +fn do_bool_and(vm: &VirtualMachine, lhs: &PyObjectRef, rhs: &PyObjectRef) -> PyResult { + if objtype::isinstance(lhs, &vm.ctx.bool_type()) + && objtype::isinstance(rhs, &vm.ctx.bool_type()) + { + let lhs = get_value(lhs); + let rhs = get_value(rhs); + (lhs && rhs).into_pyobject(vm) + } else { + Ok(lhs.payload::().unwrap().and(rhs.clone(), vm)) + } +} + +fn bool_and(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + do_bool_and(vm, lhs, rhs) +} + +fn bool_rand(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(rhs, None), (lhs, None)]); + do_bool_and(vm, lhs, rhs) +} + +fn do_bool_xor(vm: &VirtualMachine, lhs: &PyObjectRef, rhs: &PyObjectRef) -> PyResult { + if objtype::isinstance(lhs, &vm.ctx.bool_type()) + && objtype::isinstance(rhs, &vm.ctx.bool_type()) + { + let lhs = get_value(lhs); + let rhs = get_value(rhs); + (lhs ^ rhs).into_pyobject(vm) + } else { + Ok(lhs.payload::().unwrap().xor(rhs.clone(), vm)) + } +} + +fn bool_xor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(lhs, None), (rhs, None)]); + do_bool_xor(vm, lhs, rhs) +} + +fn bool_rxor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(rhs, None), (lhs, None)]); + do_bool_xor(vm, lhs, rhs) +} + fn bool_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, diff --git a/vm/src/obj/objbuiltinfunc.rs b/vm/src/obj/objbuiltinfunc.rs index 259c94b627..928f240b01 100644 --- a/vm/src/obj/objbuiltinfunc.rs +++ b/vm/src/obj/objbuiltinfunc.rs @@ -1,7 +1,8 @@ use std::fmt; use crate::function::PyNativeFunc; -use crate::pyobject::{PyObjectRef, PyValue}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyValue; use crate::vm::VirtualMachine; pub struct PyBuiltinFunction { @@ -10,7 +11,7 @@ pub struct PyBuiltinFunction { } impl PyValue for PyBuiltinFunction { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.builtin_function_or_method_type() } } diff --git a/vm/src/obj/objbytearray.rs b/vm/src/obj/objbytearray.rs index c7038fe9ac..51d608b095 100644 --- a/vm/src/obj/objbytearray.rs +++ b/vm/src/obj/objbytearray.rs @@ -1,356 +1,598 @@ //! Implementation of the python bytearray object. -use std::cell::RefCell; -use std::fmt::Write; -use std::ops::{Deref, DerefMut}; - -use num_traits::ToPrimitive; - -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::obj::objbyteinner::{ + ByteInnerExpandtabsOptions, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerPosition, ByteInnerSplitOptions, ByteInnerSplitlinesOptions, + ByteInnerTranslateOptions, ByteOr, PyByteInner, +}; +use crate::obj::objint::PyIntRef; +use crate::obj::objslice::PySliceRef; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtuple::PyTupleRef; +use crate::pyobject::{ + Either, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, +}; use crate::vm::VirtualMachine; - -use super::objint; -use super::objtype::{self, PyClassRef}; - -#[derive(Debug)] +use num_traits::ToPrimitive; +use std::cell::{Cell, RefCell}; +use std::convert::TryFrom; + +use super::objiter; +use super::objtype::PyClassRef; + +/// "bytearray(iterable_of_ints) -> bytearray\n\ +/// bytearray(string, encoding[, errors]) -> bytearray\n\ +/// bytearray(bytes_or_buffer) -> mutable copy of bytes_or_buffer\n\ +/// bytearray(int) -> bytes array of size given by the parameter initialized with null bytes\n\ +/// bytearray() -> empty bytes array\n\n\ +/// Construct a mutable bytearray object from:\n \ +/// - an iterable yielding integers in range(256)\n \ +/// - a text string encoded using the specified encoding\n \ +/// - a bytes or a buffer object\n \ +/// - any object implementing the buffer API.\n \ +/// - an integer"; +#[pyclass(name = "bytearray")] +#[derive(Clone, Debug)] pub struct PyByteArray { - // TODO: shouldn't be public - pub value: RefCell>, + pub inner: RefCell, } type PyByteArrayRef = PyRef; impl PyByteArray { pub fn new(data: Vec) -> Self { PyByteArray { - value: RefCell::new(data), + inner: RefCell::new(PyByteInner { elements: data }), + } + } + + pub fn from_inner(inner: PyByteInner) -> Self { + PyByteArray { + inner: RefCell::new(inner), } } + + // pub fn get_value(&self) -> Vec { + // self.inner.borrow().clone().elements + // } + + // pub fn get_value_mut(&self) -> Vec { + // self.inner.borrow_mut().clone().elements + // } } impl PyValue for PyByteArray { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.bytearray_type() } } -pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - obj.payload::().unwrap().value.borrow() -} - -pub fn get_mut_value<'a>(obj: &'a PyObjectRef) -> impl DerefMut> + 'a { - obj.payload::().unwrap().value.borrow_mut() -} +// pub fn get_value(obj: &PyObjectRef) -> Vec { +// obj.payload::().unwrap().get_value() +// } -// Binary data support +// pub fn get_value_mut(obj: &PyObjectRef) -> Vec { +// obj.payload::().unwrap().get_value_mut() +// } /// Fill bytearray class methods dictionary. pub fn init(context: &PyContext) { + PyByteArrayRef::extend_class(context, &context.bytearray_type); let bytearray_type = &context.bytearray_type; + extend_class!(context, bytearray_type, { + "fromhex" => context.new_rustfunc(PyByteArrayRef::fromhex), + "maketrans" => context.new_rustfunc(PyByteInner::maketrans), + }); - let bytearray_doc = - "bytearray(iterable_of_ints) -> bytearray\n\ - bytearray(string, encoding[, errors]) -> bytearray\n\ - bytearray(bytes_or_buffer) -> mutable copy of bytes_or_buffer\n\ - bytearray(int) -> bytes array of size given by the parameter initialized with null bytes\n\ - bytearray() -> empty bytes array\n\n\ - Construct a mutable bytearray object from:\n \ - - an iterable yielding integers in range(256)\n \ - - a text string encoded using the specified encoding\n \ - - a bytes or a buffer object\n \ - - any object implementing the buffer API.\n \ - - an integer"; - - context.set_attr( - &bytearray_type, - "__eq__", - context.new_rustfunc(bytearray_eq), - ); - context.set_attr( - &bytearray_type, - "__new__", - context.new_rustfunc(bytearray_new), - ); - context.set_attr( - &bytearray_type, - "__repr__", - context.new_rustfunc(bytearray_repr), - ); - context.set_attr( - &bytearray_type, - "__len__", - context.new_rustfunc(bytesarray_len), - ); - context.set_attr( - &bytearray_type, - "__doc__", - context.new_str(bytearray_doc.to_string()), - ); - context.set_attr( - &bytearray_type, - "isalnum", - context.new_rustfunc(bytearray_isalnum), - ); - context.set_attr( - &bytearray_type, - "isalpha", - context.new_rustfunc(bytearray_isalpha), - ); - context.set_attr( - &bytearray_type, - "isascii", - context.new_rustfunc(bytearray_isascii), - ); - context.set_attr( - &bytearray_type, - "isdigit", - context.new_rustfunc(bytearray_isdigit), - ); - context.set_attr( - &bytearray_type, - "islower", - context.new_rustfunc(bytearray_islower), - ); - context.set_attr( - &bytearray_type, - "isspace", - context.new_rustfunc(bytearray_isspace), - ); - context.set_attr( - &bytearray_type, - "isupper", - context.new_rustfunc(bytearray_isupper), - ); - context.set_attr( - &bytearray_type, - "istitle", - context.new_rustfunc(bytearray_istitle), - ); - context.set_attr( - &bytearray_type, - "clear", - context.new_rustfunc(bytearray_clear), - ); - context.set_attr(&bytearray_type, "pop", context.new_rustfunc(bytearray_pop)); - context.set_attr( - &bytearray_type, - "lower", - context.new_rustfunc(bytearray_lower), - ); - context.set_attr( - &bytearray_type, - "upper", - context.new_rustfunc(bytearray_upper), - ); + PyByteArrayIterator::extend_class(context, &context.bytearrayiterator_type); } -fn bytearray_new( - cls: PyClassRef, - val_option: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - if let Some(i) = v.to_u8() { - data_bytes.push(i); - } else { - return Err(vm.new_value_error("byte must be in range(0, 256)".to_string())); - } +#[pyimpl] +impl PyByteArrayRef { + #[pymethod(name = "__new__")] + fn bytearray_new( + cls: PyClassRef, + options: ByteInnerNewOptions, + vm: &VirtualMachine, + ) -> PyResult { + PyByteArray::from_inner(options.get_value(vm)?).into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__repr__")] + fn repr(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_str(format!("bytearray(b'{}')", self.inner.borrow().repr()?))) + } + + #[pymethod(name = "__len__")] + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.borrow().len() + } + + #[pymethod(name = "__eq__")] + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().eq(other, vm) + } + + #[pymethod(name = "__ge__")] + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().ge(other, vm) + } + + #[pymethod(name = "__le__")] + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().le(other, vm) + } + + #[pymethod(name = "__gt__")] + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().gt(other, vm) + } + + #[pymethod(name = "__lt__")] + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().lt(other, vm) + } + + #[pymethod(name = "__hash__")] + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("unhashable type: bytearray".to_string())) + } + + #[pymethod(name = "__iter__")] + fn iter(self, _vm: &VirtualMachine) -> PyByteArrayIterator { + PyByteArrayIterator { + position: Cell::new(0), + bytearray: self, } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; - PyByteArray::new(value).into_ref_with_type(vm, cls.clone()) -} + } -fn bytesarray_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytearray_type()))]); + #[pymethod(name = "__add__")] + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.ctx.new_bytearray(self.inner.borrow().add(other))) + } else { + Ok(vm.ctx.not_implemented()) + } + } - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) -} + #[pymethod(name = "__contains__")] + fn contains(self, needle: Either, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(needle, vm) + } -fn bytearray_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytearray_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytearray_type()) { - get_value(a).to_vec() == get_value(b).to_vec() - } else { - false - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__getitem__")] + fn getitem(self, needle: Either, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().getitem(needle, vm) + } -fn bytearray_isalnum(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphanumeric()))) -} + #[pymethod(name = "__setitem__")] + fn setitem( + self, + needle: Either, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.borrow_mut().setitem(needle, value, vm) + } -fn bytearray_isalpha(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_alphabetic()))) -} + #[pymethod(name = "isalnum")] + fn isalnum(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isalnum(vm) + } -fn bytearray_isascii(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_ascii()))) -} + #[pymethod(name = "isalpha")] + fn isalpha(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isalpha(vm) + } -fn bytearray_isdigit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_digit(10)))) -} + #[pymethod(name = "isascii")] + fn isascii(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isascii(vm) + } -fn bytearray_islower(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool( - !bytes.is_empty() - && bytes - .iter() - .filter(|x| !char::from(**x).is_whitespace()) - .all(|x| char::from(*x).is_lowercase()), - )) -} + #[pymethod(name = "isdigit")] + fn isdigit(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isdigit(vm) + } -fn bytearray_isspace(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool(!bytes.is_empty() && bytes.iter().all(|x| char::from(*x).is_whitespace()))) -} + #[pymethod(name = "islower")] + fn islower(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().islower(vm) + } -fn bytearray_isupper(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - Ok(vm.new_bool( - !bytes.is_empty() - && bytes - .iter() - .filter(|x| !char::from(**x).is_whitespace()) - .all(|x| char::from(*x).is_uppercase()), - )) -} + #[pymethod(name = "isspace")] + fn isspace(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isspace(vm) + } + + #[pymethod(name = "isupper")] + fn isupper(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isupper(vm) + } + + #[pymethod(name = "istitle")] + fn istitle(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().istitle(vm) + } + + #[pymethod(name = "lower")] + fn lower(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().lower(vm))) + } + + #[pymethod(name = "upper")] + fn upper(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().upper(vm))) + } + + #[pymethod(name = "capitalize")] + fn capitalize(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().capitalize(vm))) + } + + #[pymethod(name = "swapcase")] + fn swapcase(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().swapcase(vm))) + } + + #[pymethod(name = "hex")] + fn hex(self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().hex(vm) + } + + fn fromhex(string: PyStringRef, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(PyByteInner::fromhex(string.as_str(), vm)?)) + } + + #[pymethod(name = "center")] + fn center(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().center(options, vm)?)) + } + + #[pymethod(name = "ljust")] + fn ljust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().ljust(options, vm)?)) + } + + #[pymethod(name = "rjust")] + fn rjust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().rjust(options, vm)?)) + } + + #[pymethod(name = "count")] + fn count(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().count(options, vm) + } -fn bytearray_istitle(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - let bytes = get_value(zelf); - - if bytes.is_empty() { - Ok(vm.new_bool(false)) - } else { - let mut iter = bytes.iter().peekable(); - let mut prev_cased = false; - - while let Some(c) = iter.next() { - let current = char::from(*c); - let next = if let Some(k) = iter.peek() { - char::from(**k) - } else if current.is_uppercase() { - return Ok(vm.new_bool(!prev_cased)); - } else { - return Ok(vm.new_bool(prev_cased)); - }; - - if (is_cased(current) && next.is_uppercase() && !prev_cased) - || (!is_cased(current) && next.is_lowercase()) - { - return Ok(vm.new_bool(false)); - } - - prev_cased = is_cased(current); + #[pymethod(name = "join")] + fn join(self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().join(iter, vm) + } + + #[pymethod(name = "endswith")] + fn endswith( + self, + suffix: Either, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner + .borrow() + .startsendswith(suffix, start, end, true, vm) + } + + #[pymethod(name = "startswith")] + fn startswith( + self, + prefix: Either, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner + .borrow() + .startsendswith(prefix, start, end, false, vm) + } + + #[pymethod(name = "find")] + fn find(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().find(options, false, vm) + } + + #[pymethod(name = "index")] + fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.borrow().find(options, false, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); } + Ok(res) + } - Ok(vm.new_bool(true)) + #[pymethod(name = "rfind")] + fn rfind(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().find(options, true, vm) } -} -// helper function for istitle -fn is_cased(c: char) -> bool { - c.to_uppercase().next().unwrap() != c || c.to_lowercase().next().unwrap() != c -} + #[pymethod(name = "rindex")] + fn rindex(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.borrow().find(options, true, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(res) + } -/* -fn bytearray_getitem(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(obj, Some(vm.ctx.bytearray_type())), (needle, None)] - ); - let elements = get_elements(obj); - get_item(vm, list, &, needle.clone()) -} -*/ -/* -fn set_value(obj: &PyObjectRef, value: Vec) { - obj.borrow_mut().kind = PyObjectPayload::Bytes { value }; -} -*/ - -/// Return a lowercase hex representation of a bytearray -fn bytearray_to_hex(bytearray: &[u8]) -> String { - bytearray.iter().fold(String::new(), |mut s, b| { - let _ = write!(s, "\\x{:02x}", b); - s - }) -} + #[pymethod(name = "remove")] + fn remove(self, x: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + let x = x.as_bigint().byte_or(vm)?; -fn bytearray_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let value = get_value(obj); - let data = - String::from_utf8(value.to_vec()).unwrap_or_else(|_| bytearray_to_hex(&value.to_vec())); - Ok(vm.new_str(format!("bytearray(b'{}')", data))) -} + let bytes = &mut self.inner.borrow_mut().elements; + let pos = bytes + .iter() + .position(|b| *b == x) + .ok_or_else(|| vm.new_value_error("value not found in bytearray".to_string()))?; -fn bytearray_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytearray_type()))]); - get_mut_value(zelf).clear(); - Ok(vm.get_none()) -} + bytes.remove(pos); + + Ok(()) + } + + #[pymethod(name = "translate")] + fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().translate(options, vm) + } + + #[pymethod(name = "strip")] + fn strip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes( + self.inner + .borrow() + .strip(chars, ByteInnerPosition::All, vm)?, + )) + } + + #[pymethod(name = "lstrip")] + fn lstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes( + self.inner + .borrow() + .strip(chars, ByteInnerPosition::Left, vm)?, + )) + } + + #[pymethod(name = "rstrip")] + fn rstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes( + self.inner + .borrow() + .strip(chars, ByteInnerPosition::Right, vm)?, + )) + } + + #[pymethod(name = "split")] + fn split(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .borrow() + .split(options, false)? + .iter() + .map(|x| vm.ctx.new_bytearray(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "rsplit")] + fn rsplit(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .borrow() + .split(options, true)? + .iter() + .map(|x| vm.ctx.new_bytearray(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "partition")] + fn partition(self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { + // sep ALWAYS converted to bytearray even it's bytes or memoryview + // so its ok to accept PyByteInner + let (left, right) = self.inner.borrow().partition(&sep, false)?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytearray(left), + vm.ctx.new_bytearray(sep.elements), + vm.ctx.new_bytearray(right), + ])) + } + + #[pymethod(name = "rpartition")] + fn rpartition(self, sep: PyByteInner, vm: &VirtualMachine) -> PyResult { + let (left, right) = self.inner.borrow().partition(&sep, true)?; + Ok(vm.ctx.new_tuple(vec![ + vm.ctx.new_bytearray(left), + vm.ctx.new_bytearray(sep.elements), + vm.ctx.new_bytearray(right), + ])) + } + + #[pymethod(name = "expandtabs")] + fn expandtabs(self, options: ByteInnerExpandtabsOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().expandtabs(options))) + } + + #[pymethod(name = "splitlines")] + fn splitlines(self, options: ByteInnerSplitlinesOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .borrow() + .splitlines(options) + .iter() + .map(|x| vm.ctx.new_bytearray(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } -fn bytearray_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let mut value = get_mut_value(obj); + #[pymethod(name = "zfill")] + fn zfill(self, width: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().zfill(width))) + } + + #[pymethod(name = "replace")] + fn replace( + self, + old: PyByteInner, + new: PyByteInner, + count: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm + .ctx + .new_bytearray(self.inner.borrow().replace(old, new, count)?)) + } + + #[pymethod(name = "clear")] + fn clear(self, _vm: &VirtualMachine) { + self.inner.borrow_mut().elements.clear(); + } + + #[pymethod(name = "copy")] + fn copy(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().elements.clone())) + } + + #[pymethod(name = "append")] + fn append(self, x: PyIntRef, vm: &VirtualMachine) -> Result<(), PyObjectRef> { + self.inner + .borrow_mut() + .elements + .push(x.as_bigint().byte_or(vm)?); + Ok(()) + } + + #[pymethod(name = "extend")] + fn extend(self, iterable_of_ints: PyIterable, vm: &VirtualMachine) -> Result<(), PyObjectRef> { + let mut inner = self.inner.borrow_mut(); + + for x in iterable_of_ints.iter(vm)? { + let x = x?; + let x = PyIntRef::try_from_object(vm, x)?; + let x = x.as_bigint().byte_or(vm)?; + inner.elements.push(x); + } + + Ok(()) + } + + #[pymethod(name = "insert")] + fn insert(self, index: PyIntRef, x: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + let bytes = &mut self.inner.borrow_mut().elements; + let len = isize::try_from(bytes.len()) + .map_err(|_e| vm.new_overflow_error("bytearray too big".to_string()))?; + + let x = x.as_bigint().byte_or(vm)?; + + let mut index = index + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("index too big".to_string()))?; + + if index >= len { + bytes.push(x); + return Ok(()); + } + + if index < 0 { + index += len; + index = index.max(0); + } + + let index = usize::try_from(index) + .map_err(|_e| vm.new_overflow_error("overflow in index calculation".to_string()))?; + + bytes.insert(index, x); + + Ok(()) + } + + #[pymethod(name = "pop")] + fn pop(self, vm: &VirtualMachine) -> PyResult { + let bytes = &mut self.inner.borrow_mut().elements; + bytes + .pop() + .ok_or_else(|| vm.new_index_error("pop from empty bytearray".to_string())) + } + + #[pymethod(name = "title")] + fn title(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().title())) + } + + #[pymethod(name = "__mul__")] + fn repeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytearray(self.inner.borrow().repeat(n, vm)?)) + } + + #[pymethod(name = "__rmul__")] + fn rmul(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + self.repeat(n, vm) + } + + #[pymethod(name = "__imul__")] + fn irepeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().irepeat(n, vm) + } - if let Some(i) = value.pop() { - Ok(vm.ctx.new_int(i)) - } else { - Err(vm.new_index_error("pop from empty bytearray".to_string())) + #[pymethod(name = "reverse")] + fn reverse(self, _vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().elements.reverse(); + Ok(()) } } -fn bytearray_lower(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let value = get_value(obj).to_vec().to_ascii_lowercase(); - Ok(vm.ctx.new_bytearray(value)) +// fn set_value(obj: &PyObjectRef, value: Vec) { +// obj.borrow_mut().kind = PyObjectPayload::Bytes { value }; +// } + +#[pyclass] +#[derive(Debug)] +pub struct PyByteArrayIterator { + position: Cell, + bytearray: PyByteArrayRef, } -fn bytearray_upper(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytearray_type()))]); - let value = get_value(obj).to_vec().to_ascii_uppercase(); - Ok(vm.ctx.new_bytearray(value)) +impl PyValue for PyByteArrayIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytearrayiterator_type() + } } -#[cfg(test)] -mod tests { - use super::*; +#[pyimpl] +impl PyByteArrayIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytearray.inner.borrow().len() { + let ret = self.bytearray.inner.borrow().elements[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } - #[test] - fn bytearray_to_hex_formatting() { - assert_eq!(&bytearray_to_hex(&[11u8, 222u8]), "\\x0b\\xde"); + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } diff --git a/vm/src/obj/objbyteinner.rs b/vm/src/obj/objbyteinner.rs new file mode 100644 index 0000000000..32a58255f8 --- /dev/null +++ b/vm/src/obj/objbyteinner.rs @@ -0,0 +1,1340 @@ +use crate::obj::objint::PyIntRef; +use crate::obj::objnone::PyNoneRef; +use crate::obj::objslice::PySliceRef; +use crate::obj::objtuple::PyTupleRef; +use crate::pyhash; +use crate::pyobject::Either; +use crate::pyobject::PyRef; +use crate::pyobject::PyValue; +use crate::pyobject::TryFromObject; +use crate::pyobject::{PyIterable, PyObjectRef}; +use core::convert::TryFrom; +use core::ops::Range; +use num_bigint::BigInt; + +use crate::function::OptionalArg; +use crate::pyobject::{PyResult, TypeProtocol}; +use crate::vm::VirtualMachine; + +use super::objint; +use super::objsequence::{is_valid_slice_arg, PySliceableSequence}; +use super::objstr::{PyString, PyStringRef}; + +use crate::obj::objint::PyInt; +use num_integer::Integer; +use num_traits::ToPrimitive; + +use super::objbytearray::PyByteArray; +use super::objbytes::PyBytes; +use super::objmemory::PyMemoryView; + +use super::objsequence; + +#[derive(Debug, Default, Clone)] +pub struct PyByteInner { + pub elements: Vec, +} + +impl TryFromObject for PyByteInner { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + + i @ PyBytes => Ok(PyByteInner{elements: i.get_value().to_vec()}), + j @ PyByteArray => Ok(PyByteInner{elements: j.inner.borrow().elements.to_vec()}), + k @ PyMemoryView => Ok(PyByteInner{elements: k.get_obj_value().unwrap()}), + obj => Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + obj.class() + ))) + ) + } +} + +impl TryFromObject for Either> { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match PyByteInner::try_from_object(vm, obj.clone()) { + Ok(a) => Ok(Either::A(a)), + Err(_) => match obj.clone().downcast::() { + Ok(b) => Ok(Either::B(b)), + Err(_) => Err(vm.new_type_error(format!( + "a bytes-like object or {} is required, not {}", + B::class(vm), + obj.class() + ))), + }, + } + } +} + +#[derive(FromArgs)] +pub struct ByteInnerNewOptions { + #[pyarg(positional_only, optional = true)] + val_option: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + encoding: OptionalArg, +} + +//same algorithm as cpython +pub fn normalize_encoding(encoding: &str) -> String { + let mut res = String::new(); + let mut punct = false; + + for c in encoding.chars() { + if c.is_alphanumeric() || c == '.' { + if punct && !res.is_empty() { + res.push('_') + } + res.push(c.to_ascii_lowercase()); + punct = false; + } else { + punct = true; + } + } + res +} + +impl ByteInnerNewOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult { + // First handle bytes(string, encoding[, errors]) + if let OptionalArg::Present(enc) = self.encoding { + if let OptionalArg::Present(eval) = self.val_option { + if let Ok(input) = eval.downcast::() { + let inner = PyByteInner::from_string(&input.value, enc.as_str(), vm)?; + return Ok(inner); + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + } else { + return Err(vm.new_type_error("encoding without a string argument".to_string())); + } + // Only one argument + } else { + let value = if let OptionalArg::Present(ival) = self.val_option { + match_class!(ival.clone(), + i @ PyInt => { + let size = objint::get_value(&i.into_object()).to_usize().unwrap(); + Ok(vec![0; size])}, + _l @ PyString=> {return Err(vm.new_type_error("string argument without an encoding".to_string()));}, + obj => { + let elements = vm.extract_elements(&obj).or_else(|_| {Err(vm.new_type_error(format!( + "cannot convert {} object to bytes", obj.class().name)))}); + + let mut data_bytes = vec![]; + for elem in elements.unwrap(){ + let v = objint::to_int(vm, &elem, 10)?; + if let Some(i) = v.to_u8() { + data_bytes.push(i); + } else { + return Err(vm.new_value_error("bytes must be in range(0, 256)".to_string())); + } + + } + Ok(data_bytes) + } + ) + } else { + Ok(vec![]) + }; + match value { + Ok(val) => Ok(PyByteInner { elements: val }), + Err(err) => Err(err), + } + } + } +} + +#[derive(FromArgs)] +pub struct ByteInnerFindOptions { + #[pyarg(positional_only, optional = false)] + sub: Either, + #[pyarg(positional_only, optional = true)] + start: OptionalArg>, + #[pyarg(positional_only, optional = true)] + end: OptionalArg>, +} + +impl ByteInnerFindOptions { + pub fn get_value( + self, + elements: &[u8], + vm: &VirtualMachine, + ) -> PyResult<(Vec, Range)> { + let sub = match self.sub { + Either::A(v) => v.elements.to_vec(), + Either::B(int) => vec![int.as_bigint().byte_or(vm)?], + }; + + let start = match self.start { + OptionalArg::Present(Some(int)) => Some(int.as_bigint().clone()), + _ => None, + }; + + let end = match self.end { + OptionalArg::Present(Some(int)) => Some(int.as_bigint().clone()), + _ => None, + }; + + let range = elements.to_vec().get_slice_range(&start, &end); + + Ok((sub, range)) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerPaddingOptions { + #[pyarg(positional_only, optional = false)] + width: PyIntRef, + #[pyarg(positional_only, optional = true)] + fillbyte: OptionalArg, +} +impl ByteInnerPaddingOptions { + fn get_value(self, fn_name: &str, len: usize, vm: &VirtualMachine) -> PyResult<(u8, usize)> { + let fillbyte = if let OptionalArg::Present(v) = &self.fillbyte { + match try_as_byte(&v) { + Some(x) => { + if x.len() == 1 { + x[0] + } else { + return Err(vm.new_type_error(format!( + "{}() argument 2 must be a byte string of length 1, not {}", + fn_name, &v + ))); + } + } + None => { + return Err(vm.new_type_error(format!( + "{}() argument 2 must be a byte string of length 1, not {}", + fn_name, &v + ))); + } + } + } else { + b' ' // default is space + }; + + // <0 = no change + let width = if let Some(x) = self.width.as_bigint().to_usize() { + if x <= len { + 0 + } else { + x + } + } else { + 0 + }; + + let diff: usize = if width != 0 { width - len } else { 0 }; + + Ok((fillbyte, diff)) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerTranslateOptions { + #[pyarg(positional_only, optional = false)] + table: Either, + #[pyarg(positional_or_keyword, optional = true)] + delete: OptionalArg, +} + +impl ByteInnerTranslateOptions { + pub fn get_value(self, vm: &VirtualMachine) -> PyResult<(Vec, Vec)> { + let table = match self.table { + Either::A(v) => v.elements.to_vec(), + Either::B(_) => (0..=255).collect::>(), + }; + + if table.len() != 256 { + return Err( + vm.new_value_error("translation table must be 256 characters long".to_string()) + ); + } + + let delete = match self.delete { + OptionalArg::Present(byte) => byte.elements, + _ => vec![], + }; + + Ok((table, delete)) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerSplitOptions { + #[pyarg(positional_or_keyword, optional = true)] + sep: OptionalArg>, + #[pyarg(positional_or_keyword, optional = true)] + maxsplit: OptionalArg, +} + +impl ByteInnerSplitOptions { + pub fn get_value(self) -> PyResult<(Vec, i32)> { + let sep = match self.sep.into_option() { + Some(Some(bytes)) => bytes.elements, + _ => vec![], + }; + + let maxsplit = if let OptionalArg::Present(value) = self.maxsplit { + value.as_bigint().to_i32().unwrap() + } else { + -1 + }; + + Ok((sep.clone(), maxsplit)) + } +} + +#[derive(FromArgs)] +pub struct ByteInnerExpandtabsOptions { + #[pyarg(positional_or_keyword, optional = true)] + tabsize: OptionalArg, +} + +impl ByteInnerExpandtabsOptions { + pub fn get_value(self) -> usize { + match self.tabsize.into_option() { + Some(int) => int.as_bigint().to_usize().unwrap_or(0), + None => 8, + } + } +} + +#[derive(FromArgs)] +pub struct ByteInnerSplitlinesOptions { + #[pyarg(positional_or_keyword, optional = true)] + keepends: OptionalArg, +} + +impl ByteInnerSplitlinesOptions { + pub fn get_value(self) -> bool { + match self.keepends.into_option() { + Some(x) => x, + None => false, + } + // if let OptionalArg::Present(value) = self.keepends { + // Ok(bool::try_from_object(vm, value)?) + // } else { + // Ok(false) + // } + } +} + +impl PyByteInner { + pub fn from_string(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult { + let normalized = normalize_encoding(encoding); + if normalized == "utf_8" || normalized == "utf8" || normalized == "u8" { + Ok(PyByteInner { + elements: value.as_bytes().to_vec(), + }) + } else { + // TODO: different encoding + Err( + vm.new_value_error(format!("unknown encoding: {}", encoding)), // should be lookup error + ) + } + } + + pub fn repr(&self) -> PyResult { + let mut res = String::with_capacity(self.elements.len()); + for i in self.elements.iter() { + match i { + 0..=8 => res.push_str(&format!("\\x0{}", i)), + 9 => res.push_str("\\t"), + 10 => res.push_str("\\n"), + 11 => res.push_str(&format!("\\x0{:x}", i)), + 13 => res.push_str("\\r"), + 32..=126 => res.push(*(i) as char), + _ => res.push_str(&format!("\\x{:x}", i)), + } + } + Ok(res) + } + + pub fn len(&self) -> usize { + self.elements.len() + } + + pub fn is_empty(&self) -> bool { + self.elements.len() == 0 + } + + pub fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements == other.elements)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + pub fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements >= other.elements)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + pub fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements <= other.elements)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + pub fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements > other.elements)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + pub fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.new_bool(self.elements < other.elements)) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + pub fn hash(&self) -> pyhash::PyHash { + pyhash::hash_value(&self.elements) + } + + pub fn add(&self, other: PyByteInner) -> Vec { + self.elements + .iter() + .chain(other.elements.iter()) + .cloned() + .collect::>() + } + + pub fn contains(&self, needle: Either, vm: &VirtualMachine) -> PyResult { + match needle { + Either::A(byte) => { + let other = &byte.elements[..]; + for (n, i) in self.elements.iter().enumerate() { + if n + other.len() <= self.len() + && *i == other[0] + && &self.elements[n..n + other.len()] == other + { + return Ok(vm.new_bool(true)); + } + } + Ok(vm.new_bool(false)) + } + Either::B(int) => { + if self.elements.contains(&int.as_bigint().byte_or(vm)?) { + Ok(vm.new_bool(true)) + } else { + Ok(vm.new_bool(false)) + } + } + } + } + + pub fn getitem(&self, needle: Either, vm: &VirtualMachine) -> PyResult { + match needle { + Either::A(int) => { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + Ok(vm.new_int(self.elements[idx])) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + Either::B(slice) => Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)), + } + } + + fn setindex(&mut self, int: PyIntRef, object: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(idx) = self.elements.get_pos(int.as_bigint().to_i32().unwrap()) { + let result = match_class!(object, + i @ PyInt => { + if let Some(value) = i.as_bigint().to_u8() { + Ok(value) + }else{ + Err(vm.new_value_error("byte must be in range(0, 256)".to_string())) + } + }, + _ => {Err(vm.new_type_error("an integer is required".to_string()))} + ); + let value = result?; + self.elements[idx] = value; + Ok(vm.new_int(value)) + } else { + Err(vm.new_index_error("index out of range".to_string())) + } + } + + fn setslice( + &mut self, + slice: PySliceRef, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let sec = match PyIterable::try_from_object(vm, object.clone()) { + Ok(sec) => { + let items: Result, _> = sec.iter(vm)?.collect(); + Ok(items? + .into_iter() + .map(|obj| u8::try_from_object(vm, obj)) + .collect::>>()?) + } + _ => match_class!(object, + i @ PyMemoryView => { + Ok(i.get_obj_value().unwrap()) + }, + _ => Err(vm.new_index_error( + "can assign only bytes, buffers, or iterables of ints in range(0, 256)" + .to_string()))), + }; + let items = sec?; + let range = self + .elements + .get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); + self.elements.splice(range, items); + Ok(vm + .ctx + .new_bytes(self.elements.get_slice_items(vm, slice.as_object())?)) + } + + pub fn setitem( + &mut self, + needle: Either, + object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match needle { + Either::A(int) => self.setindex(int, object, vm), + Either::B(slice) => self.setslice(slice, object, vm), + } + } + + pub fn isalnum(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .all(|x| char::from(*x).is_alphanumeric()), + )) + } + + pub fn isalpha(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_alphabetic()), + )) + } + + pub fn isascii(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_ascii()), + )) + } + + pub fn isdigit(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() && self.elements.iter().all(|x| char::from(*x).is_digit(10)), + )) + } + + pub fn islower(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_lowercase()), + )) + } + + pub fn isspace(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self.elements.iter().all(|x| char::from(*x).is_whitespace()), + )) + } + + pub fn isupper(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_bool( + !self.elements.is_empty() + && self + .elements + .iter() + .filter(|x| !char::from(**x).is_whitespace()) + .all(|x| char::from(*x).is_uppercase()), + )) + } + + pub fn istitle(&self, vm: &VirtualMachine) -> PyResult { + if self.elements.is_empty() { + return Ok(vm.new_bool(false)); + } + + let mut iter = self.elements.iter().peekable(); + let mut prev_cased = false; + + while let Some(c) = iter.next() { + let current = char::from(*c); + let next = if let Some(k) = iter.peek() { + char::from(**k) + } else if current.is_uppercase() { + return Ok(vm.new_bool(!prev_cased)); + } else { + return Ok(vm.new_bool(prev_cased)); + }; + + let is_cased = current.to_uppercase().next().unwrap() != current + || current.to_lowercase().next().unwrap() != current; + if (is_cased && next.is_uppercase() && !prev_cased) + || (!is_cased && next.is_lowercase()) + { + return Ok(vm.new_bool(false)); + } + + prev_cased = is_cased; + } + + Ok(vm.new_bool(true)) + } + + pub fn lower(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_lowercase() + } + + pub fn upper(&self, _vm: &VirtualMachine) -> Vec { + self.elements.to_ascii_uppercase() + } + + pub fn capitalize(&self, _vm: &VirtualMachine) -> Vec { + let mut new: Vec = Vec::new(); + if let Some((first, second)) = self.elements.split_first() { + new.push(first.to_ascii_uppercase()); + second.iter().for_each(|x| new.push(x.to_ascii_lowercase())); + } + new + } + + pub fn swapcase(&self, _vm: &VirtualMachine) -> Vec { + let mut new: Vec = Vec::with_capacity(self.elements.len()); + for w in &self.elements { + match w { + 65..=90 => new.push(w.to_ascii_lowercase()), + 97..=122 => new.push(w.to_ascii_uppercase()), + x => new.push(*x), + } + } + new + } + + pub fn hex(&self, vm: &VirtualMachine) -> PyResult { + let bla = self + .elements + .iter() + .map(|x| format!("{:02x}", x)) + .collect::(); + Ok(vm.ctx.new_str(bla)) + } + + pub fn fromhex(string: &str, vm: &VirtualMachine) -> PyResult> { + // first check for invalid character + for (i, c) in string.char_indices() { + if !c.is_digit(16) && !c.is_whitespace() { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + i + ))); + } + } + + // strip white spaces + let stripped = string.split_whitespace().collect::(); + + // Hex is evaluated on 2 digits + if stripped.len() % 2 != 0 { + return Err(vm.new_value_error(format!( + "non-hexadecimal number found in fromhex() arg at position {}", + stripped.len() - 1 + ))); + } + + // parse even string + Ok(stripped + .chars() + .collect::>() + .chunks(2) + .map(|x| x.to_vec().iter().collect::()) + .map(|x| u8::from_str_radix(&x, 16).unwrap()) + .collect::>()) + } + + pub fn center( + &self, + options: ByteInnerPaddingOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let (fillbyte, diff) = options.get_value("center", self.len(), vm)?; + + let mut ln: usize = diff / 2; + let mut rn: usize = ln; + + if diff.is_odd() && self.len() % 2 == 0 { + ln += 1 + } + + if diff.is_odd() && self.len() % 2 != 0 { + rn += 1 + } + + // merge all + let mut res = vec![fillbyte; ln]; + res.extend_from_slice(&self.elements[..]); + res.extend_from_slice(&vec![fillbyte; rn][..]); + + Ok(res) + } + + pub fn ljust( + &self, + options: ByteInnerPaddingOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let (fillbyte, diff) = options.get_value("ljust", self.len(), vm)?; + + // merge all + let mut res = vec![]; + res.extend_from_slice(&self.elements[..]); + res.extend_from_slice(&vec![fillbyte; diff][..]); + + Ok(res) + } + + pub fn rjust( + &self, + options: ByteInnerPaddingOptions, + vm: &VirtualMachine, + ) -> PyResult> { + let (fillbyte, diff) = options.get_value("rjust", self.len(), vm)?; + + // merge all + let mut res = vec![fillbyte; diff]; + res.extend_from_slice(&self.elements[..]); + + Ok(res) + } + + pub fn count(&self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let (sub, range) = options.get_value(&self.elements, vm)?; + + if sub.is_empty() { + return Ok(self.len() + 1); + } + + let mut total: usize = 0; + let mut i_start = range.start; + let i_end = range.end; + + for i in self.elements.do_slice(range) { + if i_start + sub.len() <= i_end + && i == sub[0] + && &self.elements[i_start..(i_start + sub.len())] == sub.as_slice() + { + total += 1; + } + i_start += 1; + } + Ok(total) + } + + pub fn join(&self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut refs = vec![]; + for v in iter.iter(vm)? { + let v = v?; + refs.extend(PyByteInner::try_from_object(vm, v)?.elements) + } + + Ok(vm.ctx.new_bytes(refs)) + } + + pub fn startsendswith( + &self, + arg: Either, + start: OptionalArg, + end: OptionalArg, + endswith: bool, // true for endswith, false for startswith + vm: &VirtualMachine, + ) -> PyResult { + let suff = match arg { + Either::A(byte) => byte.elements, + Either::B(tuple) => { + let mut flatten = vec![]; + for v in objsequence::get_elements_tuple(tuple.as_object()).to_vec() { + flatten.extend(PyByteInner::try_from_object(vm, v)?.elements) + } + flatten + } + }; + + if suff.is_empty() { + return Ok(vm.new_bool(true)); + } + let range = self.elements.get_slice_range( + &is_valid_slice_arg(start, vm)?, + &is_valid_slice_arg(end, vm)?, + ); + + if range.end - range.start < suff.len() { + return Ok(vm.new_bool(false)); + } + + let offset = if endswith { + (range.end - suff.len())..range.end + } else { + 0..suff.len() + }; + + Ok(vm.new_bool(suff.as_slice() == &self.elements.do_slice(range)[offset])) + } + + pub fn find( + &self, + options: ByteInnerFindOptions, + reverse: bool, + vm: &VirtualMachine, + ) -> PyResult { + let (sub, range) = options.get_value(&self.elements, vm)?; + // not allowed for this method + if range.end < range.start { + return Ok(-1isize); + } + + let start = range.start; + let end = range.end; + + if reverse { + let slice = self.elements.do_slice_reverse(range); + for (n, _) in slice.iter().enumerate() { + if n + sub.len() <= slice.len() && &slice[n..n + sub.len()] == sub.as_slice() { + return Ok((end - n - 1) as isize); + } + } + } else { + let slice = self.elements.do_slice(range); + for (n, _) in slice.iter().enumerate() { + if n + sub.len() <= slice.len() && &slice[n..n + sub.len()] == sub.as_slice() { + return Ok((start + n) as isize); + } + } + }; + Ok(-1isize) + } + + pub fn maketrans(from: PyByteInner, to: PyByteInner, vm: &VirtualMachine) -> PyResult { + let mut res = vec![]; + + for i in 0..=255 { + res.push( + if let Some(position) = from.elements.iter().position(|&x| x == i) { + to.elements[position] + } else { + i + }, + ); + } + + Ok(vm.ctx.new_bytes(res)) + } + + pub fn translate(&self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { + let (table, delete) = options.get_value(vm)?; + + let mut res = vec![]; + + for i in self.elements.iter() { + if !delete.contains(&i) { + res.push(table[*i as usize]); + } + } + + Ok(vm.ctx.new_bytes(res)) + } + + pub fn strip( + &self, + chars: OptionalArg, + position: ByteInnerPosition, + _vm: &VirtualMachine, + ) -> PyResult> { + let chars = if let OptionalArg::Present(bytes) = chars { + bytes.elements + } else { + vec![b' '] + }; + + let mut start = 0; + let mut end = self.len(); + + if let ByteInnerPosition::Left | ByteInnerPosition::All = position { + for (n, i) in self.elements.iter().enumerate() { + if !chars.contains(i) { + start = n; + break; + } + } + } + + if let ByteInnerPosition::Right | ByteInnerPosition::All = position { + for (n, i) in self.elements.iter().rev().enumerate() { + if !chars.contains(i) { + end = self.len() - n; + break; + } + } + } + Ok(self.elements[start..end].to_vec()) + } + + pub fn split(&self, options: ByteInnerSplitOptions, reverse: bool) -> PyResult> { + let (sep, maxsplit) = options.get_value()?; + + if self.elements.is_empty() { + if !sep.is_empty() { + return Ok(vec![&[]]); + } + return Ok(vec![]); + } + + if reverse { + Ok(split_slice_reverse(&self.elements, &sep, maxsplit)) + } else { + Ok(split_slice(&self.elements, &sep, maxsplit)) + } + } + + pub fn partition(&self, sep: &PyByteInner, reverse: bool) -> PyResult<(Vec, Vec)> { + let splitted = if reverse { + split_slice_reverse(&self.elements, &sep.elements, 1) + } else { + split_slice(&self.elements, &sep.elements, 1) + }; + Ok((splitted[0].to_vec(), splitted[1].to_vec())) + } + + pub fn expandtabs(&self, options: ByteInnerExpandtabsOptions) -> Vec { + let tabsize = options.get_value(); + let mut counter: usize = 0; + let mut res = vec![]; + + if tabsize == 0 { + return self + .elements + .iter() + .cloned() + .filter(|x| *x != b'\t') + .collect::>(); + } + + for i in &self.elements { + if *i == b'\t' { + let len = tabsize - counter % tabsize; + res.extend_from_slice(&vec![b' '; len]); + counter += len; + } else { + res.push(*i); + if *i == b'\r' || *i == b'\n' { + counter = 0; + } else { + counter += 1; + } + } + } + + res + } + + pub fn splitlines(&self, options: ByteInnerSplitlinesOptions) -> Vec<&[u8]> { + let keepends = options.get_value(); + + let mut res = vec![]; + + if self.elements.is_empty() { + return vec![]; + } + + let mut prev_index = 0; + let mut index = 0; + let keep = if keepends { 1 } else { 0 }; + let slice = &self.elements; + + while index < slice.len() { + match slice[index] { + b'\n' => { + res.push(&slice[prev_index..index + keep]); + index += 1; + prev_index = index; + } + b'\r' => { + if index + 2 <= slice.len() && slice[index + 1] == b'\n' { + res.push(&slice[prev_index..index + keep + keep]); + index += 2; + } else { + res.push(&slice[prev_index..index + keep]); + index += 1; + } + prev_index = index; + } + _x => { + if index == slice.len() - 1 { + res.push(&slice[prev_index..=index]); + break; + } + index += 1 + } + } + } + + res + } + + pub fn zfill(&self, width: PyIntRef) -> Vec { + if let Some(value) = width.as_bigint().to_usize() { + if value < self.elements.len() { + return self.elements.to_vec(); + } + let mut res = vec![]; + if self.elements.starts_with(&[b'-']) { + res.push(b'-'); + res.extend_from_slice(&vec![b'0'; value - self.elements.len()]); + res.extend_from_slice(&self.elements[1..]); + } else { + res.extend_from_slice(&vec![b'0'; value - self.elements.len()]); + res.extend_from_slice(&self.elements[0..]); + } + res + } else { + self.elements.to_vec() + } + } + + pub fn replace( + &self, + old: PyByteInner, + new: PyByteInner, + count: OptionalArg, + ) -> PyResult> { + let count = match count.into_option() { + Some(int) => int + .as_bigint() + .to_u32() + .unwrap_or(self.elements.len() as u32), + None => self.elements.len() as u32, + }; + + let mut res = vec![]; + let mut index = 0; + let mut done = 0; + + let slice = &self.elements; + while index <= slice.len() - old.len() { + if done == count { + res.extend_from_slice(&slice[index..]); + break; + } + if &slice[index..index + old.len()] == old.elements.as_slice() { + res.extend_from_slice(&new.elements); + index += old.len(); + done += 1; + } else { + res.push(slice[index]); + index += 1 + } + } + + Ok(res) + } + + pub fn title(&self) -> Vec { + let mut res = vec![]; + let mut spaced = true; + + for i in self.elements.iter() { + match i { + 65..=90 | 97..=122 => { + if spaced { + res.push(i.to_ascii_uppercase()); + spaced = false + } else { + res.push(i.to_ascii_lowercase()); + } + } + _ => { + res.push(*i); + spaced = true + } + } + } + + res + } + + pub fn repeat(&self, n: PyIntRef, vm: &VirtualMachine) -> PyResult> { + if self.elements.is_empty() { + // We can multiple an empty vector by any integer, even if it doesn't fit in an isize. + return Ok(vec![]); + } + + let n = n.as_bigint().to_isize().ok_or_else(|| { + vm.new_overflow_error("can't multiply bytes that many times".to_string()) + })?; + + if n <= 0 { + Ok(vec![]) + } else { + let n = usize::try_from(n).unwrap(); + + let mut new_value = Vec::with_capacity(n * self.elements.len()); + for _ in 0..n { + new_value.extend(&self.elements); + } + + Ok(new_value) + } + } + + pub fn irepeat(&mut self, n: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + if self.elements.is_empty() { + // We can multiple an empty vector by any integer, even if it doesn't fit in an isize. + return Ok(()); + } + + let n = n.as_bigint().to_isize().ok_or_else(|| { + vm.new_overflow_error("can't multiply bytes that many times".to_string()) + })?; + + if n <= 0 { + self.elements.clear(); + } else { + let n = usize::try_from(n).unwrap(); + + let old = self.elements.clone(); + + self.elements.reserve((n - 1) * old.len()); + for _ in 1..n { + self.elements.extend(&old); + } + } + + Ok(()) + } +} + +pub fn try_as_byte(obj: &PyObjectRef) -> Option> { + match_class!(obj.clone(), + + i @ PyBytes => Some(i.get_value().to_vec()), + j @ PyByteArray => Some(j.inner.borrow().elements.to_vec()), + _ => None) +} + +pub trait ByteOr: ToPrimitive { + fn byte_or(&self, vm: &VirtualMachine) -> Result { + match self.to_u8() { + Some(value) => Ok(value), + None => Err(vm.new_value_error("byte must be in range(0, 256)".to_string())), + } + } +} + +impl ByteOr for BigInt {} + +pub enum ByteInnerPosition { + Left, + Right, + All, +} + +fn split_slice<'a>(slice: &'a [u8], sep: &[u8], maxsplit: i32) -> Vec<&'a [u8]> { + let mut splitted: Vec<&[u8]> = vec![]; + let mut prev_index = 0; + let mut index = 0; + let mut count = 0; + let mut in_string = false; + + // No sep given, will split for any \t \n \r and space = [9, 10, 13, 32] + if sep.is_empty() { + // split wihtout sep always trim left spaces for any maxsplit + // so we have to ignore left spaces. + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + index += 1 + } else { + prev_index = index; + break; + } + } + + // most simple case + if maxsplit == 0 { + splitted.push(&slice[index..slice.len()]); + return splitted; + } + + // main loop. in_string means previous char is ascii char(true) or space(false) + // loop from left to right + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + if in_string { + splitted.push(&slice[prev_index..index]); + in_string = false; + count += 1; + if count == maxsplit { + // while index < slice.len() + splitted.push(&slice[index + 1..slice.len()]); + break; + } + } + } else if !in_string { + prev_index = index; + in_string = true; + } + + index += 1; + + // handle last item in slice + if index == slice.len() { + if in_string { + if [9, 10, 13, 32].contains(&slice[index - 1]) { + splitted.push(&slice[prev_index..index - 1]); + } else { + splitted.push(&slice[prev_index..index]); + } + } + break; + } + } + } else { + // sep is given, we match exact slice + while index != slice.len() { + if index + sep.len() >= slice.len() { + if &slice[index..slice.len()] == sep { + splitted.push(&slice[prev_index..index]); + splitted.push(&[]); + break; + } + splitted.push(&slice[prev_index..slice.len()]); + break; + } + + if &slice[index..index + sep.len()] == sep { + splitted.push(&slice[prev_index..index]); + index += sep.len(); + prev_index = index; + count += 1; + if count == maxsplit { + // maxsplit reached, append, the remaing + splitted.push(&slice[prev_index..slice.len()]); + break; + } + continue; + } + + index += 1; + } + } + splitted +} + +fn split_slice_reverse<'a>(slice: &'a [u8], sep: &[u8], maxsplit: i32) -> Vec<&'a [u8]> { + let mut splitted: Vec<&[u8]> = vec![]; + let mut prev_index = slice.len(); + let mut index = slice.len(); + let mut count = 0; + + // No sep given, will split for any \t \n \r and space = [9, 10, 13, 32] + if sep.is_empty() { + //adjust index + index -= 1; + + // rsplit without sep always trim right spaces for any maxsplit + // so we have to ignore right spaces. + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + index -= 1 + } else { + break; + } + } + prev_index = index + 1; + + // most simple case + if maxsplit == 0 { + splitted.push(&slice[0..=index]); + return splitted; + } + + // main loop. in_string means previous char is ascii char(true) or space(false) + // loop from right to left and reverse result the end + let mut in_string = true; + loop { + if [9, 10, 13, 32].contains(&slice[index]) { + if in_string { + splitted.push(&slice[index + 1..prev_index]); + count += 1; + if count == maxsplit { + // maxsplit reached, append, the remaing + splitted.push(&slice[0..index]); + break; + } + in_string = false; + index -= 1; + continue; + } + } else if !in_string { + in_string = true; + if index == 0 { + splitted.push(&slice[0..1]); + break; + } + prev_index = index + 1; + } + if index == 0 { + break; + } + index -= 1; + } + } else { + // sep is give, we match exact slice going backwards + while index != 0 { + if index <= sep.len() { + if &slice[0..index] == sep { + splitted.push(&slice[index..prev_index]); + splitted.push(&[]); + break; + } + splitted.push(&slice[0..prev_index]); + break; + } + if &slice[(index - sep.len())..index] == sep { + splitted.push(&slice[index..prev_index]); + index -= sep.len(); + prev_index = index; + count += 1; + if count == maxsplit { + // maxsplit reached, append, the remaing + splitted.push(&slice[0..prev_index]); + break; + } + continue; + } + + index -= 1; + } + } + splitted.reverse(); + splitted +} diff --git a/vm/src/obj/objbytes.rs b/vm/src/obj/objbytes.rs index dc1f930a46..33016e511a 100644 --- a/vm/src/obj/objbytes.rs +++ b/vm/src/obj/objbytes.rs @@ -1,27 +1,59 @@ -use std::cell::Cell; -use std::hash::{Hash, Hasher}; -use std::ops::Deref; +use crate::obj::objint::PyIntRef; +use crate::obj::objslice::PySliceRef; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtuple::PyTupleRef; +use crate::pyhash; -use num_traits::ToPrimitive; +use crate::pyobject::Either; +use crate::vm::VirtualMachine; +use core::cell::Cell; +use std::ops::Deref; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, }; -use crate::vm::VirtualMachine; -use super::objint; -use super::objtype::{self, PyClassRef}; +use super::objbyteinner::{ + ByteInnerExpandtabsOptions, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, + ByteInnerPosition, ByteInnerSplitOptions, ByteInnerSplitlinesOptions, + ByteInnerTranslateOptions, PyByteInner, +}; +use super::objiter; -#[derive(Debug)] +use super::objtype::PyClassRef; + +/// "bytes(iterable_of_ints) -> bytes\n\ +/// bytes(string, encoding[, errors]) -> bytes\n\ +/// bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ +/// bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ +/// bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ +/// - an iterable yielding integers in range(256)\n \ +/// - a text string encoded using the specified encoding\n \ +/// - any object implementing the buffer API.\n \ +/// - an integer"; +#[pyclass(name = "bytes")] +#[derive(Clone, Debug)] pub struct PyBytes { - value: Vec, + inner: PyByteInner, } -type PyBytesRef = PyRef; +pub type PyBytesRef = PyRef; impl PyBytes { - pub fn new(data: Vec) -> Self { - PyBytes { value: data } + pub fn new(elements: Vec) -> Self { + PyBytes { + inner: PyByteInner { elements }, + } + } + + pub fn from_string(value: &str, encoding: &str, vm: &VirtualMachine) -> PyResult { + Ok(PyBytes { + inner: PyByteInner::from_string(value, encoding, vm)?, + }) + } + + pub fn get_value(&self) -> &[u8] { + &self.inner.elements } } @@ -29,177 +61,395 @@ impl Deref for PyBytes { type Target = [u8]; fn deref(&self) -> &[u8] { - &self.value + &self.inner.elements } } impl PyValue for PyBytes { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.bytes_type() } } -// Binary data support +pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { + &obj.payload::().unwrap().inner.elements +} -// Fill bytes class methods: pub fn init(context: &PyContext) { + PyBytesRef::extend_class(context, &context.bytes_type); let bytes_type = &context.bytes_type; + extend_class!(context, bytes_type, { + "fromhex" => context.new_rustfunc(PyBytesRef::fromhex), + "maketrans" => context.new_rustfunc(PyByteInner::maketrans), - let bytes_doc = - "bytes(iterable_of_ints) -> bytes\n\ - bytes(string, encoding[, errors]) -> bytes\n\ - bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n\ - bytes(int) -> bytes object of size given by the parameter initialized with null bytes\n\ - bytes() -> empty bytes object\n\nConstruct an immutable array of bytes from:\n \ - - an iterable yielding integers in range(256)\n \ - - a text string encoded using the specified encoding\n \ - - any object implementing the buffer API.\n \ - - an integer"; - - context.set_attr(bytes_type, "__eq__", context.new_rustfunc(bytes_eq)); - context.set_attr(bytes_type, "__lt__", context.new_rustfunc(bytes_lt)); - context.set_attr(bytes_type, "__le__", context.new_rustfunc(bytes_le)); - context.set_attr(bytes_type, "__gt__", context.new_rustfunc(bytes_gt)); - context.set_attr(bytes_type, "__ge__", context.new_rustfunc(bytes_ge)); - context.set_attr(bytes_type, "__hash__", context.new_rustfunc(bytes_hash)); - context.set_attr(bytes_type, "__new__", context.new_rustfunc(bytes_new)); - context.set_attr(bytes_type, "__repr__", context.new_rustfunc(bytes_repr)); - context.set_attr(bytes_type, "__len__", context.new_rustfunc(bytes_len)); - context.set_attr(bytes_type, "__iter__", context.new_rustfunc(bytes_iter)); - context.set_attr( - bytes_type, - "__doc__", - context.new_str(bytes_doc.to_string()), - ); + }); + PyBytesIterator::extend_class(context, &context.bytesiterator_type); } -fn bytes_new( - cls: PyClassRef, - val_option: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - // Create bytes data: - let value = if let OptionalArg::Present(ival) = val_option { - let elements = vm.extract_elements(&ival)?; - let mut data_bytes = vec![]; - for elem in elements.iter() { - let v = objint::to_int(vm, elem, 10)?; - data_bytes.push(v.to_u8().unwrap()); +#[pyimpl] +impl PyBytesRef { + #[pymethod(name = "__new__")] + fn bytes_new( + cls: PyClassRef, + options: ByteInnerNewOptions, + vm: &VirtualMachine, + ) -> PyResult { + PyBytes { + inner: options.get_value(vm)?, } - data_bytes - // return Err(vm.new_type_error("Cannot construct bytes".to_string())); - } else { - vec![] - }; + .into_ref_with_type(vm, cls) + } - PyBytes::new(value).into_ref_with_type(vm, cls) -} + #[pymethod(name = "__repr__")] + fn repr(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.new_str(format!("b'{}'", self.inner.repr()?))) + } -fn bytes_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() == get_value(b).to_vec() - } else { - false - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__len__")] + fn len(self, _vm: &VirtualMachine) -> usize { + self.inner.len() + } -fn bytes_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() >= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__eq__")] + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.eq(other, vm) + } + #[pymethod(name = "__ge__")] + fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.ge(other, vm) + } + #[pymethod(name = "__le__")] + fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.le(other, vm) + } + #[pymethod(name = "__gt__")] + fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.gt(other, vm) + } + #[pymethod(name = "__lt__")] + fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.lt(other, vm) + } -fn bytes_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() > get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '>='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__hash__")] + fn hash(self, _vm: &VirtualMachine) -> pyhash::PyHash { + self.inner.hash() + } -fn bytes_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() <= get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<'", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__iter__")] + fn iter(self, _vm: &VirtualMachine) -> PyBytesIterator { + PyBytesIterator { + position: Cell::new(0), + bytes: self, + } + } -fn bytes_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(a, Some(vm.ctx.bytes_type())), (b, None)] - ); - - let result = if objtype::isinstance(b, &vm.ctx.bytes_type()) { - get_value(a).to_vec() < get_value(b).to_vec() - } else { - return Err(vm.new_type_error(format!("Cannot compare {} and {} using '<='", a, b))); - }; - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__add__")] + fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(other) = PyByteInner::try_from_object(vm, other) { + Ok(vm.ctx.new_bytearray(self.inner.add(other))) + } else { + Ok(vm.ctx.not_implemented()) + } + } -fn bytes_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(a, Some(vm.ctx.bytes_type()))]); + #[pymethod(name = "__contains__")] + fn contains(self, needle: Either, vm: &VirtualMachine) -> PyResult { + self.inner.contains(needle, vm) + } - let byte_vec = get_value(a).to_vec(); - Ok(vm.ctx.new_int(byte_vec.len())) -} + #[pymethod(name = "__getitem__")] + fn getitem(self, needle: Either, vm: &VirtualMachine) -> PyResult { + self.inner.getitem(needle, vm) + } + + #[pymethod(name = "isalnum")] + fn isalnum(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalnum(vm) + } + + #[pymethod(name = "isalpha")] + fn isalpha(self, vm: &VirtualMachine) -> PyResult { + self.inner.isalpha(vm) + } + + #[pymethod(name = "isascii")] + fn isascii(self, vm: &VirtualMachine) -> PyResult { + self.inner.isascii(vm) + } + + #[pymethod(name = "isdigit")] + fn isdigit(self, vm: &VirtualMachine) -> PyResult { + self.inner.isdigit(vm) + } + + #[pymethod(name = "islower")] + fn islower(self, vm: &VirtualMachine) -> PyResult { + self.inner.islower(vm) + } + + #[pymethod(name = "isspace")] + fn isspace(self, vm: &VirtualMachine) -> PyResult { + self.inner.isspace(vm) + } + + #[pymethod(name = "isupper")] + fn isupper(self, vm: &VirtualMachine) -> PyResult { + self.inner.isupper(vm) + } + + #[pymethod(name = "istitle")] + fn istitle(self, vm: &VirtualMachine) -> PyResult { + self.inner.istitle(vm) + } + + #[pymethod(name = "lower")] + fn lower(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.lower(vm))) + } + + #[pymethod(name = "upper")] + fn upper(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.upper(vm))) + } + + #[pymethod(name = "capitalize")] + fn capitalize(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.capitalize(vm))) + } + + #[pymethod(name = "swapcase")] + fn swapcase(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.swapcase(vm))) + } + + #[pymethod(name = "hex")] + fn hex(self, vm: &VirtualMachine) -> PyResult { + self.inner.hex(vm) + } -fn bytes_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.bytes_type()))]); - let data = get_value(zelf); - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - Ok(vm.ctx.new_int(hash)) + fn fromhex(string: PyStringRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(PyByteInner::fromhex(string.as_str(), vm)?)) + } + + #[pymethod(name = "center")] + fn center(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.center(options, vm)?)) + } + + #[pymethod(name = "ljust")] + fn ljust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.ljust(options, vm)?)) + } + + #[pymethod(name = "rjust")] + fn rjust(self, options: ByteInnerPaddingOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.rjust(options, vm)?)) + } + + #[pymethod(name = "count")] + fn count(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.count(options, vm) + } + + #[pymethod(name = "join")] + fn join(self, iter: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.join(iter, vm) + } + + #[pymethod(name = "endswith")] + fn endswith( + self, + suffix: Either, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.startsendswith(suffix, start, end, true, vm) + } + + #[pymethod(name = "startswith")] + fn startswith( + self, + prefix: Either, + start: OptionalArg, + end: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + self.inner.startsendswith(prefix, start, end, false, vm) + } + + #[pymethod(name = "find")] + fn find(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.find(options, false, vm) + } + + #[pymethod(name = "index")] + fn index(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.find(options, false, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(res) + } + + #[pymethod(name = "rfind")] + fn rfind(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + self.inner.find(options, true, vm) + } + + #[pymethod(name = "rindex")] + fn rindex(self, options: ByteInnerFindOptions, vm: &VirtualMachine) -> PyResult { + let res = self.inner.find(options, true, vm)?; + if res == -1 { + return Err(vm.new_value_error("substring not found".to_string())); + } + Ok(res) + } + + #[pymethod(name = "translate")] + fn translate(self, options: ByteInnerTranslateOptions, vm: &VirtualMachine) -> PyResult { + self.inner.translate(options, vm) + } + + #[pymethod(name = "strip")] + fn strip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.inner.strip(chars, ByteInnerPosition::All, vm)?)) + } + + #[pymethod(name = "lstrip")] + fn lstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.inner.strip(chars, ByteInnerPosition::Left, vm)?)) + } + + #[pymethod(name = "rstrip")] + fn rstrip(self, chars: OptionalArg, vm: &VirtualMachine) -> PyResult { + Ok(vm + .ctx + .new_bytes(self.inner.strip(chars, ByteInnerPosition::Right, vm)?)) + } + + #[pymethod(name = "split")] + fn split(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .split(options, false)? + .iter() + .map(|x| vm.ctx.new_bytes(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "rsplit")] + fn rsplit(self, options: ByteInnerSplitOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .split(options, true)? + .iter() + .map(|x| vm.ctx.new_bytes(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "partition")] + fn partition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let sepa = PyByteInner::try_from_object(vm, sep.clone())?; + + let (left, right) = self.inner.partition(&sepa, false)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_bytes(left), sep, vm.ctx.new_bytes(right)])) + } + #[pymethod(name = "rpartition")] + fn rpartition(self, sep: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let sepa = PyByteInner::try_from_object(vm, sep.clone())?; + + let (left, right) = self.inner.partition(&sepa, true)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_bytes(left), sep, vm.ctx.new_bytes(right)])) + } + + #[pymethod(name = "expandtabs")] + fn expandtabs(self, options: ByteInnerExpandtabsOptions, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.expandtabs(options))) + } + + #[pymethod(name = "splitlines")] + fn splitlines(self, options: ByteInnerSplitlinesOptions, vm: &VirtualMachine) -> PyResult { + let as_bytes = self + .inner + .splitlines(options) + .iter() + .map(|x| vm.ctx.new_bytes(x.to_vec())) + .collect::>(); + Ok(vm.ctx.new_list(as_bytes)) + } + + #[pymethod(name = "zfill")] + fn zfill(self, width: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.zfill(width))) + } + + #[pymethod(name = "replace")] + fn replace( + self, + old: PyByteInner, + new: PyByteInner, + count: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.replace(old, new, count)?)) + } + + #[pymethod(name = "title")] + fn title(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.title())) + } + + #[pymethod(name = "__mul__")] + fn repeat(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.inner.repeat(n, vm)?)) + } + + #[pymethod(name = "__rmul__")] + fn rmul(self, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + self.repeat(n, vm) + } } -pub fn get_value<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { - &obj.payload::().unwrap().value +#[pyclass] +#[derive(Debug)] +pub struct PyBytesIterator { + position: Cell, + bytes: PyBytesRef, } -fn bytes_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.bytes_type()))]); - let value = get_value(obj); - let data = String::from_utf8(value.to_vec()).unwrap(); - Ok(vm.new_str(format!("b'{}'", data))) +impl PyValue for PyBytesIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.bytesiterator_type() + } } -fn bytes_iter(obj: PyBytesRef, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { - position: Cell::new(0), - iterated_obj: obj.into_object(), +#[pyimpl] +impl PyBytesIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.bytes.inner.len() { + let ret = self.bytes[self.position.get()]; + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } diff --git a/vm/src/obj/objclassmethod.rs b/vm/src/obj/objclassmethod.rs index 725f508e98..10b3106ec9 100644 --- a/vm/src/obj/objclassmethod.rs +++ b/vm/src/obj/objclassmethod.rs @@ -1,7 +1,28 @@ use super::objtype::PyClassRef; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +/// classmethod(function) -> method +/// +/// Convert a function to be a class method. +/// +/// A class method receives the class as implicit first argument, +/// just like an instance method receives the instance. +/// To declare a class method, use this idiom: +/// +/// class C: +/// @classmethod +/// def f(cls, arg1, arg2, ...): +/// ... +/// +/// It can be called either on the class (e.g. C.f()) or on an instance +/// (e.g. C().f()). The instance is ignored except for its class. +/// If a class method is called for a derived class, the derived class +/// object is passed as the implied first argument. +/// +/// Class methods are different than C++ or Java static methods. +/// If you want those, see the staticmethod builtin. +#[pyclass] #[derive(Clone, Debug)] pub struct PyClassMethod { pub callable: PyObjectRef, @@ -9,12 +30,16 @@ pub struct PyClassMethod { pub type PyClassMethodRef = PyRef; impl PyValue for PyClassMethod { - fn class(vm: &VirtualMachine) -> PyObjectRef { + const HAVE_DICT: bool = true; + + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.classmethod_type() } } -impl PyClassMethodRef { +#[pyimpl] +impl PyClassMethod { + #[pymethod(name = "__new__")] fn new( cls: PyClassRef, callable: PyObjectRef, @@ -26,17 +51,19 @@ impl PyClassMethodRef { .into_ref_with_type(vm, cls) } - fn get(self, _inst: PyObjectRef, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__get__")] + fn get(&self, _inst: PyObjectRef, owner: PyObjectRef, vm: &VirtualMachine) -> PyResult { Ok(vm .ctx .new_bound_method(self.callable.clone(), owner.clone())) } + + #[pyproperty(name = "__func__")] + fn func(&self, _vm: &VirtualMachine) -> PyObjectRef { + self.callable.clone() + } } pub fn init(context: &PyContext) { - let classmethod_type = &context.classmethod_type; - extend_class!(context, classmethod_type, { - "__get__" => context.new_rustfunc(PyClassMethodRef::get), - "__new__" => context.new_rustfunc(PyClassMethodRef::new) - }); + PyClassMethod::extend_class(context, &context.classmethod_type); } diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index 21df7f2cc1..51a73d6141 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -5,12 +5,14 @@ use std::fmt; use crate::bytecode; -use crate::function::PyFuncArgs; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +pub type PyCodeRef = PyRef; + pub struct PyCode { - code: bytecode::CodeObject, + pub code: bytecode::CodeObject, } impl PyCode { @@ -26,94 +28,68 @@ impl fmt::Debug for PyCode { } impl PyValue for PyCode { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.code_type() } } -pub fn init(context: &PyContext) { - let code_type = &context.code_type; - context.set_attr(code_type, "__new__", context.new_rustfunc(code_new)); - context.set_attr(code_type, "__repr__", context.new_rustfunc(code_repr)); - - for (name, f) in &[ - ( - "co_argcount", - code_co_argcount as fn(&VirtualMachine, PyFuncArgs) -> PyResult, - ), - ("co_consts", code_co_consts), - ("co_filename", code_co_filename), - ("co_firstlineno", code_co_firstlineno), - ("co_kwonlyargcount", code_co_kwonlyargcount), - ("co_name", code_co_name), - ] { - context.set_attr(code_type, name, context.new_property(f)) +impl PyCodeRef { + #[allow(clippy::new_ret_no_self)] + fn new(_cls: PyClassRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot directly create code object".to_string())) } -} -pub fn get_value(obj: &PyObjectRef) -> bytecode::CodeObject { - if let Some(code) = obj.payload::() { - code.code.clone() - } else { - panic!("Inner error getting code {:?}", obj) + fn repr(self, _vm: &VirtualMachine) -> String { + let code = &self.code; + format!( + "", + code.obj_name, + self.get_id(), + code.source_path, + code.first_line_number + ) } -} - -fn code_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_cls, None)]); - Err(vm.new_type_error("Cannot directly create code object".to_string())) -} - -fn code_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.code_type()))]); - - let code = get_value(o); - let repr = format!( - "", - code.obj_name, - o.get_id(), - code.source_path, - code.first_line_number - ); - Ok(vm.new_str(repr)) -} -fn member_code_obj(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.code_type()))]); - Ok(get_value(zelf)) -} + fn co_argcount(self, _vm: &VirtualMachine) -> usize { + self.code.arg_names.len() + } -fn code_co_argcount(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let code_obj = member_code_obj(vm, args)?; - Ok(vm.ctx.new_int(code_obj.arg_names.len())) -} + fn co_filename(self, _vm: &VirtualMachine) -> String { + self.code.source_path.clone() + } -fn code_co_filename(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let code_obj = member_code_obj(vm, args)?; - let source_path = code_obj.source_path; - Ok(vm.new_str(source_path)) -} + fn co_firstlineno(self, _vm: &VirtualMachine) -> usize { + self.code.first_line_number + } -fn code_co_firstlineno(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let code_obj = member_code_obj(vm, args)?; - Ok(vm.ctx.new_int(code_obj.first_line_number)) -} + fn co_kwonlyargcount(self, _vm: &VirtualMachine) -> usize { + self.code.kwonlyarg_names.len() + } -fn code_co_kwonlyargcount(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let code_obj = member_code_obj(vm, args)?; - Ok(vm.ctx.new_int(code_obj.kwonlyarg_names.len())) -} + fn co_consts(self, vm: &VirtualMachine) -> PyObjectRef { + let consts = self + .code + .get_constants() + .map(|x| vm.ctx.unwrap_constant(x)) + .collect(); + vm.ctx.new_tuple(consts) + } -fn code_co_consts(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let code_obj = member_code_obj(vm, args)?; - let consts = code_obj - .get_constants() - .map(|x| vm.ctx.unwrap_constant(x)) - .collect(); - Ok(vm.ctx.new_tuple(consts)) + fn co_name(self, _vm: &VirtualMachine) -> String { + self.code.obj_name.clone() + } } -fn code_co_name(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let code_obj = member_code_obj(vm, args)?; - Ok(vm.new_str(code_obj.obj_name)) +pub fn init(context: &PyContext) { + extend_class!(context, &context.code_type, { + "__new__" => context.new_rustfunc(PyCodeRef::new), + "__repr__" => context.new_rustfunc(PyCodeRef::repr), + + "co_argcount" => context.new_property(PyCodeRef::co_argcount), + "co_consts" => context.new_property(PyCodeRef::co_consts), + "co_filename" => context.new_property(PyCodeRef::co_filename), + "co_firstlineno" => context.new_property(PyCodeRef::co_firstlineno), + "co_kwonlyargcount" => context.new_property(PyCodeRef::co_kwonlyargcount), + "co_name" => context.new_property(PyCodeRef::co_name), + }); } diff --git a/vm/src/obj/objcomplex.rs b/vm/src/obj/objcomplex.rs index 7443306512..436634e9c4 100644 --- a/vm/src/obj/objcomplex.rs +++ b/vm/src/obj/objcomplex.rs @@ -1,14 +1,20 @@ use num_complex::Complex64; -use num_traits::ToPrimitive; +use num_traits::Zero; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyhash; +use crate::pyobject::{ + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, +}; use crate::vm::VirtualMachine; -use super::objfloat; -use super::objint; +use super::objfloat::{self, PyFloat}; use super::objtype::{self, PyClassRef}; +/// Create a complex number from a real part and an optional imaginary part. +/// +/// This is equivalent to (real + imag*1j) where imag defaults to 0. +#[pyclass(name = "complex")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyComplex { value: Complex64, @@ -16,11 +22,17 @@ pub struct PyComplex { type PyComplexRef = PyRef; impl PyValue for PyComplex { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.complex_type() } } +impl IntoPyObject for Complex64 { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_complex(self)) + } +} + impl From for PyComplex { fn from(value: Complex64) -> Self { PyComplex { value } @@ -28,167 +40,221 @@ impl From for PyComplex { } pub fn init(context: &PyContext) { - let complex_type = &context.complex_type; - - let complex_doc = - "Create a complex number from a real part and an optional imaginary part.\n\n\ - This is equivalent to (real + imag*1j) where imag defaults to 0."; - - context.set_attr(&complex_type, "__abs__", context.new_rustfunc(complex_abs)); - context.set_attr(&complex_type, "__add__", context.new_rustfunc(complex_add)); - context.set_attr( - &complex_type, - "__radd__", - context.new_rustfunc(complex_radd), - ); - context.set_attr(&complex_type, "__eq__", context.new_rustfunc(complex_eq)); - context.set_attr(&complex_type, "__neg__", context.new_rustfunc(complex_neg)); - context.set_attr(&complex_type, "__new__", context.new_rustfunc(complex_new)); - context.set_attr(&complex_type, "real", context.new_property(complex_real)); - context.set_attr(&complex_type, "imag", context.new_property(complex_imag)); - context.set_attr( - &complex_type, - "__doc__", - context.new_str(complex_doc.to_string()), - ); - context.set_attr( - &complex_type, - "__repr__", - context.new_rustfunc(complex_repr), - ); - context.set_attr( - &complex_type, - "conjugate", - context.new_rustfunc(complex_conjugate), - ); + PyComplex::extend_class(context, &context.complex_type); } pub fn get_value(obj: &PyObjectRef) -> Complex64 { obj.payload::().unwrap().value } -fn complex_new( - cls: PyClassRef, - real: OptionalArg, - imag: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let real = match real { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; - - let imag = match imag { - OptionalArg::Missing => 0.0, - OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, - }; - - let value = Complex64::new(real, imag); - PyComplex { value }.into_ref_with_type(vm, cls) +fn try_complex(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { + Ok(if objtype::isinstance(&value, &vm.ctx.complex_type()) { + Some(get_value(&value)) + } else if let Some(float) = objfloat::try_float(value, vm)? { + Some(Complex64::new(float, 0.0)) + } else { + None + }) } -fn complex_real(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { re, .. } = get_value(zelf); - Ok(vm.ctx.new_float(re)) -} +#[pyimpl] +impl PyComplex { + #[pyproperty(name = "real")] + fn real(&self, _vm: &VirtualMachine) -> PyFloat { + self.value.re.into() + } -fn complex_imag(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - let Complex64 { im, .. } = get_value(zelf); - Ok(vm.ctx.new_float(im)) -} + #[pyproperty(name = "imag")] + fn imag(&self, _vm: &VirtualMachine) -> PyFloat { + self.value.im.into() + } -fn complex_abs(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); + #[pymethod(name = "__abs__")] + fn abs(&self, _vm: &VirtualMachine) -> PyFloat { + let Complex64 { im, re } = self.value; + re.hypot(im).into() + } - let Complex64 { re, im } = get_value(zelf); - Ok(vm.ctx.new_float(re.hypot(im))) -} + #[pymethod(name = "__add__")] + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value + other).into_pyobject(vm), + ) + } -fn complex_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); - - let v1 = get_value(i); - if objtype::isinstance(i2, &vm.ctx.complex_type()) { - Ok(vm.ctx.new_complex(v1 + get_value(i2))) - } else if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.add(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value - other).into_pyobject(vm), + ) } -} -fn complex_radd(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(i, Some(vm.ctx.complex_type())), (i2, None)] - ); + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other - self.value).into_pyobject(vm), + ) + } + + #[pymethod(name = "conjugate")] + fn conjugate(&self, _vm: &VirtualMachine) -> Complex64 { + self.value.conj() + } - let v1 = get_value(i); + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + let result = if objtype::isinstance(&other, &vm.ctx.complex_type()) { + self.value == get_value(&other) + } else { + match objfloat::try_float(&other, vm) { + Ok(Some(other)) => self.value.im == 0.0f64 && self.value.re == other, + Err(_) => false, + Ok(None) => return vm.ctx.not_implemented(), + } + }; - if objtype::isinstance(i2, &vm.ctx.int_type()) { - Ok(vm.ctx.new_complex(Complex64::new( - v1.re + objint::get_value(i2).to_f64().unwrap(), - v1.im, - ))) - } else { - Err(vm.new_type_error(format!("Cannot add {} and {}", i, i2))) + vm.ctx.new_bool(result) } -} -fn complex_conjugate(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(i, Some(vm.ctx.complex_type()))]); + #[pymethod(name = "__float__")] + fn float(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(String::from("Can't convert complex to float"))) + } - let v1 = get_value(i); - Ok(vm.ctx.new_complex(v1.conj())) -} + #[pymethod(name = "__int__")] + fn int(&self, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error(String::from("Can't convert complex to int"))) + } + + #[pymethod(name = "__mul__")] + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value * other).into_pyobject(vm), + ) + } + + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mul(other, vm) + } + + #[pymethod(name = "__truediv__")] + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value / other).into_pyobject(vm), + ) + } + + #[pymethod(name = "__rtruediv__")] + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other / self.value).into_pyobject(vm), + ) + } + + #[pymethod(name = "__mod__")] + fn mod_(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("can't mod complex numbers.".to_string())) + } + + #[pymethod(name = "__rmod__")] + fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mod_(other, vm) + } + + #[pymethod(name = "__floordiv__")] + fn floordiv(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("can't take floor of complex number.".to_string())) + } + + #[pymethod(name = "__rfloordiv__")] + fn rfloordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.floordiv(other, vm) + } + + #[pymethod(name = "__divmod__")] + fn divmod(&self, _other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("can't take floor or mod of complex number.".to_string())) + } + + #[pymethod(name = "__rdivmod__")] + fn rdivmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.divmod(other, vm) + } + + #[pymethod(name = "__neg__")] + fn neg(&self, _vm: &VirtualMachine) -> Complex64 { + -self.value + } -fn complex_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.complex_type())), (other, None)] - ); - - let z = get_value(zelf); - - let result = if objtype::isinstance(other, &vm.ctx.complex_type()) { - z == get_value(other) - } else if objtype::isinstance(other, &vm.ctx.int_type()) { - match objint::get_value(other).to_f64() { - Some(f) => z.im == 0.0f64 && z.re == f, - None => false, + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { + let Complex64 { re, im } = self.value; + if re == 0.0 { + format!("{}j", im) + } else { + format!("({}{:+}j)", re, im) } - } else if objtype::isinstance(other, &vm.ctx.float_type()) { - z.im == 0.0 && z.re == objfloat::get_value(other) - } else { - return Ok(vm.ctx.not_implemented()); - }; + } - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__pow__")] + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value.powc(other)).into_pyobject(vm), + ) + } -fn complex_neg(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.complex_type()))]); - Ok(vm.ctx.new_complex(-get_value(zelf))) -} + #[pymethod(name = "__rpow__")] + fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_complex(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other.powc(self.value)).into_pyobject(vm), + ) + } -fn complex_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.complex_type()))]); - let v = get_value(obj); - let repr = if v.re == 0. { - format!("{}j", v.im) - } else { - format!("({}+{}j)", v.re, v.im) - }; - Ok(vm.new_str(repr)) + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { + !Complex64::is_zero(&self.value) + } + + #[pymethod(name = "__new__")] + fn complex_new( + cls: PyClassRef, + real: OptionalArg, + imag: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let real = match real { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; + + let imag = match imag { + OptionalArg::Missing => 0.0, + OptionalArg::Present(ref value) => objfloat::make_float(vm, value)?, + }; + + let value = Complex64::new(real, imag); + PyComplex { value }.into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + let re_hash = pyhash::hash_float(self.value.re); + let im_hash = pyhash::hash_float(self.value.im); + + re_hash + im_hash * pyhash::IMAG + } } diff --git a/vm/src/obj/objdict.rs b/vm/src/obj/objdict.rs index ad134f5aeb..7b708611e8 100644 --- a/vm/src/obj/objdict.rs +++ b/vm/src/obj/objdict.rs @@ -1,24 +1,25 @@ use std::cell::{Cell, RefCell}; -use std::collections::HashMap; use std::fmt; -use std::ops::{Deref, DerefMut}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::{KwArgs, OptionalArg}; use crate::pyobject::{ - PyAttributes, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, IntoPyObject, ItemProtocol, PyAttributes, PyContext, PyIterable, PyObjectRef, + PyRef, PyResult, PyValue, }; use crate::vm::{ReprGuard, VirtualMachine}; +use super::objbool; use super::objiter; -use super::objstr::{self, PyStringRef}; -use super::objtype; +use super::objstr; +use crate::dictdatatype; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::PyClassImpl; -pub type DictContentType = HashMap; +pub type DictContentType = dictdatatype::Dict; #[derive(Default)] pub struct PyDict { - // TODO: should be private - pub entries: RefCell, + entries: RefCell, } pub type PyDictRef = PyRef; @@ -30,159 +31,128 @@ impl fmt::Debug for PyDict { } impl PyValue for PyDict { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.dict_type() } } -pub fn get_elements<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { - obj.payload::().unwrap().entries.borrow() -} - -pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut + 'a { - obj.payload::().unwrap().entries.borrow_mut() -} - -pub fn set_item( - dict: &PyObjectRef, - _vm: &VirtualMachine, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // TODO: use vm to call eventual __hash__ and __eq__methods. - let mut elements = get_mut_elements(dict); - set_item_in_content(&mut elements, needle, value); -} - -pub fn set_item_in_content( - elements: &mut DictContentType, - needle: &PyObjectRef, - value: &PyObjectRef, -) { - // XXX: Currently, we only support String keys, so we have to unwrap the - // PyObject (and ensure it is a String). - - // TODO: invoke __hash__ function here! - let needle_str = objstr::get_value(needle); - elements.insert(needle_str, (needle.clone(), value.clone())); -} +// Python dict methods: +impl PyDictRef { + fn new( + class: PyClassRef, + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let dict = DictContentType::default(); -pub fn get_key_value_pairs(dict: &PyObjectRef) -> Vec<(PyObjectRef, PyObjectRef)> { - let dict_elements = get_elements(dict); - get_key_value_pairs_from_content(&dict_elements) -} + let entries = RefCell::new(dict); + // it's unfortunate that we can't abstract over RefCall, as we should be able to use dict + // directly here, but that would require generic associated types + PyDictRef::merge(&entries, dict_obj, kwargs, vm)?; -pub fn get_key_value_pairs_from_content( - dict_content: &DictContentType, -) -> Vec<(PyObjectRef, PyObjectRef)> { - let mut pairs: Vec<(PyObjectRef, PyObjectRef)> = Vec::new(); - for (_str_key, pair) in dict_content.iter() { - let (key, obj) = pair; - pairs.push((key.clone(), obj.clone())); + PyDict { entries }.into_ref_with_type(vm, class) } - pairs -} - -pub fn get_item(dict: &PyObjectRef, key: &PyObjectRef) -> Option { - let needle_str = objstr::get_value(key); - get_key_str(dict, &needle_str) -} -// Special case for the case when requesting a str key from a dict: -pub fn get_key_str(dict: &PyObjectRef, key: &str) -> Option { - let elements = get_elements(dict); - content_get_key_str(&elements, key) -} - -/// Retrieve a key from dict contents: -pub fn content_get_key_str(elements: &DictContentType, key: &str) -> Option { - // TODO: let hash: usize = key; - match elements.get(key) { - Some(v) => Some(v.1.clone()), - None => None, + fn merge( + dict: &RefCell, + dict_obj: OptionalArg, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let OptionalArg::Present(dict_obj) = dict_obj { + let dicted: PyResult = dict_obj.clone().downcast(); + if let Ok(dict_obj) = dicted { + for (key, value) in dict_obj { + dict.borrow_mut().insert(vm, &key, value)?; + } + } else { + let iter = objiter::get_iter(vm, &dict_obj)?; + loop { + fn err(vm: &VirtualMachine) -> PyObjectRef { + vm.new_type_error("Iterator must have exactly two elements".to_string()) + } + let element = match objiter::get_next_object(vm, &iter)? { + Some(obj) => obj, + None => break, + }; + let elem_iter = objiter::get_iter(vm, &element)?; + let key = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; + if objiter::get_next_object(vm, &elem_iter)?.is_some() { + return Err(err(vm)); + } + dict.borrow_mut().insert(vm, &key, value)?; + } + } + } + let mut dict_borrowed = dict.borrow_mut(); + for (key, value) in kwargs.into_iter() { + dict_borrowed.insert(vm, &vm.new_str(key), value)?; + } + Ok(()) } -} - -pub fn contains_key_str(dict: &PyObjectRef, key: &str) -> bool { - let elements = get_elements(dict); - content_contains_key_str(&elements, key) -} - -pub fn content_contains_key_str(elements: &DictContentType, key: &str) -> bool { - // TODO: let hash: usize = key; - elements.get(key).is_some() -} -/// Take a python dictionary and convert it to attributes. -pub fn py_dict_to_attributes(dict: &PyObjectRef) -> PyAttributes { - let mut attrs = PyAttributes::new(); - for (key, value) in get_key_value_pairs(dict) { - let key = objstr::get_value(&key); - attrs.insert(key, value); + fn fromkeys( + class: PyClassRef, + iterable: PyIterable, + value: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let mut dict = DictContentType::default(); + let value = value.unwrap_or_else(|| vm.ctx.none()); + for elem in iterable.iter(vm)? { + let elem = elem?; + dict.insert(vm, &elem, value.clone())?; + } + let entries = RefCell::new(dict); + PyDict { entries }.into_ref_with_type(vm, class) } - attrs -} -pub fn attributes_to_py_dict(vm: &VirtualMachine, attributes: PyAttributes) -> PyResult { - let dict = vm.ctx.new_dict(); - for (key, value) in attributes { - let key = vm.ctx.new_str(key); - set_item(&dict, vm, &key, &value); + fn bool(self, _vm: &VirtualMachine) -> bool { + !self.entries.borrow().is_empty() } - Ok(dict) -} -// Python dict methods: -fn dict_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_ty, Some(vm.ctx.type_type()))], - optional = [(dict_obj, None)] - ); - let dict = vm.ctx.new_dict(); - if let Some(dict_obj) = dict_obj { - if objtype::isinstance(&dict_obj, &vm.ctx.dict_type()) { - for (needle, value) in get_key_value_pairs(&dict_obj) { - set_item(&dict, vm, &needle, &value); - } - } else { - let iter = objiter::get_iter(vm, dict_obj)?; - loop { - fn err(vm: &VirtualMachine) -> PyObjectRef { - vm.new_type_error("Iterator must have exactly two elements".to_string()) + fn inner_eq(self, other: &PyDict, vm: &VirtualMachine) -> PyResult { + if other.entries.borrow().len() != self.entries.borrow().len() { + return Ok(false); + } + for (k, v1) in self { + match other.entries.borrow().get(vm, &k)? { + Some(v2) => { + if v1.is(&v2) { + continue; + } + let value = objbool::boolval(vm, vm._eq(v1, v2)?)?; + if !value { + return Ok(false); + } } - let element = match objiter::get_next_object(vm, &iter)? { - Some(obj) => obj, - None => break, - }; - let elem_iter = objiter::get_iter(vm, &element)?; - let needle = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; - let value = objiter::get_next_object(vm, &elem_iter)?.ok_or_else(|| err(vm))?; - if objiter::get_next_object(vm, &elem_iter)?.is_some() { - return Err(err(vm)); + None => { + return Ok(false); } - set_item(&dict, vm, &needle, &value); } } + Ok(true) } - for (needle, value) in args.kwargs { - let py_needle = vm.new_str(needle); - set_item(&dict, vm, &py_needle, &value); + + fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(other) = other.payload::() { + let eq = self.inner_eq(other, vm)?; + Ok(vm.ctx.new_bool(eq)) + } else { + Ok(vm.ctx.not_implemented()) + } } - Ok(dict) -} -impl PyDictRef { fn len(self, _vm: &VirtualMachine) -> usize { self.entries.borrow().len() } - fn repr(self, vm: &VirtualMachine) -> PyResult { + fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { - let elements = get_key_value_pairs(self.as_object()); let mut str_parts = vec![]; - for (key, value) in elements { + for (key, value) in self { let key_repr = vm.to_repr(&key)?; let value_repr = vm.to_repr(&value)?; str_parts.push(format!("{}: {}", key_repr.value, value_repr.value)); @@ -192,125 +162,355 @@ impl PyDictRef { } else { "{...}".to_string() }; - Ok(vm.new_str(s)) + Ok(s) } - fn contains(self, key: PyStringRef, _vm: &VirtualMachine) -> bool { - self.entries.borrow().contains_key(&key.value) + fn contains(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow().contains(vm, &key) } - fn delitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - let key = &key.value; - // Delete the item: - let mut elements = self.entries.borrow_mut(); - match elements.remove(key) { - Some(_) => Ok(()), - None => Err(vm.new_value_error(format!("Key not found: {}", key))), - } + fn inner_delitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.entries.borrow_mut().delete(vm, &key) } fn clear(self, _vm: &VirtualMachine) { self.entries.borrow_mut().clear() } - /// When iterating over a dictionary, we iterate over the keys of it. - fn iter(self, vm: &VirtualMachine) -> PyIteratorValue { - let keys = self - .entries - .borrow() - .values() - .map(|(k, _v)| k.clone()) - .collect(); - let key_list = vm.ctx.new_list(keys); + fn iter(self, _vm: &VirtualMachine) -> PyDictKeyIterator { + PyDictKeyIterator::new(self) + } - PyIteratorValue { - position: Cell::new(0), - iterated_obj: key_list, - } + fn keys(self, _vm: &VirtualMachine) -> PyDictKeys { + PyDictKeys::new(self) } - fn values(self, vm: &VirtualMachine) -> PyIteratorValue { - let values = self - .entries - .borrow() - .values() - .map(|(_k, v)| v.clone()) - .collect(); - let values_list = vm.ctx.new_list(values); + fn values(self, _vm: &VirtualMachine) -> PyDictValues { + PyDictValues::new(self) + } - PyIteratorValue { - position: Cell::new(0), - iterated_obj: values_list, - } + fn items(self, _vm: &VirtualMachine) -> PyDictItems { + PyDictItems::new(self) } - fn items(self, vm: &VirtualMachine) -> PyIteratorValue { - let items = self - .entries - .borrow() - .values() - .map(|(k, v)| vm.ctx.new_tuple(vec![k.clone(), v.clone()])) - .collect(); - let items_list = vm.ctx.new_list(items); + fn inner_setitem( + self, + key: PyObjectRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + self.entries.borrow_mut().insert(vm, &key, value) + } - PyIteratorValue { - position: Cell::new(0), - iterated_obj: items_list, + fn inner_getitem(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(value) = self.entries.borrow().get(vm, &key)? { + return Ok(value); } + if let Some(method_or_err) = vm.get_method(self.clone().into_object(), "__missing__") { + let method = method_or_err?; + return vm.invoke(method, vec![key]); + } + Err(vm.new_key_error(key.clone())) } - fn setitem(self, needle: PyObjectRef, value: PyObjectRef, _vm: &VirtualMachine) { - let mut elements = self.entries.borrow_mut(); - set_item_in_content(&mut elements, &needle, &value) + fn get( + self, + key: PyObjectRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + match self.entries.borrow().get(vm, &key)? { + Some(value) => Ok(value), + None => Ok(default.unwrap_or_else(|| vm.ctx.none())), + } } - fn getitem(self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { - let key = &key.value; + fn setdefault( + self, + key: PyObjectRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let mut entries = self.entries.borrow_mut(); + match entries.get(vm, &key)? { + Some(value) => Ok(value), + None => { + let set_value = default.unwrap_or_else(|| vm.ctx.none()); + entries.insert(vm, &key, set_value.clone())?; + Ok(set_value) + } + } + } - // What we are looking for: - let elements = self.entries.borrow(); - if elements.contains_key(key) { - Ok(elements[key].1.clone()) - } else { - Err(vm.new_value_error(format!("Key not found: {}", key))) + fn copy(self, _vm: &VirtualMachine) -> PyDict { + PyDict { + entries: self.entries.clone(), } } - fn get( + fn update( self, - key: PyStringRef, - default: OptionalArg, + dict_obj: OptionalArg, + kwargs: KwArgs, vm: &VirtualMachine, - ) -> PyObjectRef { - // What we are looking for: - let key = &key.value; + ) -> PyResult<()> { + PyDictRef::merge(&self.entries, dict_obj, kwargs, vm) + } + + fn pop(self, key: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.entries.borrow_mut().pop(vm, &key) + } - let elements = self.entries.borrow(); - if elements.contains_key(key) { - elements[key].1.clone() + fn popitem(self, vm: &VirtualMachine) -> PyResult { + let mut entries = self.entries.borrow_mut(); + if let Some((key, value)) = entries.pop_front() { + Ok(vm.ctx.new_tuple(vec![key, value])) } else { - match default { - OptionalArg::Present(value) => value, - OptionalArg::Missing => vm.ctx.none(), - } + let err_msg = vm.new_str("popitem(): dictionary is empty".to_string()); + Err(vm.new_key_error(err_msg)) + } + } + + /// Take a python dictionary and convert it to attributes. + pub fn to_attributes(self) -> PyAttributes { + let mut attrs = PyAttributes::new(); + for (key, value) in self { + let key = objstr::get_value(&key); + attrs.insert(key, value); + } + attrs + } + + pub fn from_attributes(attrs: PyAttributes, vm: &VirtualMachine) -> PyResult { + let dict = DictContentType::default(); + let entries = RefCell::new(dict); + + for (key, value) in attrs { + entries + .borrow_mut() + .insert(vm, &vm.ctx.new_str(key), value)?; + } + + Ok(PyDict { entries }.into_ref(vm)) + } + + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("unhashable type".to_string())) + } + + pub fn contains_key(&self, key: T, vm: &VirtualMachine) -> bool { + let key = key.into_pyobject(vm).unwrap(); + self.entries.borrow().contains(vm, &key).unwrap() + } + + pub fn size(&self) -> dictdatatype::DictSize { + self.entries.borrow().size() + } +} + +impl ItemProtocol for PyDictRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + self.as_object().get_item(key, vm) + } + + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + self.as_object().set_item(key, value, vm) + } + + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + self.as_object().del_item(key, vm) + } +} + +// Implement IntoIterator so that we can easily iterate dictionaries from rust code. +impl IntoIterator for PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self) + } +} + +impl IntoIterator for &PyDictRef { + type Item = (PyObjectRef, PyObjectRef); + type IntoIter = DictIter; + + fn into_iter(self) -> Self::IntoIter { + DictIter::new(self.clone()) + } +} + +pub struct DictIter { + dict: PyDictRef, + position: usize, +} + +impl DictIter { + pub fn new(dict: PyDictRef) -> DictIter { + DictIter { dict, position: 0 } + } +} + +impl Iterator for DictIter { + type Item = (PyObjectRef, PyObjectRef); + + fn next(&mut self) -> Option { + match self.dict.entries.borrow().next_entry(&mut self.position) { + Some((key, value)) => Some((key.clone(), value.clone())), + None => None, } } } +macro_rules! dict_iterator { + ( $name: ident, $iter_name: ident, $class: ident, $iter_class: ident, $class_name: literal, $iter_class_name: literal, $result_fn: expr) => { + #[pyclass(name = $class_name)] + #[derive(Debug)] + struct $name { + pub dict: PyDictRef, + } + + #[pyimpl] + impl $name { + fn new(dict: PyDictRef) -> Self { + $name { dict: dict } + } + + #[pymethod(name = "__iter__")] + fn iter(&self, _vm: &VirtualMachine) -> $iter_name { + $iter_name::new(self.dict.clone()) + } + + #[pymethod(name = "__len__")] + fn len(&self, vm: &VirtualMachine) -> usize { + self.dict.clone().len(vm) + } + } + + impl PyValue for $name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$class.clone() + } + } + + #[pyclass(name = $iter_class_name)] + #[derive(Debug)] + struct $iter_name { + pub dict: PyDictRef, + pub size: dictdatatype::DictSize, + pub position: Cell, + } + + #[pyimpl] + impl $iter_name { + fn new(dict: PyDictRef) -> Self { + $iter_name { + position: Cell::new(0), + size: dict.size(), + dict, + } + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let mut position = self.position.get(); + let dict = self.dict.entries.borrow(); + if dict.has_changed_size(&self.size) { + return Err(vm.new_exception( + vm.ctx.exceptions.runtime_error.clone(), + "dictionary changed size during iteration".to_string(), + )); + } + match dict.next_entry(&mut position) { + Some((key, value)) => { + self.position.set(position); + Ok($result_fn(vm, key, value)) + } + None => Err(objiter::new_stop_iteration(vm)), + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } + } + + impl PyValue for $iter_name { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.$iter_class.clone() + } + } + }; +} + +dict_iterator! { + PyDictKeys, + PyDictKeyIterator, + dictkeys_type, + dictkeyiterator_type, + "dictkeys", + "dictkeyiterator", + |_vm: &VirtualMachine, key: &PyObjectRef, _value: &PyObjectRef| key.clone() +} + +dict_iterator! { + PyDictValues, + PyDictValueIterator, + dictvalues_type, + dictvalueiterator_type, + "dictvalues", + "dictvalueiterator", + |_vm: &VirtualMachine, _key: &PyObjectRef, value: &PyObjectRef| value.clone() +} + +dict_iterator! { + PyDictItems, + PyDictItemIterator, + dictitems_type, + dictitemiterator_type, + "dictitems", + "dictitemiterator", + |vm: &VirtualMachine, key: &PyObjectRef, value: &PyObjectRef| + vm.ctx.new_tuple(vec![key.clone(), value.clone()]) +} + pub fn init(context: &PyContext) { extend_class!(context, &context.dict_type, { + "__bool__" => context.new_rustfunc(PyDictRef::bool), "__len__" => context.new_rustfunc(PyDictRef::len), "__contains__" => context.new_rustfunc(PyDictRef::contains), - "__delitem__" => context.new_rustfunc(PyDictRef::delitem), - "__getitem__" => context.new_rustfunc(PyDictRef::getitem), + "__delitem__" => context.new_rustfunc(PyDictRef::inner_delitem), + "__eq__" => context.new_rustfunc(PyDictRef::eq), + "__getitem__" => context.new_rustfunc(PyDictRef::inner_getitem), "__iter__" => context.new_rustfunc(PyDictRef::iter), - "__new__" => context.new_rustfunc(dict_new), + "__new__" => context.new_rustfunc(PyDictRef::new), "__repr__" => context.new_rustfunc(PyDictRef::repr), - "__setitem__" => context.new_rustfunc(PyDictRef::setitem), + "__setitem__" => context.new_rustfunc(PyDictRef::inner_setitem), + "__hash__" => context.new_rustfunc(PyDictRef::hash), "clear" => context.new_rustfunc(PyDictRef::clear), "values" => context.new_rustfunc(PyDictRef::values), "items" => context.new_rustfunc(PyDictRef::items), - "keys" => context.new_rustfunc(PyDictRef::iter), + "keys" => context.new_rustfunc(PyDictRef::keys), + "fromkeys" => context.new_classmethod(PyDictRef::fromkeys), "get" => context.new_rustfunc(PyDictRef::get), + "setdefault" => context.new_rustfunc(PyDictRef::setdefault), + "copy" => context.new_rustfunc(PyDictRef::copy), + "update" => context.new_rustfunc(PyDictRef::update), + "pop" => context.new_rustfunc(PyDictRef::pop), + "popitem" => context.new_rustfunc(PyDictRef::popitem), }); + + PyDictKeys::extend_class(context, &context.dictkeys_type); + PyDictKeyIterator::extend_class(context, &context.dictkeyiterator_type); + PyDictValues::extend_class(context, &context.dictvalues_type); + PyDictValueIterator::extend_class(context, &context.dictvalueiterator_type); + PyDictItems::extend_class(context, &context.dictitems_type); + PyDictItemIterator::extend_class(context, &context.dictitemiterator_type); } diff --git a/vm/src/obj/objellipsis.rs b/vm/src/obj/objellipsis.rs index 03c9d69e8a..cb188e1f11 100644 --- a/vm/src/obj/objellipsis.rs +++ b/vm/src/obj/objellipsis.rs @@ -1,15 +1,12 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyResult, TypeProtocol}; +use crate::pyobject::{PyContext, PyResult}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { - let ellipsis_type = &context.ellipsis_type; - context.set_attr(ellipsis_type, "__new__", context.new_rustfunc(ellipsis_new)); - context.set_attr( - ellipsis_type, - "__repr__", - context.new_rustfunc(ellipsis_repr), - ); + extend_class!(context, &context.ellipsis_type, { + "__new__" => context.new_rustfunc(ellipsis_new), + "__repr__" => context.new_rustfunc(ellipsis_repr) + }); } fn ellipsis_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/obj/objenumerate.rs b/vm/src/obj/objenumerate.rs index 5a66b7b5e2..56d232272b 100644 --- a/vm/src/obj/objenumerate.rs +++ b/vm/src/obj/objenumerate.rs @@ -4,14 +4,15 @@ use std::ops::AddAssign; use num_bigint::BigInt; use num_traits::Zero; -use crate::function::{OptionalArg, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::OptionalArg; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objint::PyIntRef; use super::objiter; use super::objtype::PyClassRef; +#[pyclass] #[derive(Debug)] pub struct PyEnumerate { counter: RefCell, @@ -20,7 +21,7 @@ pub struct PyEnumerate { type PyEnumerateRef = PyRef; impl PyValue for PyEnumerate { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.enumerate_type() } } @@ -32,7 +33,7 @@ fn enumerate_new( vm: &VirtualMachine, ) -> PyResult { let counter = match start { - OptionalArg::Present(start) => start.value.clone(), + OptionalArg::Present(start) => start.as_bigint().clone(), OptionalArg::Missing => BigInt::zero(), }; @@ -44,18 +45,12 @@ fn enumerate_new( .into_ref_with_type(vm, cls) } -fn enumerate_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(enumerate, Some(vm.ctx.enumerate_type()))] - ); - - if let Some(PyEnumerate { - ref counter, - ref iterator, - }) = enumerate.payload() - { +#[pyimpl] +impl PyEnumerate { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let iterator = &self.iterator; + let counter = &self.counter; let next_obj = objiter::call_next(vm, iterator)?; let result = vm .ctx @@ -64,22 +59,17 @@ fn enumerate_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { AddAssign::add_assign(&mut counter.borrow_mut() as &mut BigInt, 1); Ok(result) - } else { - panic!("enumerate doesn't have correct payload"); + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let enumerate_type = &context.enumerate_type; - objiter::iter_type_init(context, enumerate_type); - context.set_attr( - enumerate_type, - "__new__", - context.new_rustfunc(enumerate_new), - ); - context.set_attr( - enumerate_type, - "__next__", - context.new_rustfunc(enumerate_next), - ); + PyEnumerate::extend_class(context, &context.enumerate_type); + extend_class!(context, &context.enumerate_type, { + "__new__" => context.new_rustfunc(enumerate_new), + }); } diff --git a/vm/src/obj/objfilter.rs b/vm/src/obj/objfilter.rs index d83f70cc5c..24800d6aaf 100644 --- a/vm/src/obj/objfilter.rs +++ b/vm/src/obj/objfilter.rs @@ -1,12 +1,17 @@ -use crate::function::PyFuncArgs; -use crate::pyobject::{ - IdProtocol, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol, -}; +use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; // Required for arg_check! to use isinstance use super::objbool; use super::objiter; +use crate::obj::objtype::PyClassRef; +pub type PyFilterRef = PyRef; + +/// filter(function or None, iterable) --> filter object +/// +/// Return an iterator yielding those items of iterable for which function(item) +/// is true. If function is None, return the items that are true. +#[pyclass] #[derive(Debug)] pub struct PyFilter { predicate: PyObjectRef, @@ -14,35 +19,32 @@ pub struct PyFilter { } impl PyValue for PyFilter { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.filter_type() } } -fn filter_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(cls, None), (function, None), (iterable, None)] - ); - let iterator = objiter::get_iter(vm, iterable)?; - Ok(PyObject::new( - PyFilter { - predicate: function.clone(), - iterator, - }, - cls.clone(), - )) -} +fn filter_new( + cls: PyClassRef, + function: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + let iterator = objiter::get_iter(vm, &iterable)?; -fn filter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(filter, Some(vm.ctx.filter_type()))]); + PyFilter { + predicate: function.clone(), + iterator, + } + .into_ref_with_type(vm, cls) +} - if let Some(PyFilter { - ref predicate, - ref iterator, - }) = filter.payload() - { +#[pyimpl] +impl PyFilter { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let predicate = &self.predicate; + let iterator = &self.iterator; loop { let next_obj = objiter::call_next(vm, iterator)?; let predicate_value = if predicate.is(&vm.get_none()) { @@ -56,24 +58,17 @@ fn filter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { return Ok(next_obj); } } - } else { - panic!("filter doesn't have correct payload"); + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let filter_type = &context.filter_type; - - objiter::iter_type_init(context, filter_type); - - let filter_doc = - "filter(function or None, iterable) --> filter object\n\n\ - Return an iterator yielding those items of iterable for which function(item)\n\ - is true. If function is None, return the items that are true."; - - extend_class!(context, filter_type, { + PyFilter::extend_class(context, &context.filter_type); + extend_class!(context, &context.filter_type, { "__new__" => context.new_rustfunc(filter_new), - "__doc__" => context.new_str(filter_doc.to_string()), - "__next__" => context.new_rustfunc(filter_next) }); } diff --git a/vm/src/obj/objfloat.rs b/vm/src/obj/objfloat.rs index 849b31d408..e8e05465ae 100644 --- a/vm/src/obj/objfloat.rs +++ b/vm/src/obj/objfloat.rs @@ -2,21 +2,35 @@ use super::objbytes; use super::objint; use super::objstr; use super::objtype; +use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ - IntoPyObject, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, + TypeProtocol, }; use crate::vm::VirtualMachine; -use num_bigint::ToBigInt; +use hexf; +use num_bigint::{BigInt, ToBigInt}; use num_rational::Ratio; -use num_traits::ToPrimitive; +use num_traits::{float::Float, sign::Signed, ToPrimitive, Zero}; +/// Convert a string or number to a floating point number, if possible. +#[pyclass(name = "float")] #[derive(Debug, Copy, Clone, PartialEq)] pub struct PyFloat { value: f64, } +impl PyFloat { + pub fn to_f64(&self) -> f64 { + self.value + } +} + impl PyValue for PyFloat { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.float_type() } } @@ -33,10 +47,100 @@ impl From for PyFloat { } } -pub type PyFloatRef = PyRef; +pub fn try_float(value: &PyObjectRef, vm: &VirtualMachine) -> PyResult> { + Ok(if objtype::isinstance(&value, &vm.ctx.float_type()) { + Some(get_value(&value)) + } else if objtype::isinstance(&value, &vm.ctx.int_type()) { + Some(objint::get_float_value(&value, vm)?) + } else { + None + }) +} + +fn inner_div(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v2 != 0.0 { + Ok(v1 / v2) + } else { + Err(vm.new_zero_division_error("float division by zero".to_string())) + } +} + +fn inner_mod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v2 != 0.0 { + Ok(v1 % v2) + } else { + Err(vm.new_zero_division_error("float mod by zero".to_string())) + } +} -impl PyFloatRef { - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +fn try_to_bigint(value: f64, vm: &VirtualMachine) -> PyResult { + match value.to_bigint() { + Some(int) => Ok(int), + None => { + if value.is_infinite() { + Err(vm.new_overflow_error( + "OverflowError: cannot convert float infinity to integer".to_string(), + )) + } else if value.is_nan() { + Err(vm + .new_value_error("ValueError: cannot convert float NaN to integer".to_string())) + } else { + // unreachable unless BigInt has a bug + unreachable!( + "A finite float value failed to be converted to bigint: {}", + value + ) + } + } + } +} + +fn inner_floordiv(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult { + if v2 != 0.0 { + Ok((v1 / v2).floor()) + } else { + Err(vm.new_zero_division_error("float floordiv by zero".to_string())) + } +} + +fn inner_divmod(v1: f64, v2: f64, vm: &VirtualMachine) -> PyResult<(f64, f64)> { + if v2 != 0.0 { + Ok(((v1 / v2).floor(), v1 % v2)) + } else { + Err(vm.new_zero_division_error("float divmod()".to_string())) + } +} + +fn inner_lt_int(value: f64, other_int: &BigInt) -> bool { + match (value.to_bigint(), other_int.to_f64()) { + (Some(self_int), Some(other_float)) => value < other_float || self_int < *other_int, + // finite float, other_int too big for float, + // the result depends only on other_int’s sign + (Some(_), None) => other_int.is_positive(), + // infinite float must be bigger or lower than any int, depending on its sign + _ if value.is_infinite() => value.is_sign_negative(), + // NaN, always false + _ => false, + } +} + +fn inner_gt_int(value: f64, other_int: &BigInt) -> bool { + match (value.to_bigint(), other_int.to_f64()) { + (Some(self_int), Some(other_float)) => value > other_float || self_int > *other_int, + // finite float, other_int too big for float, + // the result depends only on other_int’s sign + (Some(_), None) => other_int.is_negative(), + // infinite float must be bigger or lower than any int, depending on its sign + _ if value.is_infinite() => value.is_sign_positive(), + // NaN, always false + _ => false, + } +} + +#[pyimpl] +impl PyFloat { + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let value = self.value; let result = if objtype::isinstance(&other, &vm.ctx.float_type()) { let other = get_value(&other); @@ -55,120 +159,158 @@ impl PyFloatRef { vm.ctx.new_bool(result) } - fn lt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__lt__")] + fn lt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 < get_value(&i2)) } else if objtype::isinstance(&i2, &vm.ctx.int_type()) { - vm.ctx - .new_bool(v1 < objint::get_value(&i2).to_f64().unwrap()) + let other_int = objint::get_value(&i2); + + vm.ctx.new_bool(inner_lt_int(self.value, other_int)) } else { vm.ctx.not_implemented() } } - fn le(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__le__")] + fn le(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 <= get_value(&i2)) } else if objtype::isinstance(&i2, &vm.ctx.int_type()) { - vm.ctx - .new_bool(v1 <= objint::get_value(&i2).to_f64().unwrap()) + let other_int = objint::get_value(&i2); + + let result = if let (Some(self_int), Some(other_float)) = + (self.value.to_bigint(), other_int.to_f64()) + { + self.value <= other_float && self_int <= *other_int + } else { + // certainly not equal, forward to inner_lt_int + inner_lt_int(self.value, other_int) + }; + + vm.ctx.new_bool(result) } else { vm.ctx.not_implemented() } } - fn gt(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__gt__")] + fn gt(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 > get_value(&i2)) } else if objtype::isinstance(&i2, &vm.ctx.int_type()) { - vm.ctx - .new_bool(v1 > objint::get_value(&i2).to_f64().unwrap()) + let other_int = objint::get_value(&i2); + + vm.ctx.new_bool(inner_gt_int(self.value, other_int)) } else { vm.ctx.not_implemented() } } - fn ge(self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__ge__")] + fn ge(&self, i2: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { let v1 = self.value; if objtype::isinstance(&i2, &vm.ctx.float_type()) { vm.ctx.new_bool(v1 >= get_value(&i2)) } else if objtype::isinstance(&i2, &vm.ctx.int_type()) { - vm.ctx - .new_bool(v1 >= objint::get_value(&i2).to_f64().unwrap()) + let other_int = objint::get_value(&i2); + + let result = if let (Some(self_int), Some(other_float)) = + (self.value.to_bigint(), other_int.to_f64()) + { + self.value >= other_float && self_int >= *other_int + } else { + // certainly not equal, forward to inner_gt_int + inner_gt_int(self.value, other_int) + }; + + vm.ctx.new_bool(result) } else { vm.ctx.not_implemented() } } - fn abs(self, _vm: &VirtualMachine) -> f64 { + #[pymethod(name = "__abs__")] + fn abs(&self, _vm: &VirtualMachine) -> f64 { self.value.abs() } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - vm.ctx.new_float(v1 + get_value(&other)) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - vm.ctx - .new_float(v1 + objint::get_value(&other).to_f64().unwrap()) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__add__")] + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value + other).into_pyobject(vm), + ) } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&other, &vm.ctx.float_type()) - || objtype::isinstance(&other, &vm.ctx.int_type()) - { - let r1 = PyFloatRef::floordiv(self.clone(), other.clone(), vm)?; - let r2 = PyFloatRef::mod_(self, other, vm)?; - Ok(vm.ctx.new_tuple(vec![r1, r2])) - } else { - Ok(vm.ctx.not_implemented()) - } + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.add(other, vm) } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? - } else { - return Ok(vm.ctx.not_implemented()); - }; + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { + self.value != 0.0 + } - if v2 != 0.0 { - Ok(vm.ctx.new_float((v1 / v2).floor())) - } else { - Err(vm.new_zero_division_error("float floordiv by zero".to_string())) - } + #[pymethod(name = "__divmod__")] + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| { + let (r1, r2) = inner_divmod(self.value, other, vm)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(r1), vm.ctx.new_float(r2)])) + }, + ) + } + + #[pymethod(name = "__rdivmod__")] + fn rdivmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| { + let (r1, r2) = inner_divmod(other, self.value, vm)?; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(r1), vm.ctx.new_float(r2)])) + }, + ) + } + + #[pymethod(name = "__floordiv__")] + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_floordiv(self.value, other, vm)?.into_pyobject(vm), + ) + } + + #[pymethod(name = "__rfloordiv__")] + fn rfloordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_floordiv(other, self.value, vm)?.into_pyobject(vm), + ) } - fn new_float(cls: PyObjectRef, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__new__")] + fn float_new(cls: PyClassRef, arg: PyObjectRef, vm: &VirtualMachine) -> PyResult { let value = if objtype::isinstance(&arg, &vm.ctx.float_type()) { get_value(&arg) } else if objtype::isinstance(&arg, &vm.ctx.int_type()) { - match objint::get_value(&arg).to_f64() { - Some(f) => f, - None => { - return Err( - vm.new_overflow_error("int too large to convert to float".to_string()) - ); - } - } + objint::get_float_value(&arg, vm)? } else if objtype::isinstance(&arg, &vm.ctx.str_type()) { match lexical::try_parse(objstr::get_value(&arg)) { Ok(f) => f, Err(_) => { let arg_repr = vm.to_pystr(&arg)?; return Err(vm.new_value_error(format!( - "could not convert string to float: {}", + "could not convert string to float: '{}'", arg_repr ))); } @@ -179,144 +321,198 @@ impl PyFloatRef { Err(_) => { let arg_repr = vm.to_pystr(&arg)?; return Err(vm.new_value_error(format!( - "could not convert string to float: {}", + "could not convert string to float: '{}'", arg_repr ))); } } } else { - let type_name = objtype::get_type_name(&arg.typ()); - return Err(vm.new_type_error(format!("can't convert {} to float", type_name))); + return Err(vm.new_type_error(format!("can't convert {} to float", arg.class().name))); }; - Ok(PyObject::new(PyFloat { value }, cls.clone())) + PyFloat { value }.into_ref_with_type(vm, cls) } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? - } else { - return Ok(vm.ctx.not_implemented()); - }; + #[pymethod(name = "__mod__")] + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_mod(self.value, other, vm)?.into_pyobject(vm), + ) + } - if v2 != 0.0 { - Ok(vm.ctx.new_float(v1 % v2)) - } else { - Err(vm.new_zero_division_error("float mod by zero".to_string())) - } + #[pymethod(name = "__rmod__")] + fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_mod(other, self.value, vm)?.into_pyobject(vm), + ) } - fn neg(self, _vm: &VirtualMachine) -> f64 { + #[pymethod(name = "__pos__")] + fn pos(&self, _vm: &VirtualMachine) -> f64 { + self.value + } + + #[pymethod(name = "__neg__")] + fn neg(&self, _vm: &VirtualMachine) -> f64 { -self.value } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - vm.ctx.new_float(v1.powf(get_value(&other))) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - let result = v1.powf(objint::get_value(&other).to_f64().unwrap()); - vm.ctx.new_float(result) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__pow__")] + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| self.value.powf(other).into_pyobject(vm), + ) } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - Ok(vm.ctx.new_float(v1 - get_value(&other))) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - Ok(vm - .ctx - .new_float(v1 - objint::get_value(&other).to_f64().unwrap())) - } else { - Ok(vm.ctx.not_implemented()) - } + #[pymethod(name = "__rpow__")] + fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| other.powf(self.value).into_pyobject(vm), + ) } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type()) { - Ok(vm.ctx.new_float(get_value(&other) - v1)) - } else if objtype::isinstance(&other, &vm.ctx.int_type()) { - Ok(vm - .ctx - .new_float(objint::get_value(&other).to_f64().unwrap() - v1)) + #[pymethod(name = "__sub__")] + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value - other).into_pyobject(vm), + ) + } + + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (other - self.value).into_pyobject(vm), + ) + } + + #[pymethod(name = "__repr__")] + fn repr(&self, vm: &VirtualMachine) -> String { + if self.is_integer(vm) { + format!("{:.1}", self.value) } else { - Ok(vm.ctx.not_implemented()) + self.value.to_string() } } - fn repr(self, _vm: &VirtualMachine) -> String { - self.value.to_string() + #[pymethod(name = "__truediv__")] + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_div(self.value, other, vm)?.into_pyobject(vm), + ) } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? - } else { - return Ok(vm.ctx.not_implemented()); - }; + #[pymethod(name = "__rtruediv__")] + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| inner_div(other, self.value, vm)?.into_pyobject(vm), + ) + } - if v2 != 0.0 { - Ok(vm.ctx.new_float(v1 / v2)) - } else { - Err(vm.new_zero_division_error("float division by zero".to_string())) - } + #[pymethod(name = "__mul__")] + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_float(&other, vm)?.map_or_else( + || Ok(vm.ctx.not_implemented()), + |other| (self.value * other).into_pyobject(vm), + ) } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - let v2 = if objtype::isinstance(&other, &vm.ctx.float_type) { - get_value(&other) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - objint::get_value(&other).to_f64().ok_or_else(|| { - vm.new_overflow_error("int too large to convert to float".to_string()) - })? - } else { - return Ok(vm.ctx.not_implemented()); - }; + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mul(other, vm) + } - if v1 != 0.0 { - Ok(vm.ctx.new_float(v2 / v1)) - } else { - Err(vm.new_zero_division_error("float division by zero".to_string())) - } + #[pymethod(name = "__trunc__")] + fn trunc(&self, vm: &VirtualMachine) -> PyResult { + try_to_bigint(self.value, vm) } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let v1 = self.value; - if objtype::isinstance(&other, &vm.ctx.float_type) { - Ok(vm.ctx.new_float(v1 * get_value(&other))) - } else if objtype::isinstance(&other, &vm.ctx.int_type) { - Ok(vm - .ctx - .new_float(v1 * objint::get_value(&other).to_f64().unwrap())) + #[pymethod(name = "__round__")] + fn round(&self, ndigits: OptionalArg, vm: &VirtualMachine) -> PyResult { + let ndigits = match ndigits { + OptionalArg::Missing => None, + OptionalArg::Present(ref value) => { + if !vm.get_none().is(value) { + let ndigits = if objtype::isinstance(value, &vm.ctx.int_type()) { + objint::get_value(value) + } else { + return Err(vm.new_type_error(format!( + "TypeError: '{}' object cannot be interpreted as an integer", + value.class().name + ))); + }; + if ndigits.is_zero() { + None + } else { + Some(ndigits) + } + } else { + None + } + } + }; + if ndigits.is_none() { + let fract = self.value.fract(); + let value = if (fract.abs() - 0.5).abs() < std::f64::EPSILON { + if self.value.trunc() % 2.0 == 0.0 { + self.value - fract + } else { + self.value + fract + } + } else { + self.value.round() + }; + let int = try_to_bigint(value, vm)?; + Ok(vm.ctx.new_int(int)) } else { Ok(vm.ctx.not_implemented()) } } - fn is_integer(self, _vm: &VirtualMachine) -> bool { - let v = self.value; - (v - v.round()).abs() < std::f64::EPSILON + #[pymethod(name = "__int__")] + fn int(&self, vm: &VirtualMachine) -> PyResult { + self.trunc(vm) } - fn real(self, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__float__")] + fn float(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf } - fn as_integer_ratio(self, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + pyhash::hash_float(self.value) + } + + #[pyproperty(name = "real")] + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf + } + + #[pyproperty(name = "imag")] + fn imag(&self, _vm: &VirtualMachine) -> f64 { + 0.0f64 + } + + #[pymethod(name = "conjugate")] + fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyFloatRef { + zelf + } + + #[pymethod(name = "is_integer")] + fn is_integer(&self, _vm: &VirtualMachine) -> bool { + let v = self.value; + (v - v.round()).abs() < std::f64::EPSILON + } + + #[pymethod(name = "as_integer_ratio")] + fn as_integer_ratio(&self, vm: &VirtualMachine) -> PyResult { let value = self.value; if value.is_infinite() { return Err( @@ -332,8 +528,74 @@ impl PyFloatRef { let denom = vm.ctx.new_int(ratio.denom().clone()); Ok(vm.ctx.new_tuple(vec![numer, denom])) } + + #[pymethod] + fn fromhex(repr: PyStringRef, vm: &VirtualMachine) -> PyResult { + hexf::parse_hexf64(&repr.value, false).or_else(|_| match repr.value.as_ref() { + "nan" => Ok(std::f64::NAN), + "inf" => Ok(std::f64::INFINITY), + "-inf" => Ok(std::f64::NEG_INFINITY), + _ => Err(vm.new_value_error("invalid hexadecimal floating-point string".to_string())), + }) + } + + #[pymethod] + fn hex(&self, _vm: &VirtualMachine) -> String { + to_hex(self.value) + } } +fn to_hex(value: f64) -> String { + let (mantissa, exponent, sign) = value.integer_decode(); + let sign_fmt = if sign < 0 { "-" } else { "" }; + match value { + value if value.is_zero() => format!("{}0x0.0p+0", sign_fmt), + value if value.is_infinite() => format!("{}inf", sign_fmt), + value if value.is_nan() => "nan".to_string(), + _ => { + const BITS: i16 = 52; + const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff; + format!( + "{}0x{:x}.{:013x}p{:+}", + sign_fmt, + mantissa >> BITS, + mantissa & FRACT_MASK, + exponent + BITS + ) + } + } +} + +#[test] +fn test_to_hex() { + use rand::Rng; + for _ in 0..20000 { + let bytes = rand::thread_rng().gen::<[u64; 1]>(); + let f = f64::from_bits(bytes[0]); + if !f.is_finite() { + continue; + } + let hex = to_hex(f); + // println!("{} -> {}", f, hex); + let roundtrip = hexf::parse_hexf64(&hex, false).unwrap(); + // println!(" -> {}", roundtrip); + assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip); + } +} + +pub fn ufrexp(value: f64) -> (f64, i32) { + if 0.0 == value { + (0.0, 0i32) + } else { + let bits = value.to_bits(); + let exponent: i32 = ((bits >> 52) & 0x7ff) as i32 - 1022; + let mantissa_bits = bits & (0x000f_ffff_ffff_ffff) | (1022 << 52); + (f64::from_bits(mantissa_bits), exponent) + } +} + +pub type PyFloatRef = PyRef; + // Retrieve inner float value: pub fn get_value(obj: &PyObjectRef) -> f64 { obj.payload::().unwrap().value @@ -342,43 +604,19 @@ pub fn get_value(obj: &PyObjectRef) -> f64 { pub fn make_float(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { if objtype::isinstance(obj, &vm.ctx.float_type()) { Ok(get_value(obj)) - } else if let Ok(method) = vm.get_method(obj.clone(), "__float__") { - let res = vm.invoke(method, vec![])?; - Ok(get_value(&res)) } else { - Err(vm.new_type_error(format!("Cannot cast {} to float", obj))) + let method = vm.get_method_or_type_error(obj.clone(), "__float__", || { + format!( + "float() argument must be a string or a number, not '{}'", + obj.class().name + ) + })?; + let result = vm.invoke(method, vec![])?; + Ok(get_value(&result)) } } #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { - let float_type = &context.float_type; - - let float_doc = "Convert a string or number to a floating point number, if possible."; - - context.set_attr(&float_type, "__eq__", context.new_rustfunc(PyFloatRef::eq)); - context.set_attr(&float_type, "__lt__", context.new_rustfunc(PyFloatRef::lt)); - context.set_attr(&float_type, "__le__", context.new_rustfunc(PyFloatRef::le)); - context.set_attr(&float_type, "__gt__", context.new_rustfunc(PyFloatRef::gt)); - context.set_attr(&float_type, "__ge__", context.new_rustfunc(PyFloatRef::ge)); - context.set_attr(&float_type, "__abs__", context.new_rustfunc(PyFloatRef::abs)); - context.set_attr(&float_type, "__add__", context.new_rustfunc(PyFloatRef::add)); - context.set_attr(&float_type, "__radd__", context.new_rustfunc(PyFloatRef::add)); - context.set_attr(&float_type, "__divmod__", context.new_rustfunc(PyFloatRef::divmod)); - context.set_attr(&float_type, "__floordiv__", context.new_rustfunc(PyFloatRef::floordiv)); - context.set_attr(&float_type, "__new__", context.new_rustfunc(PyFloatRef::new_float)); - context.set_attr(&float_type, "__mod__", context.new_rustfunc(PyFloatRef::mod_)); - context.set_attr(&float_type, "__neg__", context.new_rustfunc(PyFloatRef::neg)); - context.set_attr(&float_type, "__pow__", context.new_rustfunc(PyFloatRef::pow)); - context.set_attr(&float_type, "__sub__", context.new_rustfunc(PyFloatRef::sub)); - context.set_attr(&float_type, "__rsub__", context.new_rustfunc(PyFloatRef::rsub)); - context.set_attr(&float_type, "__repr__", context.new_rustfunc(PyFloatRef::repr)); - context.set_attr(&float_type, "__doc__", context.new_str(float_doc.to_string())); - context.set_attr(&float_type, "__truediv__", context.new_rustfunc(PyFloatRef::truediv)); - context.set_attr(&float_type, "__rtruediv__", context.new_rustfunc(PyFloatRef::rtruediv)); - context.set_attr(&float_type, "__mul__", context.new_rustfunc(PyFloatRef::mul)); - context.set_attr(&float_type, "__rmul__", context.new_rustfunc(PyFloatRef::mul)); - context.set_attr(&float_type, "real", context.new_property(PyFloatRef::real)); - context.set_attr(&float_type, "is_integer", context.new_rustfunc(PyFloatRef::is_integer)); - context.set_attr(&float_type, "as_integer_ratio", context.new_rustfunc(PyFloatRef::as_integer_ratio)); + PyFloat::extend_class(context, &context.float_type); } diff --git a/vm/src/obj/objframe.rs b/vm/src/obj/objframe.rs index 2d179fa24d..6669e3bb9a 100644 --- a/vm/src/obj/objframe.rs +++ b/vm/src/obj/objframe.rs @@ -2,41 +2,36 @@ */ -use crate::frame::Frame; -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use super::objcode::PyCodeRef; +use super::objdict::PyDictRef; +use crate::frame::FrameRef; +use crate::pyobject::{PyContext, PyResult}; use crate::vm::VirtualMachine; pub fn init(context: &PyContext) { - let frame_type = &context.frame_type; - context.set_attr(&frame_type, "__new__", context.new_rustfunc(frame_new)); - context.set_attr(&frame_type, "__repr__", context.new_rustfunc(frame_repr)); - context.set_attr(&frame_type, "f_locals", context.new_property(frame_flocals)); - context.set_attr(&frame_type, "f_code", context.new_property(frame_fcode)); + extend_class!(context, &context.frame_type, { + "__new__" => context.new_rustfunc(FrameRef::new), + "__repr__" => context.new_rustfunc(FrameRef::repr), + "f_locals" => context.new_property(FrameRef::flocals), + "f_code" => context.new_property(FrameRef::fcode), + }); } -fn frame_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_cls, None)]); - Err(vm.new_type_error("Cannot directly create frame object".to_string())) -} +impl FrameRef { + #[allow(clippy::new_ret_no_self)] + fn new(_class: FrameRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("Cannot directly create frame object".to_string())) + } -fn frame_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_frame, Some(vm.ctx.frame_type()))]); - let repr = "".to_string(); - Ok(vm.new_str(repr)) -} + fn repr(self, _vm: &VirtualMachine) -> String { + "".to_string() + } -fn frame_flocals(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(frame, Some(vm.ctx.frame_type()))]); - let frame = get_value(frame); - Ok(frame.scope.get_locals().clone()) -} - -fn frame_fcode(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(frame, Some(vm.ctx.frame_type()))]); - Ok(vm.ctx.new_code_object(get_value(frame).code.clone())) -} + fn flocals(self, _vm: &VirtualMachine) -> PyDictRef { + self.scope.get_locals() + } -pub fn get_value(obj: &PyObjectRef) -> &Frame { - &obj.payload::().unwrap() + fn fcode(self, vm: &VirtualMachine) -> PyCodeRef { + vm.ctx.new_code_object(self.code.clone()) + } } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index c4c480d67b..05b66c3460 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -1,32 +1,63 @@ use crate::frame::Scope; -use crate::function::PyFuncArgs; -use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::function::{Args, KwArgs}; +use crate::obj::objcode::PyCodeRef; +use crate::obj::objdict::PyDictRef; +use crate::obj::objtuple::PyTupleRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; +pub type PyFunctionRef = PyRef; + #[derive(Debug)] pub struct PyFunction { // TODO: these shouldn't be public - pub code: PyObjectRef, + pub code: PyCodeRef, pub scope: Scope, - pub defaults: PyObjectRef, + pub defaults: Option, + pub kw_only_defaults: Option, } impl PyFunction { - pub fn new(code: PyObjectRef, scope: Scope, defaults: PyObjectRef) -> Self { + pub fn new( + code: PyCodeRef, + scope: Scope, + defaults: Option, + kw_only_defaults: Option, + ) -> Self { PyFunction { code, scope, defaults, + kw_only_defaults, } } } impl PyValue for PyFunction { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.function_type() } } +impl PyFunctionRef { + fn call(self, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { + vm.invoke(self.into_object(), (&args, &kwargs)) + } + + fn code(self, _vm: &VirtualMachine) -> PyCodeRef { + self.code.clone() + } + + fn defaults(self, _vm: &VirtualMachine) -> Option { + self.defaults.clone() + } + + fn kwdefaults(self, _vm: &VirtualMachine) -> Option { + self.kw_only_defaults.clone() + } +} + #[derive(Debug)] pub struct PyMethod { // TODO: these shouldn't be public @@ -41,7 +72,7 @@ impl PyMethod { } impl PyValue for PyMethod { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.bound_method_type() } } @@ -50,7 +81,10 @@ pub fn init(context: &PyContext) { let function_type = &context.function_type; extend_class!(context, function_type, { "__get__" => context.new_rustfunc(bind_method), - "__code__" => context.new_property(function_code) + "__call__" => context.new_rustfunc(PyFunctionRef::call), + "__code__" => context.new_property(PyFunctionRef::code), + "__defaults__" => context.new_property(PyFunctionRef::defaults), + "__kwdefaults__" => context.new_property(PyFunctionRef::kwdefaults), }); let builtin_function_or_method_type = &context.builtin_function_or_method_type; @@ -59,23 +93,15 @@ pub fn init(context: &PyContext) { }); } -fn bind_method(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(function, None), (obj, None), (cls, None)] - ); - - if obj.is(&vm.get_none()) && !cls.is(&obj.typ()) { - Ok(function.clone()) +fn bind_method( + function: PyObjectRef, + obj: PyObjectRef, + cls: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + if obj.is(&vm.get_none()) && !cls.is(&obj.class()) { + Ok(function) } else { - Ok(vm.ctx.new_bound_method(function.clone(), obj.clone())) - } -} - -fn function_code(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - match args.args[0].payload() { - Some(PyFunction { ref code, .. }) => Ok(code.clone()), - None => Err(vm.new_type_error("no code".to_string())), + Ok(vm.ctx.new_bound_method(function, obj)) } } diff --git a/vm/src/obj/objgenerator.rs b/vm/src/obj/objgenerator.rs index 208bcb43f9..2c51778fde 100644 --- a/vm/src/obj/objgenerator.rs +++ b/vm/src/obj/objgenerator.rs @@ -2,83 +2,78 @@ * The mythical generator. */ -use crate::frame::{ExecutionResult, Frame}; -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::frame::{ExecutionResult, FrameRef}; +use crate::obj::objtype::{isinstance, PyClassRef}; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +pub type PyGeneratorRef = PyRef; + +#[pyclass(name = "generator")] #[derive(Debug)] pub struct PyGenerator { - frame: PyObjectRef, + frame: FrameRef, } -type PyGeneratorRef = PyRef; impl PyValue for PyGenerator { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.generator_type() } } -pub fn init(context: &PyContext) { - let generator_type = &context.generator_type; - context.set_attr( - &generator_type, - "__iter__", - context.new_rustfunc(generator_iter), - ); - context.set_attr( - &generator_type, - "__next__", - context.new_rustfunc(generator_next), - ); - context.set_attr( - &generator_type, - "send", - context.new_rustfunc(generator_send), - ); -} +#[pyimpl] +impl PyGenerator { + pub fn new(frame: FrameRef, vm: &VirtualMachine) -> PyGeneratorRef { + PyGenerator { frame }.into_ref(vm) + } -pub fn new_generator(frame: PyObjectRef, vm: &VirtualMachine) -> PyGeneratorRef { - PyGenerator { frame }.into_ref(vm) -} + #[pymethod(name = "__iter__")] + fn iter(zelf: PyGeneratorRef, _vm: &VirtualMachine) -> PyGeneratorRef { + zelf + } -fn generator_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.generator_type()))]); - Ok(o.clone()) -} + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + self.send(vm.get_none(), vm) + } -fn generator_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.generator_type()))]); - let value = vm.get_none(); - send(vm, o, &value) -} + #[pymethod] + fn send(&self, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.frame.push_value(value.clone()); -fn generator_send(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(o, Some(vm.ctx.generator_type())), (value, None)] - ); - send(vm, o, value) -} + let result = vm.run_frame(self.frame.clone())?; + handle_execution_result(result, vm) + } -fn send(vm: &VirtualMachine, gen: &PyObjectRef, value: &PyObjectRef) -> PyResult { - if let Some(PyGenerator { ref frame }) = gen.payload() { - if let Some(frame) = frame.payload::() { - frame.push_value(value.clone()); - } else { - panic!("Generator frame isn't a frame."); + #[pymethod] + fn throw( + &self, + _exc_type: PyObjectRef, + exc_val: PyObjectRef, + _exc_tb: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + // TODO what should we do with the other parameters? CPython normalises them with + // PyErr_NormalizeException, do we want to do the same. + if !isinstance(&exc_val, &vm.ctx.exceptions.base_exception_type) { + return Err(vm.new_type_error("Can't throw non exception".to_string())); } + let result = vm.frame_throw(self.frame.clone(), exc_val)?; + handle_execution_result(result, vm) + } +} - match vm.run_frame(frame.clone())? { - ExecutionResult::Yield(value) => Ok(value), - ExecutionResult::Return(_value) => { - // Stop iteration! - let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); - Err(vm.new_exception(stop_iteration, "End of generator".to_string())) - } +fn handle_execution_result(result: ExecutionResult, vm: &VirtualMachine) -> PyResult { + match result { + ExecutionResult::Yield(value) => Ok(value), + ExecutionResult::Return(_value) => { + // Stop iteration! + let stop_iteration = vm.ctx.exceptions.stop_iteration.clone(); + Err(vm.new_exception(stop_iteration, "End of generator".to_string())) } - } else { - panic!("Cannot extract frame from non-generator"); } } + +pub fn init(ctx: &PyContext) { + PyGenerator::extend_class(ctx, &ctx.generator_type); +} diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 553042ed5f..88e0233b2f 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -1,25 +1,48 @@ -use std::hash::{Hash, Hasher}; +use std::fmt; -use num_bigint::{BigInt, ToBigInt}; +use num_bigint::{BigInt, Sign}; use num_integer::Integer; -use num_traits::{Pow, Signed, ToPrimitive, Zero}; +use num_traits::{One, Pow, Signed, ToPrimitive, Zero}; use crate::format::FormatSpec; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::function::{KwArgs, OptionalArg, PyFuncArgs}; +use crate::pyhash; use crate::pyobject::{ - IntoPyObject, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + IntoPyObject, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; -use super::objfloat; -use super::objstr; +use super::objbyteinner::PyByteInner; +use super::objbytes::PyBytes; +use super::objstr::{PyString, PyStringRef}; use super::objtype; - +use crate::obj::objtype::PyClassRef; + +/// int(x=0) -> integer +/// int(x, base=10) -> integer +/// +/// Convert a number or string to an integer, or return 0 if no arguments +/// are given. If x is a number, return x.__int__(). For floating point +/// numbers, this truncates towards zero. +/// +/// If x is not a number or if base is given, then x must be a string, +/// bytes, or bytearray instance representing an integer literal in the +/// given base. The literal can be preceded by '+' or '-' and be surrounded +/// by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. +/// Base 0 means to interpret the base from the string as an integer literal. +/// >>> int('0b100', base=0) +/// 4 +#[pyclass] #[derive(Debug)] pub struct PyInt { - // TODO: shouldn't be public - pub value: BigInt, + value: BigInt, +} + +impl fmt::Display for PyInt { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + BigInt::fmt(&self.value, f) + } } pub type PyIntRef = PyRef; @@ -28,6 +51,10 @@ impl PyInt { pub fn new>(i: T) -> Self { PyInt { value: i.into() } } + + pub fn as_bigint(&self) -> &BigInt { + &self.value + } } impl IntoPyObject for BigInt { @@ -37,7 +64,7 @@ impl IntoPyObject for BigInt { } impl PyValue for PyInt { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.int_type() } } @@ -85,12 +112,44 @@ impl_try_from_object_int!( (u64, to_u64), ); -impl PyIntRef { - fn pass_value(self, _vm: &VirtualMachine) -> Self { - self +#[allow(clippy::collapsible_if)] +fn inner_pow(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { + let result = if int2.value.is_negative() { + let v1 = int1.float(vm)?; + let v2 = int2.float(vm)?; + vm.ctx.new_float(v1.pow(v2)) + } else { + if let Some(v2) = int2.value.to_u64() { + vm.ctx.new_int(int1.value.pow(v2)) + } else if int1.value.is_one() || int1.value.is_zero() { + vm.ctx.new_int(int1.value.clone()) + } else if int1.value == BigInt::from(-1) { + if int2.value.is_odd() { + vm.ctx.new_int(-1) + } else { + vm.ctx.new_int(1) + } + } else { + // missing feature: BigInt exp + // practically, exp over u64 is not possible to calculate anyway + vm.ctx.not_implemented() + } + }; + Ok(result) +} + +fn inner_mod(int1: &PyInt, int2: &PyInt, vm: &VirtualMachine) -> PyResult { + if int2.value != BigInt::zero() { + Ok(vm.ctx.new_int(&int1.value % &int2.value)) + } else { + Err(vm.new_zero_division_error("integer modulo by zero".to_string())) } +} - fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { +#[pyimpl] +impl PyInt { + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value == *get_value(&other)) } else { @@ -98,7 +157,8 @@ impl PyIntRef { } } - fn ne(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__ne__")] + fn ne(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value != *get_value(&other)) } else { @@ -106,7 +166,8 @@ impl PyIntRef { } } - fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value < *get_value(&other)) } else { @@ -114,7 +175,8 @@ impl PyIntRef { } } - fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value <= *get_value(&other)) } else { @@ -122,7 +184,8 @@ impl PyIntRef { } } - fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value > *get_value(&other)) } else { @@ -130,7 +193,8 @@ impl PyIntRef { } } - fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_bool(self.value >= *get_value(&other)) } else { @@ -138,7 +202,8 @@ impl PyIntRef { } } - fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__add__")] + fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) + get_value(&other)) } else { @@ -146,7 +211,13 @@ impl PyIntRef { } } - fn sub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__radd__")] + fn radd(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.add(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) - get_value(&other)) } else { @@ -154,7 +225,8 @@ impl PyIntRef { } } - fn rsub(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int(get_value(&other) - (&self.value)) } else { @@ -162,7 +234,8 @@ impl PyIntRef { } } - fn mul(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__mul__")] + fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) * get_value(&other)) } else { @@ -170,7 +243,13 @@ impl PyIntRef { } } - fn truediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rmul__")] + fn rmul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.mul(other, vm) + } + + #[pymethod(name = "__truediv__")] + fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &self.value, &get_value(&other)) } else { @@ -178,7 +257,8 @@ impl PyIntRef { } } - fn rtruediv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rtruediv__")] + fn rtruediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { div_ints(vm, &get_value(&other), &self.value) } else { @@ -186,11 +266,13 @@ impl PyIntRef { } } - fn floordiv(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__floordiv__")] + fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { - Ok(vm.ctx.new_int((&self.value) / v2)) + let modulo = (&self.value % v2 + v2) % v2; + Ok(vm.ctx.new_int((&self.value - modulo) / v2)) } else { Err(vm.new_zero_division_error("integer floordiv by zero".to_string())) } @@ -199,13 +281,10 @@ impl PyIntRef { } } - fn lshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__lshift__")] + fn lshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { - return Err(vm.new_type_error(format!( - "unsupported operand type(s) for << '{}' and '{}'", - objtype::get_type_name(&self.as_object().typ()), - objtype::get_type_name(&other.typ()) - ))); + return Ok(vm.ctx.not_implemented()); } if let Some(n_bits) = get_value(&other).to_usize() { @@ -222,13 +301,10 @@ impl PyIntRef { } } - fn rshift(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rshift__")] + fn rshift(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if !objtype::isinstance(&other, &vm.ctx.int_type()) { - return Err(vm.new_type_error(format!( - "unsupported operand type(s) for >> '{}' and '{}'", - objtype::get_type_name(&self.as_object().typ()), - objtype::get_type_name(&other.typ()) - ))); + return Ok(vm.ctx.not_implemented()); } if let Some(n_bits) = get_value(&other).to_usize() { @@ -245,7 +321,8 @@ impl PyIntRef { } } - fn xor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__xor__")] + pub fn xor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) ^ get_value(&other)) } else { @@ -253,15 +330,13 @@ impl PyIntRef { } } - fn rxor(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - if objtype::isinstance(&other, &vm.ctx.int_type()) { - vm.ctx.new_int(get_value(&other) ^ (&self.value)) - } else { - vm.ctx.not_implemented() - } + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + self.xor(other, vm) } - fn or(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__or__")] + pub fn or(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { vm.ctx.new_int((&self.value) | get_value(&other)) } else { @@ -269,7 +344,8 @@ impl PyIntRef { } } - fn and(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__and__")] + pub fn and(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); vm.ctx.new_int((&self.value) & v2) @@ -278,32 +354,48 @@ impl PyIntRef { } } - fn pow(self, other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod(name = "__pow__")] + fn pow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { - let v2 = get_value(&other).to_u32().unwrap(); - vm.ctx.new_int(self.value.pow(v2)) - } else if objtype::isinstance(&other, &vm.ctx.float_type()) { - let v2 = objfloat::get_value(&other); - vm.ctx.new_float((self.value.to_f64().unwrap()).powf(v2)) + let other = other.payload::().unwrap(); + inner_pow(self, &other, vm) } else { - vm.ctx.not_implemented() + Ok(vm.ctx.not_implemented()) } } - fn mod_(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__rpow__")] + fn rpow(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { - let v2 = get_value(&other); - if *v2 != BigInt::zero() { - Ok(vm.ctx.new_int((&self.value) % v2)) - } else { - Err(vm.new_zero_division_error("integer modulo by zero".to_string())) - } + let other = other.payload::().unwrap(); + inner_pow(&other, self, vm) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + #[pymethod(name = "__mod__")] + fn mod_(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&other, &vm.ctx.int_type()) { + let other = other.payload::().unwrap(); + inner_mod(self, &other, vm) + } else { + Ok(vm.ctx.not_implemented()) + } + } + + #[pymethod(name = "__rmod__")] + fn rmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::isinstance(&other, &vm.ctx.int_type()) { + let other = other.payload::().unwrap(); + inner_mod(&other, self, vm) } else { Ok(vm.ctx.not_implemented()) } } - fn divmod(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__divmod__")] + fn divmod(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.int_type()) { let v2 = get_value(&other); if *v2 != BigInt::zero() { @@ -319,37 +411,82 @@ impl PyIntRef { } } - fn neg(self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__neg__")] + fn neg(&self, _vm: &VirtualMachine) -> BigInt { -(&self.value) } - fn hash(self, _vm: &VirtualMachine) -> u64 { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() + #[pymethod(name = "__hash__")] + pub fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + match self.value.to_i64() { + Some(value) => (value % pyhash::MODULUS as i64), + None => (&self.value % pyhash::MODULUS).to_i64().unwrap(), + } } - fn abs(self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__abs__")] + fn abs(&self, _vm: &VirtualMachine) -> BigInt { self.value.abs() } - fn round(self, _precision: OptionalArg, _vm: &VirtualMachine) -> Self { - self + #[pymethod(name = "__round__")] + fn round( + zelf: PyRef, + _precision: OptionalArg, + _vm: &VirtualMachine, + ) -> PyIntRef { + zelf + } + + #[pymethod(name = "__int__")] + fn int(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__pos__")] + fn pos(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__float__")] + fn float(&self, vm: &VirtualMachine) -> PyResult { + self.value + .to_f64() + .ok_or_else(|| vm.new_overflow_error("int too large to convert to float".to_string())) + } + + #[pymethod(name = "__trunc__")] + fn trunc(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__floor__")] + fn floor(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod(name = "__ceil__")] + fn ceil(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf } - fn float(self, _vm: &VirtualMachine) -> f64 { - self.value.to_f64().unwrap() + #[pymethod(name = "__index__")] + fn index(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf } - fn invert(self, _vm: &VirtualMachine) -> BigInt { + #[pymethod(name = "__invert__")] + fn invert(&self, _vm: &VirtualMachine) -> BigInt { !(&self.value) } - fn repr(self, _vm: &VirtualMachine) -> String { + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { self.value.to_string() } - fn format(self, spec: PyRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__format__")] + fn format(&self, spec: PyStringRef, vm: &VirtualMachine) -> PyResult { let format_spec = FormatSpec::parse(&spec.value); match format_spec.format_int(&self.value) { Ok(string) => Ok(string), @@ -357,67 +494,194 @@ impl PyIntRef { } } - fn bool(self, _vm: &VirtualMachine) -> bool { + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { !self.value.is_zero() } - fn bit_length(self, _vm: &VirtualMachine) -> usize { + #[pymethod] + fn bit_length(&self, _vm: &VirtualMachine) -> usize { self.value.bits() } - fn imag(self, _vm: &VirtualMachine) -> usize { + #[pymethod] + fn conjugate(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pymethod] + #[allow(clippy::match_bool)] + fn from_bytes( + bytes: PyByteInner, + byteorder: PyStringRef, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let mut signed = false; + for (key, value) in kwargs.into_iter() { + if key == "signed" { + signed = match_class!(value, + + b @ PyInt => !b.as_bigint().is_zero(), + _ => false, + ); + } + } + let x; + if byteorder.value == "big" { + x = match signed { + true => BigInt::from_signed_bytes_be(&bytes.elements), + false => BigInt::from_bytes_be(Sign::Plus, &bytes.elements), + } + } else if byteorder.value == "little" { + x = match signed { + true => BigInt::from_signed_bytes_le(&bytes.elements), + false => BigInt::from_bytes_le(Sign::Plus, &bytes.elements), + } + } else { + return Err( + vm.new_value_error("byteorder must be either 'little' or 'big'".to_string()) + ); + } + Ok(x) + } + #[pymethod] + #[allow(clippy::match_bool)] + fn to_bytes( + &self, + length: PyIntRef, + byteorder: PyStringRef, + kwargs: KwArgs, + vm: &VirtualMachine, + ) -> PyResult { + let mut signed = false; + let value = self.as_bigint(); + for (key, value) in kwargs.into_iter() { + if key == "signed" { + signed = match_class!(value, + + b @ PyInt => !b.as_bigint().is_zero(), + _ => false, + ); + } + } + if value.sign() == Sign::Minus && !signed { + return Err(vm.new_overflow_error("can't convert negative int to unsigned".to_string())); + } + let byte_len; + if let Some(temp) = length.as_bigint().to_usize() { + byte_len = temp; + } else { + return Err(vm.new_value_error("length parameter is illegal".to_string())); + } + + let mut origin_bytes = match byteorder.value.as_str() { + "big" => match signed { + true => value.to_signed_bytes_be(), + false => value.to_bytes_be().1, + }, + "little" => match signed { + true => value.to_signed_bytes_le(), + false => value.to_bytes_le().1, + }, + _ => { + return Err( + vm.new_value_error("byteorder must be either 'little' or 'big'".to_string()) + ); + } + }; + let origin_len = origin_bytes.len(); + if origin_len > byte_len { + return Err(vm.new_value_error("int too big to convert".to_string())); + } + + let mut append_bytes = match value.sign() { + Sign::Minus => vec![255u8; byte_len - origin_len], + _ => vec![0u8; byte_len - origin_len], + }; + let mut bytes = vec![]; + match byteorder.value.as_str() { + "big" => { + bytes = append_bytes; + bytes.append(&mut origin_bytes); + } + "little" => { + bytes = origin_bytes; + bytes.append(&mut append_bytes); + } + _ => (), + } + + Ok(PyBytes::new(bytes)) + } + #[pyproperty] + fn real(zelf: PyRef, _vm: &VirtualMachine) -> PyIntRef { + zelf + } + + #[pyproperty] + fn imag(&self, _vm: &VirtualMachine) -> usize { 0 } } -fn int_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(cls, None)], - optional = [(val_option, None)] - ); - if !objtype::issubclass(cls, &vm.ctx.int_type()) { - return Err(vm.new_type_error(format!("{:?} is not a subtype of int", cls))); +#[derive(FromArgs)] +struct IntOptions { + #[pyarg(positional_only, optional = true)] + val_options: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + base: OptionalArg, +} + +impl IntOptions { + fn get_int_value(self, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(val) = self.val_options { + let base = if let OptionalArg::Present(base) = self.base { + if !objtype::isinstance(&val, &vm.ctx.str_type) { + return Err(vm.new_type_error( + "int() can't convert non-string with explicit base".to_string(), + )); + } + base + } else { + 10 + }; + to_int(vm, &val, base) + } else if let OptionalArg::Present(_) = self.base { + Err(vm.new_type_error("int() missing string argument".to_string())) + } else { + Ok(Zero::zero()) + } } +} - let base = match args.get_optional_kwarg("base") { - Some(argument) => get_value(&argument).to_u32().unwrap(), - None => 10, - }; - let val = match val_option { - Some(val) => to_int(vm, val, base)?, - None => Zero::zero(), - }; - Ok(PyObject::new(PyInt::new(val), cls.clone())) +fn int_new(cls: PyClassRef, options: IntOptions, vm: &VirtualMachine) -> PyResult { + PyInt::new(options.get_int_value(vm)?).into_ref_with_type(vm, cls) } // Casting function: pub fn to_int(vm: &VirtualMachine, obj: &PyObjectRef, base: u32) -> PyResult { - let val = if objtype::isinstance(obj, &vm.ctx.int_type()) { - get_value(obj).clone() - } else if objtype::isinstance(obj, &vm.ctx.float_type()) { - objfloat::get_value(obj).to_bigint().unwrap() - } else if objtype::isinstance(obj, &vm.ctx.str_type()) { - let s = objstr::get_value(obj); - match i32::from_str_radix(&s, base) { - Ok(v) => v.to_bigint().unwrap(), - Err(err) => { - trace!("Error occurred during int conversion {:?}", err); - return Err(vm.new_value_error(format!( + match_class!(obj.clone(), + s @ PyString => { + i32::from_str_radix(s.as_str(), base) + .map(BigInt::from) + .map_err(|_|vm.new_value_error(format!( "invalid literal for int() with base {}: '{}'", base, s - ))); + ))) + }, + obj => { + let method = vm.get_method_or_type_error(obj.clone(), "__int__", || { + format!("int() argument must be a string or a number, not '{}'", obj.class().name) + })?; + let result = vm.invoke(method, PyFuncArgs::default())?; + match result.payload::() { + Some(int_obj) => Ok(int_obj.as_bigint().clone()), + None => Err(vm.new_type_error(format!( + "TypeError: __int__ returned non-int (type '{}')", result.class().name))), } } - } else { - let type_name = objtype::get_type_name(&obj.typ()); - return Err(vm.new_type_error(format!( - "int() argument must be a string or a number, not '{}'", - type_name - ))); - }; - Ok(val) + ) } // Retrieve inner int value: @@ -425,6 +689,10 @@ pub fn get_value(obj: &PyObjectRef) -> &BigInt { &obj.payload::().unwrap().value } +pub fn get_float_value(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + obj.payload::().unwrap().float(vm) +} + #[inline] fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { if i2.is_zero() { @@ -457,67 +725,9 @@ fn div_ints(vm: &VirtualMachine, i1: &BigInt, i2: &BigInt) -> PyResult { } } -#[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { - let int_doc = "int(x=0) -> integer -int(x, base=10) -> integer - -Convert a number or string to an integer, or return 0 if no arguments -are given. If x is a number, return x.__int__(). For floating point -numbers, this truncates towards zero. - -If x is not a number or if base is given, then x must be a string, -bytes, or bytearray instance representing an integer literal in the -given base. The literal can be preceded by '+' or '-' and be surrounded -by whitespace. The base defaults to 10. Valid bases are 0 and 2-36. -Base 0 means to interpret the base from the string as an integer literal. ->>> int('0b100', base=0) -4"; - let int_type = &context.int_type; - - context.set_attr(&int_type, "__doc__", context.new_str(int_doc.to_string())); - context.set_attr(&int_type, "__eq__", context.new_rustfunc(PyIntRef::eq)); - context.set_attr(&int_type, "__ne__", context.new_rustfunc(PyIntRef::ne)); - context.set_attr(&int_type, "__lt__", context.new_rustfunc(PyIntRef::lt)); - context.set_attr(&int_type, "__le__", context.new_rustfunc(PyIntRef::le)); - context.set_attr(&int_type, "__gt__", context.new_rustfunc(PyIntRef::gt)); - context.set_attr(&int_type, "__ge__", context.new_rustfunc(PyIntRef::ge)); - context.set_attr(&int_type, "__abs__", context.new_rustfunc(PyIntRef::abs)); - context.set_attr(&int_type, "__add__", context.new_rustfunc(PyIntRef::add)); - context.set_attr(&int_type, "__radd__", context.new_rustfunc(PyIntRef::add)); - context.set_attr(&int_type, "__and__", context.new_rustfunc(PyIntRef::and)); - context.set_attr(&int_type, "__divmod__", context.new_rustfunc(PyIntRef::divmod)); - context.set_attr(&int_type, "__float__", context.new_rustfunc(PyIntRef::float)); - context.set_attr(&int_type, "__round__", context.new_rustfunc(PyIntRef::round)); - context.set_attr(&int_type, "__ceil__", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "__floor__", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "__index__", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "__trunc__", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "__int__", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "__floordiv__", context.new_rustfunc(PyIntRef::floordiv)); - context.set_attr(&int_type, "__hash__", context.new_rustfunc(PyIntRef::hash)); - context.set_attr(&int_type, "__lshift__", context.new_rustfunc(PyIntRef::lshift)); - context.set_attr(&int_type, "__rshift__", context.new_rustfunc(PyIntRef::rshift)); - context.set_attr(&int_type, "__new__", context.new_rustfunc(int_new)); - context.set_attr(&int_type, "__mod__", context.new_rustfunc(PyIntRef::mod_)); - context.set_attr(&int_type, "__mul__", context.new_rustfunc(PyIntRef::mul)); - context.set_attr(&int_type, "__rmul__", context.new_rustfunc(PyIntRef::mul)); - context.set_attr(&int_type, "__or__", context.new_rustfunc(PyIntRef::or)); - context.set_attr(&int_type, "__neg__", context.new_rustfunc(PyIntRef::neg)); - context.set_attr(&int_type, "__pos__", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "__pow__", context.new_rustfunc(PyIntRef::pow)); - context.set_attr(&int_type, "__repr__", context.new_rustfunc(PyIntRef::repr)); - context.set_attr(&int_type, "__sub__", context.new_rustfunc(PyIntRef::sub)); - context.set_attr(&int_type, "__rsub__", context.new_rustfunc(PyIntRef::rsub)); - context.set_attr(&int_type, "__format__", context.new_rustfunc(PyIntRef::format)); - context.set_attr(&int_type, "__truediv__", context.new_rustfunc(PyIntRef::truediv)); - context.set_attr(&int_type, "__rtruediv__", context.new_rustfunc(PyIntRef::rtruediv)); - context.set_attr(&int_type, "__xor__", context.new_rustfunc(PyIntRef::xor)); - context.set_attr(&int_type, "__rxor__", context.new_rustfunc(PyIntRef::rxor)); - context.set_attr(&int_type, "__bool__", context.new_rustfunc(PyIntRef::bool)); - context.set_attr(&int_type, "__invert__", context.new_rustfunc(PyIntRef::invert)); - context.set_attr(&int_type, "bit_length", context.new_rustfunc(PyIntRef::bit_length)); - context.set_attr(&int_type, "conjugate", context.new_rustfunc(PyIntRef::pass_value)); - context.set_attr(&int_type, "real", context.new_property(PyIntRef::pass_value)); - context.set_attr(&int_type, "imag", context.new_property(PyIntRef::imag)); + PyInt::extend_class(context, &context.int_type); + extend_class!(context, &context.int_type, { + "__new__" => context.new_rustfunc(int_new), + }); } diff --git a/vm/src/obj/objiter.rs b/vm/src/obj/objiter.rs index 1b7709d748..0fcd078c7c 100644 --- a/vm/src/obj/objiter.rs +++ b/vm/src/obj/objiter.rs @@ -2,16 +2,15 @@ * Various types to support iteration. */ -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyIteratorValue, PyObjectRef, PyResult, TypeProtocol}; +use std::cell::Cell; + +use crate::pyobject::{ + PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, +}; use crate::vm::VirtualMachine; -use super::objbool; -use super::objbytearray::PyByteArray; -use super::objbytes::PyBytes; -use super::objrange::PyRange; -use super::objsequence; use super::objtype; +use super::objtype::PyClassRef; /* * This helper function is called at multiple places. First, it is called @@ -19,10 +18,20 @@ use super::objtype; * function 'iter' is called. */ pub fn get_iter(vm: &VirtualMachine, iter_target: &PyObjectRef) -> PyResult { - vm.call_method(iter_target, "__iter__", vec![]) - // let type_str = objstr::get_value(&vm.to_str(iter_target.typ()).unwrap()); - // let type_error = vm.new_type_error(format!("Cannot iterate over {}", type_str)); - // return Err(type_error); + if let Some(method_or_err) = vm.get_method(iter_target.clone(), "__iter__") { + let method = method_or_err?; + vm.invoke(method, vec![]) + } else { + vm.get_method_or_type_error(iter_target.clone(), "__getitem__", || { + format!("Cannot iterate over {}", iter_target.class().name) + })?; + let obj_iterator = PySequenceIterator { + position: Cell::new(0), + obj: iter_target.clone(), + reversed: false, + }; + Ok(obj_iterator.into_ref(vm).into_object()) + } } pub fn call_next(vm: &VirtualMachine, iter_obj: &PyObjectRef) -> PyResult { @@ -69,116 +78,49 @@ pub fn new_stop_iteration(vm: &VirtualMachine) -> PyObjectRef { vm.new_exception(stop_iteration_type, "End of iterator".to_string()) } -fn contains(vm: &VirtualMachine, args: PyFuncArgs, iter_type: PyObjectRef) -> PyResult { - arg_check!( - vm, - args, - required = [(iter, Some(iter_type)), (needle, None)] - ); - loop { - if let Some(element) = get_next_object(vm, iter)? { - let equal = vm._eq(needle.clone(), element.clone())?; - if objbool::get_value(&equal) { - return Ok(vm.new_bool(true)); - } else { - continue; - } - } else { - return Ok(vm.new_bool(false)); - } - } -} - -/// Common setup for iter types, adds __iter__ and __contains__ methods -pub fn iter_type_init(context: &PyContext, iter_type: &PyObjectRef) { - let contains_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| contains(vm, args, cloned_iter_type.clone()) - }; - context.set_attr( - &iter_type, - "__contains__", - context.new_rustfunc(contains_func), - ); - let iter_func = { - let cloned_iter_type = iter_type.clone(); - move |vm: &VirtualMachine, args: PyFuncArgs| { - arg_check!( - vm, - args, - required = [(iter, Some(cloned_iter_type.clone()))] - ); - // Return self: - Ok(iter.clone()) - } - }; - context.set_attr(&iter_type, "__iter__", context.new_rustfunc(iter_func)); +#[pyclass] +#[derive(Debug)] +pub struct PySequenceIterator { + pub position: Cell, + pub obj: PyObjectRef, + pub reversed: bool, } -// Sequence iterator: -fn iter_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter_target, None)]); - - get_iter(vm, iter_target) +impl PyValue for PySequenceIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.iter_type() + } } -fn iter_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(iter, Some(vm.ctx.iter_type()))]); - - if let Some(PyIteratorValue { - ref position, - iterated_obj: ref iterated_obj_ref, - }) = iter.payload() - { - if let Some(range) = iterated_obj_ref.payload::() { - if let Some(int) = range.get(position.get()) { - position.set(position.get() + 1); - Ok(vm.ctx.new_int(int)) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.len() { - let obj_ref = vm.ctx.new_int(bytes[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } - } else if let Some(bytes) = iterated_obj_ref.payload::() { - if position.get() < bytes.value.borrow().len() { - let obj_ref = vm.ctx.new_int(bytes.value.borrow()[position.get()]); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) +#[pyimpl] +impl PySequenceIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.position.get() >= 0 { + let step: isize = if self.reversed { -1 } else { 1 }; + let number = vm.ctx.new_int(self.position.get()); + match vm.call_method(&self.obj, "__getitem__", vec![number]) { + Ok(val) => { + self.position.set(self.position.get() + step); + Ok(val) + } + Err(ref e) if objtype::isinstance(&e, &vm.ctx.exceptions.index_error) => { + Err(new_stop_iteration(vm)) + } + // also catches stop_iteration => stop_iteration + Err(e) => Err(e), } } else { - let elements = objsequence::get_elements(iterated_obj_ref); - if position.get() < elements.len() { - let obj_ref = elements[position.get()].clone(); - position.set(position.get() + 1); - Ok(obj_ref) - } else { - Err(new_stop_iteration(vm)) - } + Err(new_stop_iteration(vm)) } - } else { - panic!("NOT IMPL"); + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let iter_type = &context.iter_type; - - let iter_doc = "iter(iterable) -> iterator\n\ - iter(callable, sentinel) -> iterator\n\n\ - Get an iterator from an object. In the first form, the argument must\n\ - supply its own iterator, or be a sequence.\n\ - In the second form, the callable is called until it returns the sentinel."; - - iter_type_init(context, iter_type); - context.set_attr(&iter_type, "__new__", context.new_rustfunc(iter_new)); - context.set_attr(&iter_type, "__next__", context.new_rustfunc(iter_next)); - context.set_attr(&iter_type, "__doc__", context.new_str(iter_doc.to_string())); + PySequenceIterator::extend_class(context, &context.iter_type); } diff --git a/vm/src/obj/objlist.rs b/vm/src/obj/objlist.rs index 1b3a9b4fa9..77c1b4c9f8 100644 --- a/vm/src/obj/objlist.rs +++ b/vm/src/obj/objlist.rs @@ -1,22 +1,28 @@ use std::cell::{Cell, RefCell}; use std::fmt; -use num_traits::ToPrimitive; +use std::ops::Range; + +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, ToPrimitive, Zero}; use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObject, PyObjectRef, PyRef, PyResult, PyValue, - TypeProtocol, + IdProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, + TryFromObject, }; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; +//use super::objint; +use super::objiter; use super::objsequence::{ - get_elements, get_elements_cell, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, - PySliceableSequence, + get_elements_cell, get_elements_list, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, + seq_mul, SequenceIndex, }; +use super::objslice::PySliceRef; use super::objtype; +use crate::obj::objtype::PyClassRef; #[derive(Default)] pub struct PyList { @@ -40,11 +46,59 @@ impl From> for PyList { } impl PyValue for PyList { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.list_type() } } +impl PyList { + pub fn get_len(&self) -> usize { + self.elements.borrow().len() + } + + pub fn get_pos(&self, p: i32) -> Option { + // convert a (potentially negative) positon into a real index + if p < 0 { + if -p as usize > self.get_len() { + None + } else { + Some(self.get_len() - ((-p) as usize)) + } + } else if p as usize >= self.get_len() { + None + } else { + Some(p as usize) + } + } + + pub fn get_slice_pos(&self, slice_pos: &BigInt) -> usize { + if let Some(pos) = slice_pos.to_i32() { + if let Some(index) = self.get_pos(pos) { + // within bounds + return index; + } + } + + if slice_pos.is_negative() { + // slice past start bound, round to start + 0 + } else { + // slice past end bound, round to end + self.get_len() + } + } + + pub fn get_slice_range(&self, start: &Option, stop: &Option) -> Range { + let start = start.as_ref().map(|x| self.get_slice_pos(x)).unwrap_or(0); + let stop = stop + .as_ref() + .map(|x| self.get_slice_pos(x)) + .unwrap_or_else(|| self.get_len()); + + start..stop + } +} + pub type PyListRef = PyRef; impl PyListRef { @@ -75,7 +129,7 @@ impl PyListRef { fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let e1 = self.elements.borrow(); - let e2 = get_elements(&other); + let e2 = get_elements_list(&other); let elements = e1.iter().chain(e2.iter()).cloned().collect(); Ok(vm.ctx.new_list(elements)) } else { @@ -87,13 +141,17 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { self.elements .borrow_mut() - .extend_from_slice(&get_elements(&other)); + .extend_from_slice(&get_elements_list(&other)); Ok(self.into_object()) } else { Ok(vm.ctx.not_implemented()) } } + fn bool(self, _vm: &VirtualMachine) -> bool { + !self.elements.borrow().is_empty() + } + fn clear(self, _vm: &VirtualMachine) { self.elements.borrow_mut().clear(); } @@ -119,29 +177,199 @@ impl PyListRef { ) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyListIterator { + PyListIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + list: self, } } - fn setitem(self, key: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let mut elements = self.elements.borrow_mut(); + fn setitem( + self, + subscript: SequenceIndex, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.setindex(index, value, vm), + SequenceIndex::Slice(slice) => { + if let Ok(sec) = PyIterable::try_from_object(vm, value) { + return self.setslice(slice, sec, vm); + } + Err(vm.new_type_error("can only assign an iterable to a slice".to_string())) + } + } + } + + fn setindex(self, index: i32, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut()[pos_index] = value; + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("list assignment index out of range".to_string())) + } + } + + fn setslice(self, slice: PySliceRef, sec: PyIterable, vm: &VirtualMachine) -> PyResult { + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&slice.start_index(vm)?, &slice.stop_index(vm)?); + if range.start < range.end { + match step.to_i32() { + Some(1) => self._set_slice(range, sec, vm), + Some(num) => { + // assign to extended slice + self._set_stepped_slice(range, num as usize, sec, vm) + } + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list, no question + #[allow(clippy::range_plus_one)] + self._set_stepped_slice(range.start..(range.start + 1), 1, sec, vm) + } + } + } else { + // this functions as an insert of sec before range.start + self._set_slice(range.start..range.start, sec, vm) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = &slice.start_index(vm)?.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = &slice.stop_index(vm)?.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); + let range = self.get_slice_range(&stop, &start); + match (-step).to_i32() { + Some(num) => self._set_stepped_slice_reverse(range, num as usize, sec, vm), + None => { + // not sure how this is reached, step too big for i32? + // then step is bigger than the than len of the list no question + self._set_stepped_slice_reverse(range.end - 1..range.end, 1, sec, vm) + } + } + } + } + + fn _set_slice(self, range: Range, sec: PyIterable, vm: &VirtualMachine) -> PyResult { + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; + + // replace the range of elements with the full sequence + self.elements.borrow_mut().splice(range, items); + + Ok(vm.get_none()) + } + + fn _set_stepped_slice( + self, + range: Range, + step: usize, + sec: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; + + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.step_by(step); + self._replace_indexes(indexes, &items); + Ok(vm.get_none()) + } else { + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) + } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) + } else { + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } + + fn _set_stepped_slice_reverse( + self, + range: Range, + step: usize, + sec: PyIterable, + vm: &VirtualMachine, + ) -> PyResult { + let slicelen = if range.end > range.start { + ((range.end - range.start - 1) / step) + 1 + } else { + 0 + }; - if objtype::isinstance(&key, &vm.ctx.int_type()) { - let idx = objint::get_value(&key).to_i32().unwrap(); - if let Some(pos_index) = elements.get_pos(idx) { - elements[pos_index] = value; + // consume the iter, we need it's size + // and if it's going to fail we want that to happen *before* we start modifing + let items: Result, _> = sec.iter(vm)?.collect(); + let items = items?; + + let n = items.len(); + + if range.start < range.end { + if n == slicelen { + let indexes = range.rev().step_by(step); + self._replace_indexes(indexes, &items); Ok(vm.get_none()) } else { - Err(vm.new_index_error("list index out of range".to_string())) + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size {}", + n, slicelen + ))) } + } else if n == 0 { + // slice is empty but so is sequence + Ok(vm.get_none()) } else { - panic!( - "TypeError: indexing type {:?} with index {:?} is not supported (yet?)", - elements, key - ) + // empty slice but this is an error because stepped slice + Err(vm.new_value_error(format!( + "attempt to assign sequence of size {} to extended slice of size 0", + n + ))) + } + } + + fn _replace_indexes(self, indexes: I, items: &[PyObjectRef]) + where + I: Iterator, + { + let mut elements = self.elements.borrow_mut(); + + for (i, value) in indexes.zip(items) { + // clone for refrence count + elements[i] = value.clone(); } } @@ -159,11 +387,21 @@ impl PyListRef { Ok(s) } + fn hash(self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("unhashable type".to_string())) + } + fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { let new_elements = seq_mul(&self.elements.borrow(), counter); vm.ctx.new_list(new_elements) } + fn imul(self, counter: isize, _vm: &VirtualMachine) -> Self { + let new_elements = seq_mul(&self.elements.borrow(), counter); + self.elements.replace(new_elements); + self + } + fn count(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut count: usize = 0; for element in self.elements.borrow().iter() { @@ -252,7 +490,7 @@ impl PyListRef { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_equal(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -263,7 +501,7 @@ impl PyListRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_lt(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -274,7 +512,7 @@ impl PyListRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_gt(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -285,7 +523,7 @@ impl PyListRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_ge(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { @@ -296,31 +534,158 @@ impl PyListRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.list_type()) { let zelf = self.elements.borrow(); - let other = get_elements(&other); + let other = get_elements_list(&other); let res = seq_le(vm, &zelf, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) } } + + fn delitem(self, subscript: SequenceIndex, vm: &VirtualMachine) -> PyResult { + match subscript { + SequenceIndex::Int(index) => self.delindex(index, vm), + SequenceIndex::Slice(slice) => self.delslice(slice, vm), + } + } + + fn delindex(self, index: i32, vm: &VirtualMachine) -> PyResult { + if let Some(pos_index) = self.get_pos(index) { + self.elements.borrow_mut().remove(pos_index); + Ok(vm.get_none()) + } else { + Err(vm.new_index_error("Index out of bounds!".to_string())) + } + } + + fn delslice(self, slice: PySliceRef, vm: &VirtualMachine) -> PyResult { + let start = slice.start_index(vm)?; + let stop = slice.stop_index(vm)?; + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); + + if step.is_zero() { + Err(vm.new_value_error("slice step cannot be zero".to_string())) + } else if step.is_positive() { + let range = self.get_slice_range(&start, &stop); + if range.start < range.end { + #[allow(clippy::range_plus_one)] + match step.to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.start..range.start + 1); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } else { + // calculate the range for the reverse slice, first the bounds needs to be made + // exclusive around stop, the lower number + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.get_len().to_bigint().unwrap() + } else { + x + 1 + } + }); + let range = self.get_slice_range(&stop, &start); + if range.start < range.end { + match (-step).to_i32() { + Some(1) => { + self._del_slice(range); + Ok(vm.get_none()) + } + Some(num) => { + self._del_stepped_slice_reverse(range, num as usize); + Ok(vm.get_none()) + } + None => { + self._del_slice(range.end - 1..range.end); + Ok(vm.get_none()) + } + } + } else { + // no del to do + Ok(vm.get_none()) + } + } + } + + fn _del_slice(self, range: Range) { + self.elements.borrow_mut().drain(range); + } + + fn _del_stepped_slice(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; + let mut elements = self.elements.borrow_mut(); + let mut indexes = range.clone().step_by(step).peekable(); + + for i in range.clone() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards front + elements.swap(i - deleted, i); + } + } + // then drain (the values to delete should now be contiguous at the end of the range) + elements.drain((range.end - deleted)..range.end); + } + + fn _del_stepped_slice_reverse(self, range: Range, step: usize) { + // no easy way to delete stepped indexes so here is what we'll do + let mut deleted = 0; + let mut elements = self.elements.borrow_mut(); + let mut indexes = range.clone().rev().step_by(step).peekable(); + + for i in range.clone().rev() { + // is this an index to delete? + if indexes.peek() == Some(&i) { + // record and move on + indexes.next(); + deleted += 1; + } else { + // swap towards back + elements.swap(i + deleted, i); + } + } + // then drain (the values to delete should now be contiguous at teh start of the range) + elements.drain(range.start..(range.start + deleted)); + } } fn list_new( - cls: PyRef, + cls: PyClassRef, iterable: OptionalArg, vm: &VirtualMachine, -) -> PyResult { - if !objtype::issubclass(cls.as_object(), &vm.ctx.list_type()) { - return Err(vm.new_type_error(format!("{} is not a subtype of list", cls))); - } - +) -> PyResult { let elements = if let OptionalArg::Present(iterable) = iterable { vm.extract_elements(&iterable)? } else { vec![] }; - Ok(PyObject::new(PyList::from(elements), cls.into_object())) + PyList::from(elements).into_ref_with_type(vm, cls) } fn quicksort( @@ -412,6 +777,38 @@ fn list_sort(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +#[pyclass] +#[derive(Debug)] +pub struct PyListIterator { + pub position: Cell, + pub list: PyListRef, +} + +impl PyValue for PyListIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.listiterator_type() + } +} + +#[pyimpl] +impl PyListIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.list.elements.borrow().len() { + let ret = self.list.elements.borrow()[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let list_type = &context.list_type; @@ -420,31 +817,39 @@ pub fn init(context: &PyContext) { If no argument is given, the constructor creates a new empty list.\n\ The argument must be an iterable if specified."; - context.set_attr(&list_type, "__add__", context.new_rustfunc(PyListRef::add)); - context.set_attr(&list_type, "__iadd__", context.new_rustfunc(PyListRef::iadd)); - context.set_attr(&list_type, "__contains__", context.new_rustfunc(PyListRef::contains)); - context.set_attr(&list_type, "__eq__", context.new_rustfunc(PyListRef::eq)); - context.set_attr(&list_type, "__lt__", context.new_rustfunc(PyListRef::lt)); - context.set_attr(&list_type, "__gt__", context.new_rustfunc(PyListRef::gt)); - context.set_attr(&list_type, "__le__", context.new_rustfunc(PyListRef::le)); - context.set_attr(&list_type, "__ge__", context.new_rustfunc(PyListRef::ge)); - context.set_attr(&list_type, "__getitem__", context.new_rustfunc(PyListRef::getitem)); - context.set_attr(&list_type, "__iter__", context.new_rustfunc(PyListRef::iter)); - context.set_attr(&list_type, "__setitem__", context.new_rustfunc(PyListRef::setitem)); - context.set_attr(&list_type, "__mul__", context.new_rustfunc(PyListRef::mul)); - context.set_attr(&list_type, "__len__", context.new_rustfunc(PyListRef::len)); - context.set_attr(&list_type, "__new__", context.new_rustfunc(list_new)); - context.set_attr(&list_type, "__repr__", context.new_rustfunc(PyListRef::repr)); - context.set_attr(&list_type, "__doc__", context.new_str(list_doc.to_string())); - context.set_attr(&list_type, "append", context.new_rustfunc(PyListRef::append)); - context.set_attr(&list_type, "clear", context.new_rustfunc(PyListRef::clear)); - context.set_attr(&list_type, "copy", context.new_rustfunc(PyListRef::copy)); - context.set_attr(&list_type, "count", context.new_rustfunc(PyListRef::count)); - context.set_attr(&list_type, "extend", context.new_rustfunc(PyListRef::extend)); - context.set_attr(&list_type, "index", context.new_rustfunc(PyListRef::index)); - context.set_attr(&list_type, "insert", context.new_rustfunc(PyListRef::insert)); - context.set_attr(&list_type, "reverse", context.new_rustfunc(PyListRef::reverse)); - context.set_attr(&list_type, "sort", context.new_rustfunc(list_sort)); - context.set_attr(&list_type, "pop", context.new_rustfunc(PyListRef::pop)); - context.set_attr(&list_type, "remove", context.new_rustfunc(PyListRef::remove)); + extend_class!(context, list_type, { + "__add__" => context.new_rustfunc(PyListRef::add), + "__iadd__" => context.new_rustfunc(PyListRef::iadd), + "__bool__" => context.new_rustfunc(PyListRef::bool), + "__contains__" => context.new_rustfunc(PyListRef::contains), + "__delitem__" => context.new_rustfunc(PyListRef::delitem), + "__eq__" => context.new_rustfunc(PyListRef::eq), + "__lt__" => context.new_rustfunc(PyListRef::lt), + "__gt__" => context.new_rustfunc(PyListRef::gt), + "__le__" => context.new_rustfunc(PyListRef::le), + "__ge__" => context.new_rustfunc(PyListRef::ge), + "__getitem__" => context.new_rustfunc(PyListRef::getitem), + "__iter__" => context.new_rustfunc(PyListRef::iter), + "__setitem__" => context.new_rustfunc(PyListRef::setitem), + "__mul__" => context.new_rustfunc(PyListRef::mul), + "__imul__" => context.new_rustfunc(PyListRef::imul), + "__len__" => context.new_rustfunc(PyListRef::len), + "__new__" => context.new_rustfunc(list_new), + "__repr__" => context.new_rustfunc(PyListRef::repr), + "__hash__" => context.new_rustfunc(PyListRef::hash), + "__doc__" => context.new_str(list_doc.to_string()), + "append" => context.new_rustfunc(PyListRef::append), + "clear" => context.new_rustfunc(PyListRef::clear), + "copy" => context.new_rustfunc(PyListRef::copy), + "count" => context.new_rustfunc(PyListRef::count), + "extend" => context.new_rustfunc(PyListRef::extend), + "index" => context.new_rustfunc(PyListRef::index), + "insert" => context.new_rustfunc(PyListRef::insert), + "reverse" => context.new_rustfunc(PyListRef::reverse), + "sort" => context.new_rustfunc(list_sort), + "pop" => context.new_rustfunc(PyListRef::pop), + "remove" => context.new_rustfunc(PyListRef::remove) + }); + + PyListIterator::extend_class(context, &context.listiterator_type); } diff --git a/vm/src/obj/objmap.rs b/vm/src/obj/objmap.rs index 74aabe1072..61fe67c0d1 100644 --- a/vm/src/obj/objmap.rs +++ b/vm/src/obj/objmap.rs @@ -1,10 +1,15 @@ -use crate::function::{Args, PyFuncArgs}; -use crate::pyobject::{PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; +use crate::function::Args; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objiter; use super::objtype::PyClassRef; +/// map(func, *iterables) --> map object +/// +/// Make an iterator that computes the function using arguments from +/// each of the iterables. Stops when the shortest iterable is exhausted. +#[pyclass] #[derive(Debug)] pub struct PyMap { mapper: PyObjectRef, @@ -13,7 +18,7 @@ pub struct PyMap { type PyMapRef = PyRef; impl PyValue for PyMap { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.map_type() } } @@ -35,35 +40,29 @@ fn map_new( .into_ref_with_type(vm, cls.clone()) } -fn map_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(map, Some(vm.ctx.map_type()))]); - - if let Some(PyMap { - ref mapper, - ref iterators, - }) = map.payload() - { - let next_objs = iterators +#[pyimpl] +impl PyMap { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let next_objs = self + .iterators .iter() .map(|iterator| objiter::call_next(vm, iterator)) .collect::, _>>()?; // the mapper itself can raise StopIteration which does stop the map iteration - vm.invoke(mapper.clone(), next_objs) - } else { - panic!("map doesn't have correct payload"); + vm.invoke(self.mapper.clone(), next_objs) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let map_type = &context.map_type; - - let map_doc = "map(func, *iterables) --> map object\n\n\ - Make an iterator that computes the function using arguments from\n\ - each of the iterables. Stops when the shortest iterable is exhausted."; - - objiter::iter_type_init(context, map_type); - context.set_attr(&map_type, "__new__", context.new_rustfunc(map_new)); - context.set_attr(&map_type, "__next__", context.new_rustfunc(map_next)); - context.set_attr(&map_type, "__doc__", context.new_str(map_doc.to_string())); + PyMap::extend_class(context, &context.map_type); + extend_class!(context, &context.map_type, { + "__new__" => context.new_rustfunc(map_new), + }); } diff --git a/vm/src/obj/objmappingproxy.rs b/vm/src/obj/objmappingproxy.rs new file mode 100644 index 0000000000..243a8c89cc --- /dev/null +++ b/vm/src/obj/objmappingproxy.rs @@ -0,0 +1,42 @@ +use super::objstr::PyStringRef; +use super::objtype::{self, PyClassRef}; +use crate::pyobject::{PyClassImpl, PyContext, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[pyclass] +#[derive(Debug)] +pub struct PyMappingProxy { + class: PyClassRef, +} + +pub type PyMappingProxyRef = PyRef; + +impl PyValue for PyMappingProxy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.mappingproxy_type.clone() + } +} + +#[pyimpl] +impl PyMappingProxy { + pub fn new(class: PyClassRef) -> PyMappingProxy { + PyMappingProxy { class } + } + + #[pymethod(name = "__getitem__")] + pub fn getitem(&self, key: PyStringRef, vm: &VirtualMachine) -> PyResult { + if let Some(value) = objtype::class_get_attr(&self.class, key.as_str()) { + return Ok(value); + } + Err(vm.new_key_error(key.into_object())) + } + + #[pymethod(name = "__contains__")] + pub fn contains(&self, attr: PyStringRef, _vm: &VirtualMachine) -> bool { + objtype::class_has_attr(&self.class, attr.as_str()) + } +} + +pub fn init(context: &PyContext) { + PyMappingProxy::extend_class(context, &context.mappingproxy_type) +} diff --git a/vm/src/obj/objmemory.rs b/vm/src/obj/objmemory.rs index aa67985514..7e1628d13d 100644 --- a/vm/src/obj/objmemory.rs +++ b/vm/src/obj/objmemory.rs @@ -1,34 +1,51 @@ -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::obj::objbyteinner::try_as_byte; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; +#[pyclass(name = "memoryview")] #[derive(Debug)] pub struct PyMemoryView { - obj: PyObjectRef, + obj_ref: PyObjectRef, } -impl PyValue for PyMemoryView { - fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.memoryview_type() +pub type PyMemoryViewRef = PyRef; + +#[pyimpl] +impl PyMemoryView { + pub fn get_obj_value(&self) -> Option> { + try_as_byte(&self.obj_ref) } -} -pub fn new_memory_view(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(cls, None), (bytes_object, None)]); - vm.ctx.set_attr(&cls, "obj", bytes_object.clone()); - Ok(PyObject::new( + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + bytes_object: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyMemoryView { - obj: bytes_object.clone(), - }, - cls.clone(), - )) + obj_ref: bytes_object.clone(), + } + .into_ref_with_type(vm, cls) + } + + #[pyproperty] + fn obj(&self, __vm: &VirtualMachine) -> PyObjectRef { + self.obj_ref.clone() + } + + #[pymethod(name = "__getitem__")] + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.call_method(&self.obj_ref, "__getitem__", vec![needle]) + } +} + +impl PyValue for PyMemoryView { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.memoryview_type() + } } pub fn init(ctx: &PyContext) { - let memoryview_type = &ctx.memoryview_type; - ctx.set_attr( - &memoryview_type, - "__new__", - ctx.new_rustfunc(new_memory_view), - ); + PyMemoryView::extend_class(ctx, &ctx.memoryview_type) } diff --git a/vm/src/obj/objmodule.rs b/vm/src/obj/objmodule.rs index fe6e3fca1d..99dff3b267 100644 --- a/vm/src/obj/objmodule.rs +++ b/vm/src/obj/objmodule.rs @@ -1,39 +1,31 @@ use crate::obj::objstr::PyStringRef; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyContext, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyModule { pub name: String, - pub dict: PyObjectRef, } pub type PyModuleRef = PyRef; impl PyValue for PyModule { - fn class(vm: &VirtualMachine) -> PyObjectRef { + const HAVE_DICT: bool = true; + + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.module_type() } } impl PyModuleRef { - fn dir(self: PyModuleRef, vm: &VirtualMachine) -> PyResult { - let keys = self - .dict - .get_key_value_pairs() - .iter() - .map(|(k, _v)| k.clone()) - .collect(); - Ok(vm.ctx.new_list(keys)) - } - - fn set_attr(self, attr: PyStringRef, value: PyObjectRef, vm: &VirtualMachine) { - self.dict.set_item(&vm.ctx, &attr.value, value) + fn init(self, name: PyStringRef, vm: &VirtualMachine) -> PyResult { + vm.set_attr(&self.into_object(), "__name__", name)?; + Ok(vm.get_none()) } } pub fn init(context: &PyContext) { extend_class!(&context, &context.module_type, { - "__dir__" => context.new_rustfunc(PyModuleRef::dir), - "__setattr__" => context.new_rustfunc(PyModuleRef::set_attr) + "__init__" => context.new_rustfunc(PyModuleRef::init), }); } diff --git a/vm/src/obj/objnamespace.rs b/vm/src/obj/objnamespace.rs new file mode 100644 index 0000000000..f8bfc93696 --- /dev/null +++ b/vm/src/obj/objnamespace.rs @@ -0,0 +1,32 @@ +use crate::function::KwArgs; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyContext, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +/// A simple attribute-based namespace. +/// +/// SimpleNamespace(**kwargs) +#[pyclass(name = "SimpleNamespace")] +#[derive(Debug)] +pub struct PyNamespace; + +impl PyValue for PyNamespace { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.namespace_type() + } +} + +#[pyimpl] +impl PyNamespace { + #[pymethod(name = "__init__")] + fn init(zelf: PyRef, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult<()> { + for (name, value) in kwargs.into_iter() { + vm.set_attr(zelf.as_object(), name, value)?; + } + Ok(()) + } +} + +pub fn init(context: &PyContext) { + PyNamespace::extend_class(context, &context.namespace_type); +} diff --git a/vm/src/obj/objnone.rs b/vm/src/obj/objnone.rs index 6bee900312..aff20f465f 100644 --- a/vm/src/obj/objnone.rs +++ b/vm/src/obj/objnone.rs @@ -1,19 +1,18 @@ -use crate::function::PyFuncArgs; use crate::obj::objproperty::PyPropertyRef; use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::{class_get_attr, class_has_attr, PyClassRef}; use crate::pyobject::{ - AttributeProtocol, IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TypeProtocol, + IntoPyObject, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyNone; pub type PyNoneRef = PyRef; impl PyValue for PyNone { - fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.none().typ() + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.none().class() } } @@ -45,7 +44,7 @@ impl PyNoneRef { fn get_attribute(self, name: PyStringRef, vm: &VirtualMachine) -> PyResult { trace!("None.__getattribute__({:?}, {:?})", self, name); - let cls = self.typ().into_object(); + let cls = self.class(); // Properties use a comparision with None to determine if they are either invoked by am // instance binding or a class binding. But if the object itself is None then this detection @@ -69,25 +68,33 @@ impl PyNoneRef { } } - if let Some(attr) = cls.get_attr(&name.value) { - let attr_class = attr.typ(); - if attr_class.has_attr("__set__") { - if let Some(get_func) = attr_class.get_attr("__get__") { - return call_descriptor(attr, get_func, self.into_object(), cls.clone(), vm); + if let Some(attr) = class_get_attr(&cls, &name.value) { + let attr_class = attr.class(); + if class_has_attr(&attr_class, "__set__") { + if let Some(get_func) = class_get_attr(&attr_class, "__get__") { + return call_descriptor( + attr, + get_func, + self.into_object(), + cls.into_object(), + vm, + ); } } } - if let Some(obj_attr) = self.as_object().get_attr(&name.value) { - Ok(obj_attr) - } else if let Some(attr) = cls.get_attr(&name.value) { - let attr_class = attr.typ(); - if let Some(get_func) = attr_class.get_attr("__get__") { - call_descriptor(attr, get_func, self.into_object(), cls.clone(), vm) + // None has no attributes and cannot have attributes set on it. + // if let Some(obj_attr) = self.as_object().get_attr(&name.value) { + // Ok(obj_attr) + // } else + if let Some(attr) = class_get_attr(&cls, &name.value) { + let attr_class = attr.class(); + if let Some(get_func) = class_get_attr(&attr_class, "__get__") { + call_descriptor(attr, get_func, self.into_object(), cls.into_object(), vm) } else { Ok(attr) } - } else if let Some(getter) = cls.get_attr("__getattr__") { + } else if let Some(getter) = class_get_attr(&cls, "__getattr__") { vm.invoke(getter, vec![self.into_object(), name.into_object()]) } else { Err(vm.new_attribute_error(format!("{} has no attribute '{}'", self.as_object(), name))) @@ -95,17 +102,12 @@ impl PyNoneRef { } } -fn none_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.type_type.clone()))] - ); - Ok(vm.get_none()) +fn none_new(_: PyClassRef, vm: &VirtualMachine) -> PyNoneRef { + vm.ctx.none.clone() } pub fn init(context: &PyContext) { - extend_class!(context, &context.none.typ(), { + extend_class!(context, &context.none.class(), { "__new__" => context.new_rustfunc(none_new), "__repr__" => context.new_rustfunc(PyNoneRef::repr), "__bool__" => context.new_rustfunc(PyNoneRef::bool), diff --git a/vm/src/obj/objobject.rs b/vm/src/obj/objobject.rs index 86c5469022..77830a402c 100644 --- a/vm/src/obj/objobject.rs +++ b/vm/src/obj/objobject.rs @@ -1,174 +1,139 @@ +use super::objdict::PyDictRef; use super::objlist::PyList; -use super::objstr::{self, PyStringRef}; +use super::objstr::PyStringRef; use super::objtype; use crate::function::PyFuncArgs; use crate::obj::objproperty::PropertyBuilder; +use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ - AttributeProtocol, DictProtocol, IdProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, - PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, ItemProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, PyResult, PyValue, + TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyInstance; impl PyValue for PyInstance { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.object() } } -pub type PyInstanceRef = PyRef; - pub fn new_instance(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { // more or less __new__ operator - let cls = args.shift(); - Ok(if cls.is(&vm.ctx.object) { - PyObject::new_without_dict(PyInstance, cls) + let cls = PyClassRef::try_from_object(vm, args.shift())?; + let dict = if cls.is(&vm.ctx.object) { + None } else { - PyObject::new(PyInstance, cls) - }) + Some(vm.ctx.new_dict()) + }; + Ok(PyObject::new(PyInstance, cls, dict)) } -fn object_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - Ok(vm.ctx.not_implemented()) +fn object_eq(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_ne(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_ne(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_lt(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_le(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_gt(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, Some(vm.ctx.object())), (_other, None)] - ); - - Ok(vm.ctx.not_implemented()) +fn object_ge(_zelf: PyObjectRef, _other: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.not_implemented() } -fn object_hash(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(_zelf, Some(vm.ctx.object()))]); - - // For now default to non hashable - Err(vm.new_type_error("unhashable type".to_string())) +fn object_hash(zelf: PyObjectRef, _vm: &VirtualMachine) -> pyhash::PyHash { + zelf.get_id() as pyhash::PyHash } fn object_setattr( - obj: PyInstanceRef, + obj: PyObjectRef, attr_name: PyStringRef, value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult<()> { trace!("object.__setattr__({:?}, {}, {:?})", obj, attr_name, value); - let cls = obj.as_object().typ(); + let cls = obj.class(); - if let Some(attr) = cls.get_attr(&attr_name.value) { - let attr_class = attr.typ(); - if let Some(descriptor) = attr_class.get_attr("__set__") { + if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) { + if let Some(descriptor) = objtype::class_get_attr(&attr.class(), "__set__") { return vm - .invoke(descriptor, vec![attr, obj.into_object(), value]) + .invoke(descriptor, vec![attr, obj.clone(), value]) .map(|_| ()); } } - if let Some(ref dict) = obj.as_object().dict { - dict.borrow_mut().insert(attr_name.value.clone(), value); + if let Some(ref dict) = obj.clone().dict { + dict.set_item(attr_name, value, vm)?; Ok(()) } else { - let type_name = objtype::get_type_name(&obj.as_object().typ()); Err(vm.new_attribute_error(format!( "'{}' object has no attribute '{}'", - type_name, &attr_name.value + obj.class().name, + &attr_name.value ))) } } -fn object_delattr(obj: PyInstanceRef, attr_name: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { - let cls = obj.as_object().typ(); +fn object_delattr(obj: PyObjectRef, attr_name: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + let cls = obj.class(); - if let Some(attr) = cls.get_attr(&attr_name.value) { - let attr_class = attr.typ(); - if let Some(descriptor) = attr_class.get_attr("__delete__") { - return vm - .invoke(descriptor, vec![attr, obj.into_object()]) - .map(|_| ()); + if let Some(attr) = objtype::class_get_attr(&cls, &attr_name.value) { + if let Some(descriptor) = objtype::class_get_attr(&attr.class(), "__delete__") { + return vm.invoke(descriptor, vec![attr, obj.clone()]).map(|_| ()); } } - if let Some(ref dict) = obj.as_object().dict { - dict.borrow_mut().remove(&attr_name.value); + if let Some(ref dict) = obj.dict { + dict.del_item(attr_name, vm)?; Ok(()) } else { - let type_name = objtype::get_type_name(&obj.as_object().typ()); Err(vm.new_attribute_error(format!( "'{}' object has no attribute '{}'", - type_name, &attr_name.value + obj.class().name, + &attr_name.value ))) } } -fn object_str(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.object()))]); - vm.call_method(zelf, "__repr__", vec![]) +fn object_str(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.call_method(&zelf, "__repr__", vec![]) } -fn object_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, Some(vm.ctx.object()))]); - let type_name = objtype::get_type_name(&obj.typ()); - let address = obj.get_id(); - Ok(vm.new_str(format!("<{} object at 0x{:x}>", type_name, address))) +fn object_repr(zelf: PyObjectRef, _vm: &VirtualMachine) -> String { + format!("<{} object at 0x{:x}>", zelf.class().name, zelf.get_id()) } -pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyList { - let attributes = get_attributes(&obj); - let attributes: Vec = attributes - .keys() - .map(|k| vm.ctx.new_str(k.to_string())) - .collect(); - PyList::from(attributes) +pub fn object_dir(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let attributes: PyAttributes = objtype::get_attributes(obj.class()); + + let dict = PyDictRef::from_attributes(attributes, vm)?; + + // Get instance attributes: + if let Some(object_dict) = &obj.dict { + vm.invoke( + vm.get_attribute(dict.clone().into_object(), "update")?, + object_dict.clone().into_object(), + )?; + } + + let attributes: Vec<_> = dict.into_iter().map(|(k, _v)| k.clone()).collect(); + + Ok(PyList::from(attributes)) } fn object_format( @@ -187,36 +152,35 @@ pub fn init(context: &PyContext) { let object = &context.object; let object_doc = "The most base type"; - context.set_attr(&object, "__new__", context.new_rustfunc(new_instance)); - context.set_attr(&object, "__init__", context.new_rustfunc(object_init)); - context.set_attr( - &object, - "__class__", + extend_class!(context, object, { + "__new__" => context.new_rustfunc(new_instance), + "__init__" => context.new_rustfunc(object_init), + "__class__" => PropertyBuilder::new(context) .add_getter(object_class) .add_setter(object_class_setter) .create(), - ); - context.set_attr(&object, "__eq__", context.new_rustfunc(object_eq)); - context.set_attr(&object, "__ne__", context.new_rustfunc(object_ne)); - context.set_attr(&object, "__lt__", context.new_rustfunc(object_lt)); - context.set_attr(&object, "__le__", context.new_rustfunc(object_le)); - context.set_attr(&object, "__gt__", context.new_rustfunc(object_gt)); - context.set_attr(&object, "__ge__", context.new_rustfunc(object_ge)); - context.set_attr(&object, "__setattr__", context.new_rustfunc(object_setattr)); - context.set_attr(&object, "__delattr__", context.new_rustfunc(object_delattr)); - context.set_attr(&object, "__dict__", context.new_property(object_dict)); - context.set_attr(&object, "__dir__", context.new_rustfunc(object_dir)); - context.set_attr(&object, "__hash__", context.new_rustfunc(object_hash)); - context.set_attr(&object, "__str__", context.new_rustfunc(object_str)); - context.set_attr(&object, "__repr__", context.new_rustfunc(object_repr)); - context.set_attr(&object, "__format__", context.new_rustfunc(object_format)); - context.set_attr( - &object, - "__getattribute__", - context.new_rustfunc(object_getattribute), - ); - context.set_attr(&object, "__doc__", context.new_str(object_doc.to_string())); + "__eq__" => context.new_rustfunc(object_eq), + "__ne__" => context.new_rustfunc(object_ne), + "__lt__" => context.new_rustfunc(object_lt), + "__le__" => context.new_rustfunc(object_le), + "__gt__" => context.new_rustfunc(object_gt), + "__ge__" => context.new_rustfunc(object_ge), + "__setattr__" => context.new_rustfunc(object_setattr), + "__delattr__" => context.new_rustfunc(object_delattr), + "__dict__" => + PropertyBuilder::new(context) + .add_getter(object_dict) + .add_setter(object_dict_setter) + .create(), + "__dir__" => context.new_rustfunc(object_dir), + "__hash__" => context.new_rustfunc(object_hash), + "__str__" => context.new_rustfunc(object_str), + "__repr__" => context.new_rustfunc(object_repr), + "__format__" => context.new_rustfunc(object_format), + "__getattribute__" => context.new_rustfunc(object_getattribute), + "__doc__" => context.new_str(object_doc.to_string()) + }); } fn object_init(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { @@ -224,7 +188,7 @@ fn object_init(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { } fn object_class(obj: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { - obj.typ() + obj.class().into_object() } fn object_class_setter( @@ -232,65 +196,61 @@ fn object_class_setter( _value: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { - let type_repr = vm.to_pystr(&instance.typ())?; + let type_repr = vm.to_pystr(&instance.class())?; Err(vm.new_type_error(format!("can't change class of type '{}'", type_repr))) } -fn object_dict(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if let Some(ref dict) = args.args[0].dict { - let new_dict = vm.new_dict(); - for (attr, value) in dict.borrow().iter() { - new_dict.set_item(&vm.ctx, &attr, value.clone()); - } - Ok(new_dict) +fn object_dict(object: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Some(ref dict) = object.dict { + Ok(dict.clone()) } else { Err(vm.new_type_error("TypeError: no dictionary.".to_string())) } } -fn object_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (obj, Some(vm.ctx.object())), - (name_str, Some(vm.ctx.str_type())) - ] - ); - let name = objstr::get_value(&name_str); +fn object_dict_setter( + _instance: PyObjectRef, + _value: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on an object isn't yet implemented".to_string(), + )) +} + +fn object_getattribute(obj: PyObjectRef, name_str: PyStringRef, vm: &VirtualMachine) -> PyResult { + let name = &name_str.value; trace!("object.__getattribute__({:?}, {:?})", obj, name); - let cls = obj.typ(); + let cls = obj.class(); - if let Some(attr) = cls.get_attr(&name) { - let attr_class = attr.typ(); - if attr_class.has_attr("__set__") { - if let Some(descriptor) = attr_class.get_attr("__get__") { - return vm.invoke(descriptor, vec![attr, obj.clone(), cls]); + if let Some(attr) = objtype::class_get_attr(&cls, &name) { + let attr_class = attr.class(); + if objtype::class_has_attr(&attr_class, "__set__") { + if let Some(descriptor) = objtype::class_get_attr(&attr_class, "__get__") { + return vm.invoke(descriptor, vec![attr, obj, cls.into_object()]); } } } - if let Some(obj_attr) = obj.get_attr(&name) { + if let Some(obj_attr) = object_getattr(&obj, &name, &vm)? { Ok(obj_attr) - } else if let Some(attr) = cls.get_attr(&name) { - vm.call_get_descriptor(attr, obj.clone()) - } else if let Some(getter) = cls.get_attr("__getattr__") { - vm.invoke(getter, vec![obj.clone(), name_str.clone()]) + } else if let Some(attr) = objtype::class_get_attr(&cls, &name) { + vm.call_get_descriptor(attr, obj) + } else if let Some(getter) = objtype::class_get_attr(&cls, "__getattr__") { + vm.invoke(getter, vec![obj, name_str.into_object()]) } else { Err(vm.new_attribute_error(format!("{} has no attribute '{}'", obj, name))) } } -pub fn get_attributes(obj: &PyObjectRef) -> PyAttributes { - // Get class attributes: - let mut attributes = objtype::get_attributes(obj.type_pyref()); - - // Get instance attributes: - if let Some(dict) = &obj.dict { - for (name, value) in dict.borrow().iter() { - attributes.insert(name.to_string(), value.clone()); - } +fn object_getattr( + obj: &PyObjectRef, + attr_name: &str, + vm: &VirtualMachine, +) -> PyResult> { + if let Some(ref dict) = obj.dict { + dict.get_item_option(attr_name, vm) + } else { + Ok(None) } - - attributes } diff --git a/vm/src/obj/objproperty.rs b/vm/src/obj/objproperty.rs index d1791f2485..9c8394a4cc 100644 --- a/vm/src/obj/objproperty.rs +++ b/vm/src/obj/objproperty.rs @@ -2,71 +2,125 @@ */ -use crate::function::IntoPyNativeFunc; -use crate::function::OptionalArg; -use crate::obj::objstr::PyStringRef; +use crate::function::{IntoPyNativeFunc, OptionalArg, PyFuncArgs}; use crate::obj::objtype::PyClassRef; -use crate::pyobject::{IdProtocol, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::pyobject::{ + IdProtocol, PyClassImpl, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, + TypeProtocol, +}; use crate::vm::VirtualMachine; -/// Read-only property, doesn't have __set__ or __delete__ +// Read-only property, doesn't have __set__ or __delete__ +#[pyclass] #[derive(Debug)] pub struct PyReadOnlyProperty { getter: PyObjectRef, } impl PyValue for PyReadOnlyProperty { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.readonly_property_type() } } pub type PyReadOnlyPropertyRef = PyRef; -impl PyReadOnlyPropertyRef { +#[pyimpl] +impl PyReadOnlyProperty { + #[pymethod(name = "__get__")] fn get( - self, + zelf: PyRef, obj: PyObjectRef, _owner: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - if obj.is(&vm.ctx.none) { - Ok(self.into_object()) + if obj.is(vm.ctx.none.as_object()) { + Ok(zelf.into_object()) } else { - vm.invoke(self.getter.clone(), obj) + vm.invoke(zelf.getter.clone(), obj) } } } -/// Fully fledged property +/// Property attribute. +/// +/// fget +/// function to be used for getting an attribute value +/// fset +/// function to be used for setting an attribute value +/// fdel +/// function to be used for del'ing an attribute +/// doc +/// docstring +/// +/// Typical use is to define a managed attribute x: +/// +/// class C(object): +/// def getx(self): return self._x +/// def setx(self, value): self._x = value +/// def delx(self): del self._x +/// x = property(getx, setx, delx, "I'm the 'x' property.") +/// +/// Decorators make defining new properties or modifying existing ones easy: +/// +/// class C(object): +/// @property +/// def x(self): +/// "I am the 'x' property." +/// return self._x +/// @x.setter +/// def x(self, value): +/// self._x = value +/// @x.deleter +/// def x(self): +/// del self._x +#[pyclass] #[derive(Debug)] pub struct PyProperty { getter: Option, setter: Option, deleter: Option, + doc: Option, } impl PyValue for PyProperty { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.property_type() } } pub type PyPropertyRef = PyRef; -impl PyPropertyRef { +#[pyimpl] +impl PyProperty { + #[pymethod(name = "__new__")] fn new_property( cls: PyClassRef, - fget: OptionalArg, - fset: OptionalArg, - fdel: OptionalArg, - _doc: OptionalArg, + args: PyFuncArgs, vm: &VirtualMachine, ) -> PyResult { + arg_check!( + vm, + args, + required = [], + optional = [(fget, None), (fset, None), (fdel, None), (doc, None)] + ); + + fn into_option(vm: &VirtualMachine, arg: Option<&PyObjectRef>) -> Option { + arg.and_then(|arg| { + if vm.ctx.none().is(arg) { + None + } else { + Some(arg.clone()) + } + }) + } + PyProperty { - getter: fget.into_option(), - setter: fset.into_option(), - deleter: fdel.into_option(), + getter: into_option(vm, fget), + setter: into_option(vm, fset), + deleter: into_option(vm, fdel), + doc: into_option(vm, doc), } .into_ref_with_type(vm, cls) } @@ -74,7 +128,7 @@ impl PyPropertyRef { // Descriptor methods // specialised version that doesn't check for None - pub(crate) fn instance_binding_get(self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + pub(crate) fn instance_binding_get(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(getter) = self.getter.as_ref() { vm.invoke(getter.clone(), obj) } else { @@ -82,15 +136,16 @@ impl PyPropertyRef { } } + #[pymethod(name = "__get__")] fn get( - self, + zelf: PyRef, obj: PyObjectRef, _owner: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - if let Some(getter) = self.getter.as_ref() { - if obj.is(&vm.ctx.none) { - Ok(self.into_object()) + if let Some(getter) = zelf.getter.as_ref() { + if obj.is(vm.ctx.none.as_object()) { + Ok(zelf.into_object()) } else { vm.invoke(getter.clone(), obj) } @@ -99,7 +154,8 @@ impl PyPropertyRef { } } - fn set(self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__set__")] + fn set(&self, obj: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(setter) = self.setter.as_ref() { vm.invoke(setter.clone(), vec![obj, value]) } else { @@ -107,7 +163,8 @@ impl PyPropertyRef { } } - fn delete(self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__delete__")] + fn delete(&self, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Some(deleter) = self.deleter.as_ref() { vm.invoke(deleter.clone(), obj) } else { @@ -117,45 +174,66 @@ impl PyPropertyRef { // Access functions - fn fget(self, _vm: &VirtualMachine) -> Option { + #[pyproperty] + fn fget(&self, _vm: &VirtualMachine) -> Option { self.getter.clone() } - fn fset(self, _vm: &VirtualMachine) -> Option { + #[pyproperty] + fn fset(&self, _vm: &VirtualMachine) -> Option { self.setter.clone() } - fn fdel(self, _vm: &VirtualMachine) -> Option { + #[pyproperty] + fn fdel(&self, _vm: &VirtualMachine) -> Option { self.deleter.clone() } // Python builder functions - fn getter(self, getter: Option, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn getter( + zelf: PyRef, + getter: Option, + vm: &VirtualMachine, + ) -> PyResult { PyProperty { - getter: getter.or_else(|| self.getter.clone()), - setter: self.setter.clone(), - deleter: self.deleter.clone(), + getter: getter.or_else(|| zelf.getter.clone()), + setter: zelf.setter.clone(), + deleter: zelf.deleter.clone(), + doc: None, } - .into_ref_with_type(vm, self.typ()) + .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } - fn setter(self, setter: Option, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn setter( + zelf: PyRef, + setter: Option, + vm: &VirtualMachine, + ) -> PyResult { PyProperty { - getter: self.getter.clone(), - setter: setter.or_else(|| self.setter.clone()), - deleter: self.deleter.clone(), + getter: zelf.getter.clone(), + setter: setter.or_else(|| zelf.setter.clone()), + deleter: zelf.deleter.clone(), + doc: None, } - .into_ref_with_type(vm, self.typ()) + .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } - fn deleter(self, deleter: Option, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn deleter( + zelf: PyRef, + deleter: Option, + vm: &VirtualMachine, + ) -> PyResult { PyProperty { - getter: self.getter.clone(), - setter: self.setter.clone(), - deleter: deleter.or_else(|| self.deleter.clone()), + getter: zelf.getter.clone(), + setter: zelf.setter.clone(), + deleter: deleter.or_else(|| zelf.deleter.clone()), + doc: None, } - .into_ref_with_type(vm, self.typ()) + .into_ref_with_type(vm, TypeProtocol::class(&zelf)) } } @@ -198,9 +276,10 @@ impl<'a> PropertyBuilder<'a> { getter: self.getter.clone(), setter: self.setter.clone(), deleter: None, + doc: None, }; - PyObject::new(payload, self.ctx.property_type()) + PyObject::new(payload, self.ctx.property_type(), None) } else { let payload = PyReadOnlyProperty { getter: self.getter.expect( @@ -208,58 +287,12 @@ impl<'a> PropertyBuilder<'a> { ), }; - PyObject::new(payload, self.ctx.readonly_property_type()) + PyObject::new(payload, self.ctx.readonly_property_type(), None) } } } pub fn init(context: &PyContext) { - extend_class!(context, &context.readonly_property_type, { - "__get__" => context.new_rustfunc(PyReadOnlyPropertyRef::get), - }); - - let property_doc = - "Property attribute.\n\n \ - fget\n \ - function to be used for getting an attribute value\n \ - fset\n \ - function to be used for setting an attribute value\n \ - fdel\n \ - function to be used for del\'ing an attribute\n \ - doc\n \ - docstring\n\n\ - Typical use is to define a managed attribute x:\n\n\ - class C(object):\n \ - def getx(self): return self._x\n \ - def setx(self, value): self._x = value\n \ - def delx(self): del self._x\n \ - x = property(getx, setx, delx, \"I\'m the \'x\' property.\")\n\n\ - Decorators make defining new properties or modifying existing ones easy:\n\n\ - class C(object):\n \ - @property\n \ - def x(self):\n \"I am the \'x\' property.\"\n \ - return self._x\n \ - @x.setter\n \ - def x(self, value):\n \ - self._x = value\n \ - @x.deleter\n \ - def x(self):\n \ - del self._x"; - - extend_class!(context, &context.property_type, { - "__new__" => context.new_rustfunc(PyPropertyRef::new_property), - "__doc__" => context.new_str(property_doc.to_string()), - - "__get__" => context.new_rustfunc(PyPropertyRef::get), - "__set__" => context.new_rustfunc(PyPropertyRef::set), - "__delete__" => context.new_rustfunc(PyPropertyRef::delete), - - "fget" => context.new_property(PyPropertyRef::fget), - "fset" => context.new_property(PyPropertyRef::fset), - "fdel" => context.new_property(PyPropertyRef::fdel), - - "getter" => context.new_rustfunc(PyPropertyRef::getter), - "setter" => context.new_rustfunc(PyPropertyRef::setter), - "deleter" => context.new_rustfunc(PyPropertyRef::deleter), - }); + PyReadOnlyProperty::extend_class(context, &context.readonly_property_type); + PyProperty::extend_class(context, &context.property_type); } diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 0730762a84..de937535fa 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -1,150 +1,101 @@ use std::cell::Cell; -use std::ops::Mul; use num_bigint::{BigInt, Sign}; use num_integer::Integer; -use num_traits::{One, Signed, ToPrimitive, Zero}; +use num_traits::{One, Signed, Zero}; -use crate::function::PyFuncArgs; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::pyobject::{ - PyContext, PyIteratorValue, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol, + PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; -use super::objint::{self, PyInt}; -use super::objslice::PySlice; -use super::objtype; - +use super::objint::{PyInt, PyIntRef}; +use super::objiter; +use super::objslice::{PySlice, PySliceRef}; +use super::objtype::{self, PyClassRef}; + +/// range(stop) -> range object +/// range(start, stop[, step]) -> range object +/// +/// Return an object that produces a sequence of integers from start (inclusive) +/// to stop (exclusive) by step. range(i, j) produces i, i+1, i+2, ..., j-1. +/// start defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3. +/// These are exactly the valid indices for a list of 4 elements. +/// When step is given, it specifies the increment (or decrement). +#[pyclass] #[derive(Debug, Clone)] pub struct PyRange { - // Unfortunately Rust's built in range type doesn't support things like indexing - // or ranges where start > end so we need to roll our own. - pub start: BigInt, - pub end: BigInt, - pub step: BigInt, + pub start: PyIntRef, + pub stop: PyIntRef, + pub step: PyIntRef, } impl PyValue for PyRange { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.range_type() } } impl PyRange { - #[inline] - pub fn try_len(&self) -> Option { - match self.step.sign() { - Sign::Plus if self.start < self.end => ((&self.end - &self.start - 1usize) - / &self.step) - .to_usize() - .map(|sz| sz + 1), - Sign::Minus if self.start > self.end => ((&self.start - &self.end - 1usize) - / (-&self.step)) - .to_usize() - .map(|sz| sz + 1), - _ => Some(0), - } - } - - #[inline] - pub fn len(&self) -> usize { - self.try_len().unwrap() - } - #[inline] fn offset(&self, value: &BigInt) -> Option { - match self.step.sign() { - Sign::Plus if *value >= self.start && *value < self.end => Some(value - &self.start), - Sign::Minus if *value <= self.start && *value > self.end => Some(&self.start - value), + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + match step.sign() { + Sign::Plus if value >= start && value < stop => Some(value - start), + Sign::Minus if value <= self.start.as_bigint() && value > stop => Some(start - value), _ => None, } } - #[inline] - pub fn contains(&self, value: &BigInt) -> bool { - match self.offset(value) { - Some(ref offset) => offset.is_multiple_of(&self.step), - None => false, - } - } - #[inline] pub fn index_of(&self, value: &BigInt) -> Option { + let step = self.step.as_bigint(); match self.offset(value) { - Some(ref offset) if offset.is_multiple_of(&self.step) => { - Some((offset / &self.step).abs()) - } + Some(ref offset) if offset.is_multiple_of(step) => Some((offset / step).abs()), Some(_) | None => None, } } - #[inline] - pub fn count(&self, value: &BigInt) -> usize { - if self.index_of(value).is_some() { - 1 - } else { - 0 - } - } - #[inline] pub fn is_empty(&self) -> bool { - (self.start <= self.end && self.step.is_negative()) - || (self.start >= self.end && self.step.is_positive()) + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + (start <= stop && step.is_negative()) || (start >= stop && step.is_positive()) } #[inline] pub fn forward(&self) -> bool { - self.start < self.end - } - - #[inline] - pub fn get<'a, T>(&'a self, index: T) -> Option - where - &'a BigInt: Mul, - { - let result = &self.start + &self.step * index; - - if (self.forward() && !self.is_empty() && result < self.end) - || (!self.forward() && !self.is_empty() && result > self.end) - { - Some(result) - } else { - None - } + self.start.as_bigint() < self.stop.as_bigint() } #[inline] - pub fn reversed(&self) -> Self { - // compute the last element that is actually contained within the range - // this is the new start - let remainder = ((&self.end - &self.start) % &self.step).abs(); - let start = if remainder.is_zero() { - &self.end - &self.step + pub fn get(&self, index: &BigInt) -> Option { + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); + + let index = if index < &BigInt::zero() { + let index = stop + index; + if index < BigInt::zero() { + return None; + } + index } else { - &self.end - &remainder + index.clone() }; - match self.step.sign() { - Sign::Plus => PyRange { - start, - end: &self.start - 1, - step: -&self.step, - }, - Sign::Minus => PyRange { - start, - end: &self.start + 1, - step: -&self.step, - }, - Sign::NoSign => unreachable!(), - } - } + let result = start + step * &index; - pub fn repr(&self) -> String { - if self.step == BigInt::one() { - format!("range({}, {})", self.start, self.end) + if (self.forward() && !self.is_empty() && &result < stop) + || (!self.forward() && !self.is_empty() && &result > stop) + { + Some(result) } else { - format!("range({}, {}, {})", self.start, self.end, self.step) + None } } } @@ -154,262 +105,281 @@ pub fn get_value(obj: &PyObjectRef) -> PyRange { } pub fn init(context: &PyContext) { - let range_type = &context.range_type; - - let range_doc = "range(stop) -> range object\n\ - range(start, stop[, step]) -> range object\n\n\ - Return an object that produces a sequence of integers from start (inclusive)\n\ - to stop (exclusive) by step. range(i, j) produces i, i+1, i+2, ..., j-1.\n\ - start defaults to 0, and stop is omitted! range(4) produces 0, 1, 2, 3.\n\ - These are exactly the valid indices for a list of 4 elements.\n\ - When step is given, it specifies the increment (or decrement)."; - - context.set_attr(&range_type, "__new__", context.new_rustfunc(range_new)); - context.set_attr(&range_type, "__iter__", context.new_rustfunc(range_iter)); - context.set_attr( - &range_type, - "__reversed__", - context.new_rustfunc(range_reversed), - ); - context.set_attr( - &range_type, - "__doc__", - context.new_str(range_doc.to_string()), - ); - context.set_attr(&range_type, "__len__", context.new_rustfunc(range_len)); - context.set_attr( - &range_type, - "__getitem__", - context.new_rustfunc(range_getitem), - ); - context.set_attr(&range_type, "__repr__", context.new_rustfunc(range_repr)); - context.set_attr(&range_type, "__bool__", context.new_rustfunc(range_bool)); - context.set_attr( - &range_type, - "__contains__", - context.new_rustfunc(range_contains), - ); - context.set_attr(&range_type, "index", context.new_rustfunc(range_index)); - context.set_attr(&range_type, "count", context.new_rustfunc(range_count)); - context.set_attr(&range_type, "start", context.new_property(range_start)); - context.set_attr(&range_type, "stop", context.new_property(range_stop)); - context.set_attr(&range_type, "step", context.new_property(range_step)); + PyRange::extend_class(context, &context.range_type); + PyRangeIterator::extend_class(context, &context.rangeiterator_type); } -fn range_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(cls, None), (first, Some(vm.ctx.int_type()))], - optional = [ - (second, Some(vm.ctx.int_type())), - (step, Some(vm.ctx.int_type())) - ] - ); - - let start = if second.is_some() { - objint::get_value(first).clone() - } else { - BigInt::zero() - }; - - let end = if let Some(pyint) = second { - objint::get_value(pyint).clone() - } else { - objint::get_value(first).clone() - }; - - let step = if let Some(pyint) = step { - objint::get_value(pyint).clone() - } else { - BigInt::one() - }; - - if step.is_zero() { - Err(vm.new_value_error("range with 0 step size".to_string())) - } else { - Ok(PyObject::new(PyRange { start, end, step }, cls.clone())) +type PyRangeRef = PyRef; + +#[pyimpl] +impl PyRange { + #[pymethod(name = "__new__")] + fn new(cls: PyClassRef, stop: PyIntRef, vm: &VirtualMachine) -> PyResult { + PyRange { + start: PyInt::new(BigInt::zero()).into_ref(vm), + stop: stop.clone(), + step: PyInt::new(BigInt::one()).into_ref(vm), + } + .into_ref_with_type(vm, cls) } -} -fn range_iter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(range, Some(vm.ctx.range_type()))]); + fn new_from( + cls: PyClassRef, + start: PyIntRef, + stop: PyIntRef, + step: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let step = step.unwrap_or_else(|| PyInt::new(BigInt::one()).into_ref(vm)); + if step.as_bigint().is_zero() { + return Err(vm.new_value_error("range() arg 3 must not be zero".to_string())); + } + PyRange { start, stop, step }.into_ref_with_type(vm, cls) + } - Ok(PyObject::new( - PyIteratorValue { - position: Cell::new(0), - iterated_obj: range.clone(), - }, - vm.ctx.iter_type(), - )) -} + #[pyproperty(name = "start")] + fn start(&self, _vm: &VirtualMachine) -> PyIntRef { + self.start.clone() + } -fn range_reversed(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); + #[pyproperty(name = "stop")] + fn stop(&self, _vm: &VirtualMachine) -> PyIntRef { + self.stop.clone() + } - let range = get_value(zelf).reversed(); + #[pyproperty(name = "step")] + fn step(&self, _vm: &VirtualMachine) -> PyIntRef { + self.step.clone() + } - Ok(PyObject::new( - PyIteratorValue { + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIterator { + PyRangeIterator { position: Cell::new(0), - iterated_obj: PyObject::new(range, vm.ctx.range_type()), - }, - vm.ctx.iter_type(), - )) -} - -fn range_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - - if let Some(len) = get_value(zelf).try_len() { - Ok(vm.ctx.new_int(len)) - } else { - Err(vm.new_overflow_error("Python int too large to convert to Rust usize".to_string())) + range: zelf, + } } -} -fn range_getitem(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (subscript, None)] - ); + #[pymethod(name = "__reversed__")] + fn reversed(&self, vm: &VirtualMachine) -> PyRangeIterator { + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); - let range = get_value(zelf); - - if let Some(i) = subscript.payload::() { - if let Some(int) = range.get(i.value.clone()) { - Ok(vm.ctx.new_int(int)) - } else { - Err(vm.new_index_error("range object index out of range".to_string())) - } - } else if let Some(PySlice { - ref start, - ref stop, - ref step, - }) = subscript.payload() - { - let new_start = if let Some(int) = start { - if let Some(i) = range.get(int) { - i - } else { - range.start.clone() - } + // compute the last element that is actually contained within the range + // this is the new start + let remainder = ((stop - start) % step).abs(); + let new_start = if remainder.is_zero() { + stop - step } else { - range.start.clone() + stop - &remainder }; - let new_end = if let Some(int) = stop { - if let Some(i) = range.get(int) { - i - } else { - range.end - } - } else { - range.end + let new_stop: BigInt = match step.sign() { + Sign::Plus => start - 1, + Sign::Minus => start + 1, + Sign::NoSign => unreachable!(), }; - let new_step = if let Some(int) = step { - int * range.step - } else { - range.step + let reversed = PyRange { + start: PyInt::new(new_start).into_ref(vm), + stop: PyInt::new(new_stop).into_ref(vm), + step: PyInt::new(-step).into_ref(vm), }; - Ok(PyObject::new( - PyRange { - start: new_start, - end: new_end, - step: new_step, - }, - vm.ctx.range_type(), - )) - } else { - Err(vm.new_type_error("range indices must be integer or slice".to_string())) + PyRangeIterator { + position: Cell::new(0), + range: reversed.into_ref(vm), + } } -} - -fn range_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - let repr = get_value(zelf).repr(); + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> PyInt { + let start = self.start.as_bigint(); + let stop = self.stop.as_bigint(); + let step = self.step.as_bigint(); - Ok(vm.ctx.new_str(repr)) -} + match step.sign() { + Sign::Plus if start < stop => PyInt::new((stop - start - 1usize) / step + 1), + Sign::Minus if start > stop => PyInt::new((start - stop - 1usize) / (-step) + 1), + Sign::Plus | Sign::Minus => PyInt::new(0), + Sign::NoSign => unreachable!(), + } + } -fn range_bool(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { + if self.step.as_bigint().is_one() { + format!("range({}, {})", self.start, self.stop) + } else { + format!("range({}, {}, {})", self.start, self.stop, self.step) + } + } - let len = get_value(zelf).len(); + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { + !self.is_empty() + } - Ok(vm.ctx.new_bool(len > 0)) -} + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, _vm: &VirtualMachine) -> bool { + if let Ok(int) = needle.downcast::() { + match self.offset(int.as_bigint()) { + Some(ref offset) => offset.is_multiple_of(self.step.as_bigint()), + None => false, + } + } else { + false + } + } -fn range_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (needle, None)] - ); + #[pymethod(name = "__eq__")] + fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + if objtype::isinstance(&rhs, &vm.ctx.range_type()) { + let rhs = get_value(&rhs); + self.start.as_bigint() == rhs.start.as_bigint() + && self.stop.as_bigint() == rhs.stop.as_bigint() + && self.step.as_bigint() == rhs.step.as_bigint() + } else { + false + } + } - let range = get_value(zelf); + #[pymethod(name = "index")] + fn index(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if let Ok(int) = needle.downcast::() { + match self.index_of(int.as_bigint()) { + Some(idx) => Ok(PyInt::new(idx)), + None => Err(vm.new_value_error(format!("{} is not in range", int))), + } + } else { + Err(vm.new_value_error("sequence.index(x): x not in sequence".to_string())) + } + } - let result = if objtype::isinstance(needle, &vm.ctx.int_type()) { - range.contains(&objint::get_value(needle)) - } else { - false - }; + #[pymethod(name = "count")] + fn count(&self, item: PyObjectRef, _vm: &VirtualMachine) -> PyInt { + if let Ok(int) = item.downcast::() { + if self.index_of(int.as_bigint()).is_some() { + PyInt::new(1) + } else { + PyInt::new(0) + } + } else { + PyInt::new(0) + } + } - Ok(vm.ctx.new_bool(result)) -} + #[pymethod(name = "__getitem__")] + fn getitem(&self, subscript: RangeIndex, vm: &VirtualMachine) -> PyResult { + match subscript { + RangeIndex::Int(index) => { + if let Some(value) = self.get(index.as_bigint()) { + Ok(PyInt::new(value).into_ref(vm).into_object()) + } else { + Err(vm.new_index_error("range object index out of range".to_string())) + } + } + RangeIndex::Slice(slice) => { + let new_start = if let Some(int) = slice.start_index(vm)? { + if let Some(i) = self.get(&int) { + PyInt::new(i).into_ref(vm) + } else { + self.start.clone() + } + } else { + self.start.clone() + }; + + let new_end = if let Some(int) = slice.stop_index(vm)? { + if let Some(i) = self.get(&int) { + PyInt::new(i).into_ref(vm) + } else { + self.stop.clone() + } + } else { + self.stop.clone() + }; + + let new_step = if let Some(int) = slice.step_index(vm)? { + PyInt::new(int * self.step.as_bigint()).into_ref(vm) + } else { + self.step.clone() + }; + + Ok(PyRange { + start: new_start, + stop: new_end, + step: new_step, + } + .into_ref(vm) + .into_object()) + } + } + } -fn range_index(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (needle, None)] - ); + #[pymethod(name = "__new__")] + fn range_new(args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let range = if args.args.len() <= 2 { + let (cls, stop) = args.bind(vm)?; + PyRange::new(cls, stop, vm) + } else { + let (cls, start, stop, step) = args.bind(vm)?; + PyRange::new_from(cls, start, stop, step, vm) + }?; - let range = get_value(zelf); + Ok(range.into_object()) + } +} - if objtype::isinstance(needle, &vm.ctx.int_type()) { - let needle = objint::get_value(needle); +#[pyclass] +#[derive(Debug)] +pub struct PyRangeIterator { + position: Cell, + range: PyRangeRef, +} - match range.index_of(&needle) { - Some(idx) => Ok(vm.ctx.new_int(idx)), - None => Err(vm.new_value_error(format!("{} is not in range", needle))), - } - } else { - Err(vm.new_value_error("sequence.index(x): x not in sequence".to_string())) +impl PyValue for PyRangeIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.rangeiterator_type() } } -fn range_count(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (item, None)] - ); +type PyRangeIteratorRef = PyRef; - let range = get_value(zelf); - - if objtype::isinstance(item, &vm.ctx.int_type()) { - Ok(vm.ctx.new_int(range.count(&objint::get_value(item)))) - } else { - Ok(vm.ctx.new_int(0)) +#[pyimpl] +impl PyRangeIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let position = BigInt::from(self.position.get()); + if let Some(int) = self.range.get(&position) { + self.position.set(self.position.get() + 1); + Ok(int) + } else { + Err(objiter::new_stop_iteration(vm)) + } } -} -fn range_start(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - Ok(vm.ctx.new_int(get_value(zelf).start)) + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRangeIteratorRef { + zelf + } } -fn range_stop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - Ok(vm.ctx.new_int(get_value(zelf).end)) +pub enum RangeIndex { + Int(PyIntRef), + Slice(PySliceRef), } -fn range_step(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, Some(vm.ctx.range_type()))]); - Ok(vm.ctx.new_int(get_value(zelf).step)) +impl TryFromObject for RangeIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(RangeIndex::Int(i)), + s @ PySlice => Ok(RangeIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) + } } diff --git a/vm/src/obj/objsequence.rs b/vm/src/obj/objsequence.rs index ad9c7bf361..d4226d51cd 100644 --- a/vm/src/obj/objsequence.rs +++ b/vm/src/obj/objsequence.rs @@ -1,17 +1,19 @@ +use crate::function::OptionalArg; +use crate::obj::objnone::PyNone; use std::cell::RefCell; use std::marker::Sized; use std::ops::{Deref, DerefMut, Range}; -use num_bigint::BigInt; -use num_traits::{One, Signed, ToPrimitive, Zero}; +use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TryFromObject, TypeProtocol}; -use crate::pyobject::{IdProtocol, PyObject, PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; +use num_bigint::{BigInt, ToBigInt}; +use num_traits::{One, Signed, ToPrimitive, Zero}; use super::objbool; use super::objint::PyInt; use super::objlist::PyList; -use super::objslice::PySlice; +use super::objslice::{PySlice, PySliceRef}; use super::objtuple::PyTuple; pub trait PySliceableSequence { @@ -65,14 +67,15 @@ pub trait PySliceableSequence { where Self: Sized, { - // TODO: we could potentially avoid this copy and use slice - match slice.payload() { - Some(PySlice { start, stop, step }) => { - let step = step.clone().unwrap_or_else(BigInt::one); + match slice.clone().downcast::() { + Ok(slice) => { + let start = slice.start_index(vm)?; + let stop = slice.stop_index(vm)?; + let step = slice.step_index(vm)?.unwrap_or_else(BigInt::one); if step.is_zero() { Err(vm.new_value_error("slice step cannot be zero".to_string())) } else if step.is_positive() { - let range = self.get_slice_range(start, stop); + let range = self.get_slice_range(&start, &stop); if range.start < range.end { #[allow(clippy::range_plus_one)] match step.to_i32() { @@ -86,8 +89,20 @@ pub trait PySliceableSequence { } else { // calculate the range for the reverse slice, first the bounds needs to be made // exclusive around stop, the lower number - let start = start.as_ref().map(|x| x + 1); - let stop = stop.as_ref().map(|x| x + 1); + let start = start.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len() + BigInt::one() //.to_bigint().unwrap() + } else { + x + 1 + } + }); + let stop = stop.as_ref().map(|x| { + if *x == (-1).to_bigint().unwrap() { + self.len().to_bigint().unwrap() + } else { + x + 1 + } + }); let range = self.get_slice_range(&stop, &start); if range.start < range.end { match (-step).to_i32() { @@ -137,6 +152,24 @@ impl PySliceableSequence for Vec { } } +pub enum SequenceIndex { + Int(i32), + Slice(PySliceRef), +} + +impl TryFromObject for SequenceIndex { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match_class!(obj, + i @ PyInt => Ok(SequenceIndex::Int(i32::try_from_object(vm, i.into_object())?)), + s @ PySlice => Ok(SequenceIndex::Slice(s)), + obj => Err(vm.new_type_error(format!( + "sequence indices be integers or slices, not {}", + obj.class(), + ))) + ) + } +} + pub fn get_item( vm: &VirtualMachine, sequence: &PyObjectRef, @@ -144,7 +177,7 @@ pub fn get_item( subscript: PyObjectRef, ) -> PyResult { if let Some(i) = subscript.payload::() { - return match i.value.to_i32() { + return match i.as_bigint().to_i32() { Some(value) => { if let Some(pos_index) = elements.to_vec().get_pos(value) { let obj = elements[pos_index].clone(); @@ -163,12 +196,14 @@ pub fn get_item( if sequence.payload::().is_some() { Ok(PyObject::new( PyList::from(elements.to_vec().get_slice_items(vm, &subscript)?), - sequence.typ(), + sequence.class(), + None, )) } else if sequence.payload::().is_some() { Ok(PyObject::new( PyTuple::from(elements.to_vec().get_slice_items(vm, &subscript)?), - sequence.typ(), + sequence.class(), + None, )) } else { panic!("sequence get_item called for non-sequence") @@ -313,18 +348,19 @@ pub fn get_elements_cell<'a>(obj: &'a PyObjectRef) -> &'a RefCell() { return &list.elements; } - if let Some(tuple) = obj.payload::() { - return &tuple.elements; - } panic!("Cannot extract elements from non-sequence"); } -pub fn get_elements<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { +pub fn get_elements_list<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { if let Some(list) = obj.payload::() { return list.elements.borrow(); } + panic!("Cannot extract elements from non-sequence"); +} + +pub fn get_elements_tuple<'a>(obj: &'a PyObjectRef) -> impl Deref> + 'a { if let Some(tuple) = obj.payload::() { - return tuple.elements.borrow(); + return &tuple.elements; } panic!("Cannot extract elements from non-sequence"); } @@ -333,8 +369,22 @@ pub fn get_mut_elements<'a>(obj: &'a PyObjectRef) -> impl DerefMut() { return list.elements.borrow_mut(); } - if let Some(tuple) = obj.payload::() { - return tuple.elements.borrow_mut(); - } panic!("Cannot extract elements from non-sequence"); } + +//Check if given arg could be used with PySciceableSequance.get_slice_range() +pub fn is_valid_slice_arg( + arg: OptionalArg, + vm: &VirtualMachine, +) -> Result, PyObjectRef> { + if let OptionalArg::Present(value) = arg { + match_class!(value, + i @ PyInt => Ok(Some(i.as_bigint().clone())), + _obj @ PyNone => Ok(None), + _=> {return Err(vm.new_type_error("slice indices must be integers or None or have an __index__ method".to_string()));} + // TODO: check for an __index__ method + ) + } else { + Ok(None) + } +} diff --git a/vm/src/obj/objset.rs b/vm/src/obj/objset.rs index b11c21aa39..ede6029f32 100644 --- a/vm/src/obj/objset.rs +++ b/vm/src/obj/objset.rs @@ -3,27 +3,44 @@ */ use std::cell::{Cell, RefCell}; -use std::collections::{hash_map::DefaultHasher, HashMap}; use std::fmt; -use std::hash::{Hash, Hasher}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::dictdatatype; +use crate::function::OptionalArg; use crate::pyobject::{ - PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, + PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, + TypeProtocol, }; use crate::vm::{ReprGuard, VirtualMachine}; -use super::objbool; -use super::objint; -use super::objiter; -use super::objtype::{self, PyClassRef}; +use super::objlist::PyListIterator; +use super::objtype; +use super::objtype::PyClassRef; +pub type SetContentType = dictdatatype::Dict<()>; + +/// set() -> new empty set object +/// set(iterable) -> new set object +/// +/// Build an unordered collection of unique elements. +#[pyclass] #[derive(Default)] pub struct PySet { - elements: RefCell>, + inner: RefCell, } pub type PySetRef = PyRef; +/// frozenset() -> empty frozenset object +/// frozenset(iterable) -> frozenset object +/// +/// Build an immutable unordered collection of unique elements. +#[pyclass] +#[derive(Default)] +pub struct PyFrozenSet { + inner: PySetInner, +} +pub type PyFrozenSetRef = PyRef; + impl fmt::Debug for PySet { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // TODO: implement more detailed, non-recursive Debug formatter @@ -31,615 +48,706 @@ impl fmt::Debug for PySet { } } +impl fmt::Debug for PyFrozenSet { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // TODO: implement more detailed, non-recursive Debug formatter + f.write_str("frozenset") + } +} + impl PyValue for PySet { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.set_type() } } -pub fn get_elements(obj: &PyObjectRef) -> HashMap { - obj.payload::().unwrap().elements.borrow().clone() +impl PyValue for PyFrozenSet { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.frozenset_type() + } } -fn perform_action_with_hash( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, - f: &Fn(&VirtualMachine, &mut HashMap, u64, &PyObjectRef) -> PyResult, -) -> PyResult { - let hash: PyObjectRef = vm.call_method(item, "__hash__", vec![])?; - - let hash_value = objint::get_value(&hash); - let mut hasher = DefaultHasher::new(); - hash_value.hash(&mut hasher); - let key = hasher.finish(); - f(vm, elements, key, item) +#[derive(Default, Clone)] +struct PySetInner { + content: SetContentType, } -fn insert_into_set( - vm: &VirtualMachine, - elements: &mut HashMap, - item: &PyObjectRef, -) -> PyResult { - fn insert( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - elements.insert(key, value.clone()); - Ok(vm.get_none()) +impl PySetInner { + fn new(iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut set = PySetInner::default(); + for item in iterable.iter(vm)? { + set.add(&item?, vm)?; + } + Ok(set) } - perform_action_with_hash(vm, elements, item, &insert) -} -fn set_add(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.add called with: {:?}", args); - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (item, None)] - ); - match zelf.payload::() { - Some(set) => insert_into_set(vm, &mut set.elements.borrow_mut(), item), - _ => Err(vm.new_type_error("set.add is called with no item".to_string())), + fn from_arg(iterable: OptionalArg, vm: &VirtualMachine) -> PyResult { + if let OptionalArg::Present(iterable) = iterable { + Self::new(iterable, vm) + } else { + Ok(PySetInner::default()) + } } -} -fn set_remove(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.remove called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn remove( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - value: &PyObjectRef, - ) -> PyResult { - match elements.remove(&key) { - None => { - let item_str = format!("{:?}", value); - Err(vm.new_key_error(item_str)) - } - Some(_) => Ok(vm.get_none()), - } - } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &remove) + fn len(&self) -> usize { + self.content.len() + } + + fn copy(&self) -> PySetInner { + PySetInner { + content: self.content.clone(), } - _ => Err(vm.new_type_error("set.remove is called with no item".to_string())), } -} -fn set_discard(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.discard called with: {:?}", args); - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.set_type())), (item, None)] - ); - match s.payload::() { - Some(set) => { - fn discard( - vm: &VirtualMachine, - elements: &mut HashMap, - key: u64, - _value: &PyObjectRef, - ) -> PyResult { - elements.remove(&key); - Ok(vm.get_none()) + fn contains(&self, needle: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.content.contains(vm, needle) + } + + #[inline] + fn _compare_inner( + &self, + other: &PySetInner, + size_func: &Fn(usize, usize) -> bool, + swap: bool, + vm: &VirtualMachine, + ) -> PyResult { + let (zelf, other) = if swap { (other, self) } else { (self, other) }; + + if size_func(zelf.len(), other.len()) { + return Ok(vm.new_bool(false)); + } + for key in other.content.keys() { + if !zelf.contains(&key, vm)? { + return Ok(vm.new_bool(false)); } - perform_action_with_hash(vm, &mut set.elements.borrow_mut(), item, &discard) } - None => Err(vm.new_type_error("set.discard is called with no item".to_string())), + Ok(vm.new_bool(true)) } -} -fn set_clear(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.clear called"); - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - match s.payload::() { - Some(set) => { - set.elements.borrow_mut().clear(); - Ok(vm.get_none()) + fn eq(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf != other }, + false, + vm, + ) + } + + fn ge(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + false, + vm, + ) + } + + fn gt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + false, + vm, + ) + } + + fn le(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf < other }, + true, + vm, + ) + } + + fn lt(&self, other: &PySetInner, vm: &VirtualMachine) -> PyResult { + self._compare_inner( + other, + &|zelf: usize, other: usize| -> bool { zelf <= other }, + true, + vm, + ) + } + + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut set = self.clone(); + for item in other.iter(vm)? { + set.add(&item?, vm)?; } - None => Err(vm.new_type_error("".to_string())), + + Ok(set) } -} -/* Create a new object of sub-type of set */ -fn set_new( - cls: PyClassRef, - iterable: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - if !objtype::issubclass(cls.as_object(), &vm.ctx.set_type()) { - return Err(vm.new_type_error(format!("{} is not a subtype of set", cls))); - } - - let elements: HashMap = match iterable { - OptionalArg::Missing => HashMap::new(), - OptionalArg::Present(iterable) => { - let mut elements = HashMap::new(); - let iterator = objiter::get_iter(vm, &iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut elements, &v)?; + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut set = PySetInner::default(); + for item in other.iter(vm)? { + let obj = item?; + if self.contains(&obj, vm)? { + set.add(&obj, vm)?; } - elements } - }; + Ok(set) + } - PySet { - elements: RefCell::new(elements), + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut set = self.copy(); + for item in other.iter(vm)? { + set.content.delete_if_exists(vm, &item?)?; + } + Ok(set) } - .into_ref_with_type(vm, cls) -} -fn set_len(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("set.len called with: {:?}", args); - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); - let elements = get_elements(s); - Ok(vm.context().new_int(elements.len())) -} + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let mut new_inner = self.clone(); -fn set_copy(obj: PySetRef, _vm: &VirtualMachine) -> PySet { - trace!("set.copy called with: {:?}", obj); - let elements = obj.elements.borrow().clone(); - PySet { - elements: RefCell::new(elements), - } -} + for item in other.iter(vm)? { + new_inner.content.delete_or_insert(vm, &item?, ())? + } -fn set_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.set_type()))]); + Ok(new_inner) + } - let elements = get_elements(o); - let s = if elements.is_empty() { - "set()".to_string() - } else if let Some(_guard) = ReprGuard::enter(o) { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in other.iter(vm)? { + if !self.contains(&item?, vm)? { + return Ok(false); + } } + Ok(true) + } - format!("{{{}}}", str_parts.join(", ")) - } else { - "set(...)".to_string() - }; - Ok(vm.new_str(s)) -} + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + let other_set = PySetInner::new(other, vm)?; + self.le(&other_set, vm) + } -pub fn set_contains(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(set, Some(vm.ctx.set_type())), (needle, None)] - ); - for element in get_elements(set).iter() { - match vm._eq(needle.clone(), element.1.clone()) { - Ok(value) => { - if objbool::get_value(&value) { - return Ok(vm.new_bool(true)); - } + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + for item in other.iter(vm)? { + if self.contains(&item?, vm)? { + return Ok(false); } - Err(_) => return Err(vm.new_type_error("".to_string())), } + Ok(true) } - Ok(vm.new_bool(false)) -} + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { + let items = self.content.keys().collect(); + let set_list = vm.ctx.new_list(items); + PyListIterator { + position: Cell::new(0), + list: set_list.downcast().unwrap(), + } + } -fn set_eq(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf != other }, - false, - ) -} + fn repr(&self, vm: &VirtualMachine) -> PyResult { + let mut str_parts = vec![]; + for key in self.content.keys() { + let part = vm.to_repr(&key)?; + str_parts.push(part.value.clone()); + } -fn set_ge(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - false, - ) -} + Ok(format!("{{{}}}", str_parts.join(", "))) + } -fn set_gt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - false, - ) -} + fn add(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.insert(vm, item, ()) + } -fn set_le(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf < other }, - true, - ) -} + fn remove(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.content.delete(vm, item) + } -fn set_lt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_compare_inner( - vm, - args, - &|zelf: usize, other: usize| -> bool { zelf <= other }, - true, - ) -} + fn discard(&mut self, item: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.content.delete_if_exists(vm, item) + } -fn set_compare_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - size_func: &Fn(usize, usize) -> bool, - swap: bool, -) -> PyResult { - arg_check!( - vm, - args, - required = [ - (zelf, Some(vm.ctx.set_type())), - (other, Some(vm.ctx.set_type())) - ] - ); - - let get_zelf = |swap: bool| -> &PyObjectRef { - if swap { - other + fn clear(&mut self) { + self.content.clear() + } + + fn pop(&mut self, vm: &VirtualMachine) -> PyResult { + if let Some((key, _)) = self.content.pop_front() { + Ok(key) } else { - zelf + let err_msg = vm.new_str("pop from an empty set".to_string()); + Err(vm.new_key_error(err_msg)) } - }; - let get_other = |swap: bool| -> &PyObjectRef { - if swap { - zelf - } else { - other + } + + fn update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + for item in iterable.iter(vm)? { + self.add(&item?, vm)?; } - }; + Ok(()) + } - let zelf_elements = get_elements(get_zelf(swap)); - let other_elements = get_elements(get_other(swap)); - if size_func(zelf_elements.len(), other_elements.len()) { - return Ok(vm.new_bool(false)); - } - for element in other_elements.iter() { - match vm.call_method(get_zelf(swap), "__contains__", vec![element.1.clone()]) { - Ok(value) => { - if !objbool::get_value(&value) { - return Ok(vm.new_bool(false)); - } + fn intersection_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + let temp_inner = self.copy(); + self.clear(); + for item in iterable.iter(vm)? { + let obj = item?; + if temp_inner.contains(&obj, vm)? { + self.add(&obj, vm)?; } - Err(_) => return Err(vm.new_type_error("".to_string())), } + Ok(()) } - Ok(vm.new_bool(true)) -} - -fn set_union(zelf: PySetRef, other: PySetRef, _vm: &VirtualMachine) -> PySet { - let mut elements = zelf.elements.borrow().clone(); - elements.extend(other.elements.borrow().clone()); - PySet { - elements: RefCell::new(elements), + fn difference_update(&mut self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult<()> { + for item in iterable.iter(vm)? { + self.content.delete_if_exists(vm, &item?)?; + } + Ok(()) } -} -fn set_intersection(zelf: PySetRef, other: PySetRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Intersection) + fn symmetric_difference_update( + &mut self, + iterable: PyIterable, + vm: &VirtualMachine, + ) -> PyResult<()> { + for item in iterable.iter(vm)? { + self.content.delete_or_insert(vm, &item?, ())?; + } + Ok(()) + } } -fn set_difference(zelf: PySetRef, other: PySetRef, vm: &VirtualMachine) -> PyResult { - set_combine_inner(zelf, other, vm, SetCombineOperation::Difference) +macro_rules! try_set_inner { + ($vm:expr, $other:expr, $op:expr) => { + match_class!($other, + set @ PySet => $op(&*set.inner.borrow()), + frozen @ PyFrozenSet => $op(&frozen.inner), + _ => Ok($vm.ctx.not_implemented()), + ); + }; } -fn set_symmetric_difference( - zelf: PySetRef, - other: PySetRef, - vm: &VirtualMachine, -) -> PyResult { - let mut elements = HashMap::new(); - - for element in zelf.elements.borrow().iter() { - let value = vm.call_method(other.as_object(), "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); +#[pyimpl] +impl PySet { + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Self { + inner: RefCell::new(PySetInner::from_arg(iterable, vm)?), } + .into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { + self.inner.borrow().len() } - for element in other.elements.borrow().iter() { - let value = vm.call_method(zelf.as_object(), "__contains__", vec![element.1.clone()])?; - if !objbool::get_value(&value) { - elements.insert(element.0.clone(), element.1.clone()); + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> Self { + Self { + inner: RefCell::new(self.inner.borrow().copy()), } } - Ok(PySet { - elements: RefCell::new(elements), - }) -} + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().contains(&needle, vm) + } -enum SetCombineOperation { - Intersection, - Difference, -} + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().eq(other, vm)) + } + + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().ge(other, vm)) + } + + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().gt(other, vm)) + } + + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().le(other, vm)) + } + + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.borrow().lt(other, vm)) + } + + #[pymethod] + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().union(other, vm)?), + }) + } + + #[pymethod] + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().intersection(other, vm)?), + }) + } + + #[pymethod] + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().difference(other, vm)?), + }) + } + + #[pymethod] + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: RefCell::new(self.inner.borrow().symmetric_difference(other, vm)?), + }) + } + + #[pymethod] + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().issuperset(other, vm) + } -fn set_combine_inner( - zelf: PySetRef, - other: PySetRef, - vm: &VirtualMachine, - op: SetCombineOperation, -) -> PyResult { - let mut elements = HashMap::new(); - - for element in zelf.elements.borrow().iter() { - let value = vm.call_method(other.as_object(), "__contains__", vec![element.1.clone()])?; - let should_add = match op { - SetCombineOperation::Intersection => objbool::get_value(&value), - SetCombineOperation::Difference => !objbool::get_value(&value), + #[pymethod] + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow().isdisjoint(other, vm) + } + + #[pymethod(name = "__or__")] + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + + #[pymethod(name = "__ror__")] + fn ror(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.or(other, vm) + } + + #[pymethod(name = "__and__")] + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + #[pymethod(name = "__rand__")] + fn rand(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.and(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.sub(other, vm) + } + + #[pymethod(name = "__xor__")] + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.xor(other, vm) + } + + #[pymethod(name = "__iter__")] + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { + self.inner.borrow().iter(vm) + } + + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let inner = zelf.inner.borrow(); + let s = if inner.len() == 0 { + "set()".to_string() + } else if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { + inner.repr(vm)? + } else { + "set(...)".to_string() }; - if should_add { - elements.insert(element.0.clone(), element.1.clone()); - } + Ok(vm.new_str(s)) } - Ok(PySet { - elements: RefCell::new(elements), - }) -} + #[pymethod] + pub fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().add(&item, vm)?; + Ok(()) + } -fn set_pop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(s, Some(vm.ctx.set_type()))]); + #[pymethod] + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().remove(&item, vm) + } - match s.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - match elements.clone().keys().next() { - Some(key) => Ok(elements.remove(key).unwrap()), - None => Err(vm.new_key_error("pop from an empty set".to_string())), - } - } - _ => Err(vm.new_type_error("".to_string())), + #[pymethod] + fn discard(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + self.inner.borrow_mut().discard(&item, vm)?; + Ok(()) } -} -fn set_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ior(vm, args)?; - Ok(vm.get_none()) -} + #[pymethod] + fn clear(&self, _vm: &VirtualMachine) { + self.inner.borrow_mut().clear() + } -fn set_ior(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - } - _ => return Err(vm.new_type_error("set.update is called with no other".to_string())), + #[pymethod] + fn pop(&self, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().pop(vm) } - Ok(zelf.clone()) -} -fn set_intersection_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection)?; - Ok(vm.get_none()) -} + #[pymethod(name = "__ior__")] + fn ior(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner.borrow_mut().update(iterable.iterable, vm)?; + Ok(zelf.as_object().clone()) + } -fn set_iand(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Intersection) -} + #[pymethod] + fn update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().update(iterable, vm)?; + Ok(vm.get_none()) + } -fn set_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference)?; - Ok(vm.get_none()) -} + #[pymethod] + fn intersection_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().intersection_update(iterable, vm)?; + Ok(vm.get_none()) + } + + #[pymethod(name = "__iand__")] + fn iand(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner + .borrow_mut() + .intersection_update(iterable.iterable, vm)?; + Ok(zelf.as_object().clone()) + } -fn set_isub(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_combine_update_inner(vm, args, SetCombineOperation::Difference) + #[pymethod] + fn difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.borrow_mut().difference_update(iterable, vm)?; + Ok(vm.get_none()) + } + + #[pymethod(name = "__isub__")] + fn isub(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner + .borrow_mut() + .difference_update(iterable.iterable, vm)?; + Ok(zelf.as_object().clone()) + } + + #[pymethod] + fn symmetric_difference_update(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner + .borrow_mut() + .symmetric_difference_update(iterable, vm)?; + Ok(vm.get_none()) + } + + #[pymethod(name = "__ixor__")] + fn ixor(zelf: PyRef, iterable: SetIterable, vm: &VirtualMachine) -> PyResult { + zelf.inner + .borrow_mut() + .symmetric_difference_update(iterable.iterable, vm)?; + Ok(zelf.as_object().clone()) + } + + #[pymethod(name = "__hash__")] + fn hash(&self, vm: &VirtualMachine) -> PyResult<()> { + Err(vm.new_type_error("unhashable type".to_string())) + } } -fn set_combine_update_inner( - vm: &VirtualMachine, - args: PyFuncArgs, - op: SetCombineOperation, -) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let mut elements = set.elements.borrow_mut(); - for element in elements.clone().iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - let should_remove = match op { - SetCombineOperation::Intersection => !objbool::get_value(&value), - SetCombineOperation::Difference => objbool::get_value(&value), - }; - if should_remove { - elements.remove(&element.0.clone()); - } - } +#[pyimpl] +impl PyFrozenSet { + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + iterable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + Self { + inner: PySetInner::from_arg(iterable, vm)?, } - _ => return Err(vm.new_type_error("".to_string())), + .into_ref_with_type(vm, cls) } - Ok(zelf.clone()) -} -fn set_symmetric_difference_update(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - set_ixor(vm, args)?; - Ok(vm.get_none()) -} + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { + self.inner.len() + } -fn set_ixor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.set_type())), (iterable, None)] - ); - - match zelf.payload::() { - Some(set) => { - let elements_original = set.elements.borrow().clone(); - let iterator = objiter::get_iter(vm, iterable)?; - while let Ok(v) = vm.call_method(&iterator, "__next__", vec![]) { - insert_into_set(vm, &mut set.elements.borrow_mut(), &v)?; - } - for element in elements_original.iter() { - let value = vm.call_method(iterable, "__contains__", vec![element.1.clone()])?; - if objbool::get_value(&value) { - set.elements.borrow_mut().remove(&element.0.clone()); - } - } + #[pymethod] + fn copy(&self, _vm: &VirtualMachine) -> Self { + Self { + inner: self.inner.copy(), } - _ => return Err(vm.new_type_error("".to_string())), } - Ok(zelf.clone()) -} + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.inner.contains(&needle, vm) + } + + #[pymethod(name = "__eq__")] + fn eq(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.eq(other, vm)) + } -fn set_iter(zelf: PySetRef, vm: &VirtualMachine) -> PyIteratorValue { - let items = zelf.elements.borrow().values().cloned().collect(); - let set_list = vm.ctx.new_list(items); - PyIteratorValue { - position: Cell::new(0), - iterated_obj: set_list, + #[pymethod(name = "__ge__")] + fn ge(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.ge(other, vm)) + } + + #[pymethod(name = "__gt__")] + fn gt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.gt(other, vm)) + } + + #[pymethod(name = "__le__")] + fn le(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.le(other, vm)) + } + + #[pymethod(name = "__lt__")] + fn lt(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + try_set_inner!(vm, other, |other| self.inner.lt(other, vm)) + } + + #[pymethod] + fn union(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.union(other, vm)?, + }) + } + + #[pymethod] + fn intersection(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.intersection(other, vm)?, + }) + } + + #[pymethod] + fn difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.difference(other, vm)?, + }) + } + + #[pymethod] + fn symmetric_difference(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: self.inner.symmetric_difference(other, vm)?, + }) + } + + #[pymethod] + fn issubset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.issubset(other, vm) + } + + #[pymethod] + fn issuperset(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.issuperset(other, vm) + } + + #[pymethod] + fn isdisjoint(&self, other: PyIterable, vm: &VirtualMachine) -> PyResult { + self.inner.isdisjoint(other, vm) + } + + #[pymethod(name = "__or__")] + fn or(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.union(other.iterable, vm) + } + + #[pymethod(name = "__ror__")] + fn ror(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.or(other, vm) + } + + #[pymethod(name = "__and__")] + fn and(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.intersection(other.iterable, vm) + } + + #[pymethod(name = "__rand__")] + fn rand(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.and(other, vm) + } + + #[pymethod(name = "__sub__")] + fn sub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.difference(other.iterable, vm) + } + + #[pymethod(name = "__rsub__")] + fn rsub(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.sub(other, vm) + } + + #[pymethod(name = "__xor__")] + fn xor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.symmetric_difference(other.iterable, vm) + } + + #[pymethod(name = "__rxor__")] + fn rxor(&self, other: SetIterable, vm: &VirtualMachine) -> PyResult { + self.xor(other, vm) + } + + #[pymethod(name = "__iter__")] + fn iter(&self, vm: &VirtualMachine) -> PyListIterator { + self.inner.iter(vm) + } + + #[pymethod(name = "__repr__")] + fn repr(zelf: PyRef, vm: &VirtualMachine) -> PyResult { + let inner = &zelf.inner; + let s = if inner.len() == 0 { + "frozenset()".to_string() + } else if let Some(_guard) = ReprGuard::enter(zelf.as_object()) { + format!("frozenset({})", inner.repr(vm)?) + } else { + "frozenset(...)".to_string() + }; + Ok(vm.new_str(s)) } } -fn frozenset_repr(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(o, Some(vm.ctx.frozenset_type()))]); +struct SetIterable { + iterable: PyIterable, +} - let elements = get_elements(o); - let s = if elements.is_empty() { - "frozenset()".to_string() - } else { - let mut str_parts = vec![]; - for elem in elements.values() { - let part = vm.to_repr(elem)?; - str_parts.push(part.value.clone()); +impl TryFromObject for SetIterable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if objtype::issubclass(&obj.class(), &vm.ctx.set_type()) + || objtype::issubclass(&obj.class(), &vm.ctx.frozenset_type()) + { + Ok(SetIterable { + iterable: PyIterable::try_from_object(vm, obj)?, + }) + } else { + Err(vm.new_type_error(format!( + "{} is not a subtype of set or frozenset", + obj.class() + ))) } - - format!("frozenset({{{}}})", str_parts.join(", ")) - }; - Ok(vm.new_str(s)) + } } pub fn init(context: &PyContext) { - let set_type = &context.set_type; - - let set_doc = "set() -> new empty set object\n\ - set(iterable) -> new set object\n\n\ - Build an unordered collection of unique elements."; - - context.set_attr( - &set_type, - "__contains__", - context.new_rustfunc(set_contains), - ); - context.set_attr(&set_type, "__len__", context.new_rustfunc(set_len)); - context.set_attr(&set_type, "__new__", context.new_rustfunc(set_new)); - context.set_attr(&set_type, "__repr__", context.new_rustfunc(set_repr)); - context.set_attr(&set_type, "__eq__", context.new_rustfunc(set_eq)); - context.set_attr(&set_type, "__ge__", context.new_rustfunc(set_ge)); - context.set_attr(&set_type, "__gt__", context.new_rustfunc(set_gt)); - context.set_attr(&set_type, "__le__", context.new_rustfunc(set_le)); - context.set_attr(&set_type, "__lt__", context.new_rustfunc(set_lt)); - context.set_attr(&set_type, "issubset", context.new_rustfunc(set_le)); - context.set_attr(&set_type, "issuperset", context.new_rustfunc(set_ge)); - context.set_attr(&set_type, "union", context.new_rustfunc(set_union)); - context.set_attr(&set_type, "__or__", context.new_rustfunc(set_union)); - context.set_attr( - &set_type, - "intersection", - context.new_rustfunc(set_intersection), - ); - context.set_attr(&set_type, "__and__", context.new_rustfunc(set_intersection)); - context.set_attr( - &set_type, - "difference", - context.new_rustfunc(set_difference), - ); - context.set_attr(&set_type, "__sub__", context.new_rustfunc(set_difference)); - context.set_attr( - &set_type, - "symmetric_difference", - context.new_rustfunc(set_symmetric_difference), - ); - context.set_attr( - &set_type, - "__xor__", - context.new_rustfunc(set_symmetric_difference), - ); - context.set_attr(&set_type, "__doc__", context.new_str(set_doc.to_string())); - context.set_attr(&set_type, "add", context.new_rustfunc(set_add)); - context.set_attr(&set_type, "remove", context.new_rustfunc(set_remove)); - context.set_attr(&set_type, "discard", context.new_rustfunc(set_discard)); - context.set_attr(&set_type, "clear", context.new_rustfunc(set_clear)); - context.set_attr(&set_type, "copy", context.new_rustfunc(set_copy)); - context.set_attr(&set_type, "pop", context.new_rustfunc(set_pop)); - context.set_attr(&set_type, "update", context.new_rustfunc(set_update)); - context.set_attr(&set_type, "__ior__", context.new_rustfunc(set_ior)); - context.set_attr( - &set_type, - "intersection_update", - context.new_rustfunc(set_intersection_update), - ); - context.set_attr(&set_type, "__iand__", context.new_rustfunc(set_iand)); - context.set_attr( - &set_type, - "difference_update", - context.new_rustfunc(set_difference_update), - ); - context.set_attr(&set_type, "__isub__", context.new_rustfunc(set_isub)); - context.set_attr( - &set_type, - "symmetric_difference_update", - context.new_rustfunc(set_symmetric_difference_update), - ); - context.set_attr(&set_type, "__ixor__", context.new_rustfunc(set_ixor)); - context.set_attr(&set_type, "__iter__", context.new_rustfunc(set_iter)); - - let frozenset_type = &context.frozenset_type; - - let frozenset_doc = "frozenset() -> empty frozenset object\n\ - frozenset(iterable) -> frozenset object\n\n\ - Build an immutable unordered collection of unique elements."; - - context.set_attr( - &frozenset_type, - "__contains__", - context.new_rustfunc(set_contains), - ); - context.set_attr(&frozenset_type, "__len__", context.new_rustfunc(set_len)); - context.set_attr( - &frozenset_type, - "__doc__", - context.new_str(frozenset_doc.to_string()), - ); - context.set_attr( - &frozenset_type, - "__repr__", - context.new_rustfunc(frozenset_repr), - ); + PySet::extend_class(context, &context.set_type); + PyFrozenSet::extend_class(context, &context.frozenset_type); } diff --git a/vm/src/obj/objslice.rs b/vm/src/obj/objslice.rs index 263490565d..82e4cef8cd 100644 --- a/vm/src/obj/objslice.rs +++ b/vm/src/obj/objslice.rs @@ -1,109 +1,125 @@ -use num_bigint::BigInt; - -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::function::{OptionalArg, PyFuncArgs}; +use crate::pyobject::{IdProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol}; use crate::vm::VirtualMachine; -use super::objint; +use crate::obj::objint::PyInt; +use crate::obj::objtype::{class_has_attr, PyClassRef}; +use num_bigint::BigInt; #[derive(Debug)] pub struct PySlice { - // TODO: should be private - pub start: Option, - pub stop: Option, - pub step: Option, + pub start: Option, + pub stop: PyObjectRef, + pub step: Option, } impl PyValue for PySlice { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.slice_type() } } -fn slice_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - no_kwargs!(vm, args); - let (cls, start, stop, step): ( - &PyObjectRef, - Option<&PyObjectRef>, - Option<&PyObjectRef>, - Option<&PyObjectRef>, - ) = match args.args.len() { - 0 | 1 => Err(vm.new_type_error("slice() must have at least one arguments.".to_owned())), - 2 => { - arg_check!( - vm, - args, - required = [ - (cls, Some(vm.ctx.type_type())), - (stop, Some(vm.ctx.int_type())) - ] - ); - Ok((cls, None, Some(stop), None)) +pub type PySliceRef = PyRef; + +fn slice_new(cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let slice: PySlice = match args.args.len() { + 0 => { + return Err(vm.new_type_error("slice() must have at least one arguments.".to_owned())); + } + 1 => { + let stop = args.bind(vm)?; + PySlice { + start: None, + stop, + step: None, + } } _ => { - arg_check!( - vm, - args, - required = [ - (cls, Some(vm.ctx.type_type())), - (start, Some(vm.ctx.int_type())), - (stop, Some(vm.ctx.int_type())) - ], - optional = [(step, Some(vm.ctx.int_type()))] - ); - Ok((cls, Some(start), Some(stop), step)) + let (start, stop, step): (PyObjectRef, PyObjectRef, OptionalArg) = + args.bind(vm)?; + PySlice { + start: Some(start), + stop, + step: step.into_option(), + } } - }?; - Ok(PyObject::new( - PySlice { - start: start.map(|x| objint::get_value(x).clone()), - stop: stop.map(|x| objint::get_value(x).clone()), - step: step.map(|x| objint::get_value(x).clone()), - }, - cls.clone(), - )) + }; + slice.into_ref_with_type(vm, cls) } -fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyResult { +fn get_property_value(vm: &VirtualMachine, value: &Option) -> PyObjectRef { if let Some(value) = value { - Ok(vm.ctx.new_int(value.clone())) + value.clone() } else { - Ok(vm.get_none()) + vm.get_none() } } -fn slice_start(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(slice, Some(vm.ctx.slice_type()))]); - if let Some(PySlice { start, .. }) = &slice.payload() { - get_property_value(vm, start) - } else { - panic!("Slice has incorrect payload."); +impl PySliceRef { + fn start(self, vm: &VirtualMachine) -> PyObjectRef { + get_property_value(vm, &self.start) } -} -fn slice_stop(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(slice, Some(vm.ctx.slice_type()))]); - if let Some(PySlice { stop, .. }) = &slice.payload() { - get_property_value(vm, stop) - } else { - panic!("Slice has incorrect payload."); + fn stop(self, _vm: &VirtualMachine) -> PyObjectRef { + self.stop.clone() + } + + fn step(self, vm: &VirtualMachine) -> PyObjectRef { + get_property_value(vm, &self.step) + } + + pub fn start_index(&self, vm: &VirtualMachine) -> PyResult> { + if let Some(obj) = &self.start { + to_index_value(vm, obj) + } else { + Ok(None) + } + } + + pub fn stop_index(&self, vm: &VirtualMachine) -> PyResult> { + to_index_value(vm, &self.stop) + } + + pub fn step_index(&self, vm: &VirtualMachine) -> PyResult> { + if let Some(obj) = &self.step { + to_index_value(vm, obj) + } else { + Ok(None) + } } } -fn slice_step(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(slice, Some(vm.ctx.slice_type()))]); - if let Some(PySlice { step, .. }) = &slice.payload() { - get_property_value(vm, step) +fn to_index_value(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult> { + if obj.is(&vm.ctx.none) { + return Ok(None); + } + + if let Some(val) = obj.payload::() { + Ok(Some(val.as_bigint().clone())) } else { - panic!("Slice has incorrect payload."); + let cls = obj.class(); + if class_has_attr(&cls, "__index__") { + let index_result = vm.call_method(obj, "__index__", vec![])?; + if let Some(val) = index_result.payload::() { + Ok(Some(val.as_bigint().clone())) + } else { + Err(vm.new_type_error("__index__ method returned non integer".to_string())) + } + } else { + Err(vm.new_type_error( + "slice indices must be integers or None or have an __index__ method".to_string(), + )) + } } } pub fn init(context: &PyContext) { - let zip_type = &context.slice_type; + let slice_type = &context.slice_type; - context.set_attr(zip_type, "__new__", context.new_rustfunc(slice_new)); - context.set_attr(zip_type, "start", context.new_property(slice_start)); - context.set_attr(zip_type, "stop", context.new_property(slice_stop)); - context.set_attr(zip_type, "step", context.new_property(slice_step)); + extend_class!(context, slice_type, { + "__new__" => context.new_rustfunc(slice_new), + "start" => context.new_property(PySliceRef::start), + "stop" => context.new_property(PySliceRef::stop), + "step" => context.new_property(PySliceRef::step) + }); } diff --git a/vm/src/obj/objstaticmethod.rs b/vm/src/obj/objstaticmethod.rs index f8430979da..a33acf176d 100644 --- a/vm/src/obj/objstaticmethod.rs +++ b/vm/src/obj/objstaticmethod.rs @@ -9,7 +9,7 @@ pub struct PyStaticMethod { pub type PyStaticMethodRef = PyRef; impl PyValue for PyStaticMethod { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.staticmethod_type() } } diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 826396925c..fafd23a65d 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -1,30 +1,72 @@ +extern crate unicode_categories; +extern crate unicode_xid; + +use std::char; use std::fmt; -use std::hash::{Hash, Hasher}; use std::ops::Range; use std::str::FromStr; use std::string::ToString; use num_traits::ToPrimitive; +use unicode_casing::CharExt; use unicode_segmentation::UnicodeSegmentation; +use unicode_xid::UnicodeXID; -use crate::format::{FormatParseError, FormatPart, FormatString}; -use crate::function::{OptionalArg, PyFuncArgs}; +use crate::cformat::{ + CFormatPart, CFormatPreconversor, CFormatQuantity, CFormatSpec, CFormatString, CFormatType, + CNumberType, +}; +use crate::format::{FormatParseError, FormatPart, FormatPreconversor, FormatString}; +use crate::function::{single_or_tuple_any, OptionalArg, PyFuncArgs}; +use crate::pyhash; use crate::pyobject::{ - IdProtocol, IntoPyObject, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, - TryFromObject, TryIntoRef, TypeProtocol, + IdProtocol, IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyIterable, PyObjectRef, PyRef, + PyResult, PyValue, TryFromObject, TryIntoRef, TypeProtocol, }; use crate::vm::VirtualMachine; -use super::objint; +use super::objbytes::PyBytes; +use super::objdict::PyDict; +use super::objint::{self, PyInt}; +use super::objnone::PyNone; use super::objsequence::PySliceableSequence; use super::objslice::PySlice; +use super::objtuple; use super::objtype::{self, PyClassRef}; +use unicode_categories::UnicodeCategories; + +/// str(object='') -> str +/// str(bytes_or_buffer[, encoding[, errors]]) -> str +/// +/// Create a new string object from the given object. If encoding or +/// errors is specified, then the object must expose a data buffer +/// that will be decoded using the given encoding and error handler. +/// Otherwise, returns the result of object.__str__() (if defined) +/// or repr(object). +/// encoding defaults to sys.getdefaultencoding(). +/// errors defaults to 'strict'." +#[pyclass(name = "str")] #[derive(Clone, Debug)] pub struct PyString { // TODO: shouldn't be public pub value: String, } + +impl PyString { + pub fn as_str(&self) -> &str { + &self.value + } +} + +impl From<&str> for PyString { + fn from(s: &str) -> PyString { + PyString { + value: s.to_string(), + } + } +} + pub type PyStringRef = PyRef; impl fmt::Display for PyString { @@ -48,8 +90,30 @@ impl TryIntoRef for &str { } } -impl PyStringRef { - fn add(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { +#[pyimpl] +impl PyString { + // TODO: should with following format + // class str(object='') + // class str(object=b'', encoding='utf-8', errors='strict') + #[pymethod(name = "__new__")] + fn new( + cls: PyClassRef, + object: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let string = match object { + OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), + OptionalArg::Missing => vm.new_str("".to_string()), + }; + if string.class().is(&cls) { + TryFromObject::try_from_object(vm, string) + } else { + let payload = string.payload::().unwrap(); + payload.clone().into_ref_with_type(vm, cls) + } + } + #[pymethod(name = "__add__")] + fn add(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(format!("{}{}", self.value, get_value(&rhs))) } else { @@ -57,7 +121,13 @@ impl PyStringRef { } } - fn eq(self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { + #[pymethod(name = "__bool__")] + fn bool(&self, _vm: &VirtualMachine) -> bool { + !self.value.is_empty() + } + + #[pymethod(name = "__eq__")] + fn eq(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> bool { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { self.value == get_value(&rhs) } else { @@ -65,15 +135,18 @@ impl PyStringRef { } } - fn contains(self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { + #[pymethod(name = "__contains__")] + fn contains(&self, needle: PyStringRef, _vm: &VirtualMachine) -> bool { self.value.contains(&needle.value) } - fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__getitem__")] + fn getitem(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { subscript(vm, &self.value, needle) } - fn gt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__gt__")] + fn gt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value > get_value(&rhs)) } else { @@ -81,7 +154,8 @@ impl PyStringRef { } } - fn ge(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__ge__")] + fn ge(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value >= get_value(&rhs)) } else { @@ -89,7 +163,8 @@ impl PyStringRef { } } - fn lt(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__lt__")] + fn lt(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value < get_value(&rhs)) } else { @@ -97,7 +172,8 @@ impl PyStringRef { } } - fn le(self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { + #[pymethod(name = "__le__")] + fn le(&self, rhs: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&rhs, &vm.ctx.str_type()) { Ok(self.value <= get_value(&rhs)) } else { @@ -105,42 +181,50 @@ impl PyStringRef { } } - fn hash(self, _vm: &VirtualMachine) -> usize { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - self.value.hash(&mut hasher); - hasher.finish() as usize + #[pymethod(name = "__hash__")] + fn hash(&self, _vm: &VirtualMachine) -> pyhash::PyHash { + pyhash::hash_value(&self.value) } - fn len(self, _vm: &VirtualMachine) -> usize { + #[pymethod(name = "__len__")] + fn len(&self, _vm: &VirtualMachine) -> usize { self.value.chars().count() } - fn mul(self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { - if objtype::isinstance(&val, &vm.ctx.int_type()) { - let value = &self.value; - let multiplier = objint::get_value(&val).to_i32().unwrap(); - let mut result = String::new(); - for _x in 0..multiplier { - result.push_str(value.as_str()); - } - Ok(result) - } else { - Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))) + #[pymethod(name = "__mul__")] + fn mul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if !objtype::isinstance(&val, &vm.ctx.int_type()) { + return Err(vm.new_type_error(format!("Cannot multiply {} and {}", self, val))); } + objint::get_value(&val) + .to_isize() + .map(|multiplier| multiplier.max(0)) + .and_then(|multiplier| multiplier.to_usize()) + .map(|multiplier| self.value.repeat(multiplier)) + .ok_or_else(|| { + vm.new_overflow_error("cannot fit 'int' into an index-sized integer".to_string()) + }) } - fn str(self, _vm: &VirtualMachine) -> PyStringRef { - self + #[pymethod(name = "__rmul__")] + fn rmul(&self, val: PyObjectRef, vm: &VirtualMachine) -> PyResult { + self.mul(val, vm) } - fn repr(self, _vm: &VirtualMachine) -> String { + #[pymethod(name = "__str__")] + fn str(zelf: PyRef, _vm: &VirtualMachine) -> PyStringRef { + zelf + } + + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { let value = &self.value; let quote_char = if count_char(value, '\'') > count_char(value, '"') { '"' } else { '\'' }; - let mut formatted = String::new(); + let mut formatted = String::with_capacity(value.len()); formatted.push(quote_char); for c in value.chars() { if c == quote_char || c == '\\' { @@ -163,27 +247,32 @@ impl PyStringRef { formatted } - fn lower(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn lower(&self, _vm: &VirtualMachine) -> String { self.value.to_lowercase() } // casefold is much more aggressive than lower - fn casefold(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn casefold(&self, _vm: &VirtualMachine) -> String { caseless::default_case_fold_str(&self.value) } - fn upper(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn upper(&self, _vm: &VirtualMachine) -> String { self.value.to_uppercase() } - fn capitalize(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn capitalize(&self, _vm: &VirtualMachine) -> String { let (first_part, lower_str) = self.value.split_at(1); format!("{}{}", first_part.to_uppercase(), lower_str) } + #[pymethod] fn split( - self, - pattern: OptionalArg, + &self, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { @@ -202,9 +291,10 @@ impl PyStringRef { vm.ctx.new_list(elements) } + #[pymethod] fn rsplit( - self, - pattern: OptionalArg, + &self, + pattern: OptionalArg, num: OptionalArg, vm: &VirtualMachine, ) -> PyObjectRef { @@ -216,62 +306,111 @@ impl PyStringRef { let num_splits = num .into_option() .unwrap_or_else(|| value.split(pattern).count()); - let elements = value + let mut elements: Vec<_> = value .rsplitn(num_splits + 1, pattern) .map(|o| vm.ctx.new_str(o.to_string())) .collect(); + // Unlike Python rsplit, Rust rsplitn returns an iterator that + // starts from the end of the string. + elements.reverse(); vm.ctx.new_list(elements) } - fn strip(self, _vm: &VirtualMachine) -> String { - self.value.trim().to_string() + #[pymethod] + fn strip(&self, chars: OptionalArg, _vm: &VirtualMachine) -> String { + let chars = match chars { + OptionalArg::Present(ref chars) => &chars.value, + OptionalArg::Missing => return self.value.trim().to_string(), + }; + self.value.trim_matches(|c| chars.contains(c)).to_string() } - fn lstrip(self, _vm: &VirtualMachine) -> String { - self.value.trim_start().to_string() + #[pymethod] + fn lstrip(&self, chars: OptionalArg, _vm: &VirtualMachine) -> String { + let chars = match chars { + OptionalArg::Present(ref chars) => &chars.value, + OptionalArg::Missing => return self.value.trim_start().to_string(), + }; + self.value + .trim_start_matches(|c| chars.contains(c)) + .to_string() } - fn rstrip(self, _vm: &VirtualMachine) -> String { - self.value.trim_end().to_string() + #[pymethod] + fn rstrip(&self, chars: OptionalArg, _vm: &VirtualMachine) -> String { + let chars = match chars { + OptionalArg::Present(ref chars) => &chars.value, + OptionalArg::Missing => return self.value.trim_end().to_string(), + }; + self.value + .trim_end_matches(|c| chars.contains(c)) + .to_string() } + #[pymethod] fn endswith( - self, - suffix: PyStringRef, + &self, + suffix: PyObjectRef, start: OptionalArg, end: OptionalArg, - _vm: &VirtualMachine, - ) -> bool { + vm: &VirtualMachine, + ) -> PyResult { if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].ends_with(&suffix.value) + let value = &self.value[start..end]; + single_or_tuple_any( + suffix, + |s: PyStringRef| Ok(value.ends_with(&s.value)), + |o| { + format!( + "endswith first arg must be str or a tuple of str, not {}", + o.class(), + ) + }, + vm, + ) } else { - false + Ok(false) } } + #[pymethod] fn startswith( - self, - prefix: PyStringRef, + &self, + prefix: PyObjectRef, start: OptionalArg, end: OptionalArg, - _vm: &VirtualMachine, - ) -> bool { + vm: &VirtualMachine, + ) -> PyResult { if let Some((start, end)) = adjust_indices(start, end, self.value.len()) { - self.value[start..end].starts_with(&prefix.value) + let value = &self.value[start..end]; + single_or_tuple_any( + prefix, + |s: PyStringRef| Ok(value.starts_with(&s.value)), + |o| { + format!( + "startswith first arg must be str or a tuple of str, not {}", + o.class(), + ) + }, + vm, + ) } else { - false + Ok(false) } } - fn isalnum(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isalnum(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } - fn isnumeric(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isnumeric(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_numeric) } - fn isdigit(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isdigit(&self, _vm: &VirtualMachine) -> bool { // python's isdigit also checks if exponents are digits, these are the unicodes for exponents let valid_unicodes: [u16; 10] = [ 0x2070, 0x00B9, 0x00B2, 0x00B3, 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, @@ -287,7 +426,8 @@ impl PyStringRef { } } - fn isdecimal(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isdecimal(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { false } else { @@ -295,11 +435,74 @@ impl PyStringRef { } } - fn title(self, _vm: &VirtualMachine) -> String { - make_title(&self.value) + #[pymethod(name = "__mod__")] + fn modulo(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let format_string_text = &self.value; + let format_string = CFormatString::from_str(format_string_text) + .map_err(|err| vm.new_value_error(format!("{}", err)))?; + do_cformat(vm, format_string, values.clone()) + } + + #[pymethod] + fn format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + if args.args.is_empty() { + return Err(vm.new_type_error( + "descriptor 'format' of 'str' object needs an argument".to_string(), + )); + } + + let zelf = &args.args[0]; + if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { + let zelf_typ = zelf.class(); + let actual_type = vm.to_pystr(&zelf_typ)?; + return Err(vm.new_type_error(format!( + "descriptor 'format' requires a 'str' object but received a '{}'", + actual_type + ))); + } + let format_string_text = get_value(zelf); + match FormatString::from_str(format_string_text.as_str()) { + Ok(format_string) => perform_format(vm, &format_string, &args), + Err(err) => match err { + FormatParseError::UnmatchedBracket => { + Err(vm.new_value_error("expected '}' before end of string".to_string())) + } + _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), + }, + } + } + + /// Return a titlecased version of the string where words start with an + /// uppercase character and the remaining characters are lowercase. + #[pymethod] + fn title(&self, _vm: &VirtualMachine) -> String { + let mut title = String::with_capacity(self.value.len()); + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_lowercase() { + if !previous_is_cased { + title.extend(c.to_titlecase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else if c.is_uppercase() || c.is_titlecase() { + if previous_is_cased { + title.extend(c.to_lowercase()); + } else { + title.push(c); + } + previous_is_cased = true; + } else { + previous_is_cased = false; + title.push(c); + } + } + title } - fn swapcase(self, _vm: &VirtualMachine) -> String { + #[pymethod] + fn swapcase(&self, _vm: &VirtualMachine) -> String { let mut swapped_str = String::with_capacity(self.value.len()); for c in self.value.chars() { // to_uppercase returns an iterator, to_ascii_uppercase returns the char @@ -314,14 +517,16 @@ impl PyStringRef { swapped_str } - fn isalpha(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isalpha(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(char::is_alphanumeric) } + #[pymethod] fn replace( - self, - old: Self, - new: Self, + &self, + old: PyStringRef, + new: PyStringRef, num: OptionalArg, _vm: &VirtualMachine, ) -> String { @@ -331,13 +536,38 @@ impl PyStringRef { } } + /// Return true if all characters in the string are printable or the string is empty, + /// false otherwise. Nonprintable characters are those characters defined in the + /// Unicode character database as `Other` or `Separator`, + /// excepting the ASCII space (0x20) which is considered printable. + /// + /// All characters except those characters defined in the Unicode character + /// database as following categories are considered printable. + /// * Cc (Other, Control) + /// * Cf (Other, Format) + /// * Cs (Other, Surrogate) + /// * Co (Other, Private Use) + /// * Cn (Other, Not Assigned) + /// * Zl Separator, Line ('\u2028', LINE SEPARATOR) + /// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR) + /// * Zs (Separator, Space) other than ASCII space('\x20'). + #[pymethod] + fn isprintable(&self, _vm: &VirtualMachine) -> bool { + self.value.chars().all(|c| match c { + '\u{0020}' => true, + _ => !(c.is_other_control() | c.is_separator()), + }) + } + // cpython's isspace ignores whitespace, including \t and \n, etc, unless the whole string is empty // which is why isspace is using is_ascii_whitespace. Same for isupper & islower - fn isspace(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isspace(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii_whitespace()) } - fn isupper(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isupper(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -346,7 +576,8 @@ impl PyStringRef { .all(char::is_uppercase) } - fn islower(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn islower(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self .value @@ -355,12 +586,14 @@ impl PyStringRef { .all(char::is_lowercase) } - fn isascii(self, _vm: &VirtualMachine) -> bool { + #[pymethod] + fn isascii(&self, _vm: &VirtualMachine) -> bool { !self.value.is_empty() && self.value.chars().all(|c| c.is_ascii()) } // doesn't implement keep new line delimiter just yet - fn splitlines(self, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod] + fn splitlines(&self, vm: &VirtualMachine) -> PyObjectRef { let elements = self .value .split('\n') @@ -369,7 +602,8 @@ impl PyStringRef { vm.ctx.new_list(elements) } - fn join(self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn join(&self, iterable: PyIterable, vm: &VirtualMachine) -> PyResult { let mut joined = String::new(); for (idx, elem) in iterable.iter(vm)?.enumerate() { @@ -383,9 +617,10 @@ impl PyStringRef { Ok(joined) } + #[pymethod] fn find( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, @@ -401,9 +636,10 @@ impl PyStringRef { } } + #[pymethod] fn rfind( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, @@ -419,9 +655,10 @@ impl PyStringRef { } } + #[pymethod] fn index( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, @@ -437,9 +674,10 @@ impl PyStringRef { } } + #[pymethod] fn rindex( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, vm: &VirtualMachine, @@ -455,7 +693,8 @@ impl PyStringRef { } } - fn partition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod] + fn partition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -473,7 +712,8 @@ impl PyStringRef { vm.ctx.new_tuple(new_tup) } - fn rpartition(self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { + #[pymethod] + fn rpartition(&self, sub: PyStringRef, vm: &VirtualMachine) -> PyObjectRef { let value = &self.value; let sub = &sub.value; let mut new_tup = Vec::new(); @@ -485,24 +725,47 @@ impl PyStringRef { new_tup.swap(0, 1); // so it's in the right order new_tup.insert(1, vm.ctx.new_str(sub.clone())); } else { - new_tup.push(vm.ctx.new_str(value.clone())); new_tup.push(vm.ctx.new_str("".to_string())); new_tup.push(vm.ctx.new_str("".to_string())); + new_tup.push(vm.ctx.new_str(value.clone())); } vm.ctx.new_tuple(new_tup) } - fn istitle(self, _vm: &VirtualMachine) -> bool { + /// Return `true` if the sequence is ASCII titlecase and the sequence is not + /// empty, `false` otherwise. + #[pymethod] + fn istitle(&self, _vm: &VirtualMachine) -> bool { if self.value.is_empty() { - false - } else { - self.value.split(' ').all(|word| word == make_title(word)) + return false; + } + + let mut cased = false; + let mut previous_is_cased = false; + for c in self.value.chars() { + if c.is_uppercase() || c.is_titlecase() { + if previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else if c.is_lowercase() { + if !previous_is_cased { + return false; + } + previous_is_cased = true; + cased = true; + } else { + previous_is_cased = false; + } } + cased } + #[pymethod] fn count( - self, - sub: Self, + &self, + sub: PyStringRef, start: OptionalArg, end: OptionalArg, _vm: &VirtualMachine, @@ -515,7 +778,8 @@ impl PyStringRef { } } - fn zfill(self, len: usize, _vm: &VirtualMachine) -> String { + #[pymethod] + fn zfill(&self, len: usize, _vm: &VirtualMachine) -> String { let value = &self.value; if len <= value.len() { value.to_string() @@ -524,7 +788,10 @@ impl PyStringRef { } } - fn get_fill_char<'a>(rep: &'a OptionalArg, vm: &VirtualMachine) -> PyResult<&'a str> { + fn get_fill_char<'a>( + rep: &'a OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<&'a str> { let rep_str = match rep { OptionalArg::Present(ref st) => &st.value, OptionalArg::Missing => " ", @@ -538,23 +805,63 @@ impl PyStringRef { } } - fn ljust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn ljust( + &self, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; - Ok(format!("{}{}", value, rep_char.repeat(len))) + let rep_char = Self::get_fill_char(&rep, vm)?; + if len <= value.len() { + Ok(value.to_string()) + } else { + Ok(format!("{}{}", value, rep_char.repeat(len - value.len()))) + } } - fn rjust(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn rjust( + &self, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; - Ok(format!("{}{}", rep_char.repeat(len), value)) + let rep_char = Self::get_fill_char(&rep, vm)?; + if len <= value.len() { + Ok(value.to_string()) + } else { + Ok(format!("{}{}", rep_char.repeat(len - value.len()), value)) + } } - fn center(self, len: usize, rep: OptionalArg, vm: &VirtualMachine) -> PyResult { + #[pymethod] + fn center( + &self, + len: usize, + rep: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { let value = &self.value; - let rep_char = PyStringRef::get_fill_char(&rep, vm)?; - let left_buff: usize = (len - value.len()) / 2; - let right_buff = len - value.len() - left_buff; + let rep_char = Self::get_fill_char(&rep, vm)?; + let value_len = self.value.chars().count(); + + if len <= value_len { + return Ok(value.to_string()); + } + let diff: usize = len - value_len; + let mut left_buff: usize = diff / 2; + let mut right_buff: usize = left_buff; + + if diff % 2 != 0 && value_len % 2 == 0 { + left_buff += 1 + } + + if diff % 2 != 0 && value_len % 2 != 0 { + right_buff += 1 + } Ok(format!( "{}{}{}", rep_char.repeat(left_buff), @@ -563,9 +870,10 @@ impl PyStringRef { )) } - fn expandtabs(self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { + #[pymethod] + fn expandtabs(&self, tab_stop: OptionalArg, _vm: &VirtualMachine) -> String { let tab_stop = tab_stop.into_option().unwrap_or(8 as usize); - let mut expanded_str = String::new(); + let mut expanded_str = String::with_capacity(self.value.len()); let mut tab_size = tab_stop; let mut col_count = 0 as usize; for ch in self.value.chars() { @@ -586,26 +894,141 @@ impl PyStringRef { expanded_str } - fn isidentifier(self, _vm: &VirtualMachine) -> bool { - let value = &self.value; + #[pymethod] + fn isidentifier(&self, _vm: &VirtualMachine) -> bool { + let mut chars = self.value.chars(); + let is_identifier_start = match chars.next() { + Some('_') => true, + Some(c) => UnicodeXID::is_xid_start(c), + None => false, + }; // a string is not an identifier if it has whitespace or starts with a number - if !value.chars().any(|c| c.is_ascii_whitespace()) - && !value.chars().nth(0).unwrap().is_digit(10) - { - for c in value.chars() { - if c != "_".chars().nth(0).unwrap() && !c.is_digit(10) && !c.is_alphabetic() { - return false; + is_identifier_start && chars.all(UnicodeXID::is_xid_continue) + } + + // https://docs.python.org/3/library/stdtypes.html#str.translate + #[pymethod] + fn translate(&self, table: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.get_method_or_type_error(table.clone(), "__getitem__", || { + format!("'{}' object is not subscriptable", table.class().name) + })?; + + let mut translated = String::new(); + for c in self.value.chars() { + match table.get_item(c as u32, vm) { + Ok(value) => { + if let Some(text) = value.payload::() { + translated.push_str(&text.value); + } else if let Some(bigint) = value.payload::() { + match bigint.as_bigint().to_u32().and_then(std::char::from_u32) { + Some(ch) => translated.push(ch as char), + None => { + return Err(vm.new_value_error( + "character mapping must be in range(0x110000)".to_owned(), + )); + } + } + } else if let Some(_) = value.payload::() { + // Do Nothing + } else { + return Err(vm.new_type_error( + "character mapping must return integer, None or str".to_owned(), + )); + } } + _ => translated.push(c), + } + } + Ok(translated) + } + + #[pymethod] + fn maketrans( + dict_or_str: PyObjectRef, + to_str: OptionalArg, + none_str: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let new_dict = vm.context().new_dict(); + if let OptionalArg::Present(to_str) = to_str { + match dict_or_str.downcast::() { + Ok(from_str) => { + if to_str.len(vm) == from_str.len(vm) { + for (c1, c2) in from_str.value.chars().zip(to_str.value.chars()) { + new_dict.set_item(c1 as u32, vm.new_int(c2 as u32), vm)?; + } + if let OptionalArg::Present(none_str) = none_str { + for c in none_str.value.chars() { + new_dict.set_item(c as u32, vm.get_none(), vm)?; + } + } + new_dict.into_pyobject(vm) + } else { + Err(vm.new_value_error( + "the first two maketrans arguments must have equal length".to_owned(), + )) + } + } + _ => Err(vm.new_type_error( + "first maketrans argument must be a string if there is a second argument" + .to_owned(), + )), } - true } else { - false + // dict_str must be a dict + match dict_or_str.downcast::() { + Ok(dict) => { + for (key, val) in dict { + if let Some(num) = key.payload::() { + new_dict.set_item(num.as_bigint().to_i32(), val, vm)?; + } else if let Some(string) = key.payload::() { + if string.len(vm) == 1 { + let num_value = string.value.chars().next().unwrap() as u32; + new_dict.set_item(num_value, val, vm)?; + } else { + return Err(vm.new_value_error( + "string keys in translate table must be of length 1".to_owned(), + )); + } + } + } + new_dict.into_pyobject(vm) + } + _ => Err(vm.new_value_error( + "if you give only one argument to maketrans it must be a dict".to_owned(), + )), + } } } + + #[pymethod] + fn encode( + &self, + encoding: OptionalArg, + _errors: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let encoding = encoding.map_or_else( + || Ok("utf-8".to_string()), + |v| { + if objtype::isinstance(&v, &vm.ctx.str_type()) { + Ok(get_value(&v)) + } else { + Err(vm.new_type_error(format!( + "encode() argument 1 must be str, not {}", + v.class().name + ))) + } + }, + )?; + + let encoded = PyBytes::from_string(&self.value, &encoding, vm)?; + Ok(encoded.into_pyobject(vm)?) + } } impl PyValue for PyString { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.str_type() } } @@ -616,74 +1039,20 @@ impl IntoPyObject for String { } } -#[rustfmt::skip] // to avoid line splitting -pub fn init(context: &PyContext) { - let str_type = &context.str_type; - let str_doc = "str(object='') -> str\n\ - str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ - \n\ - Create a new string object from the given object. If encoding or\n\ - errors is specified, then the object must expose a data buffer\n\ - that will be decoded using the given encoding and error handler.\n\ - Otherwise, returns the result of object.__str__() (if defined)\n\ - or repr(object).\n\ - encoding defaults to sys.getdefaultencoding().\n\ - errors defaults to 'strict'."; - context.set_attr(&str_type, "__add__", context.new_rustfunc(PyStringRef::add)); - context.set_attr(&str_type, "__eq__", context.new_rustfunc(PyStringRef::eq)); - context.set_attr(&str_type, "__contains__", context.new_rustfunc(PyStringRef::contains)); - context.set_attr(&str_type, "__getitem__", context.new_rustfunc(PyStringRef::getitem)); - context.set_attr(&str_type, "__gt__", context.new_rustfunc(PyStringRef::gt)); - context.set_attr(&str_type, "__ge__", context.new_rustfunc(PyStringRef::ge)); - context.set_attr(&str_type, "__lt__", context.new_rustfunc(PyStringRef::lt)); - context.set_attr(&str_type, "__le__", context.new_rustfunc(PyStringRef::le)); - context.set_attr(&str_type, "__hash__", context.new_rustfunc(PyStringRef::hash)); - context.set_attr(&str_type, "__len__", context.new_rustfunc(PyStringRef::len)); - context.set_attr(&str_type, "__mul__", context.new_rustfunc(PyStringRef::mul)); - context.set_attr(&str_type, "__new__", context.new_rustfunc(str_new)); - context.set_attr(&str_type, "__str__", context.new_rustfunc(PyStringRef::str)); - context.set_attr(&str_type, "__repr__", context.new_rustfunc(PyStringRef::repr)); - context.set_attr(&str_type, "format", context.new_rustfunc(str_format)); - context.set_attr(&str_type, "lower", context.new_rustfunc(PyStringRef::lower)); - context.set_attr(&str_type, "casefold", context.new_rustfunc(PyStringRef::casefold)); - context.set_attr(&str_type, "upper", context.new_rustfunc(PyStringRef::upper)); - context.set_attr(&str_type, "capitalize", context.new_rustfunc(PyStringRef::capitalize)); - context.set_attr(&str_type, "split", context.new_rustfunc(PyStringRef::split)); - context.set_attr(&str_type, "rsplit", context.new_rustfunc(PyStringRef::rsplit)); - context.set_attr(&str_type, "strip", context.new_rustfunc(PyStringRef::strip)); - context.set_attr(&str_type, "lstrip", context.new_rustfunc(PyStringRef::lstrip)); - context.set_attr(&str_type, "rstrip", context.new_rustfunc(PyStringRef::rstrip)); - context.set_attr(&str_type, "endswith", context.new_rustfunc(PyStringRef::endswith)); - context.set_attr(&str_type, "startswith", context.new_rustfunc(PyStringRef::startswith)); - context.set_attr(&str_type, "isalnum", context.new_rustfunc(PyStringRef::isalnum)); - context.set_attr(&str_type, "isnumeric", context.new_rustfunc(PyStringRef::isnumeric)); - context.set_attr(&str_type, "isdigit", context.new_rustfunc(PyStringRef::isdigit)); - context.set_attr(&str_type, "isdecimal", context.new_rustfunc(PyStringRef::isdecimal)); - context.set_attr(&str_type, "title", context.new_rustfunc(PyStringRef::title)); - context.set_attr(&str_type, "swapcase", context.new_rustfunc(PyStringRef::swapcase)); - context.set_attr(&str_type, "isalpha", context.new_rustfunc(PyStringRef::isalpha)); - context.set_attr(&str_type, "replace", context.new_rustfunc(PyStringRef::replace)); - context.set_attr(&str_type, "isspace", context.new_rustfunc(PyStringRef::isspace)); - context.set_attr(&str_type, "isupper", context.new_rustfunc(PyStringRef::isupper)); - context.set_attr(&str_type, "islower", context.new_rustfunc(PyStringRef::islower)); - context.set_attr(&str_type, "isascii", context.new_rustfunc(PyStringRef::isascii)); - context.set_attr(&str_type, "splitlines", context.new_rustfunc(PyStringRef::splitlines)); - context.set_attr(&str_type, "join", context.new_rustfunc(PyStringRef::join)); - context.set_attr(&str_type, "find", context.new_rustfunc(PyStringRef::find)); - context.set_attr(&str_type, "rfind", context.new_rustfunc(PyStringRef::rfind)); - context.set_attr(&str_type, "index", context.new_rustfunc(PyStringRef::index)); - context.set_attr(&str_type, "rindex", context.new_rustfunc(PyStringRef::rindex)); - context.set_attr(&str_type, "partition", context.new_rustfunc(PyStringRef::partition)); - context.set_attr(&str_type, "rpartition", context.new_rustfunc(PyStringRef::rpartition)); - context.set_attr(&str_type, "istitle", context.new_rustfunc(PyStringRef::istitle)); - context.set_attr(&str_type, "count", context.new_rustfunc(PyStringRef::count)); - context.set_attr(&str_type, "zfill", context.new_rustfunc(PyStringRef::zfill)); - context.set_attr(&str_type, "ljust", context.new_rustfunc(PyStringRef::ljust)); - context.set_attr(&str_type, "rjust", context.new_rustfunc(PyStringRef::rjust)); - context.set_attr(&str_type, "center", context.new_rustfunc(PyStringRef::center)); - context.set_attr(&str_type, "expandtabs", context.new_rustfunc(PyStringRef::expandtabs)); - context.set_attr(&str_type, "isidentifier", context.new_rustfunc(PyStringRef::isidentifier)); - context.set_attr(&str_type, "__doc__", context.new_str(str_doc.to_string())); +impl IntoPyObject for &str { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.to_string())) + } +} + +impl IntoPyObject for &String { + fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_str(self.clone())) + } +} + +pub fn init(ctx: &PyContext) { + PyString::extend_class(ctx, &ctx.str_type); } pub fn get_value(obj: &PyObjectRef) -> String { @@ -698,45 +1067,234 @@ fn count_char(s: &str, c: char) -> usize { s.chars().filter(|x| *x == c).count() } -fn str_format(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - if args.args.is_empty() { - return Err( - vm.new_type_error("descriptor 'format' of 'str' object needs an argument".to_string()) - ); - } - - let zelf = &args.args[0]; - if !objtype::isinstance(&zelf, &vm.ctx.str_type()) { - let zelf_typ = zelf.typ(); - let actual_type = vm.to_pystr(&zelf_typ)?; - return Err(vm.new_type_error(format!( - "descriptor 'format' requires a 'str' object but received a '{}'", - actual_type - ))); - } - let format_string_text = get_value(zelf); - match FormatString::from_str(format_string_text.as_str()) { - Ok(format_string) => perform_format(vm, &format_string, &args), - Err(err) => match err { - FormatParseError::UnmatchedBracket => { - Err(vm.new_value_error("expected '}' before end of string".to_string())) - } - _ => Err(vm.new_value_error("Unexpected error parsing format string".to_string())), - }, - } +fn call_getitem(vm: &VirtualMachine, container: &PyObjectRef, key: &PyObjectRef) -> PyResult { + vm.call_method(container, "__getitem__", vec![key.clone()]) } fn call_object_format(vm: &VirtualMachine, argument: PyObjectRef, format_spec: &str) -> PyResult { - let returned_type = vm.ctx.new_str(format_spec.to_string()); + let (preconversor, new_format_spec) = FormatPreconversor::parse_and_consume(format_spec); + let argument = match preconversor { + Some(FormatPreconversor::Str) => vm.call_method(&argument, "__str__", vec![])?, + Some(FormatPreconversor::Repr) => vm.call_method(&argument, "__repr__", vec![])?, + Some(FormatPreconversor::Ascii) => vm.call_method(&argument, "__repr__", vec![])?, + None => argument, + }; + let returned_type = vm.ctx.new_str(new_format_spec.to_string()); + let result = vm.call_method(&argument, "__format__", vec![returned_type])?; if !objtype::isinstance(&result, &vm.ctx.str_type()) { - let result_type = result.typ(); + let result_type = result.class(); let actual_type = vm.to_pystr(&result_type)?; return Err(vm.new_type_error(format!("__format__ must return a str, not {}", actual_type))); } Ok(result) } +fn do_cformat_specifier( + vm: &VirtualMachine, + format_spec: &mut CFormatSpec, + obj: PyObjectRef, +) -> Result { + use CNumberType::*; + // do the formatting by type + let format_type = &format_spec.format_type; + + match format_type { + CFormatType::String(preconversor) => { + let result = match preconversor { + CFormatPreconversor::Str => vm.call_method(&obj.clone(), "__str__", vec![])?, + CFormatPreconversor::Repr => vm.call_method(&obj.clone(), "__repr__", vec![])?, + CFormatPreconversor::Ascii => vm.call_method(&obj.clone(), "__repr__", vec![])?, + }; + Ok(format_spec.format_string(get_value(&result))) + } + CFormatType::Number(_) => { + if !objtype::isinstance(&obj, &vm.ctx.int_type()) { + let required_type_string = match format_type { + CFormatType::Number(Decimal) => "a number", + CFormatType::Number(_) => "an integer", + _ => unreachable!(), + }; + return Err(vm.new_type_error(format!( + "%{} format: {} is required, not {}", + format_spec.format_char, + required_type_string, + obj.class() + ))); + } + Ok(format_spec.format_number(objint::get_value(&obj))) + } + CFormatType::Character => { + let char_string = { + if objtype::isinstance(&obj, &vm.ctx.int_type()) { + // BigInt truncation is fine in this case because only the unicode range is relevant + match objint::get_value(&obj).to_u32().and_then(char::from_u32) { + Some(value) => Ok(value.to_string()), + None => { + Err(vm.new_overflow_error("%c arg not in range(0x110000)".to_string())) + } + } + } else if objtype::isinstance(&obj, &vm.ctx.str_type()) { + let s: String = get_value(&obj); + let num_chars = s.chars().count(); + if num_chars != 1 { + Err(vm.new_type_error("%c requires int or char".to_string())) + } else { + Ok(s.chars().next().unwrap().to_string()) + } + } else { + // TODO re-arrange this block so this error is only created once + Err(vm.new_type_error("%c requires int or char".to_string())) + } + }?; + format_spec.precision = Some(CFormatQuantity::Amount(1)); + Ok(format_spec.format_string(char_string)) + } + _ => Err(vm.new_not_implemented_error(format!( + "Not yet implemented for %{}", + format_spec.format_char + ))), + } +} + +fn try_update_quantity_from_tuple( + vm: &VirtualMachine, + elements: &mut Iterator, + q: &mut Option, + mut tuple_index: usize, +) -> PyResult { + match q { + Some(CFormatQuantity::FromValuesTuple) => { + match elements.next() { + Some(width_obj) => { + tuple_index += 1; + if !objtype::isinstance(&width_obj, &vm.ctx.int_type()) { + Err(vm.new_type_error("* wants int".to_string())) + } else { + // TODO: handle errors when truncating BigInt to usize + *q = Some(CFormatQuantity::Amount( + objint::get_value(&width_obj).to_usize().unwrap(), + )); + Ok(tuple_index) + } + } + None => { + Err(vm.new_type_error("not enough arguments for format string".to_string())) + } + } + } + _ => Ok(tuple_index), + } +} + +fn do_cformat( + vm: &VirtualMachine, + mut format_string: CFormatString, + values_obj: PyObjectRef, +) -> PyResult { + let mut final_string = String::new(); + let num_specifiers = format_string + .format_parts + .iter() + .filter(|(_, part)| CFormatPart::is_specifier(part)) + .count(); + let mapping_required = format_string + .format_parts + .iter() + .any(|(_, part)| CFormatPart::has_key(part)) + && format_string + .format_parts + .iter() + .filter(|(_, part)| CFormatPart::is_specifier(part)) + .all(|(_, part)| CFormatPart::has_key(part)); + + let values_obj = if mapping_required { + if !objtype::isinstance(&values_obj, &vm.ctx.dict_type()) { + return Err(vm.new_type_error("format requires a mapping".to_string())); + } + values_obj.clone() + } else { + // check for only literal parts, in which case only empty tuple is allowed + if 0 == num_specifiers + && (!objtype::isinstance(&values_obj, &vm.ctx.tuple_type()) + || !objtuple::get_value(&values_obj).is_empty()) + { + return Err(vm.new_type_error( + "not all arguments converted during string formatting".to_string(), + )); + } + + // convert `values_obj` to a new tuple if it's not a tuple + if !objtype::isinstance(&values_obj, &vm.ctx.tuple_type()) { + vm.ctx.new_tuple(vec![values_obj.clone()]) + } else { + values_obj.clone() + } + }; + + let mut tuple_index: usize = 0; + for (_, part) in &mut format_string.format_parts { + let result_string: String = match part { + CFormatPart::Spec(format_spec) => { + // try to get the object + let obj: PyObjectRef = match &format_spec.mapping_key { + Some(key) => { + // TODO: change the KeyError message to match the one in cpython + call_getitem(vm, &values_obj, &vm.ctx.new_str(key.to_string()))? + } + None => { + let mut elements = objtuple::get_value(&values_obj) + .into_iter() + .skip(tuple_index); + + tuple_index = try_update_quantity_from_tuple( + vm, + &mut elements, + &mut format_spec.min_field_width, + tuple_index, + )?; + tuple_index = try_update_quantity_from_tuple( + vm, + &mut elements, + &mut format_spec.precision, + tuple_index, + )?; + + let obj = match elements.next() { + Some(obj) => Ok(obj), + None => Err(vm.new_type_error( + "not enough arguments for format string".to_string(), + )), + }?; + tuple_index += 1; + + obj + } + }; + do_cformat_specifier(vm, format_spec, obj) + } + CFormatPart::Literal(literal) => Ok(literal.clone()), + }?; + final_string.push_str(&result_string); + } + + // check that all arguments were converted + if !mapping_required { + if objtuple::get_value(&values_obj) + .into_iter() + .skip(tuple_index) + .next() + .is_some() + { + return Err(vm.new_type_error( + "not all arguments converted during string formatting".to_string(), + )); + } + } + + Ok(vm.ctx.new_str(final_string)) +} + fn perform_format( vm: &VirtualMachine, format_string: &FormatString, @@ -777,7 +1335,7 @@ fn perform_format( let result = match arguments.get_optional_kwarg(&keyword) { Some(argument) => call_object_format(vm, argument.clone(), &format_spec)?, None => { - return Err(vm.new_key_error(format!("'{}'", keyword))); + return Err(vm.new_key_error(vm.new_str(keyword.to_string()))); } }; get_value(&result) @@ -789,26 +1347,6 @@ fn perform_format( Ok(vm.ctx.new_str(final_string)) } -// TODO: should with following format -// class str(object='') -// class str(object=b'', encoding='utf-8', errors='strict') -fn str_new( - cls: PyClassRef, - object: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let string = match object { - OptionalArg::Present(ref input) => vm.to_str(input)?.into_object(), - OptionalArg::Missing => vm.new_str("".to_string()), - }; - if string.typ().is(&cls) { - TryFromObject::try_from_object(vm, string) - } else { - let payload = string.payload::().unwrap(); - payload.clone().into_ref_with_type(vm, cls) - } -} - impl PySliceableSequence for String { fn do_slice(&self, range: Range) -> Self { to_graphemes(self) @@ -926,22 +1464,83 @@ fn adjust_indices( } } -// helper function to title strings -fn make_title(s: &str) -> String { - let mut titled_str = String::new(); - let mut capitalize_char: bool = true; - for c in s.chars() { - if c.is_alphabetic() { - if !capitalize_char { - titled_str.push(c); - } else if capitalize_char { - titled_str.push(c.to_ascii_uppercase()); - capitalize_char = false; - } - } else { - titled_str.push(c); - capitalize_char = true; +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn str_title() { + let vm = VirtualMachine::new(); + + let tests = vec![ + (" Hello ", " hello "), + ("Hello ", "hello "), + ("Hello ", "Hello "), + ("Format This As Title String", "fOrMaT thIs aS titLe String"), + ("Format,This-As*Title;String", "fOrMaT,thIs-aS*titLe;String"), + ("Getint", "getInt"), + ("Greek Ωppercases ...", "greek ωppercases ..."), + ("Greek ῼitlecases ...", "greek ῳitlecases ..."), + ]; + for (title, input) in tests { + assert_eq!(PyString::from(input).title(&vm).as_str(), title); } } - titled_str + + #[test] + fn str_istitle() { + let vm = VirtualMachine::new(); + + let pos = vec![ + "A", + "A Titlecased Line", + "A\nTitlecased Line", + "A Titlecased, Line", + "Greek Ωppercases ...", + "Greek ῼitlecases ...", + ]; + + for s in pos { + assert!(PyString::from(s).istitle(&vm)); + } + + let neg = vec![ + "", + "a", + "\n", + "Not a capitalized String", + "Not\ta Titlecase String", + "Not--a Titlecase String", + "NOT", + ]; + for s in neg { + assert!(!PyString::from(s).istitle(&vm)); + } + } + + #[test] + fn str_maketrans_and_translate() { + let vm = VirtualMachine::new(); + + let table = vm.context().new_dict(); + table + .set_item("a", vm.new_str("🎅".to_owned()), &vm) + .unwrap(); + table.set_item("b", vm.get_none(), &vm).unwrap(); + table + .set_item("c", vm.new_str("xda".to_owned()), &vm) + .unwrap(); + let translated = PyString::maketrans( + table.into_object(), + OptionalArg::Missing, + OptionalArg::Missing, + &vm, + ) + .unwrap(); + let text = PyString::from("abc"); + let translated = text.translate(translated, &vm).unwrap(); + assert_eq!(translated, "🎅xda".to_owned()); + let translated = text.translate(vm.new_int(3), &vm); + assert_eq!(translated.unwrap_err().class().name, "TypeError".to_owned()); + } } diff --git a/vm/src/obj/objsuper.rs b/vm/src/obj/objsuper.rs index 2b2be02414..194212a492 100644 --- a/vm/src/obj/objsuper.rs +++ b/vm/src/obj/objsuper.rs @@ -7,16 +7,19 @@ https://github.com/python/cpython/blob/50b48572d9a90c5bb36e2bef6179548ea927a35a/ */ use crate::frame::NameProtocol; -use crate::function::PyFuncArgs; +use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objfunction::PyMethod; use crate::obj::objstr; -use crate::obj::objtype::PyClass; +use crate::obj::objtype::{PyClass, PyClassRef}; use crate::pyobject::{ - DictProtocol, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol, + ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; use crate::vm::VirtualMachine; use super::objtype; +pub type PySuperRef = PyRef; + #[derive(Debug)] pub struct PySuper { obj: PyObjectRef, @@ -24,7 +27,7 @@ pub struct PySuper { } impl PyValue for PySuper { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.super_type() } } @@ -46,17 +49,11 @@ pub fn init(context: &PyContext) { def cmeth(cls, arg):\n \ super().cmeth(arg)\n"; - context.set_attr(&super_type, "__new__", context.new_rustfunc(super_new)); - context.set_attr( - &super_type, - "__getattribute__", - context.new_rustfunc(super_getattribute), - ); - context.set_attr( - &super_type, - "__doc__", - context.new_str(super_doc.to_string()), - ); + extend_class!(context, super_type, { + "__new__" => context.new_rustfunc(super_new), + "__getattribute__" => context.new_rustfunc(super_getattribute), + "__doc__" => context.new_str(super_doc.to_string()), + }); } fn super_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -76,6 +73,10 @@ fn super_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Some(PyClass { ref mro, .. }) => { for class in mro { if let Ok(item) = vm.get_attribute(class.as_object().clone(), name_str.clone()) { + if item.payload_is::() { + // This is a classmethod + return Ok(item); + } return Ok(vm.ctx.new_bound_method(item, inst.clone())); } } @@ -89,25 +90,18 @@ fn super_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -fn super_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - trace!("super.__new__ {:?}", args.args); - arg_check!( - vm, - args, - required = [(cls, None)], - optional = [(py_type, None), (py_obj, None)] - ); - - if !objtype::issubclass(cls, &vm.ctx.super_type()) { - return Err(vm.new_type_error(format!("{:?} is not a subtype of super", cls))); - } - +fn super_new( + cls: PyClassRef, + py_type: OptionalArg, + py_obj: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { // Get the type: - let py_type = if let Some(ty) = py_type { + let py_type = if let OptionalArg::Present(ty) = py_type { ty.clone() } else { match vm.current_scope().load_cell(vm, "__class__") { - Some(obj) => obj.clone(), + Some(obj) => PyClassRef::try_from_object(vm, obj)?, _ => { return Err(vm.new_type_error( "super must be called with 1 argument or from inside class method".to_string(), @@ -117,25 +111,26 @@ fn super_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }; // Check type argument: - if !objtype::isinstance(&py_type, &vm.get_type()) { - let type_name = objtype::get_type_name(&py_type.typ()); + if !objtype::isinstance(py_type.as_object(), &vm.get_type()) { return Err(vm.new_type_error(format!( "super() argument 1 must be type, not {}", - type_name + py_type.class().name ))); } // Get the bound object: - let py_obj = if let Some(obj) = py_obj { + let py_obj = if let OptionalArg::Present(obj) = py_obj { obj.clone() } else { - let frame = vm.current_frame(); + let frame = vm.current_frame().expect("no current frame for super()"); if let Some(first_arg) = frame.code.arg_names.get(0) { - match vm.get_locals().get_item(first_arg) { + match vm.get_locals().get_item_option(first_arg, vm)? { Some(obj) => obj.clone(), _ => { - return Err(vm - .new_type_error(format!("super arguement {} was not supplied", first_arg))); + return Err(vm.new_type_error(format!( + "super arguement {} was not supplied", + first_arg + ))); } } } else { @@ -146,17 +141,22 @@ fn super_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }; // Check obj type: - if !(objtype::isinstance(&py_obj, &py_type) || objtype::issubclass(&py_obj, &py_type)) { - return Err(vm.new_type_error( - "super(type, obj): obj must be an instance or subtype of type".to_string(), - )); + if !objtype::isinstance(&py_obj, &py_type) { + let is_subclass = if let Ok(py_obj) = PyClassRef::try_from_object(vm, py_obj.clone()) { + objtype::issubclass(&py_obj, &py_type) + } else { + false + }; + if !is_subclass { + return Err(vm.new_type_error( + "super(type, obj): obj must be an instance or subtype of type".to_string(), + )); + } } - Ok(PyObject::new( - PySuper { - obj: py_obj, - typ: py_type, - }, - cls.clone(), - )) + PySuper { + obj: py_obj, + typ: py_type.into_object(), + } + .into_ref_with_type(vm, cls) } diff --git a/vm/src/obj/objtuple.rs b/vm/src/obj/objtuple.rs index 7438e1c90a..a6e181f3ba 100644 --- a/vm/src/obj/objtuple.rs +++ b/vm/src/obj/objtuple.rs @@ -1,25 +1,21 @@ -use std::cell::{Cell, RefCell}; +use std::cell::Cell; use std::fmt; -use std::hash::{Hash, Hasher}; use crate::function::OptionalArg; -use crate::pyobject::{ - IdProtocol, PyContext, PyIteratorValue, PyObjectRef, PyRef, PyResult, PyValue, -}; +use crate::pyhash; +use crate::pyobject::{IdProtocol, PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::{ReprGuard, VirtualMachine}; use super::objbool; -use super::objint; +use super::objiter; use super::objsequence::{ - get_elements, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, + get_elements_tuple, get_item, seq_equal, seq_ge, seq_gt, seq_le, seq_lt, seq_mul, }; use super::objtype::{self, PyClassRef}; -#[derive(Default)] pub struct PyTuple { // TODO: shouldn't be public - // TODO: tuples are immutable, remove this RefCell - pub elements: RefCell>, + pub elements: Vec, } impl fmt::Debug for PyTuple { @@ -31,26 +27,33 @@ impl fmt::Debug for PyTuple { impl From> for PyTuple { fn from(elements: Vec) -> Self { - PyTuple { - elements: RefCell::new(elements), - } + PyTuple { elements } } } impl PyValue for PyTuple { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.tuple_type() } } +impl PyTuple { + pub fn fast_getitem(&self, idx: usize) -> PyObjectRef { + self.elements[idx].clone() + } +} + pub type PyTupleRef = PyRef; +pub fn get_value(obj: &PyObjectRef) -> Vec { + obj.payload::().unwrap().elements.clone() +} + impl PyTupleRef { fn lt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_lt(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_lt(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -59,9 +62,8 @@ impl PyTupleRef { fn gt(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_gt(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_gt(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -70,9 +72,8 @@ impl PyTupleRef { fn ge(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_ge(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_ge(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -81,9 +82,8 @@ impl PyTupleRef { fn le(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = self.elements.borrow(); - let other = get_elements(&other); - let res = seq_le(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_le(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) @@ -92,18 +92,21 @@ impl PyTupleRef { fn add(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let e1 = self.elements.borrow(); - let e2 = get_elements(&other); - let elements = e1.iter().chain(e2.iter()).cloned().collect(); + let e2 = get_elements_tuple(&other); + let elements = self.elements.iter().chain(e2.iter()).cloned().collect(); Ok(vm.ctx.new_tuple(elements)) } else { Ok(vm.ctx.not_implemented()) } } + fn bool(self, _vm: &VirtualMachine) -> bool { + !self.elements.is_empty() + } + fn count(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { let mut count: usize = 0; - for element in self.elements.borrow().iter() { + for element in self.elements.iter() { if element.is(&needle) { count += 1; } else { @@ -118,40 +121,33 @@ impl PyTupleRef { fn eq(self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { if objtype::isinstance(&other, &vm.ctx.tuple_type()) { - let zelf = &self.elements.borrow(); - let other = get_elements(&other); - let res = seq_equal(vm, &zelf, &other)?; + let other = get_elements_tuple(&other); + let res = seq_equal(vm, &self.elements, &other)?; Ok(vm.new_bool(res)) } else { Ok(vm.ctx.not_implemented()) } } - fn hash(self, vm: &VirtualMachine) -> PyResult { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - for element in self.elements.borrow().iter() { - let hash_result = vm.call_method(element, "__hash__", vec![])?; - let element_hash = objint::get_value(&hash_result); - element_hash.hash(&mut hasher); - } - Ok(hasher.finish()) + fn hash(self, vm: &VirtualMachine) -> PyResult { + pyhash::hash_iter(self.elements.iter(), vm) } - fn iter(self, _vm: &VirtualMachine) -> PyIteratorValue { - PyIteratorValue { + fn iter(self, _vm: &VirtualMachine) -> PyTupleIterator { + PyTupleIterator { position: Cell::new(0), - iterated_obj: self.into_object(), + tuple: self, } } fn len(self, _vm: &VirtualMachine) -> usize { - self.elements.borrow().len() + self.elements.len() } fn repr(self, vm: &VirtualMachine) -> PyResult { let s = if let Some(_guard) = ReprGuard::enter(self.as_object()) { let mut str_parts = vec![]; - for elem in self.elements.borrow().iter() { + for elem in self.elements.iter() { let s = vm.to_repr(elem)?; str_parts.push(s.value.clone()); } @@ -168,21 +164,21 @@ impl PyTupleRef { } fn mul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { - let new_elements = seq_mul(&self.elements.borrow(), counter); + let new_elements = seq_mul(&self.elements, counter); + vm.ctx.new_tuple(new_elements) + } + + fn rmul(self, counter: isize, vm: &VirtualMachine) -> PyObjectRef { + let new_elements = seq_mul(&self.elements, counter); vm.ctx.new_tuple(new_elements) } fn getitem(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - get_item( - vm, - self.as_object(), - &self.elements.borrow(), - needle.clone(), - ) + get_item(vm, self.as_object(), &self.elements, needle.clone()) } fn index(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for (index, element) in self.elements.borrow().iter().enumerate() { + for (index, element) in self.elements.iter().enumerate() { if element.is(&needle) { return Ok(index); } @@ -195,7 +191,7 @@ impl PyTupleRef { } fn contains(self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { - for element in self.elements.borrow().iter() { + for element in self.elements.iter() { if element.is(&needle) { return Ok(true); } @@ -213,10 +209,6 @@ fn tuple_new( iterable: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - if !objtype::issubclass(cls.as_object(), &vm.ctx.tuple_type()) { - return Err(vm.new_type_error(format!("{} is not a subtype of tuple", cls))); - } - let elements = if let OptionalArg::Present(iterable) = iterable { vm.extract_elements(&iterable)? } else { @@ -226,6 +218,38 @@ fn tuple_new( PyTuple::from(elements).into_ref_with_type(vm, cls) } +#[pyclass] +#[derive(Debug)] +pub struct PyTupleIterator { + position: Cell, + tuple: PyTupleRef, +} + +impl PyValue for PyTupleIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.tupleiterator_type() + } +} + +#[pyimpl] +impl PyTupleIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.position.get() < self.tuple.elements.len() { + let ret = self.tuple.elements[self.position.get()].clone(); + self.position.set(self.position.get() + 1); + Ok(ret) + } else { + Err(objiter::new_stop_iteration(vm)) + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + #[rustfmt::skip] // to avoid line splitting pub fn init(context: &PyContext) { let tuple_type = &context.tuple_type; @@ -233,21 +257,27 @@ pub fn init(context: &PyContext) { tuple(iterable) -> tuple initialized from iterable's items If the argument is a tuple, the return value is the same object."; - context.set_attr(&tuple_type, "__add__", context.new_rustfunc(PyTupleRef::add)); - context.set_attr(&tuple_type, "__eq__", context.new_rustfunc(PyTupleRef::eq)); - context.set_attr(&tuple_type,"__contains__",context.new_rustfunc(PyTupleRef::contains)); - context.set_attr(&tuple_type,"__getitem__",context.new_rustfunc(PyTupleRef::getitem)); - context.set_attr(&tuple_type, "__hash__", context.new_rustfunc(PyTupleRef::hash)); - context.set_attr(&tuple_type, "__iter__", context.new_rustfunc(PyTupleRef::iter)); - context.set_attr(&tuple_type, "__len__", context.new_rustfunc(PyTupleRef::len)); - context.set_attr(&tuple_type, "__new__", context.new_rustfunc(tuple_new)); - context.set_attr(&tuple_type, "__mul__", context.new_rustfunc(PyTupleRef::mul)); - context.set_attr(&tuple_type, "__repr__", context.new_rustfunc(PyTupleRef::repr)); - context.set_attr(&tuple_type, "count", context.new_rustfunc(PyTupleRef::count)); - context.set_attr(&tuple_type, "__lt__", context.new_rustfunc(PyTupleRef::lt)); - context.set_attr(&tuple_type, "__le__", context.new_rustfunc(PyTupleRef::le)); - context.set_attr(&tuple_type, "__gt__", context.new_rustfunc(PyTupleRef::gt)); - context.set_attr(&tuple_type, "__ge__", context.new_rustfunc(PyTupleRef::ge)); - context.set_attr(&tuple_type,"__doc__",context.new_str(tuple_doc.to_string())); - context.set_attr(&tuple_type, "index", context.new_rustfunc(PyTupleRef::index)); + extend_class!(context, tuple_type, { + "__add__" => context.new_rustfunc(PyTupleRef::add), + "__bool__" => context.new_rustfunc(PyTupleRef::bool), + "__eq__" => context.new_rustfunc(PyTupleRef::eq), + "__contains__" => context.new_rustfunc(PyTupleRef::contains), + "__getitem__" => context.new_rustfunc(PyTupleRef::getitem), + "__hash__" => context.new_rustfunc(PyTupleRef::hash), + "__iter__" => context.new_rustfunc(PyTupleRef::iter), + "__len__" => context.new_rustfunc(PyTupleRef::len), + "__new__" => context.new_rustfunc(tuple_new), + "__mul__" => context.new_rustfunc(PyTupleRef::mul), + "__rmul__" => context.new_rustfunc(PyTupleRef::rmul), + "__repr__" => context.new_rustfunc(PyTupleRef::repr), + "count" => context.new_rustfunc(PyTupleRef::count), + "__lt__" => context.new_rustfunc(PyTupleRef::lt), + "__le__" => context.new_rustfunc(PyTupleRef::le), + "__gt__" => context.new_rustfunc(PyTupleRef::gt), + "__ge__" => context.new_rustfunc(PyTupleRef::ge), + "__doc__" => context.new_str(tuple_doc.to_string()), + "index" => context.new_rustfunc(PyTupleRef::index) + }); + + PyTupleIterator::extend_class(context, &context.tupleiterator_type); } diff --git a/vm/src/obj/objtype.rs b/vm/src/obj/objtype.rs index d5648f07be..90ec95ae8b 100644 --- a/vm/src/obj/objtype.rs +++ b/vm/src/obj/objtype.rs @@ -2,23 +2,27 @@ use std::cell::RefCell; use std::collections::HashMap; use std::fmt; -use crate::function::PyFuncArgs; +use crate::function::{Args, KwArgs, PyFuncArgs}; use crate::pyobject::{ - AttributeProtocol, FromPyObjectRef, IdProtocol, PyAttributes, PyContext, PyObject, PyObjectRef, - PyRef, PyResult, PyValue, TypeProtocol, + IdProtocol, PyAttributes, PyContext, PyIterable, PyObject, PyObjectRef, PyRef, PyResult, + PyValue, TypeProtocol, }; use crate::vm::VirtualMachine; -use super::objdict; +use super::objdict::PyDictRef; use super::objlist::PyList; +use super::objmappingproxy::PyMappingProxy; use super::objproperty::PropertyBuilder; -use super::objstr::{self, PyStringRef}; +use super::objstr::PyStringRef; use super::objtuple::PyTuple; +use super::objweakref::PyWeak; -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyClass { pub name: String, pub mro: Vec, + pub subclasses: RefCell>, + pub attributes: RefCell, } impl fmt::Display for PyClass { @@ -30,23 +34,11 @@ impl fmt::Display for PyClass { pub type PyClassRef = PyRef; impl PyValue for PyClass { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.type_type() } } -impl IdProtocol for PyClassRef { - fn get_id(&self) -> usize { - self.as_object().get_id() - } -} - -impl TypeProtocol for PyClassRef { - fn type_ref(&self) -> &PyObjectRef { - &self.as_object().type_ref() - } -} - struct IterMro<'a> { cls: &'a PyClassRef, offset: Option, @@ -101,19 +93,96 @@ impl PyClassRef { } fn instance_check(self, obj: PyObjectRef, _vm: &VirtualMachine) -> bool { - isinstance(&obj, self.as_object()) + isinstance(&obj, &self) } - fn subclass_check(self, subclass: PyObjectRef, _vm: &VirtualMachine) -> bool { - issubclass(&subclass, self.as_object()) + fn subclass_check(self, subclass: PyClassRef, _vm: &VirtualMachine) -> bool { + issubclass(&subclass, &self) + } + + fn name(self, _vm: &VirtualMachine) -> String { + self.name.clone() } fn repr(self, _vm: &VirtualMachine) -> String { format!("", self.name) } - fn prepare(_name: PyStringRef, _bases: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { - vm.new_dict() + fn prepare(_name: PyStringRef, _bases: PyObjectRef, vm: &VirtualMachine) -> PyDictRef { + vm.ctx.new_dict() + } + + fn getattribute(self, name_ref: PyStringRef, vm: &VirtualMachine) -> PyResult { + let name = &name_ref.value; + trace!("type.__getattribute__({:?}, {:?})", self, name); + let mcl = self.class(); + + if let Some(attr) = class_get_attr(&mcl, &name) { + let attr_class = attr.class(); + if class_has_attr(&attr_class, "__set__") { + if let Some(descriptor) = class_get_attr(&attr_class, "__get__") { + return vm.invoke( + descriptor, + vec![attr, self.into_object(), mcl.into_object()], + ); + } + } + } + + if let Some(attr) = class_get_attr(&self, &name) { + let attr_class = attr.class(); + if let Some(descriptor) = class_get_attr(&attr_class, "__get__") { + let none = vm.get_none(); + return vm.invoke(descriptor, vec![attr, none, self.into_object()]); + } + } + + if let Some(cls_attr) = class_get_attr(&self, &name) { + Ok(cls_attr) + } else if let Some(attr) = class_get_attr(&mcl, &name) { + vm.call_get_descriptor(attr, self.into_object()) + } else if let Some(getter) = class_get_attr(&self, "__getattr__") { + vm.invoke(getter, vec![mcl.into_object(), name_ref.into_object()]) + } else { + Err(vm.new_attribute_error(format!("{} has no attribute '{}'", self, name))) + } + } + + fn set_attr( + self, + attr_name: PyStringRef, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if let Some(attr) = class_get_attr(&self.class(), &attr_name.value) { + if let Some(descriptor) = class_get_attr(&attr.class(), "__set__") { + vm.invoke(descriptor, vec![attr, self.into_object(), value])?; + return Ok(()); + } + } + + self.attributes + .borrow_mut() + .insert(attr_name.to_string(), value); + Ok(()) + } + + // This is used for class initialisation where the vm is not yet available. + pub fn set_str_attr>(&self, attr_name: &str, value: V) { + self.attributes + .borrow_mut() + .insert(attr_name.to_string(), value.into()); + } + + fn subclasses(self, _vm: &VirtualMachine) -> PyList { + let mut subclasses = self.subclasses.borrow_mut(); + subclasses.retain(|x| x.upgrade().is_some()); + PyList::from( + subclasses + .iter() + .map(|x| x.upgrade().unwrap()) + .collect::>(), + ) } } @@ -128,15 +197,24 @@ pub fn init(ctx: &PyContext) { extend_class!(&ctx, &ctx.type_type, { "__call__" => ctx.new_rustfunc(type_call), + "__dict__" => + PropertyBuilder::new(ctx) + .add_getter(type_dict) + .add_setter(type_dict_setter) + .create(), "__new__" => ctx.new_rustfunc(type_new), "__mro__" => PropertyBuilder::new(ctx) .add_getter(PyClassRef::mro) .add_setter(PyClassRef::set_mro) .create(), + "__name__" => ctx.new_property(PyClassRef::name), "__repr__" => ctx.new_rustfunc(PyClassRef::repr), "__prepare__" => ctx.new_rustfunc(PyClassRef::prepare), - "__getattribute__" => ctx.new_rustfunc(type_getattribute), + "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), + "__setattr__" => ctx.new_rustfunc(PyClassRef::set_attr), + "__subclasses__" => ctx.new_rustfunc(PyClassRef::subclasses), + "__getattribute__" => ctx.new_rustfunc(PyClassRef::getattribute), "__instancecheck__" => ctx.new_rustfunc(PyClassRef::instance_check), "__subclasscheck__" => ctx.new_rustfunc(PyClassRef::subclass_check), "__doc__" => ctx.new_str(type_doc.to_string()), @@ -150,83 +228,51 @@ fn _mro(cls: &PyClassRef) -> Vec { /// Determines if `obj` actually an instance of `cls`, this doesn't call __instancecheck__, so only /// use this if `cls` is known to have not overridden the base __instancecheck__ magic method. -pub fn isinstance(obj: &PyObjectRef, cls: &PyObjectRef) -> bool { - issubclass(obj.type_ref(), &cls) +pub fn isinstance(obj: &PyObjectRef, cls: &PyClassRef) -> bool { + issubclass(&obj.class(), &cls) } /// Determines if `subclass` is actually a subclass of `cls`, this doesn't call __subclasscheck__, /// so only use this if `cls` is known to have not overridden the base __subclasscheck__ magic /// method. -pub fn issubclass(subclass: &PyObjectRef, cls: &PyObjectRef) -> bool { - let mro = &subclass.payload::().unwrap().mro; - subclass.is(cls) || mro.iter().any(|c| c.is(cls)) -} - -pub fn get_type_name(typ: &PyObjectRef) -> String { - if let Some(PyClass { name, .. }) = &typ.payload::() { - name.clone() - } else { - panic!("Cannot get type_name of non-type type {:?}", typ); - } +pub fn issubclass(subclass: &PyClassRef, cls: &PyClassRef) -> bool { + let mro = &subclass.mro; + subclass.is(cls) || mro.iter().any(|c| c.is(cls.as_object())) } pub fn type_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { debug!("type.__new__ {:?}", args); if args.args.len() == 2 { - arg_check!( - vm, - args, - required = [(_typ, Some(vm.ctx.type_type())), (obj, None)] - ); - Ok(obj.typ()) + Ok(args.args[1].class().into_object()) } else if args.args.len() == 4 { - arg_check!( - vm, - args, - required = [ - (typ, Some(vm.ctx.type_type())), - (name, Some(vm.ctx.str_type())), - (bases, None), - (dict, Some(vm.ctx.dict_type())) - ] - ); - type_new_class(vm, typ, name, bases, dict) + let (typ, name, bases, dict) = args.bind(vm)?; + type_new_class(vm, typ, name, bases, dict).map(PyRef::into_object) } else { - Err(vm.new_type_error(format!(": type_new: {:?}", args))) + Err(vm.new_type_error("type() takes 1 or 3 arguments".to_string())) } } pub fn type_new_class( vm: &VirtualMachine, - typ: &PyObjectRef, - name: &PyObjectRef, - bases: &PyObjectRef, - dict: &PyObjectRef, -) -> PyResult { - let mut bases: Vec = vm - .extract_elements(bases)? - .iter() - .map(|x| FromPyObjectRef::from_pyobj(x)) - .collect(); - bases.push(FromPyObjectRef::from_pyobj(&vm.ctx.object())); - let name = objstr::get_value(name); - new( - typ.clone(), - &name, - bases, - objdict::py_dict_to_attributes(dict), - ) + typ: PyClassRef, + name: PyStringRef, + bases: PyIterable, + dict: PyDictRef, +) -> PyResult { + let mut bases: Vec = bases.iter(vm)?.collect::, _>>()?; + bases.push(vm.ctx.object()); + new(typ.clone(), &name.value, bases, dict.to_attributes()) } -pub fn type_call(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { - debug!("type_call: {:?}", args); - let cls = args.shift(); - let new = cls.get_attr("__new__").unwrap(); - let new_wrapped = vm.call_get_descriptor(new, cls)?; - let obj = vm.invoke(new_wrapped, args.clone())?; +pub fn type_call(class: PyClassRef, args: Args, kwargs: KwArgs, vm: &VirtualMachine) -> PyResult { + debug!("type_call: {:?}", class); + let new = class_get_attr(&class, "__new__").expect("All types should have a __new__."); + let new_wrapped = vm.call_get_descriptor(new, class.into_object())?; + let obj = vm.invoke(new_wrapped, (&args, &kwargs))?; - if let Ok(init) = vm.get_method(obj.clone(), "__init__") { - let res = vm.invoke(init, args)?; + if let Some(init_method_or_err) = vm.get_method(obj.clone(), "__init__") { + let init_method = init_method_or_err?; + let res = vm.invoke(init_method, (&args, &kwargs))?; if !res.is(&vm.get_none()) { return Err(vm.new_type_error("__init__ must return None".to_string())); } @@ -234,45 +280,36 @@ pub fn type_call(vm: &VirtualMachine, mut args: PyFuncArgs) -> PyResult { Ok(obj) } -pub fn type_getattribute(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (cls, Some(vm.ctx.object())), - (name_str, Some(vm.ctx.str_type())) - ] - ); - let name = objstr::get_value(&name_str); - trace!("type.__getattribute__({:?}, {:?})", cls, name); - let mcl = cls.typ(); - - if let Some(attr) = mcl.get_attr(&name) { - let attr_class = attr.typ(); - if attr_class.has_attr("__set__") { - if let Some(descriptor) = attr_class.get_attr("__get__") { - return vm.invoke(descriptor, vec![attr, cls.clone(), mcl]); - } - } - } +fn type_dict(class: PyClassRef, _vm: &VirtualMachine) -> PyMappingProxy { + PyMappingProxy::new(class) +} - if let Some(attr) = cls.get_attr(&name) { - let attr_class = attr.typ(); - if let Some(descriptor) = attr_class.get_attr("__get__") { - let none = vm.get_none(); - return vm.invoke(descriptor, vec![attr, none, cls.clone()]); +fn type_dict_setter(_instance: PyClassRef, _value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error( + "Setting __dict__ attribute on a type isn't yet implemented".to_string(), + )) +} + +/// This is the internal get_attr implementation for fast lookup on a class. +pub fn class_get_attr(class: &PyClassRef, attr_name: &str) -> Option { + if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { + return Some(item); + } + for class in &class.mro { + if let Some(item) = class.attributes.borrow().get(attr_name).cloned() { + return Some(item); } } + None +} - if let Some(cls_attr) = cls.get_attr(&name) { - Ok(cls_attr) - } else if let Some(attr) = mcl.get_attr(&name) { - vm.call_get_descriptor(attr, cls.clone()) - } else if let Some(getter) = cls.get_attr("__getattr__") { - vm.invoke(getter, vec![mcl, name_str.clone()]) - } else { - Err(vm.new_attribute_error(format!("{} has no attribute '{}'", cls, name))) - } +// This is the internal has_attr implementation for fast lookup on a class. +pub fn class_has_attr(class: &PyClassRef, attr_name: &str) -> bool { + class.attributes.borrow().contains_key(attr_name) + || class + .mro + .iter() + .any(|c| c.attributes.borrow().contains_key(attr_name)) } pub fn get_attributes(cls: PyClassRef) -> PyAttributes { @@ -283,10 +320,8 @@ pub fn get_attributes(cls: PyClassRef) -> PyAttributes { base_classes.reverse(); for bc in base_classes { - if let Some(ref dict) = &bc.as_object().dict { - for (name, value) in dict.borrow().iter() { - attributes.insert(name.to_string(), value.clone()); - } + for (name, value) in bc.attributes.borrow().iter() { + attributes.insert(name.to_string(), value.clone()); } } @@ -300,10 +335,7 @@ fn take_next_base(mut bases: Vec>) -> Option<(PyClassRef, Vec>) -> Option<(PyClassRef, Vec>) -> Option> { } pub fn new( - typ: PyObjectRef, + typ: PyClassRef, name: &str, bases: Vec, dict: HashMap, -) -> PyResult { - let mros = bases.into_iter().map(|x| _mro(&x)).collect(); +) -> PyResult { + let mros = bases.iter().map(|x| _mro(&x)).collect(); let mro = linearise_mro(mros).unwrap(); - Ok(PyObject { - payload: Box::new(PyClass { + let new_type = PyObject { + payload: PyClass { name: String::from(name), mro, - }), - dict: Some(RefCell::new(dict)), + subclasses: RefCell::new(vec![]), + attributes: RefCell::new(dict), + }, + dict: None, typ, } - .into_ref()) + .into_ref(); + for base in bases { + base.subclasses + .borrow_mut() + .push(PyWeak::downgrade(&new_type)); + } + + Ok(new_type.downcast().unwrap()) } #[cfg(test)] mod tests { - use super::FromPyObjectRef; use super::{linearise_mro, new}; use super::{HashMap, IdProtocol, PyClassRef, PyContext}; @@ -373,15 +413,12 @@ mod tests { #[test] fn test_linearise() { let context = PyContext::new(); - let object: PyClassRef = FromPyObjectRef::from_pyobj(&context.object); + let object: PyClassRef = context.object.clone(); let type_type = &context.type_type; let a = new(type_type.clone(), "A", vec![object.clone()], HashMap::new()).unwrap(); let b = new(type_type.clone(), "B", vec![object.clone()], HashMap::new()).unwrap(); - let a: PyClassRef = FromPyObjectRef::from_pyobj(&a); - let b: PyClassRef = FromPyObjectRef::from_pyobj(&b); - assert_eq!( map_ids(linearise_mro(vec![ vec![object.clone()], diff --git a/vm/src/obj/objweakproxy.rs b/vm/src/obj/objweakproxy.rs new file mode 100644 index 0000000000..5d4f68d9d7 --- /dev/null +++ b/vm/src/obj/objweakproxy.rs @@ -0,0 +1,65 @@ +use super::objweakref::PyWeak; +use crate::function::OptionalArg; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[pyclass] +#[derive(Debug)] +pub struct PyWeakProxy { + weak: PyWeak, +} + +impl PyValue for PyWeakProxy { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.weakproxy_type() + } +} + +pub type PyWeakProxyRef = PyRef; + +#[pyimpl] +impl PyWeakProxy { + // TODO: callbacks + #[pymethod(name = "__new__")] + fn create( + cls: PyClassRef, + referent: PyObjectRef, + callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + if callback.is_present() { + panic!("Passed a callback to weakproxy, but weakproxy does not yet support proxies."); + } + PyWeakProxy { + weak: PyWeak::downgrade(&referent), + } + .into_ref_with_type(vm, cls) + } + + #[pymethod(name = "__getattr__")] + fn getattr(&self, attr_name: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.get_attribute(obj, attr_name), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } + + #[pymethod(name = "__setattr__")] + fn setattr(&self, attr_name: PyObjectRef, value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match self.weak.upgrade() { + Some(obj) => vm.set_attr(&obj, attr_name, value), + None => Err(vm.new_exception( + vm.ctx.exceptions.reference_error.clone(), + "weakly-referenced object no longer exists".to_string(), + )), + } + } +} + +pub fn init(context: &PyContext) { + PyWeakProxy::extend_class(&context, &context.weakproxy_type); +} diff --git a/vm/src/obj/objweakref.rs b/vm/src/obj/objweakref.rs index e6dd1e1e69..b3b7e03c9b 100644 --- a/vm/src/obj/objweakref.rs +++ b/vm/src/obj/objweakref.rs @@ -1,19 +1,20 @@ +use crate::function::OptionalArg; use crate::obj::objtype::PyClassRef; use crate::pyobject::PyValue; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyRef, PyResult}; +use crate::pyobject::{PyContext, PyObject, PyObjectPayload, PyObjectRef, PyRef, PyResult}; use crate::vm::VirtualMachine; use std::rc::{Rc, Weak}; #[derive(Debug)] pub struct PyWeak { - referent: Weak, + referent: Weak>, } impl PyWeak { - pub fn downgrade(obj: PyObjectRef) -> PyWeak { + pub fn downgrade(obj: &PyObjectRef) -> PyWeak { PyWeak { - referent: Rc::downgrade(&obj), + referent: Rc::downgrade(obj), } } @@ -23,7 +24,7 @@ impl PyWeak { } impl PyValue for PyWeak { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.weakref_type() } } @@ -32,8 +33,13 @@ pub type PyWeakRef = PyRef; impl PyWeakRef { // TODO callbacks - fn create(cls: PyClassRef, referent: PyObjectRef, vm: &VirtualMachine) -> PyResult { - PyWeak::downgrade(referent).into_ref_with_type(vm, cls) + fn create( + cls: PyClassRef, + referent: PyObjectRef, + _callback: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + PyWeak::downgrade(&referent).into_ref_with_type(vm, cls) } fn call(self, vm: &VirtualMachine) -> PyObjectRef { diff --git a/vm/src/obj/objzip.rs b/vm/src/obj/objzip.rs index 0a3c5453a4..d9c577aa03 100644 --- a/vm/src/obj/objzip.rs +++ b/vm/src/obj/objzip.rs @@ -1,55 +1,58 @@ -use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::function::Args; +use crate::pyobject::{PyClassImpl, PyContext, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; use super::objiter; +use crate::obj::objtype::PyClassRef; +pub type PyZipRef = PyRef; + +#[pyclass] #[derive(Debug)] pub struct PyZip { iterators: Vec, } impl PyValue for PyZip { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.zip_type() } } -fn zip_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - no_kwargs!(vm, args); - let cls = &args.args[0]; - let iterables = &args.args[1..]; +fn zip_new(cls: PyClassRef, iterables: Args, vm: &VirtualMachine) -> PyResult { let iterators = iterables - .iter() - .map(|iterable| objiter::get_iter(vm, iterable)) + .into_iter() + .map(|iterable| objiter::get_iter(vm, &iterable)) .collect::, _>>()?; - Ok(PyObject::new(PyZip { iterators }, cls.clone())) + PyZip { iterators }.into_ref_with_type(vm, cls) } -fn zip_next(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zip, Some(vm.ctx.zip_type()))]); - - if let Some(PyZip { ref iterators }) = zip.payload() { - if iterators.is_empty() { +#[pyimpl] +impl PyZip { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.iterators.is_empty() { Err(objiter::new_stop_iteration(vm)) } else { - let next_objs = iterators + let next_objs = self + .iterators .iter() .map(|iterator| objiter::call_next(vm, iterator)) .collect::, _>>()?; Ok(vm.ctx.new_tuple(next_objs)) } - } else { - panic!("zip doesn't have correct payload"); + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf } } pub fn init(context: &PyContext) { - let zip_type = &context.zip_type; - objiter::iter_type_init(context, zip_type); - extend_class!(context, zip_type, { + PyZip::extend_class(context, &context.zip_type); + extend_class!(context, &context.zip_type, { "__new__" => context.new_rustfunc(zip_new), - "__next__" => context.new_rustfunc(zip_next) }); } diff --git a/vm/src/py_serde.rs b/vm/src/py_serde.rs new file mode 100644 index 0000000000..5e43c43a7a --- /dev/null +++ b/vm/src/py_serde.rs @@ -0,0 +1,218 @@ +use std::fmt; + +use serde; +use serde::de::{DeserializeSeed, Visitor}; +use serde::ser::{Serialize, SerializeMap, SerializeSeq}; + +use crate::obj::{objbool, objdict::PyDictRef, objfloat, objint, objsequence, objstr, objtype}; +use crate::pyobject::{IdProtocol, ItemProtocol, PyObjectRef, TypeProtocol}; +use crate::VirtualMachine; +use num_traits::cast::ToPrimitive; +use num_traits::sign::Signed; + +#[inline] +pub fn serialize( + vm: &VirtualMachine, + pyobject: &PyObjectRef, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + PyObjectSerializer { vm, pyobject }.serialize(serializer) +} + +#[inline] +pub fn deserialize<'de, D>( + vm: &'de VirtualMachine, + deserializer: D, +) -> Result<::Value, D::Error> +where + D: serde::Deserializer<'de>, +{ + PyObjectDeserializer { vm }.deserialize(deserializer) +} + +// We need to have a VM available to serialise a PyObject based on its subclass, so we implement +// PyObject serialisation via a proxy object which holds a reference to a VM +pub struct PyObjectSerializer<'s> { + pyobject: &'s PyObjectRef, + vm: &'s VirtualMachine, +} + +impl<'s> PyObjectSerializer<'s> { + pub fn new(vm: &'s VirtualMachine, pyobject: &'s PyObjectRef) -> Self { + PyObjectSerializer { vm, pyobject } + } + + fn clone_with_object(&self, pyobject: &'s PyObjectRef) -> PyObjectSerializer { + PyObjectSerializer { + pyobject, + vm: self.vm, + } + } +} + +impl<'s> serde::Serialize for PyObjectSerializer<'s> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let serialize_seq_elements = + |serializer: S, elements: &Vec| -> Result { + let mut seq = serializer.serialize_seq(Some(elements.len()))?; + for e in elements.iter() { + seq.serialize_element(&self.clone_with_object(e))?; + } + seq.end() + }; + if objtype::isinstance(self.pyobject, &self.vm.ctx.str_type()) { + serializer.serialize_str(&objstr::get_value(&self.pyobject)) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.float_type()) { + serializer.serialize_f64(objfloat::get_value(self.pyobject)) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.bool_type()) { + serializer.serialize_bool(objbool::get_value(self.pyobject)) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.int_type()) { + let v = objint::get_value(self.pyobject); + let int_too_large = || serde::ser::Error::custom("int too large to serialize"); + // TODO: serialize BigInt when it does not fit into i64 + // BigInt implements serialization to a tuple of sign and a list of u32s, + // eg. -1 is [-1, [1]], 0 is [0, []], 12345678900000654321 is [1, [2710766577,2874452364]] + // CPython serializes big ints as long decimal integer literals + if v.is_positive() { + serializer.serialize_u64(v.to_u64().ok_or_else(int_too_large)?) + } else { + serializer.serialize_i64(v.to_i64().ok_or_else(int_too_large)?) + } + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.list_type()) { + let elements = objsequence::get_elements_list(self.pyobject); + serialize_seq_elements(serializer, &elements) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.tuple_type()) { + let elements = objsequence::get_elements_tuple(self.pyobject); + serialize_seq_elements(serializer, &elements) + } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { + let dict: PyDictRef = self.pyobject.clone().downcast().unwrap(); + let pairs: Vec<_> = dict.into_iter().collect(); + let mut map = serializer.serialize_map(Some(pairs.len()))?; + for (key, e) in pairs.iter() { + map.serialize_entry(&self.clone_with_object(key), &self.clone_with_object(&e))?; + } + map.end() + } else if self.pyobject.is(&self.vm.get_none()) { + serializer.serialize_none() + } else { + Err(serde::ser::Error::custom(format!( + "Object of type '{:?}' is not serializable", + self.pyobject.class() + ))) + } + } +} + +// This object is used as the seed for deserialization so we have access to the PyContext for type +// creation +#[derive(Clone)] +pub struct PyObjectDeserializer<'c> { + vm: &'c VirtualMachine, +} + +impl<'c> PyObjectDeserializer<'c> { + pub fn new(vm: &'c VirtualMachine) -> Self { + PyObjectDeserializer { vm } + } +} + +impl<'de> DeserializeSeed<'de> for PyObjectDeserializer<'de> { + type Value = PyObjectRef; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_any(self.clone()) + } +} + +impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { + type Value = PyObjectRef; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a type that can deserialise in Python") + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + Ok(self.vm.ctx.new_bool(value)) + } + + // Other signed integers delegate to this method by default, it’s the only one needed + fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + Ok(self.vm.ctx.new_int(value)) + } + + // Other unsigned integers delegate to this method by default, it’s the only one needed + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + Ok(self.vm.ctx.new_int(value)) + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + Ok(self.vm.ctx.new_float(value)) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + // Owned value needed anyway, delegate to visit_string + self.visit_string(value.to_string()) + } + + fn visit_string(self, value: String) -> Result + where + E: serde::de::Error, + { + Ok(self.vm.ctx.new_str(value)) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(self.vm.get_none()) + } + + fn visit_seq(self, mut access: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut seq = Vec::with_capacity(access.size_hint().unwrap_or(0)); + while let Some(value) = access.next_element_seed(self.clone())? { + seq.push(value); + } + Ok(self.vm.ctx.new_list(seq)) + } + + fn visit_map(self, mut access: M) -> Result + where + M: serde::de::MapAccess<'de>, + { + let dict = self.vm.ctx.new_dict(); + // Although JSON keys must be strings, implementation accepts any keys + // and can be reused by other deserializers without such limit + while let Some((key_obj, value)) = access.next_entry_seed(self.clone(), self.clone())? { + dict.set_item(key_obj, value, self.vm).unwrap(); + } + Ok(dict.into_object()) + } +} diff --git a/vm/src/pyhash.rs b/vm/src/pyhash.rs new file mode 100644 index 0000000000..8bbd82168e --- /dev/null +++ b/vm/src/pyhash.rs @@ -0,0 +1,83 @@ +use std::hash::{Hash, Hasher}; + +use crate::obj::objfloat; +use crate::pyobject::PyObjectRef; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +pub type PyHash = i64; +pub type PyUHash = u64; + +/// Prime multiplier used in string and various other hashes. +pub const MULTIPLIER: PyHash = 1_000_003; // 0xf4243 +/// Numeric hashes are based on reduction modulo the prime 2**_BITS - 1 +pub const BITS: usize = 61; +pub const MODULUS: PyUHash = (1 << BITS) - 1; +pub const INF: PyHash = 314_159; +pub const NAN: PyHash = 0; +pub const IMAG: PyHash = MULTIPLIER; + +// pub const CUTOFF: usize = 7; + +pub fn hash_float(value: f64) -> PyHash { + // cpython _Py_HashDouble + if !value.is_finite() { + return if value.is_infinite() { + if value > 0.0 { + INF + } else { + -INF + } + } else { + NAN + }; + } + + let frexp = objfloat::ufrexp(value); + + // process 28 bits at a time; this should work well both for binary + // and hexadecimal floating point. + let mut m = frexp.0; + let mut e = frexp.1; + let mut x: PyUHash = 0; + while m != 0.0 { + x = ((x << 28) & MODULUS) | x >> (BITS - 28); + m *= 268_435_456.0; // 2**28 + e -= 28; + let y = m as PyUHash; // pull out integer part + m -= y as f64; + x += y; + if x >= MODULUS { + x -= MODULUS; + } + } + + // adjust for the exponent; first reduce it modulo BITS + const BITS32: i32 = BITS as i32; + e = if e >= 0 { + e % BITS32 + } else { + BITS32 - 1 - ((-1 - e) % BITS32) + }; + x = ((x << e) & MODULUS) | x >> (BITS32 - e); + + x as PyHash * value.signum() as PyHash +} + +pub fn hash_value(data: &T) -> PyHash { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + hasher.finish() as PyHash +} + +pub fn hash_iter<'a, I: std::iter::Iterator>( + iter: I, + vm: &VirtualMachine, +) -> PyResult { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + for element in iter { + let item_hash = vm._hash(&element)?; + item_hash.hash(&mut hasher); + } + Ok(hasher.finish() as PyHash) +} diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d6e84b653b..82eb6e5287 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1,5 +1,6 @@ use std::any::Any; -use std::cell::{Cell, RefCell}; +use std::cell::Cell; +use std::cell::RefCell; use std::collections::HashMap; use std::fmt; use std::marker::PhantomData; @@ -14,16 +15,17 @@ use num_traits::{One, Zero}; use crate::bytecode; use crate::exceptions; -use crate::frame::{Frame, Scope}; +use crate::frame::Scope; use crate::function::{IntoPyNativeFunc, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; use crate::obj::objbytearray; use crate::obj::objbytes; -use crate::obj::objclassmethod; +use crate::obj::objclassmethod::{self, PyClassMethod}; use crate::obj::objcode; +use crate::obj::objcode::PyCodeRef; use crate::obj::objcomplex::{self, PyComplex}; -use crate::obj::objdict::{self, PyDict}; +use crate::obj::objdict::{self, PyDict, PyDictRef}; use crate::obj::objellipsis; use crate::obj::objenumerate; use crate::obj::objfilter; @@ -31,13 +33,15 @@ use crate::obj::objfloat::{self, PyFloat}; use crate::obj::objframe; use crate::obj::objfunction::{self, PyFunction, PyMethod}; use crate::obj::objgenerator; -use crate::obj::objint::{self, PyInt}; +use crate::obj::objint::{self, PyInt, PyIntRef}; use crate::obj::objiter; use crate::obj::objlist::{self, PyList}; use crate::obj::objmap; +use crate::obj::objmappingproxy; use crate::obj::objmemory; use crate::obj::objmodule::{self, PyModule}; -use crate::obj::objnone; +use crate::obj::objnamespace::{self, PyNamespace}; +use crate::obj::objnone::{self, PyNone, PyNoneRef}; use crate::obj::objobject; use crate::obj::objproperty; use crate::obj::objproperty::PropertyBuilder; @@ -47,11 +51,13 @@ use crate::obj::objslice; use crate::obj::objstaticmethod; use crate::obj::objstr; use crate::obj::objsuper; -use crate::obj::objtuple::{self, PyTuple}; +use crate::obj::objtuple::{self, PyTuple, PyTupleRef}; use crate::obj::objtype::{self, PyClass, PyClassRef}; +use crate::obj::objweakproxy; use crate::obj::objweakref; use crate::obj::objzip; use crate::vm::VirtualMachine; +use indexmap::IndexMap; /* Python objects and references. @@ -72,7 +78,7 @@ Basically reference counting, but then done by rust. /// this reference counting is accounted for by this type. Use the `.clone()` /// method to create a new reference and increment the amount of references /// to the python object by 1. -pub type PyObjectRef = Rc; +pub type PyObjectRef = Rc>; /// Use this type for function which return a python object or and exception. /// Both the python object and the python exception are `PyObjectRef` types @@ -81,13 +87,13 @@ pub type PyResult = Result; // A valid value, o /// For attributes we do not use a dict, but a hashmap. This is probably /// faster, unordered, and only supports strings as keys. +/// TODO: class attributes should maintain insertion order (use IndexMap here) pub type PyAttributes = HashMap; -impl fmt::Display for PyObject { +impl fmt::Display for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::TypeProtocol; if let Some(PyClass { ref name, .. }) = self.payload::() { - let type_name = objtype::get_type_name(&self.typ()); + let type_name = self.class().name.clone(); // We don't have access to a vm, so just assume that if its parent's name // is type, it's a type if type_name == "type" { @@ -100,118 +106,145 @@ impl fmt::Display for PyObject { if let Some(PyModule { ref name, .. }) = self.payload::() { return write!(f, "module '{}'", name); } - write!(f, "'{}' object", objtype::get_type_name(&self.typ())) + write!(f, "'{}' object", self.class().name) } } #[derive(Debug)] pub struct PyContext { - pub bytes_type: PyObjectRef, - pub bytearray_type: PyObjectRef, - pub bool_type: PyObjectRef, - pub classmethod_type: PyObjectRef, - pub code_type: PyObjectRef, - pub dict_type: PyObjectRef, - pub ellipsis_type: PyObjectRef, - pub enumerate_type: PyObjectRef, - pub filter_type: PyObjectRef, - pub float_type: PyObjectRef, - pub frame_type: PyObjectRef, - pub frozenset_type: PyObjectRef, - pub generator_type: PyObjectRef, - pub int_type: PyObjectRef, - pub iter_type: PyObjectRef, - pub complex_type: PyObjectRef, - pub true_value: PyObjectRef, - pub false_value: PyObjectRef, - pub list_type: PyObjectRef, - pub map_type: PyObjectRef, - pub memoryview_type: PyObjectRef, - pub none: PyObjectRef, - pub ellipsis: PyObjectRef, - pub not_implemented: PyObjectRef, - pub tuple_type: PyObjectRef, - pub set_type: PyObjectRef, - pub staticmethod_type: PyObjectRef, - pub super_type: PyObjectRef, - pub str_type: PyObjectRef, - pub range_type: PyObjectRef, - pub slice_type: PyObjectRef, - pub type_type: PyObjectRef, - pub zip_type: PyObjectRef, - pub function_type: PyObjectRef, - pub builtin_function_or_method_type: PyObjectRef, - pub property_type: PyObjectRef, - pub readonly_property_type: PyObjectRef, - pub module_type: PyObjectRef, - pub bound_method_type: PyObjectRef, - pub weakref_type: PyObjectRef, - pub object: PyObjectRef, + pub bytes_type: PyClassRef, + pub bytesiterator_type: PyClassRef, + pub bytearray_type: PyClassRef, + pub bytearrayiterator_type: PyClassRef, + pub bool_type: PyClassRef, + pub classmethod_type: PyClassRef, + pub code_type: PyClassRef, + pub dict_type: PyClassRef, + pub ellipsis_type: PyClassRef, + pub enumerate_type: PyClassRef, + pub filter_type: PyClassRef, + pub float_type: PyClassRef, + pub frame_type: PyClassRef, + pub frozenset_type: PyClassRef, + pub generator_type: PyClassRef, + pub int_type: PyClassRef, + pub iter_type: PyClassRef, + pub complex_type: PyClassRef, + pub true_value: PyIntRef, + pub false_value: PyIntRef, + pub list_type: PyClassRef, + pub listiterator_type: PyClassRef, + pub dictkeyiterator_type: PyClassRef, + pub dictvalueiterator_type: PyClassRef, + pub dictitemiterator_type: PyClassRef, + pub dictkeys_type: PyClassRef, + pub dictvalues_type: PyClassRef, + pub dictitems_type: PyClassRef, + pub map_type: PyClassRef, + pub memoryview_type: PyClassRef, + pub none: PyNoneRef, + pub ellipsis: PyEllipsisRef, + pub not_implemented: PyNotImplementedRef, + pub tuple_type: PyClassRef, + pub tupleiterator_type: PyClassRef, + pub set_type: PyClassRef, + pub staticmethod_type: PyClassRef, + pub super_type: PyClassRef, + pub str_type: PyClassRef, + pub range_type: PyClassRef, + pub rangeiterator_type: PyClassRef, + pub slice_type: PyClassRef, + pub type_type: PyClassRef, + pub zip_type: PyClassRef, + pub function_type: PyClassRef, + pub builtin_function_or_method_type: PyClassRef, + pub property_type: PyClassRef, + pub readonly_property_type: PyClassRef, + pub module_type: PyClassRef, + pub namespace_type: PyClassRef, + pub bound_method_type: PyClassRef, + pub weakref_type: PyClassRef, + pub weakproxy_type: PyClassRef, + pub mappingproxy_type: PyClassRef, + pub object: PyClassRef, pub exceptions: exceptions::ExceptionZoo, } -pub fn create_type(name: &str, type_type: &PyObjectRef, base: &PyObjectRef) -> PyObjectRef { +pub fn create_type(name: &str, type_type: &PyClassRef, base: &PyClassRef) -> PyClassRef { let dict = PyAttributes::new(); - objtype::new( - type_type.clone(), - name, - vec![FromPyObjectRef::from_pyobj(base)], - dict, - ) - .unwrap() + objtype::new(type_type.clone(), name, vec![base.clone()], dict).unwrap() } +pub type PyNotImplementedRef = PyRef; + #[derive(Debug)] pub struct PyNotImplemented; impl PyValue for PyNotImplemented { - fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.not_implemented().typ() + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.ctx.not_implemented().class() } } +pub type PyEllipsisRef = PyRef; + #[derive(Debug)] pub struct PyEllipsis; impl PyValue for PyEllipsis { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.ctx.ellipsis_type.clone() } } -fn init_type_hierarchy() -> (PyObjectRef, PyObjectRef) { +fn init_type_hierarchy() -> (PyClassRef, PyClassRef) { // `type` inherits from `object` // and both `type` and `object are instances of `type`. // to produce this circular dependency, we need an unsafe block. // (and yes, this will never get dropped. TODO?) - unsafe { + let (type_type, object_type) = unsafe { let object_type = PyObject { typ: mem::uninitialized(), // ! - dict: Some(RefCell::new(PyAttributes::new())), - payload: Box::new(PyClass { + dict: None, + payload: PyClass { name: String::from("object"), mro: vec![], - }), + subclasses: RefCell::new(vec![]), + attributes: RefCell::new(PyAttributes::new()), + }, } .into_ref(); let type_type = PyObject { typ: mem::uninitialized(), // ! - dict: Some(RefCell::new(PyAttributes::new())), - payload: Box::new(PyClass { + dict: None, + payload: PyClass { name: String::from("type"), - mro: vec![FromPyObjectRef::from_pyobj(&object_type)], - }), + mro: vec![object_type.clone().downcast().unwrap()], + subclasses: RefCell::new(vec![]), + attributes: RefCell::new(PyAttributes::new()), + }, } .into_ref(); - let object_type_ptr = PyObjectRef::into_raw(object_type.clone()) as *mut PyObject; - let type_type_ptr = PyObjectRef::into_raw(type_type.clone()) as *mut PyObject; + let object_type_ptr = PyObjectRef::into_raw(object_type.clone()) as *mut PyObject; + let type_type_ptr = PyObjectRef::into_raw(type_type.clone()) as *mut PyObject; + + let type_type: PyClassRef = type_type.downcast().unwrap(); + let object_type: PyClassRef = object_type.downcast().unwrap(); + ptr::write(&mut (*object_type_ptr).typ, type_type.clone()); ptr::write(&mut (*type_type_ptr).typ, type_type.clone()); (type_type, object_type) - } + }; + + object_type + .subclasses + .borrow_mut() + .push(objweakref::PyWeak::downgrade(&type_type.as_object())); + + (type_type, object_type) } // Basic objects: @@ -221,6 +254,7 @@ impl PyContext { let dict_type = create_type("dict", &type_type, &object_type); let module_type = create_type("module", &type_type, &object_type); + let namespace_type = create_type("SimpleNamespace", &type_type, &object_type); let classmethod_type = create_type("classmethod", &type_type, &object_type); let staticmethod_type = create_type("staticmethod", &type_type, &object_type); let function_type = create_type("function", &type_type, &object_type); @@ -230,10 +264,18 @@ impl PyContext { let readonly_property_type = create_type("readonly_property", &type_type, &object_type); let super_type = create_type("super", &type_type, &object_type); let weakref_type = create_type("ref", &type_type, &object_type); + let weakproxy_type = create_type("weakproxy", &type_type, &object_type); let generator_type = create_type("generator", &type_type, &object_type); let bound_method_type = create_type("method", &type_type, &object_type); let str_type = create_type("str", &type_type, &object_type); let list_type = create_type("list", &type_type, &object_type); + let listiterator_type = create_type("list_iterator", &type_type, &object_type); + let dictkeys_type = create_type("dict_keys", &type_type, &object_type); + let dictvalues_type = create_type("dict_values", &type_type, &object_type); + let dictitems_type = create_type("dict_items", &type_type, &object_type); + let dictkeyiterator_type = create_type("dict_keyiterator", &type_type, &object_type); + let dictvalueiterator_type = create_type("dict_valueiterator", &type_type, &object_type); + let dictitemiterator_type = create_type("dict_itemiterator", &type_type, &object_type); let set_type = create_type("set", &type_type, &object_type); let frozenset_type = create_type("frozenset", &type_type, &object_type); let int_type = create_type("int", &type_type, &object_type); @@ -241,40 +283,50 @@ impl PyContext { let frame_type = create_type("frame", &type_type, &object_type); let complex_type = create_type("complex", &type_type, &object_type); let bytes_type = create_type("bytes", &type_type, &object_type); + let bytesiterator_type = create_type("bytes_iterator", &type_type, &object_type); let bytearray_type = create_type("bytearray", &type_type, &object_type); + let bytearrayiterator_type = create_type("bytearray_iterator", &type_type, &object_type); let tuple_type = create_type("tuple", &type_type, &object_type); + let tupleiterator_type = create_type("tuple_iterator", &type_type, &object_type); let iter_type = create_type("iter", &type_type, &object_type); - let ellipsis_type = create_type("EllipsisType", &type_type, &object_type); let enumerate_type = create_type("enumerate", &type_type, &object_type); let filter_type = create_type("filter", &type_type, &object_type); let map_type = create_type("map", &type_type, &object_type); let zip_type = create_type("zip", &type_type, &object_type); let bool_type = create_type("bool", &type_type, &int_type); let memoryview_type = create_type("memoryview", &type_type, &object_type); - let code_type = create_type("code", &type_type, &int_type); + let code_type = create_type("code", &type_type, &object_type); let range_type = create_type("range", &type_type, &object_type); + let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); + let mappingproxy_type = create_type("mappingproxy", &type_type, &object_type); let exceptions = exceptions::ExceptionZoo::new(&type_type, &object_type); - let none = PyObject::new( - objnone::PyNone, - create_type("NoneType", &type_type, &object_type), - ); + fn create_object(payload: T, cls: &PyClassRef) -> PyRef { + PyRef { + obj: PyObject::new(payload, cls.clone(), None), + _payload: PhantomData, + } + } + + let none_type = create_type("NoneType", &type_type, &object_type); + let none = create_object(PyNone, &none_type); - let ellipsis = PyObject::new(PyEllipsis, ellipsis_type.clone()); + let ellipsis_type = create_type("EllipsisType", &type_type, &object_type); + let ellipsis = create_object(PyEllipsis, &ellipsis_type); - let not_implemented = PyObject::new( - PyNotImplemented, - create_type("NotImplementedType", &type_type, &object_type), - ); + let not_implemented_type = create_type("NotImplementedType", &type_type, &object_type); + let not_implemented = create_object(PyNotImplemented, ¬_implemented_type); - let true_value = PyObject::new(PyInt::new(BigInt::one()), bool_type.clone()); - let false_value = PyObject::new(PyInt::new(BigInt::zero()), bool_type.clone()); + let true_value = create_object(PyInt::new(BigInt::one()), &bool_type); + let false_value = create_object(PyInt::new(BigInt::zero()), &bool_type); let context = PyContext { bool_type, memoryview_type, bytearray_type, + bytearrayiterator_type, bytes_type, + bytesiterator_type, code_type, complex_type, classmethod_type, @@ -283,11 +335,19 @@ impl PyContext { frame_type, staticmethod_type, list_type, + listiterator_type, + dictkeys_type, + dictvalues_type, + dictitems_type, + dictkeyiterator_type, + dictvalueiterator_type, + dictitemiterator_type, set_type, frozenset_type, true_value, false_value, tuple_type, + tupleiterator_type, iter_type, ellipsis_type, enumerate_type, @@ -300,17 +360,21 @@ impl PyContext { not_implemented, str_type, range_type, + rangeiterator_type, slice_type, object: object_type, function_type, builtin_function_or_method_type, super_type, + mappingproxy_type, property_type, readonly_property_type, generator_type, module_type, + namespace_type, bound_method_type, weakref_type, + weakproxy_type, type_type, exceptions, }; @@ -346,242 +410,269 @@ impl PyContext { objcode::init(&context); objframe::init(&context); objweakref::init(&context); + objweakproxy::init(&context); objnone::init(&context); objmodule::init(&context); + objnamespace::init(&context); + objmappingproxy::init(&context); exceptions::init(&context); context } - pub fn bytearray_type(&self) -> PyObjectRef { + pub fn bytearray_type(&self) -> PyClassRef { self.bytearray_type.clone() } - pub fn bytes_type(&self) -> PyObjectRef { + pub fn bytearrayiterator_type(&self) -> PyClassRef { + self.bytearrayiterator_type.clone() + } + + pub fn bytes_type(&self) -> PyClassRef { self.bytes_type.clone() } - pub fn code_type(&self) -> PyObjectRef { + pub fn bytesiterator_type(&self) -> PyClassRef { + self.bytesiterator_type.clone() + } + + pub fn code_type(&self) -> PyClassRef { self.code_type.clone() } - pub fn complex_type(&self) -> PyObjectRef { + pub fn complex_type(&self) -> PyClassRef { self.complex_type.clone() } - pub fn dict_type(&self) -> PyObjectRef { + pub fn dict_type(&self) -> PyClassRef { self.dict_type.clone() } - pub fn float_type(&self) -> PyObjectRef { + pub fn float_type(&self) -> PyClassRef { self.float_type.clone() } - pub fn frame_type(&self) -> PyObjectRef { + pub fn frame_type(&self) -> PyClassRef { self.frame_type.clone() } - pub fn int_type(&self) -> PyObjectRef { + pub fn int_type(&self) -> PyClassRef { self.int_type.clone() } - pub fn list_type(&self) -> PyObjectRef { + pub fn list_type(&self) -> PyClassRef { self.list_type.clone() } - pub fn module_type(&self) -> PyObjectRef { + pub fn listiterator_type(&self) -> PyClassRef { + self.listiterator_type.clone() + } + + pub fn module_type(&self) -> PyClassRef { self.module_type.clone() } - pub fn set_type(&self) -> PyObjectRef { + pub fn namespace_type(&self) -> PyClassRef { + self.namespace_type.clone() + } + + pub fn set_type(&self) -> PyClassRef { self.set_type.clone() } - pub fn range_type(&self) -> PyObjectRef { + pub fn range_type(&self) -> PyClassRef { self.range_type.clone() } - pub fn slice_type(&self) -> PyObjectRef { + pub fn rangeiterator_type(&self) -> PyClassRef { + self.rangeiterator_type.clone() + } + + pub fn slice_type(&self) -> PyClassRef { self.slice_type.clone() } - pub fn frozenset_type(&self) -> PyObjectRef { + pub fn frozenset_type(&self) -> PyClassRef { self.frozenset_type.clone() } - pub fn bool_type(&self) -> PyObjectRef { + pub fn bool_type(&self) -> PyClassRef { self.bool_type.clone() } - pub fn memoryview_type(&self) -> PyObjectRef { + pub fn memoryview_type(&self) -> PyClassRef { self.memoryview_type.clone() } - pub fn tuple_type(&self) -> PyObjectRef { + pub fn tuple_type(&self) -> PyClassRef { self.tuple_type.clone() } - pub fn iter_type(&self) -> PyObjectRef { + pub fn tupleiterator_type(&self) -> PyClassRef { + self.tupleiterator_type.clone() + } + + pub fn iter_type(&self) -> PyClassRef { self.iter_type.clone() } - pub fn enumerate_type(&self) -> PyObjectRef { + pub fn enumerate_type(&self) -> PyClassRef { self.enumerate_type.clone() } - pub fn filter_type(&self) -> PyObjectRef { + pub fn filter_type(&self) -> PyClassRef { self.filter_type.clone() } - pub fn map_type(&self) -> PyObjectRef { + pub fn map_type(&self) -> PyClassRef { self.map_type.clone() } - pub fn zip_type(&self) -> PyObjectRef { + pub fn zip_type(&self) -> PyClassRef { self.zip_type.clone() } - pub fn str_type(&self) -> PyObjectRef { + pub fn str_type(&self) -> PyClassRef { self.str_type.clone() } - pub fn super_type(&self) -> PyObjectRef { + pub fn super_type(&self) -> PyClassRef { self.super_type.clone() } - pub fn function_type(&self) -> PyObjectRef { + pub fn function_type(&self) -> PyClassRef { self.function_type.clone() } - pub fn builtin_function_or_method_type(&self) -> PyObjectRef { + pub fn builtin_function_or_method_type(&self) -> PyClassRef { self.builtin_function_or_method_type.clone() } - pub fn property_type(&self) -> PyObjectRef { + pub fn property_type(&self) -> PyClassRef { self.property_type.clone() } - pub fn readonly_property_type(&self) -> PyObjectRef { + pub fn readonly_property_type(&self) -> PyClassRef { self.readonly_property_type.clone() } - pub fn classmethod_type(&self) -> PyObjectRef { + pub fn classmethod_type(&self) -> PyClassRef { self.classmethod_type.clone() } - pub fn staticmethod_type(&self) -> PyObjectRef { + pub fn staticmethod_type(&self) -> PyClassRef { self.staticmethod_type.clone() } - pub fn generator_type(&self) -> PyObjectRef { + pub fn generator_type(&self) -> PyClassRef { self.generator_type.clone() } - pub fn bound_method_type(&self) -> PyObjectRef { + pub fn bound_method_type(&self) -> PyClassRef { self.bound_method_type.clone() } - pub fn weakref_type(&self) -> PyObjectRef { + pub fn weakref_type(&self) -> PyClassRef { self.weakref_type.clone() } - pub fn type_type(&self) -> PyObjectRef { + pub fn weakproxy_type(&self) -> PyClassRef { + self.weakproxy_type.clone() + } + + pub fn type_type(&self) -> PyClassRef { self.type_type.clone() } pub fn none(&self) -> PyObjectRef { - self.none.clone() + self.none.clone().into_object() } pub fn ellipsis(&self) -> PyObjectRef { - self.ellipsis.clone() + self.ellipsis.clone().into_object() } pub fn not_implemented(&self) -> PyObjectRef { - self.not_implemented.clone() + self.not_implemented.clone().into_object() } - pub fn object(&self) -> PyObjectRef { + pub fn object(&self) -> PyClassRef { self.object.clone() } - pub fn new_object(&self) -> PyObjectRef { - self.new_instance(self.object(), None) - } - pub fn new_int>(&self, i: T) -> PyObjectRef { - PyObject::new(PyInt::new(i), self.int_type()) + PyObject::new(PyInt::new(i), self.int_type(), None) } pub fn new_float(&self, value: f64) -> PyObjectRef { - PyObject::new(PyFloat::from(value), self.float_type()) + PyObject::new(PyFloat::from(value), self.float_type(), None) } pub fn new_complex(&self, value: Complex64) -> PyObjectRef { - PyObject::new(PyComplex::from(value), self.complex_type()) + PyObject::new(PyComplex::from(value), self.complex_type(), None) } pub fn new_str(&self, s: String) -> PyObjectRef { - PyObject::new(objstr::PyString { value: s }, self.str_type()) + PyObject::new(objstr::PyString { value: s }, self.str_type(), None) } pub fn new_bytes(&self, data: Vec) -> PyObjectRef { - PyObject::new(objbytes::PyBytes::new(data), self.bytes_type()) + PyObject::new(objbytes::PyBytes::new(data), self.bytes_type(), None) } pub fn new_bytearray(&self, data: Vec) -> PyObjectRef { - PyObject::new(objbytearray::PyByteArray::new(data), self.bytearray_type()) + PyObject::new( + objbytearray::PyByteArray::new(data), + self.bytearray_type(), + None, + ) } pub fn new_bool(&self, b: bool) -> PyObjectRef { if b { - self.true_value.clone() + self.true_value.clone().into_object() } else { - self.false_value.clone() + self.false_value.clone().into_object() } } pub fn new_tuple(&self, elements: Vec) -> PyObjectRef { - PyObject::new(PyTuple::from(elements), self.tuple_type()) + PyObject::new(PyTuple::from(elements), self.tuple_type(), None) } pub fn new_list(&self, elements: Vec) -> PyObjectRef { - PyObject::new(PyList::from(elements), self.list_type()) + PyObject::new(PyList::from(elements), self.list_type(), None) } pub fn new_set(&self) -> PyObjectRef { // Initialized empty, as calling __hash__ is required for adding each object to the set // which requires a VM context - this is done in the objset code itself. - PyObject::new(PySet::default(), self.set_type()) + PyObject::new(PySet::default(), self.set_type(), None) } - pub fn new_dict(&self) -> PyObjectRef { - PyObject::new(PyDict::default(), self.dict_type()) - } - - pub fn new_class(&self, name: &str, base: PyObjectRef) -> PyObjectRef { - objtype::new( - self.type_type(), - name, - vec![FromPyObjectRef::from_pyobj(&base)], - PyAttributes::new(), - ) - .unwrap() + pub fn new_dict(&self) -> PyDictRef { + PyObject::new(PyDict::default(), self.dict_type(), None) + .downcast() + .unwrap() } - pub fn new_scope(&self) -> Scope { - Scope::new(None, self.new_dict()) + pub fn new_class(&self, name: &str, base: PyClassRef) -> PyClassRef { + objtype::new(self.type_type(), name, vec![base], PyAttributes::new()).unwrap() } - pub fn new_module(&self, name: &str, dict: PyObjectRef) -> PyObjectRef { + pub fn new_module(&self, name: &str, dict: PyDictRef) -> PyObjectRef { PyObject::new( PyModule { name: name.to_string(), - dict, }, self.module_type.clone(), + Some(dict), ) } + pub fn new_namespace(&self) -> PyObjectRef { + PyObject::new(PyNamespace, self.namespace_type(), Some(self.new_dict())) + } + pub fn new_rustfunc(&self, f: F) -> PyObjectRef where F: IntoPyNativeFunc, @@ -589,11 +680,21 @@ impl PyContext { PyObject::new( PyBuiltinFunction::new(f.into_func()), self.builtin_function_or_method_type(), + None, ) } - pub fn new_frame(&self, code: PyObjectRef, scope: Scope) -> PyObjectRef { - PyObject::new(Frame::new(code, scope), self.frame_type()) + pub fn new_classmethod(&self, f: F) -> PyObjectRef + where + F: IntoPyNativeFunc, + { + PyObject::new( + PyClassMethod { + callable: self.new_rustfunc(f), + }, + self.classmethod_type(), + None, + ) } pub fn new_property(&self, f: F) -> PyObjectRef @@ -603,63 +704,43 @@ impl PyContext { PropertyBuilder::new(self).add_getter(f).create() } - pub fn new_code_object(&self, code: bytecode::CodeObject) -> PyObjectRef { - PyObject::new(objcode::PyCode::new(code), self.code_type()) + pub fn new_code_object(&self, code: bytecode::CodeObject) -> PyCodeRef { + PyObject::new(objcode::PyCode::new(code), self.code_type(), None) + .downcast() + .unwrap() } pub fn new_function( &self, - code_obj: PyObjectRef, + code_obj: PyCodeRef, scope: Scope, - defaults: PyObjectRef, + defaults: Option, + kw_only_defaults: Option, ) -> PyObjectRef { PyObject::new( - PyFunction::new(code_obj, scope, defaults), + PyFunction::new(code_obj, scope, defaults, kw_only_defaults), self.function_type(), + Some(self.new_dict()), ) } pub fn new_bound_method(&self, function: PyObjectRef, object: PyObjectRef) -> PyObjectRef { - PyObject::new(PyMethod::new(object, function), self.bound_method_type()) + PyObject::new( + PyMethod::new(object, function), + self.bound_method_type(), + None, + ) } - pub fn new_instance(&self, class: PyObjectRef, dict: Option) -> PyObjectRef { - let dict = dict.unwrap_or_default(); + pub fn new_instance(&self, class: PyClassRef, dict: Option) -> PyObjectRef { PyObject { typ: class, - dict: Some(RefCell::new(dict)), - payload: Box::new(objobject::PyInstance), + dict, + payload: objobject::PyInstance, } .into_ref() } - // Item set/get: - pub fn set_item(&self, obj: &PyObjectRef, key: &str, v: PyObjectRef) { - if let Some(dict) = obj.payload::() { - let key = self.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else { - unimplemented!() - }; - } - - pub fn get_attr(&self, obj: &PyObjectRef, attr_name: &str) -> Option { - // This does not need to be on the PyContext. - // We do not require to make a new key as string for this function - // (yet)... - obj.get_attr(attr_name) - } - - pub fn set_attr(&self, obj: &PyObjectRef, attr_name: &str, value: PyObjectRef) { - if let Some(PyModule { ref dict, .. }) = obj.payload::() { - dict.set_item(self, attr_name, value) - } else if let Some(ref dict) = obj.dict { - dict.borrow_mut().insert(attr_name.to_string(), value); - } else { - unimplemented!("set_attr unimplemented for: {:?}", obj); - }; - } - pub fn unwrap_constant(&self, value: &bytecode::Constant) -> PyObjectRef { match *value { bytecode::Constant::Integer { ref value } => self.new_int(value.clone()), @@ -668,7 +749,9 @@ impl PyContext { bytecode::Constant::String { ref value } => self.new_str(value.clone()), bytecode::Constant::Bytes { ref value } => self.new_bytes(value.clone()), bytecode::Constant::Boolean { ref value } => self.new_bool(value.clone()), - bytecode::Constant::Code { ref code } => self.new_code_object(*code.clone()), + bytecode::Constant::Code { ref code } => { + self.new_code_object(*code.clone()).into_object() + } bytecode::Constant::Tuple { ref elements } => { let elements = elements .iter() @@ -691,10 +774,35 @@ impl Default for PyContext { /// This is an actual python object. It consists of a `typ` which is the /// python class, and carries some rust payload optionally. This rust /// payload can be a rust float or rust int in case of float and int objects. -pub struct PyObject { - pub typ: PyObjectRef, - pub dict: Option>, // __dict__ member - pub payload: Box, +pub struct PyObject +where + T: ?Sized + PyObjectPayload, +{ + pub typ: PyClassRef, + pub dict: Option, // __dict__ member + pub payload: T, +} + +impl PyObject { + /// Attempt to downcast this reference to a subclass. + /// + /// If the downcast fails, the original ref is returned in as `Err` so + /// another downcast can be attempted without unnecessary cloning. + /// + /// Note: The returned `Result` is _not_ a `PyResult`, even though the + /// types are compatible. + pub fn downcast(self: Rc) -> Result, PyObjectRef> { + if self.payload_is::() { + Ok({ + PyRef { + obj: self, + _payload: PhantomData, + } + }) + } else { + Err(self) + } + } } /// A reference to a Python object. @@ -706,24 +814,34 @@ pub struct PyObject { /// A `PyRef` can be directly returned from a built-in function to handle /// situations (such as when implementing in-place methods such as `__iadd__`) /// where a reference to the same object must be returned. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct PyRef { // invariant: this obj must always have payload of type T obj: PyObjectRef, _payload: PhantomData, } +impl Clone for PyRef { + fn clone(&self) -> Self { + Self { + obj: self.obj.clone(), + _payload: PhantomData, + } + } +} + impl PyRef { pub fn as_object(&self) -> &PyObjectRef { &self.obj } + pub fn into_object(self) -> PyObjectRef { self.obj } pub fn typ(&self) -> PyClassRef { PyRef { - obj: self.obj.typ(), + obj: self.obj.class().into_object(), _payload: PhantomData, } } @@ -753,7 +871,7 @@ where } else { let class = T::class(vm); let expected_type = vm.to_pystr(&class)?; - let actual_type = vm.to_pystr(&obj.typ())?; + let actual_type = vm.to_pystr(&obj.class())?; Err(vm.new_type_error(format!( "Expected type {}, not {}", expected_type, actual_type, @@ -768,6 +886,18 @@ impl IntoPyObject for PyRef { } } +impl<'a, T: PyValue> From<&'a PyRef> for &'a PyObjectRef { + fn from(obj: &'a PyRef) -> Self { + obj.as_object() + } +} + +impl From> for PyObjectRef { + fn from(obj: PyRef) -> Self { + obj.into_object() + } +} + impl fmt::Display for PyRef where T: PyValue + fmt::Display, @@ -778,6 +908,39 @@ where } } +#[derive(Clone)] +pub struct PyCallable { + obj: PyObjectRef, +} + +impl PyCallable { + #[inline] + pub fn invoke(&self, args: impl Into, vm: &VirtualMachine) -> PyResult { + vm.invoke(self.obj.clone(), args) + } + + #[inline] + pub fn into_object(self) -> PyObjectRef { + self.obj + } +} + +impl TryFromObject for PyCallable { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if vm.is_callable(&obj) { + Ok(PyCallable { obj }) + } else { + Err(vm.new_type_error(format!("'{}' object is not callable", obj.class().name))) + } + } +} + +impl IntoPyObject for PyCallable { + fn into_pyobject(self, _vm: &VirtualMachine) -> PyResult { + Ok(self.into_object()) + } +} + pub trait IdProtocol { fn get_id(&self) -> usize; fn is(&self, other: &T) -> bool @@ -788,154 +951,101 @@ pub trait IdProtocol { } } -impl IdProtocol for PyObjectRef { - fn get_id(&self) -> usize { - &*self as &PyObject as *const PyObject as usize +#[derive(Debug)] +enum Never {} + +impl PyValue for Never { + fn class(_vm: &VirtualMachine) -> PyClassRef { + unreachable!() } } -pub trait FromPyObjectRef { - fn from_pyobj(obj: &PyObjectRef) -> Self; +impl IdProtocol for PyObject { + fn get_id(&self) -> usize { + self as *const _ as *const PyObject as usize + } } -pub trait TypeProtocol { - fn typ(&self) -> PyObjectRef { - self.type_ref().clone() - } - fn type_pyref(&self) -> PyClassRef { - FromPyObjectRef::from_pyobj(self.type_ref()) +impl IdProtocol for Rc { + fn get_id(&self) -> usize { + (**self).get_id() } - fn type_ref(&self) -> &PyObjectRef; } -impl TypeProtocol for PyObjectRef { - fn type_ref(&self) -> &PyObjectRef { - (**self).type_ref() +impl IdProtocol for PyRef { + fn get_id(&self) -> usize { + self.obj.get_id() } } -impl TypeProtocol for PyObject { - fn type_ref(&self) -> &PyObjectRef { - &self.typ - } +pub trait TypeProtocol { + fn class(&self) -> PyClassRef; } -pub trait AttributeProtocol { - fn get_attr(&self, attr_name: &str) -> Option; - fn has_attr(&self, attr_name: &str) -> bool; +impl TypeProtocol for PyObjectRef { + fn class(&self) -> PyClassRef { + (**self).class() + } } -fn class_get_item(class: &PyObjectRef, attr_name: &str) -> Option { - if let Some(ref dict) = class.dict { - dict.borrow().get(attr_name).cloned() - } else { - panic!("Only classes should be in MRO!"); +impl TypeProtocol for PyObject +where + T: ?Sized + PyObjectPayload, +{ + fn class(&self) -> PyClassRef { + self.typ.clone() } } -fn class_has_item(class: &PyObjectRef, attr_name: &str) -> bool { - if let Some(ref dict) = class.dict { - dict.borrow().contains_key(attr_name) - } else { - panic!("Only classes should be in MRO!"); +impl TypeProtocol for PyRef { + fn class(&self) -> PyClassRef { + self.obj.typ.clone() } } -impl AttributeProtocol for PyObjectRef { - fn get_attr(&self, attr_name: &str) -> Option { - if let Some(PyClass { ref mro, .. }) = self.payload::() { - if let Some(item) = class_get_item(self, attr_name) { - return Some(item); - } - for class in mro { - if let Some(item) = class_get_item(class.as_object(), attr_name) { - return Some(item); +pub trait ItemProtocol { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult; + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult; + fn get_item_option( + &self, + key: T, + vm: &VirtualMachine, + ) -> PyResult> { + match self.get_item(key, vm) { + Ok(value) => Ok(Some(value)), + Err(exc) => { + if objtype::isinstance(&exc, &vm.ctx.exceptions.key_error) { + Ok(None) + } else { + Err(exc) } } - return None; - } - - if let Some(PyModule { ref dict, .. }) = self.payload::() { - return dict.get_item(attr_name); - } - - if let Some(ref dict) = self.dict { - dict.borrow().get(attr_name).cloned() - } else { - None } } - - fn has_attr(&self, attr_name: &str) -> bool { - if let Some(PyClass { ref mro, .. }) = self.payload::() { - return class_has_item(self, attr_name) - || mro.iter().any(|d| class_has_item(d.as_object(), attr_name)); - } - - if let Some(PyModule { ref dict, .. }) = self.payload::() { - return dict.contains_key(attr_name); - } - - if let Some(ref dict) = self.dict { - dict.borrow().contains_key(attr_name) - } else { - false - } - } -} - -pub trait DictProtocol { - fn contains_key(&self, k: &str) -> bool; - fn get_item(&self, k: &str) -> Option; - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)>; - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef); - fn del_item(&self, key: &str); } -impl DictProtocol for PyObjectRef { - fn contains_key(&self, k: &str) -> bool { - if let Some(dict) = self.payload::() { - objdict::content_contains_key_str(&dict.entries.borrow(), k) - } else { - unimplemented!() - } - } - - fn get_item(&self, k: &str) -> Option { - if let Some(dict) = self.payload::() { - objdict::content_get_key_str(&dict.entries.borrow(), k) - } else if let Some(PyModule { ref dict, .. }) = self.payload::() { - dict.get_item(k) - } else { - panic!("TODO {:?}", k) - } +impl ItemProtocol for PyObjectRef { + fn get_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__getitem__", key.into_pyobject(vm)?) } - fn get_key_value_pairs(&self) -> Vec<(PyObjectRef, PyObjectRef)> { - if self.payload_is::() { - objdict::get_key_value_pairs(self) - } else if let Some(PyModule { ref dict, .. }) = self.payload::() { - dict.get_key_value_pairs() - } else { - panic!("TODO") - } - } - - // Item set/get: - fn set_item(&self, ctx: &PyContext, key: &str, v: PyObjectRef) { - if let Some(dict) = self.payload::() { - let key = ctx.new_str(key.to_string()); - objdict::set_item_in_content(&mut dict.entries.borrow_mut(), &key, &v); - } else if let Some(PyModule { ref dict, .. }) = self.payload::() { - dict.set_item(ctx, key, v); - } else { - panic!("TODO {:?}", self); - } + fn set_item( + &self, + key: T, + value: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + vm.call_method(self, "__setitem__", vec![key.into_pyobject(vm)?, value]) } - fn del_item(&self, key: &str) { - let mut elements = objdict::get_mut_elements(self); - elements.remove(key).unwrap(); + fn del_item(&self, key: T, vm: &VirtualMachine) -> PyResult { + vm.call_method(self, "__delitem__", key.into_pyobject(vm)?) } } @@ -945,7 +1055,7 @@ pub trait BufferProtocol { impl BufferProtocol for PyObjectRef { fn readonly(&self) -> bool { - match objtype::get_type_name(&self.typ()).as_ref() { + match self.class().name.as_str() { "bytes" => false, "bytearray" | "memoryview" => true, _ => panic!("Bytes-Like type expected not {:?}", self), @@ -953,9 +1063,9 @@ impl BufferProtocol for PyObjectRef { } } -impl fmt::Debug for PyObject { +impl fmt::Debug for PyObject { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "[PyObj {:?}]", self.payload) + write!(f, "[PyObj {:?}]", &self.payload) } } @@ -981,7 +1091,7 @@ impl PyIterable { self.method.clone(), PyFuncArgs { args: vec![], - kwargs: vec![], + kwargs: IndexMap::new(), }, )?; @@ -1009,8 +1119,7 @@ where match self.vm.call_method(&self.obj, "__next__", vec![]) { Ok(value) => Some(T::try_from_object(self.vm, value)), Err(err) => { - let stop_ex = self.vm.ctx.exceptions.stop_iteration.clone(); - if objtype::isinstance(&err, &stop_ex) { + if objtype::isinstance(&err, &self.vm.ctx.exceptions.stop_iteration) { None } else { Some(Err(err)) @@ -1025,10 +1134,27 @@ where T: TryFromObject, { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - Ok(PyIterable { - method: vm.get_method(obj, "__iter__")?, - _item: std::marker::PhantomData, - }) + if let Some(method_or_err) = vm.get_method(obj.clone(), "__iter__") { + let method = method_or_err?; + Ok(PyIterable { + method, + _item: std::marker::PhantomData, + }) + } else { + vm.get_method_or_type_error(obj.clone(), "__getitem__", || { + format!("'{}' object is not iterable", obj.class().name) + })?; + Self::try_from_object( + vm, + objiter::PySequenceIterator { + position: Cell::new(0), + obj: obj.clone(), + reversed: false, + } + .into_ref(vm) + .into_object(), + ) + } } } @@ -1109,48 +1235,26 @@ where T: PyValue + Sized, { fn into_pyobject(self, vm: &VirtualMachine) -> PyResult { - Ok(PyObject::new(self, T::class(vm))) + Ok(PyObject::new(self, T::class(vm), None)) } } -// TODO: This is a workaround and shouldn't exist. -// Each iterable type should have its own distinct iterator type. -#[derive(Debug)] -pub struct PyIteratorValue { - pub position: Cell, - pub iterated_obj: PyObjectRef, -} - -impl PyValue for PyIteratorValue { - fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.ctx.iter_type() - } -} - -impl PyObject { - pub fn new(payload: T, typ: PyObjectRef) -> PyObjectRef { - PyObject { - typ, - dict: Some(RefCell::new(PyAttributes::new())), - payload: Box::new(payload), - } - .into_ref() - } - - pub fn new_without_dict(payload: T, typ: PyObjectRef) -> PyObjectRef { - PyObject { - typ, - dict: None, - payload: Box::new(payload), - } - .into_ref() +impl PyObject +where + T: Sized + PyObjectPayload, +{ + #[allow(clippy::new_ret_no_self)] + pub fn new(payload: T, typ: PyClassRef, dict: Option) -> PyObjectRef { + PyObject { typ, dict, payload }.into_ref() } // Move this object into a reference object, transferring ownership. pub fn into_ref(self) -> PyObjectRef { Rc::new(self) } +} +impl PyObject { #[inline] pub fn payload(&self) -> Option<&T> { self.payload.as_any().downcast_ref() @@ -1163,25 +1267,32 @@ impl PyObject { } pub trait PyValue: fmt::Debug + Sized + 'static { - fn class(vm: &VirtualMachine) -> PyObjectRef; + const HAVE_DICT: bool = false; + + fn class(vm: &VirtualMachine) -> PyClassRef; fn into_ref(self, vm: &VirtualMachine) -> PyRef { PyRef { - obj: PyObject::new(self, Self::class(vm)), + obj: PyObject::new(self, Self::class(vm), None), _payload: PhantomData, } } fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyClassRef) -> PyResult> { let class = Self::class(vm); - if objtype::issubclass(&cls.obj, &class) { + if objtype::issubclass(&cls, &class) { + let dict = if !Self::HAVE_DICT && cls.is(&class) { + None + } else { + Some(vm.ctx.new_dict()) + }; Ok(PyRef { - obj: PyObject::new(self, cls.obj), + obj: PyObject::new(self, cls, dict), _payload: PhantomData, }) } else { let subtype = vm.to_pystr(&cls.obj)?; - let basetype = vm.to_pystr(&class)?; + let basetype = vm.to_pystr(&class.obj)?; Err(vm.new_type_error(format!("{} is not a subtype of {}", subtype, basetype))) } } @@ -1198,22 +1309,89 @@ impl PyObjectPayload for T { } } -impl FromPyObjectRef for PyRef { - fn from_pyobj(obj: &PyObjectRef) -> Self { - if obj.payload_is::() { - PyRef { - obj: obj.clone(), - _payload: PhantomData, - } - } else { - panic!("Error getting inner type: {:?}", obj.typ) +pub enum Either { + A(A), + B(B), +} + +/// This allows a builtin method to accept arguments that may be one of two +/// types, raising a `TypeError` if it is neither. +/// +/// # Example +/// +/// ``` +/// use rustpython_vm::VirtualMachine; +/// use rustpython_vm::obj::{objstr::PyStringRef, objint::PyIntRef}; +/// use rustpython_vm::pyobject::Either; +/// +/// fn do_something(arg: Either, vm: &VirtualMachine) { +/// match arg { +/// Either::A(int)=> { +/// // do something with int +/// } +/// Either::B(string) => { +/// // do something with string +/// } +/// } +/// } +/// ``` +impl TryFromObject for Either, PyRef> +where + A: PyValue, + B: PyValue, +{ + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + obj.downcast::() + .map(Either::A) + .or_else(|obj| obj.clone().downcast::().map(Either::B)) + .map_err(|obj| { + vm.new_type_error(format!( + "must be {} or {}, not {}", + A::class(vm), + B::class(vm), + obj.class() + )) + }) + } +} + +pub trait PyClassDef { + const NAME: &'static str; + const DOC: Option<&'static str> = None; +} + +impl PyClassDef for PyRef +where + T: PyClassDef, +{ + const NAME: &'static str = T::NAME; + const DOC: Option<&'static str> = T::DOC; +} + +pub trait PyClassImpl: PyClassDef { + fn impl_extend_class(ctx: &PyContext, class: &PyClassRef); + + fn extend_class(ctx: &PyContext, class: &PyClassRef) { + Self::impl_extend_class(ctx, class); + if let Some(doc) = Self::DOC { + class.set_str_attr("__doc__", ctx.new_str(doc.into())); } } + + fn make_class(ctx: &PyContext) -> PyClassRef { + Self::make_class_with_base(ctx, ctx.object()) + } + + fn make_class_with_base(ctx: &PyContext, base: PyClassRef) -> PyClassRef { + let py_class = ctx.new_class(Self::NAME, base); + Self::extend_class(ctx, &py_class); + py_class + } } #[cfg(test)] mod tests { - use super::PyContext; + use super::*; #[test] fn test_type_type() { diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index a3010a5199..5808e303f7 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -9,76 +9,73 @@ use num_complex::Complex64; use rustpython_parser::{ast, parser}; -use crate::function::PyFuncArgs; -use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::objlist::PyListRef; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -/* - * Idea: maybe we can create a sort of struct with some helper functions? -struct AstToPyAst { - ctx: &PyContext, +#[derive(Debug)] +struct AstNode; +type AstNodeRef = PyRef; + +impl PyValue for AstNode { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("ast", "AST") + } } -impl AstToPyAst { - fn new(ctx: &PyContext) -> Self { - AstToPyAst { - ctx: ctx, +macro_rules! node { + ( $vm: expr, $node_name:ident, { $($attr_name:ident => $attr_value:expr),* $(,)* }) => { + { + let node = create_node($vm, stringify!($node_name))?; + $( + $vm.set_attr(node.as_object(), stringify!($attr_name), $attr_value)?; + )* + Ok(node) } + }; + ( $vm: expr, $node_name:ident) => { + create_node($vm, stringify!($node_name)) } - } -*/ -fn program_to_ast(ctx: &PyContext, program: &ast::Program) -> PyObjectRef { - let mut body = vec![]; - for statement in &program.statements { - body.push(statement_to_ast(ctx, statement)); - } - // TODO: create Module node: - // let ast_node = ctx.new_instance(this.Module); - let ast_node = ctx.new_object(); - let py_body = ctx.new_list(body); - ctx.set_attr(&ast_node, "body", py_body); - ast_node +fn program_to_ast(vm: &VirtualMachine, program: &ast::Program) -> PyResult { + let py_body = statements_to_ast(vm, &program.statements)?; + node!(vm, Module, { body => py_body }) } // Create a node class instance -fn create_node(ctx: &PyContext, _name: &str) -> PyObjectRef { - // TODO: instantiate a class of type given by name - // TODO: lookup in the current module? - ctx.new_object() +fn create_node(vm: &VirtualMachine, name: &str) -> PyResult { + AstNode.into_ref_with_type(vm, vm.class("ast", name)) } -fn statements_to_ast(ctx: &PyContext, statements: &[ast::LocatedStatement]) -> PyObjectRef { - let mut py_statements = vec![]; - for statement in statements { - py_statements.push(statement_to_ast(ctx, statement)); - } - ctx.new_list(py_statements) +fn statements_to_ast( + vm: &VirtualMachine, + statements: &[ast::LocatedStatement], +) -> PyResult { + let body: PyResult> = statements + .iter() + .map(|statement| Ok(statement_to_ast(&vm, statement)?.into_object())) + .collect(); + Ok(vm.ctx.new_list(body?).downcast().unwrap()) } -fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObjectRef { +fn statement_to_ast( + vm: &VirtualMachine, + statement: &ast::LocatedStatement, +) -> PyResult { let node = match &statement.node { ast::Statement::ClassDef { name, body, decorator_list, .. - } => { - let node = create_node(ctx, "ClassDef"); - - // Set name: - ctx.set_attr(&node, "name", ctx.new_str(name.to_string())); - - // Set body: - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - let py_decorator_list = expressions_to_ast(ctx, decorator_list); - ctx.set_attr(&node, "decorator_list", py_decorator_list); - node - } + } => node!(vm, ClassDef, { + name => vm.ctx.new_str(name.to_string()), + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, + }), ast::Statement::FunctionDef { name, args, @@ -86,176 +83,151 @@ fn statement_to_ast(ctx: &PyContext, statement: &ast::LocatedStatement) -> PyObj decorator_list, returns, } => { - let node = create_node(ctx, "FunctionDef"); - - // Set name: - ctx.set_attr(&node, "name", ctx.new_str(name.to_string())); - - ctx.set_attr(&node, "args", parameters_to_ast(ctx, args)); - - // Set body: - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - let py_decorator_list = expressions_to_ast(ctx, decorator_list); - ctx.set_attr(&node, "decorator_list", py_decorator_list); - let py_returns = if let Some(hint) = returns { - expression_to_ast(ctx, hint) + expression_to_ast(vm, hint)?.into_object() } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "returns", py_returns); - node - } - ast::Statement::Continue => create_node(ctx, "Continue"), - ast::Statement::Break => create_node(ctx, "Break"), - ast::Statement::Pass => create_node(ctx, "Pass"), + node!(vm, FunctionDef, { + name => vm.ctx.new_str(name.to_string()), + args => parameters_to_ast(vm, args)?, + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, + returns => py_returns + }) + } + ast::Statement::AsyncFunctionDef { + name, + args, + body, + decorator_list, + returns, + } => { + let py_returns = if let Some(hint) = returns { + expression_to_ast(vm, hint)?.into_object() + } else { + vm.ctx.none() + }; + node!(vm, AsyncFunctionDef, { + name => vm.ctx.new_str(name.to_string()), + args => parameters_to_ast(vm, args)?, + body => statements_to_ast(vm, body)?, + decorator_list => expressions_to_ast(vm, decorator_list)?, + returns => py_returns + }) + } + ast::Statement::Continue => node!(vm, Continue), + ast::Statement::Break => node!(vm, Break), + ast::Statement::Pass => node!(vm, Pass), ast::Statement::Assert { test, msg } => { - let node = create_node(ctx, "Pass"); - - ctx.set_attr(&node, "test", expression_to_ast(ctx, test)); - let py_msg = match msg { - Some(msg) => expression_to_ast(ctx, msg), - None => ctx.none(), + Some(msg) => expression_to_ast(vm, msg)?.into_object(), + None => vm.ctx.none(), }; - ctx.set_attr(&node, "msg", py_msg); - - node + node!(vm, Assert, { + test => expression_to_ast(vm, test)?, + msg => py_msg + }) } ast::Statement::Delete { targets } => { - let node = create_node(ctx, "Delete"); - - let py_targets = - ctx.new_tuple(targets.iter().map(|v| expression_to_ast(ctx, v)).collect()); - ctx.set_attr(&node, "targets", py_targets); - - node + let targets: PyResult<_> = targets + .iter() + .map(|v| Ok(expression_to_ast(vm, v)?.into_object())) + .collect(); + let py_targets = vm.ctx.new_tuple(targets?); + node!(vm, Delete, { targets => py_targets }) } ast::Statement::Return { value } => { - let node = create_node(ctx, "Return"); - let py_value = if let Some(value) = value { - ctx.new_tuple(value.iter().map(|v| expression_to_ast(ctx, v)).collect()) - } else { - ctx.none() - }; - ctx.set_attr(&node, "value", py_value); - - node - } - ast::Statement::If { test, body, orelse } => { - let node = create_node(ctx, "If"); - - let py_test = expression_to_ast(ctx, test); - ctx.set_attr(&node, "test", py_test); - - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(ctx, orelse) + expression_to_ast(vm, value)?.into_object() } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "orelse", py_orelse); - node - } + node!(vm, Return, { + value => py_value + }) + } + ast::Statement::If { test, body, orelse } => node!(vm, If, { + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() + } else { + vm.ctx.none() + } + }), ast::Statement::For { target, iter, body, orelse, - } => { - let node = create_node(ctx, "For"); - - let py_target = expression_to_ast(ctx, target); - ctx.set_attr(&node, "target", py_target); - - let py_iter = expressions_to_ast(ctx, iter); - ctx.set_attr(&node, "iter", py_iter); - - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(ctx, orelse) + } => node!(vm, For, { + target => expression_to_ast(vm, target)?, + iter => expression_to_ast(vm, iter)?, + body => statements_to_ast(vm, body)?, + or_else => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() } else { - ctx.none() - }; - ctx.set_attr(&node, "orelse", py_orelse); - - node - } - ast::Statement::While { test, body, orelse } => { - let node = create_node(ctx, "While"); - - let py_test = expression_to_ast(ctx, test); - ctx.set_attr(&node, "test", py_test); - - let py_body = statements_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - let py_orelse = if let Some(orelse) = orelse { - statements_to_ast(ctx, orelse) + vm.ctx.none() + } + }), + ast::Statement::AsyncFor { + target, + iter, + body, + orelse, + } => node!(vm, AsyncFor, { + target => expression_to_ast(vm, target)?, + iter => expression_to_ast(vm, iter)?, + body => statements_to_ast(vm, body)?, + or_else => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() } else { - ctx.none() - }; - ctx.set_attr(&node, "orelse", py_orelse); - - node - } - ast::Statement::Expression { expression } => { - let node = create_node(ctx, "Expr"); - - let value = expression_to_ast(ctx, expression); - ctx.set_attr(&node, "value", value); - - node - } + vm.ctx.none() + } + }), + ast::Statement::While { test, body, orelse } => node!(vm, While, { + test => expression_to_ast(vm, test)?, + body => statements_to_ast(vm, body)?, + orelse => if let Some(orelse) = orelse { + statements_to_ast(vm, orelse)?.into_object() + } else { + vm.ctx.none() + } + }), + ast::Statement::Expression { expression } => node!(vm, Expr, { + value => expression_to_ast(vm, expression)? + }), x => { - unimplemented!("{:?}", x); + return Err(vm.new_type_error(format!("Ast not implemented: {:?}", x))); } - }; + }?; // set lineno on node: - let lineno = ctx.new_int(statement.location.get_row()); - ctx.set_attr(&node, "lineno", lineno); + let lineno = vm.ctx.new_int(statement.location.row()); + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); - node + Ok(node) } -fn expressions_to_ast(ctx: &PyContext, expressions: &[ast::Expression]) -> PyObjectRef { - let mut py_expression_nodes = vec![]; - for expression in expressions { - py_expression_nodes.push(expression_to_ast(ctx, expression)); - } - ctx.new_list(py_expression_nodes) +fn expressions_to_ast(vm: &VirtualMachine, expressions: &[ast::Expression]) -> PyResult { + let py_expression_nodes: PyResult<_> = expressions + .iter() + .map(|expression| Ok(expression_to_ast(vm, expression)?.into_object())) + .collect(); + Ok(vm.ctx.new_list(py_expression_nodes?).downcast().unwrap()) } -fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectRef { +fn expression_to_ast(vm: &VirtualMachine, expression: &ast::Expression) -> PyResult { let node = match &expression { - ast::Expression::Call { function, args, .. } => { - let node = create_node(ctx, "Call"); - - let py_func_ast = expression_to_ast(ctx, function); - ctx.set_attr(&node, "func", py_func_ast); - - let py_args = expressions_to_ast(ctx, args); - ctx.set_attr(&node, "args", py_args); - - node - } + ast::Expression::Call { function, args, .. } => node!(vm, Call, { + func => expression_to_ast(vm, function)?, + args => expressions_to_ast(vm, args)?, + }), ast::Expression::Binop { a, op, b } => { - let node = create_node(ctx, "BinOp"); - - let py_a = expression_to_ast(ctx, a); - ctx.set_attr(&node, "left", py_a); - // Operator: - let str_op = match op { + let op = match op { ast::Operator::Add => "Add", ast::Operator::Sub => "Sub", ast::Operator::Mult => "Mult", @@ -270,56 +242,46 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR ast::Operator::BitAnd => "BitAnd", ast::Operator::FloorDiv => "FloorDiv", }; - let py_op = ctx.new_str(str_op.to_string()); - ctx.set_attr(&node, "op", py_op); - - let py_b = expression_to_ast(ctx, b); - ctx.set_attr(&node, "right", py_b); - node + node!(vm, BinOp, { + left => expression_to_ast(vm, a)?, + op => vm.ctx.new_str(op.to_string()), + right => expression_to_ast(vm, b)?, + }) } ast::Expression::Unop { op, a } => { - let node = create_node(ctx, "UnaryOp"); - - let str_op = match op { + let op = match op { ast::UnaryOperator::Not => "Not", ast::UnaryOperator::Inv => "Invert", ast::UnaryOperator::Neg => "USub", ast::UnaryOperator::Pos => "UAdd", }; - let py_op = ctx.new_str(str_op.to_string()); - ctx.set_attr(&node, "op", py_op); - - let py_a = expression_to_ast(ctx, a); - ctx.set_attr(&node, "operand", py_a); - - node + node!(vm, UnaryOp, { + op => vm.ctx.new_str(op.to_string()), + operand => expression_to_ast(vm, a)?, + }) } ast::Expression::BoolOp { a, op, b } => { - let node = create_node(ctx, "BoolOp"); - // Attach values: - let py_a = expression_to_ast(ctx, a); - let py_b = expression_to_ast(ctx, b); - let py_values = ctx.new_tuple(vec![py_a, py_b]); - ctx.set_attr(&node, "values", py_values); + let py_a = expression_to_ast(vm, a)?.into_object(); + let py_b = expression_to_ast(vm, b)?.into_object(); + let py_values = vm.ctx.new_tuple(vec![py_a, py_b]); let str_op = match op { ast::BooleanOperator::And => "And", ast::BooleanOperator::Or => "Or", }; - let py_op = ctx.new_str(str_op.to_string()); - ctx.set_attr(&node, "op", py_op); + let py_op = vm.ctx.new_str(str_op.to_string()); - node + node!(vm, BoolOp, { + op => py_op, + values => py_values, + }) } - ast::Expression::Compare { a, op, b } => { - let node = create_node(ctx, "Compare"); - - let py_a = expression_to_ast(ctx, a); - ctx.set_attr(&node, "left", py_a); + ast::Expression::Compare { vals, ops } => { + let left = expression_to_ast(vm, &vals[0])?; // Operator: - let str_op = match op { + let to_operator = |op: &ast::Comparison| match op { ast::Comparison::Equal => "Eq", ast::Comparison::NotEqual => "NotEq", ast::Comparison::Less => "Lt", @@ -331,303 +293,267 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR ast::Comparison::Is => "Is", ast::Comparison::IsNot => "IsNot", }; - let py_ops = ctx.new_list(vec![ctx.new_str(str_op.to_string())]); - ctx.set_attr(&node, "ops", py_ops); - - let py_b = ctx.new_list(vec![expression_to_ast(ctx, b)]); - ctx.set_attr(&node, "comparators", py_b); - node - } - ast::Expression::Identifier { name } => { - let node = create_node(ctx, "Identifier"); - - // Id: - let py_name = ctx.new_str(name.clone()); - ctx.set_attr(&node, "id", py_name); - node - } - ast::Expression::Lambda { args, body } => { - let node = create_node(ctx, "Lambda"); - - ctx.set_attr(&node, "args", parameters_to_ast(ctx, args)); - - let py_body = expression_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - node - } - ast::Expression::IfExpression { test, body, orelse } => { - let node = create_node(ctx, "IfExp"); - - let py_test = expression_to_ast(ctx, test); - ctx.set_attr(&node, "test", py_test); - - let py_body = expression_to_ast(ctx, body); - ctx.set_attr(&node, "body", py_body); - - let py_orelse = expression_to_ast(ctx, orelse); - ctx.set_attr(&node, "orelse", py_orelse); + let ops = vm.ctx.new_list( + ops.iter() + .map(|x| vm.ctx.new_str(to_operator(x).to_string())) + .collect(), + ); - node - } + let comparators: PyResult<_> = vals + .iter() + .skip(1) + .map(|x| Ok(expression_to_ast(vm, x)?.into_object())) + .collect(); + let comparators = vm.ctx.new_list(comparators?); + node!(vm, Compare, { + left => left, + ops => ops, + comparators => comparators, + }) + } + ast::Expression::Identifier { name } => node!(vm, Identifier, { + id => vm.ctx.new_str(name.clone()) + }), + ast::Expression::Lambda { args, body } => node!(vm, Lambda, { + args => parameters_to_ast(vm, args)?, + body => expression_to_ast(vm, body)?, + }), + ast::Expression::IfExpression { test, body, orelse } => node!(vm, IfExp, { + text => expression_to_ast(vm, test)?, + body => expression_to_ast(vm, body)?, + or_else => expression_to_ast(vm, orelse)?, + }), ast::Expression::Number { value } => { - let node = create_node(ctx, "Num"); - let py_n = match value { - ast::Number::Integer { value } => ctx.new_int(value.clone()), - ast::Number::Float { value } => ctx.new_float(*value), + ast::Number::Integer { value } => vm.ctx.new_int(value.clone()), + ast::Number::Float { value } => vm.ctx.new_float(*value), ast::Number::Complex { real, imag } => { - ctx.new_complex(Complex64::new(*real, *imag)) + vm.ctx.new_complex(Complex64::new(*real, *imag)) } }; - ctx.set_attr(&node, "n", py_n); - - node - } - ast::Expression::True => { - let node = create_node(ctx, "NameConstant"); - - ctx.set_attr(&node, "value", ctx.new_bool(true)); - - node - } - ast::Expression::False => { - let node = create_node(ctx, "NameConstant"); - - ctx.set_attr(&node, "value", ctx.new_bool(false)); - - node - } - ast::Expression::None => { - let node = create_node(ctx, "NameConstant"); - - ctx.set_attr(&node, "value", ctx.none()); - - node - } - ast::Expression::Ellipsis => create_node(ctx, "Ellipsis"), - ast::Expression::List { elements } => { - let node = create_node(ctx, "List"); - - let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); - let py_elts = ctx.new_list(elts); - ctx.set_attr(&node, "elts", py_elts); - - node - } - ast::Expression::Tuple { elements } => { - let node = create_node(ctx, "Tuple"); - - let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); - let py_elts = ctx.new_list(elts); - ctx.set_attr(&node, "elts", py_elts); - - node - } - ast::Expression::Set { elements } => { - let node = create_node(ctx, "Set"); - - let elts = elements.iter().map(|e| expression_to_ast(ctx, e)).collect(); - let py_elts = ctx.new_list(elts); - ctx.set_attr(&node, "elts", py_elts); - - node - } + node!(vm, Num, { + n => py_n + }) + } + ast::Expression::True => node!(vm, NameConstant, { + value => vm.ctx.new_bool(true) + }), + ast::Expression::False => node!(vm, NameConstant, { + value => vm.ctx.new_bool(false) + }), + ast::Expression::None => node!(vm, NameConstant, { + value => vm.ctx.none() + }), + ast::Expression::Ellipsis => node!(vm, Ellipsis), + ast::Expression::List { elements } => node!(vm, List, { + elts => expressions_to_ast(vm, &elements)? + }), + ast::Expression::Tuple { elements } => node!(vm, Tuple, { + elts => expressions_to_ast(vm, &elements)? + }), + ast::Expression::Set { elements } => node!(vm, Set, { + elts => expressions_to_ast(vm, &elements)? + }), ast::Expression::Dict { elements } => { - let node = create_node(ctx, "Dict"); - let mut keys = Vec::new(); let mut values = Vec::new(); for (k, v) in elements { - keys.push(expression_to_ast(ctx, k)); - values.push(expression_to_ast(ctx, v)); + if let Some(k) = k { + keys.push(expression_to_ast(vm, k)?.into_object()); + } else { + keys.push(vm.ctx.none()); + } + values.push(expression_to_ast(vm, v)?.into_object()); } - let py_keys = ctx.new_list(keys); - ctx.set_attr(&node, "keys", py_keys); - - let py_values = ctx.new_list(values); - ctx.set_attr(&node, "values", py_values); - - node + node!(vm, Dict, { + keys => vm.ctx.new_list(keys), + values => vm.ctx.new_list(values), + }) } ast::Expression::Comprehension { kind, generators } => { - let node = match kind.deref() { + let py_generators = map_ast(comprehension_to_ast, vm, generators)?; + + match kind.deref() { ast::ComprehensionKind::GeneratorExpression { .. } => { - create_node(ctx, "GeneratorExp") + node!(vm, GeneratorExp, {generators => py_generators}) } - ast::ComprehensionKind::List { .. } => create_node(ctx, "ListComp"), - ast::ComprehensionKind::Set { .. } => create_node(ctx, "SetComp"), - ast::ComprehensionKind::Dict { .. } => create_node(ctx, "DictComp"), - }; - - let g = generators - .iter() - .map(|g| comprehension_to_ast(ctx, g)) - .collect(); - let py_generators = ctx.new_list(g); - ctx.set_attr(&node, "generators", py_generators); - - node + ast::ComprehensionKind::List { .. } => { + node!(vm, ListComp, {generators => py_generators}) + } + ast::ComprehensionKind::Set { .. } => { + node!(vm, SetComp, {generators => py_generators}) + } + ast::ComprehensionKind::Dict { .. } => { + node!(vm, DictComp, {generators => py_generators}) + } + } + } + ast::Expression::Await { value } => { + let py_value = expression_to_ast(vm, value)?; + node!(vm, Await, { + value => py_value + }) } ast::Expression::Yield { value } => { - let node = create_node(ctx, "Yield"); - let py_value = match value { - Some(value) => expression_to_ast(ctx, value), - None => ctx.none(), + Some(value) => expression_to_ast(vm, value)?.into_object(), + None => vm.ctx.none(), }; - ctx.set_attr(&node, "value", py_value); - - node + node!(vm, Yield, { + value => py_value + }) } ast::Expression::YieldFrom { value } => { - let node = create_node(ctx, "YieldFrom"); - - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); - - node - } - ast::Expression::Subscript { a, b } => { - let node = create_node(ctx, "Subscript"); - - let py_value = expression_to_ast(ctx, a); - ctx.set_attr(&node, "value", py_value); - - let py_slice = expression_to_ast(ctx, b); - ctx.set_attr(&node, "slice", py_slice); - - node - } - ast::Expression::Attribute { value, name } => { - let node = create_node(ctx, "Attribute"); - - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); - - let py_attr = ctx.new_str(name.to_string()); - ctx.set_attr(&node, "attr", py_attr); - - node - } - ast::Expression::Starred { value } => { - let node = create_node(ctx, "Starred"); - - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); - - node - } - ast::Expression::Slice { elements } => { - let node = create_node(ctx, "Slice"); - - let py_value = expressions_to_ast(ctx, elements); - ctx.set_attr(&node, "bounds", py_value); - - node - } - ast::Expression::String { value } => string_to_ast(ctx, value), + let py_value = expression_to_ast(vm, value)?; + node!(vm, YieldFrom, { + value => py_value + }) + } + ast::Expression::Subscript { a, b } => node!(vm, Subscript, { + value => expression_to_ast(vm, a)?, + slice => expression_to_ast(vm, b)?, + }), + ast::Expression::Attribute { value, name } => node!(vm, Attribute, { + value => expression_to_ast(vm, value)?, + attr => vm.ctx.new_str(name.to_string()), + }), + ast::Expression::Starred { value } => node!(vm, Starred, { + value => expression_to_ast(vm, value)? + }), + ast::Expression::Slice { elements } => node!(vm, Slice, { + bounds => expressions_to_ast(vm, elements)? + }), + ast::Expression::String { value } => string_to_ast(vm, value), ast::Expression::Bytes { value } => { - let node = create_node(ctx, "Bytes"); - ctx.set_attr(&node, "s", ctx.new_bytes(value.clone())); - node + node!(vm, Bytes, { s => vm.ctx.new_bytes(value.clone()) }) } - }; + }?; // TODO: retrieve correct lineno: - let lineno = ctx.new_int(1); - ctx.set_attr(&node, "lineno", lineno); - - node + let lineno = vm.ctx.new_int(1); + vm.set_attr(node.as_object(), "lineno", lineno).unwrap(); + Ok(node) } -fn parameters_to_ast(ctx: &PyContext, args: &ast::Parameters) -> PyObjectRef { - let node = create_node(ctx, "arguments"); - - ctx.set_attr( - &node, - "args", - ctx.new_list(args.args.iter().map(|a| parameter_to_ast(ctx, a)).collect()), - ); - - node +fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyResult { + let args = map_ast(parameter_to_ast, vm, &args.args)?; + node!(vm, arguments, { args => args }) } -fn parameter_to_ast(ctx: &PyContext, parameter: &ast::Parameter) -> PyObjectRef { - let node = create_node(ctx, "arg"); - - let py_arg = ctx.new_str(parameter.arg.to_string()); - ctx.set_attr(&node, "arg", py_arg); - +fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult { let py_annotation = if let Some(annotation) = ¶meter.annotation { - expression_to_ast(ctx, annotation) + expression_to_ast(vm, annotation)?.into_object() } else { - ctx.none() + vm.ctx.none() }; - ctx.set_attr(&node, "annotation", py_annotation); - node + node!(vm, arg, { + arg => vm.ctx.new_str(parameter.arg.to_string()), + annotation => py_annotation + }) } -fn comprehension_to_ast(ctx: &PyContext, comprehension: &ast::Comprehension) -> PyObjectRef { - let node = create_node(ctx, "comprehension"); - - let py_target = expression_to_ast(ctx, &comprehension.target); - ctx.set_attr(&node, "target", py_target); - - let py_iter = expression_to_ast(ctx, &comprehension.iter); - ctx.set_attr(&node, "iter", py_iter); - - let py_ifs = expressions_to_ast(ctx, &comprehension.ifs); - ctx.set_attr(&node, "ifs", py_ifs); +fn map_ast( + f: fn(vm: &VirtualMachine, &T) -> PyResult, + vm: &VirtualMachine, + items: &[T], +) -> PyResult { + let list: PyResult> = + items.iter().map(|x| Ok(f(vm, x)?.into_object())).collect(); + Ok(vm.ctx.new_list(list?)) +} - node +fn comprehension_to_ast( + vm: &VirtualMachine, + comprehension: &ast::Comprehension, +) -> PyResult { + node!(vm, comprehension, { + target => expression_to_ast(vm, &comprehension.target)?, + iter => expression_to_ast(vm, &comprehension.iter)?, + ifs => expressions_to_ast(vm, &comprehension.ifs)?, + }) } -fn string_to_ast(ctx: &PyContext, string: &ast::StringGroup) -> PyObjectRef { +fn string_to_ast(vm: &VirtualMachine, string: &ast::StringGroup) -> PyResult { match string { ast::StringGroup::Constant { value } => { - let node = create_node(ctx, "Str"); - ctx.set_attr(&node, "s", ctx.new_str(value.clone())); - node + node!(vm, Str, { s => vm.ctx.new_str(value.clone()) }) } ast::StringGroup::FormattedValue { value, .. } => { - let node = create_node(ctx, "FormattedValue"); - let py_value = expression_to_ast(ctx, value); - ctx.set_attr(&node, "value", py_value); - node + node!(vm, FormattedValue, { value => expression_to_ast(vm, value)? }) } ast::StringGroup::Joined { values } => { - let node = create_node(ctx, "JoinedStr"); - let py_values = ctx.new_list( - values - .iter() - .map(|value| string_to_ast(ctx, value)) - .collect(), - ); - ctx.set_attr(&node, "values", py_values); - node + let py_values = map_ast(string_to_ast, vm, &values)?; + node!(vm, JoinedStr, { values => py_values }) } } } -fn ast_parse(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(source, Some(vm.ctx.str_type()))]); - - let source_string = objstr::get_value(source); - let internal_ast = parser::parse_program(&source_string) +fn ast_parse(source: PyStringRef, vm: &VirtualMachine) -> PyResult { + let internal_ast = parser::parse_program(&source.value) .map_err(|err| vm.new_value_error(format!("{}", err)))?; // source.clone(); - let ast_node = program_to_ast(&vm.ctx, &internal_ast); - Ok(ast_node) + program_to_ast(&vm, &internal_ast) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "ast", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let ast_base = py_class!(ctx, "_ast.AST", ctx.object(), {}); + py_module!(vm, "ast", { "parse" => ctx.new_rustfunc(ast_parse), - "Module" => py_class!(ctx, "_ast.Module", ctx.object(), {}), - "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ctx.object(), {}), - "Call" => py_class!(ctx, "_ast.Call", ctx.object(), {}) + "AST" => ast_base.clone(), + // TODO: There's got to be a better way! + "arg" => py_class!(ctx, "_ast.arg", ast_base.clone(), {}), + "arguments" => py_class!(ctx, "_ast.arguments", ast_base.clone(), {}), + "AsyncFor" => py_class!(ctx, "_ast.AsyncFor", ast_base.clone(), {}), + "AsyncFunctionDef" => py_class!(ctx, "_ast.AsyncFunctionDef", ast_base.clone(), {}), + "Assert" => py_class!(ctx, "_ast.Assert", ast_base.clone(), {}), + "Attribute" => py_class!(ctx, "_ast.Attribute", ast_base.clone(), {}), + "Await" => py_class!(ctx, "_ast.Await", ast_base.clone(), {}), + "BinOp" => py_class!(ctx, "_ast.BinOp", ast_base.clone(), {}), + "BoolOp" => py_class!(ctx, "_ast.BoolOp", ast_base.clone(), {}), + "Break" => py_class!(ctx, "_ast.Break", ast_base.clone(), {}), + "Bytes" => py_class!(ctx, "_ast.Bytes", ast_base.clone(), {}), + "Call" => py_class!(ctx, "_ast.Call", ast_base.clone(), {}), + "ClassDef" => py_class!(ctx, "_ast.ClassDef", ast_base.clone(), {}), + "Compare" => py_class!(ctx, "_ast.Compare", ast_base.clone(), {}), + "comprehension" => py_class!(ctx, "_ast.comprehension", ast_base.clone(), {}), + "Continue" => py_class!(ctx, "_ast.Continue", ast_base.clone(), {}), + "Delete" => py_class!(ctx, "_ast.Delete", ast_base.clone(), {}), + "Dict" => py_class!(ctx, "_ast.Dict", ast_base.clone(), {}), + "DictComp" => py_class!(ctx, "_ast.DictComp", ast_base.clone(), {}), + "Ellipsis" => py_class!(ctx, "_ast.Ellipsis", ast_base.clone(), {}), + "Expr" => py_class!(ctx, "_ast.Expr", ast_base.clone(), {}), + "For" => py_class!(ctx, "_ast.For", ast_base.clone(), {}), + "FormattedValue" => py_class!(ctx, "_ast.FormattedValue", ast_base.clone(), {}), + "FunctionDef" => py_class!(ctx, "_ast.FunctionDef", ast_base.clone(), {}), + "GeneratorExp" => py_class!(ctx, "_ast.GeneratorExp", ast_base.clone(), {}), + "Identifier" => py_class!(ctx, "_ast.Identifier", ast_base.clone(), {}), + "If" => py_class!(ctx, "_ast.If", ast_base.clone(), {}), + "IfExp" => py_class!(ctx, "_ast.IfExp", ast_base.clone(), {}), + "JoinedStr" => py_class!(ctx, "_ast.JoinedStr", ast_base.clone(), {}), + "Lambda" => py_class!(ctx, "_ast.Lambda", ast_base.clone(), {}), + "List" => py_class!(ctx, "_ast.List", ast_base.clone(), {}), + "ListComp" => py_class!(ctx, "_ast.ListComp", ast_base.clone(), {}), + "Module" => py_class!(ctx, "_ast.Module", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "NameConstant" => py_class!(ctx, "_ast.NameConstant", ast_base.clone(), {}), + "Num" => py_class!(ctx, "_ast.Num", ast_base.clone(), {}), + "Pass" => py_class!(ctx, "_ast.Pass", ast_base.clone(), {}), + "Return" => py_class!(ctx, "_ast.Return", ast_base.clone(), {}), + "Set" => py_class!(ctx, "_ast.Set", ast_base.clone(), {}), + "SetComp" => py_class!(ctx, "_ast.SetComp", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Starred" => py_class!(ctx, "_ast.Starred", ast_base.clone(), {}), + "Str" => py_class!(ctx, "_ast.Str", ast_base.clone(), {}), + "Subscript" => py_class!(ctx, "_ast.Subscript", ast_base.clone(), {}), + "Tuple" => py_class!(ctx, "_ast.Tuple", ast_base.clone(), {}), + "UnaryOp" => py_class!(ctx, "_ast.UnaryOp", ast_base.clone(), {}), + "While" => py_class!(ctx, "_ast.While", ast_base.clone(), {}), + "Yield" => py_class!(ctx, "_ast.Yield", ast_base.clone(), {}), + "YieldFrom" => py_class!(ctx, "_ast.YieldFrom", ast_base.clone(), {}), }) } diff --git a/vm/src/stdlib/binascii.rs b/vm/src/stdlib/binascii.rs new file mode 100644 index 0000000000..ff748aa013 --- /dev/null +++ b/vm/src/stdlib/binascii.rs @@ -0,0 +1,92 @@ +use crate::function::PyFuncArgs; +use crate::obj::objbytes; +use crate::obj::objint; +use crate::pyobject::{PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; +use crc::{crc32, Hasher32}; +use num_traits::ToPrimitive; + +fn hex_nibble(n: u8) -> u8 { + match n { + 0..=9 => b'0' + n, + 10..=15 => b'a' + n, + _ => unreachable!(), + } +} + +fn binascii_hexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(data, Some(vm.ctx.bytes_type()))]); + + let bytes = objbytes::get_value(data); + let mut hex = Vec::::with_capacity(bytes.len() * 2); + for b in bytes.iter() { + hex.push(hex_nibble(b >> 4)); + hex.push(hex_nibble(b & 0xf)); + } + + Ok(vm.ctx.new_bytes(hex)) +} + +fn unhex_nibble(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +fn binascii_unhexlify(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + // TODO: allow 'str' hexstrings as well + arg_check!(vm, args, required = [(hexstr, Some(vm.ctx.bytes_type()))]); + + let hex_bytes = &objbytes::get_value(hexstr); + if hex_bytes.len() % 2 != 0 { + return Err(vm.new_value_error("Odd-length string".to_string())); + } + + let mut unhex = Vec::::with_capacity(hex_bytes.len() / 2); + for i in (0..hex_bytes.len()).step_by(2) { + let n1 = unhex_nibble(hex_bytes[i]); + let n2 = unhex_nibble(hex_bytes[i + 1]); + if n1.is_some() && n2.is_some() { + unhex.push(n1.unwrap() << 4 | n2.unwrap()); + } else { + return Err(vm.new_value_error("Non-hexadecimal digit found".to_string())); + } + } + + Ok(vm.ctx.new_bytes(unhex)) +} + +fn binascii_crc32(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(data, Some(vm.ctx.bytes_type()))], + optional = [(value, None)] + ); + + let bytes = objbytes::get_value(data); + let crc = match value { + None => 0u32, + Some(value) => objint::get_value(&value).to_u32().unwrap(), + }; + + let mut digest = crc32::Digest::new_with_initial(crc32::IEEE, crc); + digest.write(&bytes); + + Ok(vm.ctx.new_int(digest.sum32())) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "binascii", { + "hexlify" => ctx.new_rustfunc(binascii_hexlify), + "b2a_hex" => ctx.new_rustfunc(binascii_hexlify), + "unhexlify" => ctx.new_rustfunc(binascii_unhexlify), + "a2b_hex" => ctx.new_rustfunc(binascii_unhexlify), + "crc32" => ctx.new_rustfunc(binascii_crc32), + }) +} diff --git a/vm/src/stdlib/dis.rs b/vm/src/stdlib/dis.rs index 04209d8b3b..4c35f45e99 100644 --- a/vm/src/stdlib/dis.rs +++ b/vm/src/stdlib/dis.rs @@ -1,29 +1,26 @@ -use crate::function::PyFuncArgs; -use crate::obj::objcode; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::objcode::PyCodeRef; +use crate::pyobject::{PyObjectRef, PyResult, TryFromObject}; use crate::vm::VirtualMachine; -fn dis_dis(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(obj, None)]); - +fn dis_dis(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { // Method or function: if let Ok(co) = vm.get_attribute(obj.clone(), "__code__") { - return dis_disassemble(vm, PyFuncArgs::new(vec![co], vec![])); + return dis_disassemble(co, vm); } - dis_disassemble(vm, PyFuncArgs::new(vec![obj.clone()], vec![])) + dis_disassemble(obj, vm) } -fn dis_disassemble(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(co, Some(vm.ctx.code_type()))]); - - let code = objcode::get_value(co); +fn dis_disassemble(co: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let code = &PyCodeRef::try_from_object(vm, co)?.code; print!("{}", code); Ok(vm.get_none()) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "dis", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "dis", { "dis" => ctx.new_rustfunc(dis_dis), "disassemble" => ctx.new_rustfunc(dis_disassemble) }) diff --git a/vm/src/stdlib/hashlib.rs b/vm/src/stdlib/hashlib.rs new file mode 100644 index 0000000000..719e3a3296 --- /dev/null +++ b/vm/src/stdlib/hashlib.rs @@ -0,0 +1,282 @@ +use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; +use std::cell::RefCell; +use std::fmt; + +use blake2::{Blake2b, Blake2s}; +use digest::DynDigest; +use md5::Md5; +use sha1::Sha1; +use sha2::{Sha224, Sha256, Sha384, Sha512}; +use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; // TODO: , Shake128, Shake256}; + +#[pyclass(name = "hasher")] +struct PyHasher { + name: String, + buffer: RefCell, +} + +impl fmt::Debug for PyHasher { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "hasher {}", self.name) + } +} + +impl PyValue for PyHasher { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("hashlib", "hasher") + } +} + +#[pyimpl] +impl PyHasher { + fn new(name: &str, d: HashWrapper) -> Self { + PyHasher { + name: name.to_string(), + buffer: RefCell::new(d), + } + } + + #[pymethod(name = "__new__")] + fn py_new(_cls: PyClassRef, _args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("md5", HashWrapper::md5()) + .into_ref(vm) + .into_object()) + } + + #[pyproperty(name = "name")] + fn name(&self, _vm: &VirtualMachine) -> String { + self.name.clone() + } + + #[pyproperty(name = "digest_size")] + fn digest_size(&self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_int(self.buffer.borrow().digest_size())) + } + + #[pymethod(name = "update")] + fn update(&self, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { + self.buffer.borrow_mut().input(data.get_value()); + Ok(vm.get_none()) + } + + #[pymethod(name = "digest")] + fn digest(&self, _vm: &VirtualMachine) -> PyBytes { + let result = self.get_digest(); + PyBytes::new(result) + } + + #[pymethod(name = "hexdigest")] + fn hexdigest(&self, _vm: &VirtualMachine) -> String { + let result = self.get_digest(); + hex::encode(result) + } + + fn get_digest(&self) -> Vec { + self.buffer.borrow().get_digest() + } +} + +fn hashlib_new( + name: PyStringRef, + data: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let hasher = match name.value.as_ref() { + "md5" => Ok(PyHasher::new("md5", HashWrapper::md5())), + "sha1" => Ok(PyHasher::new("sha1", HashWrapper::sha1())), + "sha224" => Ok(PyHasher::new("sha224", HashWrapper::sha224())), + "sha256" => Ok(PyHasher::new("sha256", HashWrapper::sha256())), + "sha384" => Ok(PyHasher::new("sha384", HashWrapper::sha384())), + "sha512" => Ok(PyHasher::new("sha512", HashWrapper::sha512())), + "sha3_224" => Ok(PyHasher::new("sha3_224", HashWrapper::sha3_224())), + "sha3_256" => Ok(PyHasher::new("sha3_256", HashWrapper::sha3_256())), + "sha3_384" => Ok(PyHasher::new("sha3_384", HashWrapper::sha3_384())), + "sha3_512" => Ok(PyHasher::new("sha3_512", HashWrapper::sha3_512())), + // TODO: "shake128" => Ok(PyHasher::new("shake128", HashWrapper::shake128())), + // TODO: "shake256" => Ok(PyHasher::new("shake256", HashWrapper::shake256())), + "blake2b" => Ok(PyHasher::new("blake2b", HashWrapper::blake2b())), + "blake2s" => Ok(PyHasher::new("blake2s", HashWrapper::blake2s())), + other => Err(vm.new_value_error(format!("Unknown hashing algorithm: {}", other))), + }?; + + if let OptionalArg::Present(data) = data { + hasher.update(data, vm)?; + } + + Ok(hasher) +} + +fn md5(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("md5", HashWrapper::md5())) +} + +fn sha1(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha1", HashWrapper::sha1())) +} + +fn sha224(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha224", HashWrapper::sha224())) +} + +fn sha256(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha256", HashWrapper::sha256())) +} + +fn sha384(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha384", HashWrapper::sha384())) +} + +fn sha512(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha512", HashWrapper::sha512())) +} + +fn sha3_224(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_224", HashWrapper::sha3_224())) +} + +fn sha3_256(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_256", HashWrapper::sha3_256())) +} + +fn sha3_384(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_384", HashWrapper::sha3_384())) +} + +fn sha3_512(_vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_512", HashWrapper::sha3_512())) +} + +fn shake128(vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error("shake256".to_string())) + // Ok(PyHasher::new("shake128", HashWrapper::shake128())) +} + +fn shake256(vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error("shake256".to_string())) + // TODO: Ok(PyHasher::new("shake256", HashWrapper::shake256())) +} + +fn blake2b(_vm: &VirtualMachine) -> PyResult { + // TODO: handle parameters + Ok(PyHasher::new("blake2b", HashWrapper::blake2b())) +} + +fn blake2s(_vm: &VirtualMachine) -> PyResult { + // TODO: handle parameters + Ok(PyHasher::new("blake2s", HashWrapper::blake2s())) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let hasher_type = PyHasher::make_class(ctx); + + py_module!(vm, "hashlib", { + "new" => ctx.new_rustfunc(hashlib_new), + "md5" => ctx.new_rustfunc(md5), + "sha1" => ctx.new_rustfunc(sha1), + "sha224" => ctx.new_rustfunc(sha224), + "sha256" => ctx.new_rustfunc(sha256), + "sha384" => ctx.new_rustfunc(sha384), + "sha512" => ctx.new_rustfunc(sha512), + "sha3_224" => ctx.new_rustfunc(sha3_224), + "sha3_256" => ctx.new_rustfunc(sha3_256), + "sha3_384" => ctx.new_rustfunc(sha3_384), + "sha3_512" => ctx.new_rustfunc(sha3_512), + "shake128" => ctx.new_rustfunc(shake128), + "shake256" => ctx.new_rustfunc(shake256), + "blake2b" => ctx.new_rustfunc(blake2b), + "blake2s" => ctx.new_rustfunc(blake2s), + "hasher" => hasher_type, + }) +} + +/// Generic wrapper patching around the hashing libraries. +struct HashWrapper { + inner: Box, +} + +impl HashWrapper { + fn new(d: D) -> Self + where + D: DynDigest, + D: Sized, + { + HashWrapper { inner: Box::new(d) } + } + + fn md5() -> Self { + Self::new(Md5::default()) + } + + fn sha1() -> Self { + Self::new(Sha1::default()) + } + + fn sha224() -> Self { + Self::new(Sha224::default()) + } + + fn sha256() -> Self { + Self::new(Sha256::default()) + } + + fn sha384() -> Self { + Self::new(Sha384::default()) + } + + fn sha512() -> Self { + Self::new(Sha512::default()) + } + + fn sha3_224() -> Self { + Self::new(Sha3_224::default()) + } + + fn sha3_256() -> Self { + Self::new(Sha3_256::default()) + } + + fn sha3_384() -> Self { + Self::new(Sha3_384::default()) + } + + fn sha3_512() -> Self { + Self::new(Sha3_512::default()) + } + + /* TODO: + fn shake128() -> Self { + Self::new(Shake128::default()) + } + + fn shake256() -> Self { + Self::new(Shake256::default()) + } + */ + fn blake2b() -> Self { + Self::new(Blake2b::default()) + } + + fn blake2s() -> Self { + Self::new(Blake2s::default()) + } + + fn input(&mut self, data: &[u8]) { + self.inner.input(data); + } + + fn digest_size(&self) -> usize { + self.inner.output_size() + } + + fn get_digest(&self) -> Vec { + let cloned = self.inner.clone(); + cloned.result().to_vec() + } +} diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs new file mode 100644 index 0000000000..4c0f023d9b --- /dev/null +++ b/vm/src/stdlib/imp.rs @@ -0,0 +1,102 @@ +use crate::import; +use crate::obj::objcode::PyCode; +use crate::obj::objmodule::PyModuleRef; +use crate::obj::objstr; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{ItemProtocol, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +fn imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(vec![])) +} + +fn imp_acquire_lock(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_release_lock(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_lock_held(_vm: &VirtualMachine) -> PyResult<()> { + // TODO + Ok(()) +} + +fn imp_is_builtin(name: PyStringRef, vm: &VirtualMachine) -> bool { + vm.stdlib_inits.borrow().contains_key(name.as_str()) +} + +fn imp_is_frozen(name: PyStringRef, vm: &VirtualMachine) -> bool { + vm.frozen.borrow().contains_key(name.as_str()) +} + +fn imp_create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let sys_modules = vm.get_attribute(vm.sys_module.clone(), "modules").unwrap(); + + let name = &objstr::get_value(&vm.get_attribute(spec.clone(), "name")?); + + if let Ok(module) = sys_modules.get_item(name, vm) { + Ok(module) + } else { + if let Some(make_module_func) = vm.stdlib_inits.borrow().get(name) { + Ok(make_module_func(vm)) + } else { + Ok(vm.get_none()) + } + } +} + +fn imp_exec_builtin(_mod: PyModuleRef, _vm: &VirtualMachine) -> i32 { + // TOOD: Should we do something here? + 0 +} + +fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + vm.frozen + .borrow() + .get(name.as_str()) + .map(|frozen| { + let mut frozen = frozen.clone(); + frozen.source_path = format!("frozen {}", name.as_str()); + PyCode::new(frozen) + }) + .ok_or_else(|| { + vm.new_import_error(format!("No such frozen object named {}", name.as_str())) + }) +} + +fn imp_init_frozen(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + import::import_frozen(vm, name.as_str()) +} + +fn imp_is_frozen_package(_name: PyStringRef, _vm: &VirtualMachine) -> bool { + // TODO: Support frozen package. + false +} + +fn imp_fix_co_filename(_code: PyObjectRef, _path: PyStringRef, _vm: &VirtualMachine) { + // TODO: +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let module = py_module!(vm, "_imp", { + "extension_suffixes" => ctx.new_rustfunc(imp_extension_suffixes), + "acquire_lock" => ctx.new_rustfunc(imp_acquire_lock), + "release_lock" => ctx.new_rustfunc(imp_release_lock), + "lock_held" => ctx.new_rustfunc(imp_lock_held), + "is_builtin" => ctx.new_rustfunc(imp_is_builtin), + "is_frozen" => ctx.new_rustfunc(imp_is_frozen), + "create_builtin" => ctx.new_rustfunc(imp_create_builtin), + "exec_builtin" => ctx.new_rustfunc(imp_exec_builtin), + "get_frozen_object" => ctx.new_rustfunc(imp_get_frozen_object), + "init_frozen" => ctx.new_rustfunc(imp_init_frozen), + "is_frozen_package" => ctx.new_rustfunc(imp_is_frozen_package), + "_fix_co_filename" => ctx.new_rustfunc(imp_fix_co_filename), + }); + + module +} diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 1ce303c5f3..f092953f30 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,83 +1,225 @@ /* * I/O core tools. */ - use std::cell::RefCell; -use std::collections::HashSet; -use std::fs::File; use std::io::prelude::*; -use std::io::BufReader; -use std::path::PathBuf; +use std::io::Cursor; +use std::io::SeekFrom; use num_bigint::ToBigInt; use num_traits::ToPrimitive; use super::os; -use crate::function::PyFuncArgs; -use crate::import; +use crate::function::{OptionalArg, PyFuncArgs}; use crate::obj::objbytearray::PyByteArray; use crate::obj::objbytes; +use crate::obj::objbytes::PyBytes; use crate::obj::objint; use crate::obj::objstr; -use crate::pyobject::{ - BufferProtocol, PyContext, PyObject, PyObjectRef, PyRef, PyResult, PyValue, TypeProtocol, -}; +use crate::obj::objtype; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::TypeProtocol; +use crate::pyobject::{BufferProtocol, PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; -fn compute_c_flag(mode: &str) -> u16 { - match mode { - "w" => 512, - "x" => 512, - "a" => 8, - "+" => 2, - _ => 0, +fn byte_count(bytes: OptionalArg>) -> i64 { + match bytes { + OptionalArg::Present(Some(ref int)) => objint::get_value(int).to_i64().unwrap(), + _ => (-1 as i64), + } +} + +#[derive(Debug)] +struct BufferedIO { + cursor: Cursor>, +} + +impl BufferedIO { + fn new(cursor: Cursor>) -> BufferedIO { + BufferedIO { cursor: cursor } + } + + fn write(&mut self, data: Vec) -> Option { + let length = data.len(); + + match self.cursor.write_all(&data) { + Ok(_) => Some(length as u64), + Err(_) => None, + } + } + + //return the entire contents of the underlying + fn getvalue(&self) -> Vec { + self.cursor.clone().into_inner() + } + + //skip to the jth position + fn seek(&mut self, offset: u64) -> Option { + match self.cursor.seek(SeekFrom::Start(offset.clone())) { + Ok(_) => Some(offset), + Err(_) => None, + } + } + + //Read k bytes from the object and return. + fn read(&mut self, bytes: i64) -> Option> { + let mut buffer = Vec::new(); + + //for a defined number of bytes, i.e. bytes != -1 + if bytes > 0 { + let mut handle = self.cursor.clone().take(bytes as u64); + //read handle into buffer + if let Err(_) = handle.read_to_end(&mut buffer) { + return None; + } + //the take above consumes the struct value + //we add this back in with the takes into_inner method + self.cursor = handle.into_inner(); + } else { + //read handle into buffer + if let Err(_) = self.cursor.read_to_end(&mut buffer) { + return None; + } + }; + + Some(buffer) } } #[derive(Debug)] struct PyStringIO { - data: RefCell, + buffer: RefCell, } type PyStringIORef = PyRef; impl PyValue for PyStringIO { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.class("io", "StringIO") } } impl PyStringIORef { - fn write(self, data: objstr::PyStringRef, _vm: &VirtualMachine) { - let data = data.value.clone(); - self.data.borrow_mut().push_str(&data); + //write string to underlying vector + fn write(self, data: objstr::PyStringRef, vm: &VirtualMachine) -> PyResult { + let bytes = &data.value.clone().into_bytes(); + + match self.buffer.borrow_mut().write(bytes.to_vec()) { + Some(value) => Ok(vm.ctx.new_int(value)), + None => Err(vm.new_type_error("Error Writing String".to_string())), + } + } + + //return the entire contents of the underlying + fn getvalue(self, vm: &VirtualMachine) -> PyResult { + match String::from_utf8(self.buffer.borrow().getvalue()) { + Ok(result) => Ok(vm.ctx.new_str(result)), + Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_string())), + } } - fn getvalue(self, _vm: &VirtualMachine) -> String { - self.data.borrow().clone() + //skip to the jth position + fn seek(self, offset: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let position = objint::get_value(&offset).to_u64().unwrap(); + match self.buffer.borrow_mut().seek(position) { + Some(value) => Ok(vm.ctx.new_int(value)), + None => Err(vm.new_value_error("Error Performing Operation".to_string())), + } + } + + //Read k bytes from the object and return. + //If k is undefined || k == -1, then we read all bytes until the end of the file. + //This also increments the stream position by the value of k + fn read(self, bytes: OptionalArg>, vm: &VirtualMachine) -> PyResult { + let data = match self.buffer.borrow_mut().read(byte_count(bytes)) { + Some(value) => value, + None => Vec::new(), + }; + + match String::from_utf8(data) { + Ok(value) => Ok(vm.ctx.new_str(value)), + Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_string())), + } } } -fn string_io_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(cls, None)]); +fn string_io_new( + cls: PyClassRef, + object: OptionalArg>, + vm: &VirtualMachine, +) -> PyResult { + let raw_string = match object { + OptionalArg::Present(Some(ref input)) => objstr::get_value(input), + _ => String::new(), + }; - Ok(PyObject::new( - PyStringIO { - data: RefCell::new(String::default()), - }, - cls.clone(), - )) + PyStringIO { + buffer: RefCell::new(BufferedIO::new(Cursor::new(raw_string.into_bytes()))), + } + .into_ref_with_type(vm, cls) } -fn bytes_io_init(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { - // TODO - Ok(vm.get_none()) +#[derive(Debug)] +struct PyBytesIO { + buffer: RefCell, } -fn bytes_io_getvalue(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args); - // TODO - Ok(vm.get_none()) +type PyBytesIORef = PyRef; + +impl PyValue for PyBytesIO { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("io", "BytesIO") + } +} + +impl PyBytesIORef { + fn write(self, data: objbytes::PyBytesRef, vm: &VirtualMachine) -> PyResult { + let bytes = data.get_value(); + + match self.buffer.borrow_mut().write(bytes.to_vec()) { + Some(value) => Ok(vm.ctx.new_int(value)), + None => Err(vm.new_type_error("Error Writing Bytes".to_string())), + } + } + //Retrieves the entire bytes object value from the underlying buffer + fn getvalue(self, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_bytes(self.buffer.borrow().getvalue())) + } + + //Takes an integer k (bytes) and returns them from the underlying buffer + //If k is undefined || k == -1, then we read all bytes until the end of the file. + //This also increments the stream position by the value of k + fn read(self, bytes: OptionalArg>, vm: &VirtualMachine) -> PyResult { + match self.buffer.borrow_mut().read(byte_count(bytes)) { + Some(value) => Ok(vm.ctx.new_bytes(value)), + None => Err(vm.new_value_error("Error Retrieving Value".to_string())), + } + } + + //skip to the jth position + fn seek(self, offset: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let position = objint::get_value(&offset).to_u64().unwrap(); + match self.buffer.borrow_mut().seek(position) { + Some(value) => Ok(vm.ctx.new_int(value)), + None => Err(vm.new_value_error("Error Performing Operation".to_string())), + } + } +} + +fn bytes_io_new( + cls: PyClassRef, + object: OptionalArg>, + vm: &VirtualMachine, +) -> PyResult { + let raw_bytes = match object { + OptionalArg::Present(Some(ref input)) => objbytes::get_value(input).to_vec(), + _ => vec![], + }; + + PyBytesIO { + buffer: RefCell::new(BufferedIO::new(Cursor::new(raw_bytes))), + } + .into_ref_with_type(vm, cls) } fn io_base_cm_enter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -100,9 +242,12 @@ fn io_base_cm_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +// TODO Check if closed, then if so raise ValueError +fn io_base_flush(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + fn buffered_io_base_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(buffered, None), (raw, None)]); - vm.ctx.set_attr(&buffered, "raw", raw.clone()); + vm.set_attr(buffered, "raw", raw.clone())?; Ok(vm.get_none()) } @@ -115,7 +260,7 @@ fn buffered_reader_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let mut result = vec![]; let mut length = buff_size; - let raw = vm.ctx.get_attr(&buffered, "raw").unwrap(); + let raw = vm.get_attribute(buffered.clone(), "raw").unwrap(); //Iterates through the raw class, invoking the readinto method //to obtain buff_size many bytes. Exit when less than buff_size many @@ -126,7 +271,7 @@ fn buffered_reader_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { //Copy bytes from the buffer vector into the results vector if let Some(bytes) = buffer.payload::() { - result.extend_from_slice(&bytes.value.borrow()); + result.extend_from_slice(&bytes.inner.borrow().elements); }; let py_len = vm.call_method(&buffer, "__len__", PyFuncArgs::default())?; @@ -136,51 +281,65 @@ fn buffered_reader_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bytes(result)) } +fn compute_c_flag(mode: &str) -> u32 { + let flags = match mode.chars().next() { + Some(mode) => match mode { + 'w' => os::FileCreationFlags::O_WRONLY | os::FileCreationFlags::O_CREAT, + 'x' => { + os::FileCreationFlags::O_WRONLY + | os::FileCreationFlags::O_CREAT + | os::FileCreationFlags::O_EXCL + } + 'a' => os::FileCreationFlags::O_APPEND, + '+' => os::FileCreationFlags::O_RDWR, + _ => os::FileCreationFlags::O_RDONLY, + }, + None => os::FileCreationFlags::O_RDONLY, + }; + flags.bits() +} + fn file_io_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, - required = [(file_io, None), (name, Some(vm.ctx.str_type()))], + required = [(file_io, None), (name, None)], optional = [(mode, Some(vm.ctx.str_type()))] ); - let rust_mode = mode.map_or("r".to_string(), |m| objstr::get_value(m)); - - match compute_c_flag(&rust_mode).to_bigint() { - Some(os_mode) => { - let args = vec![name.clone(), vm.ctx.new_int(os_mode)]; - let file_no = os::os_open(vm, PyFuncArgs::new(args, vec![]))?; - - vm.ctx.set_attr(&file_io, "name", name.clone()); - vm.ctx.set_attr(&file_io, "fileno", file_no); - vm.ctx.set_attr(&file_io, "closefd", vm.new_bool(false)); - vm.ctx.set_attr(&file_io, "closed", vm.new_bool(false)); + let file_no = if objtype::isinstance(&name, &vm.ctx.str_type()) { + let rust_mode = mode.map_or("r".to_string(), objstr::get_value); + let args = vec![ + name.clone(), + vm.ctx + .new_int(compute_c_flag(&rust_mode).to_bigint().unwrap()), + ]; + os::os_open(vm, PyFuncArgs::new(args, vec![]))? + } else if objtype::isinstance(&name, &vm.ctx.int_type()) { + name.clone() + } else { + return Err(vm.new_type_error("name parameter must be string or int".to_string())); + }; - Ok(vm.get_none()) - } - None => Err(vm.new_type_error(format!("invalid mode {}", rust_mode))), - } + vm.set_attr(file_io, "name", name.clone())?; + vm.set_attr(file_io, "fileno", file_no)?; + vm.set_attr(file_io, "closefd", vm.new_bool(false))?; + vm.set_attr(file_io, "closed", vm.new_bool(false))?; + Ok(vm.get_none()) } fn file_io_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(file_io, None)]); - let py_name = vm.get_attribute(file_io.clone(), "name")?; - let f = match File::open(objstr::get_value(&py_name)) { - Ok(v) => Ok(v), - Err(_) => Err(vm.new_type_error("Error opening file".to_string())), - }; - let buffer = match f { - Ok(v) => Ok(BufReader::new(v)), - Err(_) => Err(vm.new_type_error("Error reading from file".to_string())), - }; + let file_no = vm.get_attribute(file_io.clone(), "fileno")?; + let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); + + let mut handle = os::rust_file(raw_fd); let mut bytes = vec![]; - if let Ok(mut buff) = buffer { - match buff.read_to_end(&mut bytes) { - Ok(_) => {} - Err(_) => return Err(vm.new_value_error("Error reading from Buffer".to_string())), - } + match handle.read_to_end(&mut bytes) { + Ok(_) => {} + Err(_) => return Err(vm.new_value_error("Error reading from Buffer".to_string())), } Ok(vm.ctx.new_bytes(bytes)) @@ -209,25 +368,21 @@ fn file_io_readinto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(bytes) = obj.payload::() { //TODO: Implement for MemoryView - let mut value_mut = bytes.value.borrow_mut(); + let value_mut = &mut bytes.inner.borrow_mut().elements; value_mut.clear(); - match f.read_to_end(&mut value_mut) { + match f.read_to_end(value_mut) { Ok(_) => {} Err(_) => return Err(vm.new_value_error("Error reading from Take".to_string())), } }; let updated = os::raw_file_number(f.into_inner()); - vm.ctx.set_attr(&file_io, "fileno", vm.ctx.new_int(updated)); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; Ok(vm.get_none()) } fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(file_io, None), (obj, Some(vm.ctx.bytes_type()))] - ); + arg_check!(vm, args, required = [(file_io, None), (obj, None)]); let file_no = vm.get_attribute(file_io.clone(), "fileno")?; let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); @@ -237,22 +392,25 @@ fn file_io_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { //to support windows - i.e. raw file_handles let mut handle = os::rust_file(raw_fd); - match obj.payload::() { - Some(bytes) => { - let value_mut = bytes.value.borrow(); - match handle.write(&value_mut[..]) { - Ok(len) => { - //reset raw fd on the FileIO object - let updated = os::raw_file_number(handle); - vm.ctx.set_attr(&file_io, "fileno", vm.ctx.new_int(updated)); - - //return number of bytes written - Ok(vm.ctx.new_int(len)) - } - Err(_) => Err(vm.new_value_error("Error Writing Bytes to Handle".to_string())), - } + let bytes = match_class!(obj.clone(), + i @ PyBytes => Ok(i.get_value().to_vec()), + j @ PyByteArray => Ok(j.inner.borrow().elements.to_vec()), + obj => Err(vm.new_type_error(format!( + "a bytes-like object is required, not {}", + obj.class() + ))) + ); + + match handle.write(&bytes?) { + Ok(len) => { + //reset raw fd on the FileIO object + let updated = os::raw_file_number(handle); + vm.set_attr(file_io, "fileno", vm.ctx.new_int(updated))?; + + //return number of bytes written + Ok(vm.ctx.new_int(len)) } - None => Err(vm.new_value_error("Expected Bytes Object".to_string())), + Err(_) => Err(vm.new_value_error("Error Writing Bytes to Handle".to_string())), } } @@ -263,7 +421,7 @@ fn buffered_writer_write(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(buffered, None), (obj, Some(vm.ctx.bytes_type()))] ); - let raw = vm.ctx.get_attr(&buffered, "raw").unwrap(); + let raw = vm.get_attribute(buffered.clone(), "raw").unwrap(); //This should be replaced with a more appropriate chunking implementation vm.call_method(&raw, "write", vec![obj.clone()]) @@ -276,14 +434,14 @@ fn text_io_wrapper_init(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { required = [(text_io_wrapper, None), (buffer, None)] ); - vm.ctx.set_attr(&text_io_wrapper, "buffer", buffer.clone()); + vm.set_attr(text_io_wrapper, "buffer", buffer.clone())?; Ok(vm.get_none()) } fn text_io_base_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(text_io_base, None)]); - let raw = vm.ctx.get_attr(&text_io_base, "buffer").unwrap(); + let raw = vm.get_attribute(text_io_base.clone(), "buffer").unwrap(); if let Ok(bytes) = vm.call_method(&raw, "read", PyFuncArgs::default()) { let value = objbytes::get_value(&bytes).to_vec(); @@ -296,6 +454,63 @@ fn text_io_base_read(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn split_mode_string(mode_string: String) -> Result<(String, String), String> { + let mut mode: char = '\0'; + let mut typ: char = '\0'; + let mut plus_is_set = false; + + for ch in mode_string.chars() { + match ch { + '+' => { + if plus_is_set { + return Err(format!("invalid mode: '{}'", mode_string)); + } + plus_is_set = true; + } + 't' | 'b' => { + if typ != '\0' { + if typ == ch { + // no duplicates allowed + return Err(format!("invalid mode: '{}'", mode_string)); + } else { + return Err("can't have text and binary mode at once".to_string()); + } + } + typ = ch; + } + 'a' | 'r' | 'w' => { + if mode != '\0' { + if mode == ch { + // no duplicates allowed + return Err(format!("invalid mode: '{}'", mode_string)); + } else { + return Err( + "must have exactly one of create/read/write/append mode".to_string() + ); + } + } + mode = ch; + } + _ => return Err(format!("invalid mode: '{}'", mode_string)), + } + } + + if mode == '\0' { + return Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string(), + ); + } + let mut mode = mode.to_string(); + if plus_is_set { + mode.push('+'); + } + if typ == '\0' { + typ = 't'; + } + Ok((mode, typ.to_string())) +} + pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -304,83 +519,77 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { optional = [(mode, Some(vm.ctx.str_type()))] ); - let module = import::import_module(vm, PathBuf::default(), "io").unwrap(); - - //mode is optional: 'rt' is the default mode (open from reading text) - let rust_mode = mode.map_or("rt".to_string(), |m| objstr::get_value(m)); - - let mut raw_modes = HashSet::new(); - - //add raw modes - raw_modes.insert("a".to_string()); - raw_modes.insert("r".to_string()); - raw_modes.insert("x".to_string()); - raw_modes.insert("w".to_string()); - - //This is not a terribly elegant way to separate the file mode from - //the "type" flag - this should be improved. The intention here is to - //match a valid flag for the file_io_init call: - //https://docs.python.org/3/library/io.html#io.FileIO - let modes: Vec = rust_mode - .chars() - .filter(|a| raw_modes.contains(&a.to_string())) - .collect(); - - if modes.is_empty() || modes.len() > 1 { - return Err(vm.new_value_error("Invalid Mode".to_string())); - } - - //Class objects (potentially) consumed by io.open - //RawIO: FileIO - //Buffered: BufferedWriter, BufferedReader - //Text: TextIOWrapper - let file_io_class = vm.ctx.get_attr(&module, "FileIO").unwrap(); - let buffered_writer_class = vm.ctx.get_attr(&module, "BufferedWriter").unwrap(); - let buffered_reader_class = vm.ctx.get_attr(&module, "BufferedReader").unwrap(); - let text_io_wrapper_class = vm.ctx.get_attr(&module, "TextIOWrapper").unwrap(); - - //Construct a FileIO (subclass of RawIOBase) - //This is subsequently consumed by a Buffered Class. - let file_args = vec![file.clone(), vm.ctx.new_str(modes[0].to_string())]; - let file_io = vm.invoke(file_io_class, file_args)?; - - //Create Buffered class to consume FileIO. The type of buffered class depends on - //the operation in the mode. - //There are 3 possible classes here, each inheriting from the RawBaseIO + // mode is optional: 'rt' is the default mode (open from reading text) + let mode_string = mode.map_or("rt".to_string(), objstr::get_value); + + let (mode, typ) = match split_mode_string(mode_string) { + Ok((mode, typ)) => (mode, typ), + Err(error_message) => { + return Err(vm.new_value_error(error_message)); + } + }; + + let io_module = vm.import("_io", &vm.ctx.new_tuple(vec![]), 0)?; + + // Construct a FileIO (subclass of RawIOBase) + // This is subsequently consumed by a Buffered Class. + let file_io_class = vm.get_attribute(io_module.clone(), "FileIO").unwrap(); + let file_io_obj = vm.invoke( + file_io_class, + vec![file.clone(), vm.ctx.new_str(mode.clone())], + )?; + + // Create Buffered class to consume FileIO. The type of buffered class depends on + // the operation in the mode. + // There are 3 possible classes here, each inheriting from the RawBaseIO // creating || writing || appending => BufferedWriter - let buffered = if rust_mode.contains('w') { - vm.invoke(buffered_writer_class, vec![file_io.clone()]) - // reading => BufferedReader - } else { - vm.invoke(buffered_reader_class, vec![file_io.clone()]) + let buffered = match mode.chars().next().unwrap() { + 'w' => { + let buffered_writer_class = vm + .get_attribute(io_module.clone(), "BufferedWriter") + .unwrap(); + vm.invoke(buffered_writer_class, vec![file_io_obj.clone()]) + } + 'r' => { + let buffered_reader_class = vm + .get_attribute(io_module.clone(), "BufferedReader") + .unwrap(); + vm.invoke(buffered_reader_class, vec![file_io_obj.clone()]) + } //TODO: updating => PyBufferedRandom + _ => unimplemented!("'a' mode is not yet implemented"), }; - if rust_mode.contains('t') { - //If the mode is text this buffer type is consumed on construction of - //a TextIOWrapper which is subsequently returned. - vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]) - } else { + let io_obj = match typ.chars().next().unwrap() { + // If the mode is text this buffer type is consumed on construction of + // a TextIOWrapper which is subsequently returned. + 't' => { + let text_io_wrapper_class = vm.get_attribute(io_module, "TextIOWrapper").unwrap(); + vm.invoke(text_io_wrapper_class, vec![buffered.unwrap()]) + } // If the mode is binary this Buffered class is returned directly at // this point. - //For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field - buffered - } + // For Buffered class construct "raw" IO class e.g. FileIO and pass this into corresponding field + 'b' => buffered, + _ => unreachable!(), + }; + io_obj } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + //IOBase the abstract base class of the IO Module let io_base = py_class!(ctx, "IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), - "__exit__" => ctx.new_rustfunc(io_base_cm_exit) + "__exit__" => ctx.new_rustfunc(io_base_cm_exit), + "flush" => ctx.new_rustfunc(io_base_flush) }); // IOBase Subclasses - let raw_io_base = py_class!(ctx, "RawIOBase", ctx.object(), {}); + let raw_io_base = py_class!(ctx, "RawIOBase", io_base.clone(), {}); - let buffered_io_base = py_class!(ctx, "BufferedIOBase", io_base.clone(), { - "__init__" => ctx.new_rustfunc(buffered_io_base_init) - }); + let buffered_io_base = py_class!(ctx, "BufferedIOBase", io_base.clone(), {}); //TextIO Base has no public constructor let text_io_base = py_class!(ctx, "TextIOBase", io_base.clone(), { @@ -388,6 +597,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { }); // RawBaseIO Subclasses + // TODO Fix name? let file_io = py_class!(ctx, "FileIO", raw_io_base.clone(), { "__init__" => ctx.new_rustfunc(file_io_init), "name" => ctx.str_type(), @@ -398,10 +608,18 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { // BufferedIOBase Subclasses let buffered_reader = py_class!(ctx, "BufferedReader", buffered_io_base.clone(), { + //workaround till the buffered classes can be fixed up to be more + //consistent with the python model + //For more info see: https://github.com/RustPython/RustPython/issues/547 + "__init__" => ctx.new_rustfunc(buffered_io_base_init), "read" => ctx.new_rustfunc(buffered_reader_read) }); let buffered_writer = py_class!(ctx, "BufferedWriter", buffered_io_base.clone(), { + //workaround till the buffered classes can be fixed up to be more + //consistent with the python model + //For more info see: https://github.com/RustPython/RustPython/issues/547 + "__init__" => ctx.new_rustfunc(buffered_io_base_init), "write" => ctx.new_rustfunc(buffered_writer_write) }); @@ -413,27 +631,153 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { //StringIO: in-memory text let string_io = py_class!(ctx, "StringIO", text_io_base.clone(), { "__new__" => ctx.new_rustfunc(string_io_new), + "seek" => ctx.new_rustfunc(PyStringIORef::seek), + "read" => ctx.new_rustfunc(PyStringIORef::read), "write" => ctx.new_rustfunc(PyStringIORef::write), "getvalue" => ctx.new_rustfunc(PyStringIORef::getvalue) }); //BytesIO: in-memory bytes let bytes_io = py_class!(ctx, "BytesIO", buffered_io_base.clone(), { - "__init__" => ctx.new_rustfunc(bytes_io_init), - "getvalue" => ctx.new_rustfunc(bytes_io_getvalue) + "__new__" => ctx.new_rustfunc(bytes_io_new), + "read" => ctx.new_rustfunc(PyBytesIORef::read), + "read1" => ctx.new_rustfunc(PyBytesIORef::read), + "seek" => ctx.new_rustfunc(PyBytesIORef::seek), + "write" => ctx.new_rustfunc(PyBytesIORef::write), + "getvalue" => ctx.new_rustfunc(PyBytesIORef::getvalue) }); - py_module!(ctx, "io", { + py_module!(vm, "_io", { "open" => ctx.new_rustfunc(io_open), - "IOBase" => io_base.clone(), - "RawIOBase" => raw_io_base.clone(), - "BufferedIOBase" => buffered_io_base.clone(), - "TextIOBase" => text_io_base.clone(), - "FileIO" => file_io.clone(), - "BufferedReader" => buffered_reader.clone(), - "BufferedWriter" => buffered_writer.clone(), - "TextIOWrapper" => text_io_wrapper.clone(), + "IOBase" => io_base, + "RawIOBase" => raw_io_base, + "BufferedIOBase" => buffered_io_base, + "TextIOBase" => text_io_base, + "FileIO" => file_io, + "BufferedReader" => buffered_reader, + "BufferedWriter" => buffered_writer, + "TextIOWrapper" => text_io_wrapper, "StringIO" => string_io, "BytesIO" => bytes_io, }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_mode_split_into(mode_string: &str, expected_mode: &str, expected_typ: &str) { + let (mode, typ) = split_mode_string(mode_string.to_string()).unwrap(); + assert_eq!(mode, expected_mode); + assert_eq!(typ, expected_typ); + } + + #[test] + fn test_split_mode_valid_cases() { + assert_mode_split_into("r", "r", "t"); + assert_mode_split_into("rb", "r", "b"); + assert_mode_split_into("rt", "r", "t"); + assert_mode_split_into("r+t", "r+", "t"); + assert_mode_split_into("w+t", "w+", "t"); + assert_mode_split_into("r+b", "r+", "b"); + assert_mode_split_into("w+b", "w+", "b"); + } + + #[test] + fn test_invalid_mode() { + assert_eq!( + split_mode_string("rbsss".to_string()), + Err("invalid mode: 'rbsss'".to_string()) + ); + assert_eq!( + split_mode_string("rrb".to_string()), + Err("invalid mode: 'rrb'".to_string()) + ); + assert_eq!( + split_mode_string("rbb".to_string()), + Err("invalid mode: 'rbb'".to_string()) + ); + } + + #[test] + fn test_mode_not_specified() { + assert_eq!( + split_mode_string("".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + assert_eq!( + split_mode_string("b".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + assert_eq!( + split_mode_string("t".to_string()), + Err( + "Must have exactly one of create/read/write/append mode and at most one plus" + .to_string() + ) + ); + } + + #[test] + fn test_text_and_binary_at_once() { + assert_eq!( + split_mode_string("rbt".to_string()), + Err("can't have text and binary mode at once".to_string()) + ); + } + + #[test] + fn test_exactly_one_mode() { + assert_eq!( + split_mode_string("rwb".to_string()), + Err("must have exactly one of create/read/write/append mode".to_string()) + ); + } + + #[test] + fn test_at_most_one_plus() { + assert_eq!( + split_mode_string("a++".to_string()), + Err("invalid mode: 'a++'".to_string()) + ); + } + + #[test] + fn test_buffered_read() { + let data = vec![1, 2, 3, 4]; + let bytes: i64 = -1; + let mut buffered = BufferedIO { + cursor: Cursor::new(data.clone()), + }; + + assert_eq!(buffered.read(bytes).unwrap(), data); + } + + #[test] + fn test_buffered_seek() { + let data = vec![1, 2, 3, 4]; + let count: u64 = 2; + let mut buffered = BufferedIO { + cursor: Cursor::new(data.clone()), + }; + + assert_eq!(buffered.seek(count.clone()).unwrap(), count); + assert_eq!(buffered.read(count.clone() as i64).unwrap(), vec![3, 4]); + } + + #[test] + fn test_buffered_value() { + let data = vec![1, 2, 3, 4]; + let buffered = BufferedIO { + cursor: Cursor::new(data.clone()), + }; + + assert_eq!(buffered.getvalue(), data); + } +} diff --git a/vm/src/stdlib/itertools.rs b/vm/src/stdlib/itertools.rs new file mode 100644 index 0000000000..732020480f --- /dev/null +++ b/vm/src/stdlib/itertools.rs @@ -0,0 +1,445 @@ +use std::cell::RefCell; +use std::cmp::Ordering; +use std::ops::{AddAssign, SubAssign}; + +use num_bigint::BigInt; +use num_traits::ToPrimitive; + +use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objbool; +use crate::obj::objint; +use crate::obj::objint::{PyInt, PyIntRef}; +use crate::obj::objiter::{call_next, get_iter, new_stop_iteration}; +use crate::obj::objtype; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{IdProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +#[pyclass(name = "chain")] +#[derive(Debug)] +struct PyItertoolsChain { + iterables: Vec, + cur: RefCell<(usize, Option)>, +} + +impl PyValue for PyItertoolsChain { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "chain") + } +} + +#[pyimpl] +impl PyItertoolsChain { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyItertoolsChain { + iterables: args.args, + cur: RefCell::new((0, None)), + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let (ref mut cur_idx, ref mut cur_iter) = *self.cur.borrow_mut(); + while *cur_idx < self.iterables.len() { + if cur_iter.is_none() { + *cur_iter = Some(get_iter(vm, &self.iterables[*cur_idx])?); + } + + // can't be directly inside the 'match' clause, otherwise the borrows collide. + let obj = call_next(vm, cur_iter.as_ref().unwrap()); + match obj { + Ok(ok) => return Ok(ok), + Err(err) => { + if objtype::isinstance(&err, &vm.ctx.exceptions.stop_iteration) { + *cur_idx += 1; + *cur_iter = None; + } else { + return Err(err); + } + } + } + } + + Err(new_stop_iteration(vm)) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass] +#[derive(Debug)] +struct PyItertoolsCount { + cur: RefCell, + step: BigInt, +} + +impl PyValue for PyItertoolsCount { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "count") + } +} + +#[pyimpl] +impl PyItertoolsCount { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new( + _cls: PyClassRef, + start: OptionalArg, + step: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let start = match start.into_option() { + Some(int) => int.as_bigint().clone(), + None => BigInt::from(0), + }; + let step = match step.into_option() { + Some(int) => int.as_bigint().clone(), + None => BigInt::from(1), + }; + + Ok(PyItertoolsCount { + cur: RefCell::new(start), + step, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, _vm: &VirtualMachine) -> PyResult { + let result = self.cur.borrow().clone(); + AddAssign::add_assign(&mut self.cur.borrow_mut() as &mut BigInt, &self.step); + Ok(PyInt::new(result)) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass] +#[derive(Debug)] +struct PyItertoolsRepeat { + object: PyObjectRef, + times: Option>, +} + +impl PyValue for PyItertoolsRepeat { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "repeat") + } +} + +#[pyimpl] +impl PyItertoolsRepeat { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new( + _cls: PyClassRef, + object: PyObjectRef, + times: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let times = match times.into_option() { + Some(int) => Some(RefCell::new(int.as_bigint().clone())), + None => None, + }; + + Ok(PyItertoolsRepeat { + object: object.clone(), + times, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if self.times.is_some() { + match self.times.as_ref().unwrap().borrow().cmp(&BigInt::from(0)) { + Ordering::Less | Ordering::Equal => return Err(new_stop_iteration(vm)), + _ => (), + }; + + SubAssign::sub_assign( + &mut self.times.as_ref().unwrap().borrow_mut() as &mut BigInt, + &BigInt::from(1), + ); + } + + Ok(self.object.clone()) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass(name = "starmap")] +#[derive(Debug)] +struct PyItertoolsStarmap { + function: PyObjectRef, + iter: PyObjectRef, +} + +impl PyValue for PyItertoolsStarmap { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "starmap") + } +} + +#[pyimpl] +impl PyItertoolsStarmap { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new( + _cls: PyClassRef, + function: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let iter = get_iter(vm, &iterable)?; + + Ok(PyItertoolsStarmap { function, iter } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + let obj = call_next(vm, &self.iter)?; + + vm.invoke(self.function.clone(), vm.extract_elements(&obj)?) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass] +#[derive(Debug)] +struct PyItertoolsTakewhile { + predicate: PyObjectRef, + iterable: PyObjectRef, + stop_flag: RefCell, +} + +impl PyValue for PyItertoolsTakewhile { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "takewhile") + } +} + +#[pyimpl] +impl PyItertoolsTakewhile { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new( + _cls: PyClassRef, + predicate: PyObjectRef, + iterable: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let iter = get_iter(vm, &iterable)?; + + Ok(PyItertoolsTakewhile { + predicate, + iterable: iter, + stop_flag: RefCell::new(false), + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + if *self.stop_flag.borrow() { + return Err(new_stop_iteration(vm)); + } + + // might be StopIteration or anything else, which is propaged upwwards + let obj = call_next(vm, &self.iterable)?; + + let verdict = vm.invoke(self.predicate.clone(), vec![obj.clone()])?; + let verdict = objbool::boolval(vm, verdict)?; + if verdict { + Ok(obj) + } else { + *self.stop_flag.borrow_mut() = true; + Err(new_stop_iteration(vm)) + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +#[pyclass(name = "islice")] +#[derive(Debug)] +struct PyItertoolsIslice { + iterable: PyObjectRef, + cur: RefCell, + next: RefCell, + stop: Option, + step: usize, +} + +impl PyValue for PyItertoolsIslice { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("itertools", "islice") + } +} + +fn pyobject_to_opt_usize(obj: PyObjectRef, vm: &VirtualMachine) -> Option { + let is_int = objtype::isinstance(&obj, &vm.ctx.int_type()); + if is_int { + objint::get_value(&obj).to_usize() + } else { + None + } +} + +#[pyimpl] +impl PyItertoolsIslice { + #[pymethod(name = "__new__")] + #[allow(clippy::new_ret_no_self)] + fn new(_cls: PyClassRef, args: PyFuncArgs, vm: &VirtualMachine) -> PyResult { + let (iter, start, stop, step) = match args.args.len() { + 0 | 1 => { + return Err(vm.new_type_error(format!( + "islice expected at least 2 arguments, got {}", + args.args.len() + ))); + } + + 2 => { + let (iter, stop): (PyObjectRef, PyObjectRef) = args.bind(vm)?; + + (iter, 0usize, stop, 1usize) + } + _ => { + let (iter, start, stop, step): ( + PyObjectRef, + PyObjectRef, + PyObjectRef, + PyObjectRef, + ) = args.bind(vm)?; + + let start = if !start.is(&vm.get_none()) { + pyobject_to_opt_usize(start, &vm).ok_or_else(|| { + vm.new_value_error( + "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize.".to_string(), + ) + })? + } else { + 0usize + }; + + let step = if !step.is(&vm.get_none()) { + pyobject_to_opt_usize(step, &vm).ok_or_else(|| { + vm.new_value_error( + "Step for islice() must be a positive integer or None.".to_string(), + ) + })? + } else { + 1usize + }; + + (iter, start, stop, step) + } + }; + + let stop = if !stop.is(&vm.get_none()) { + Some(pyobject_to_opt_usize(stop, &vm).ok_or_else(|| { + vm.new_value_error( + "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize." + .to_string(), + ) + })?) + } else { + None + }; + + let iter = get_iter(vm, &iter)?; + + Ok(PyItertoolsIslice { + iterable: iter, + cur: RefCell::new(0), + next: RefCell::new(start), + stop, + step, + } + .into_ref(vm) + .into_object()) + } + + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + while *self.cur.borrow() < *self.next.borrow() { + call_next(vm, &self.iterable)?; + *self.cur.borrow_mut() += 1; + } + + if let Some(stop) = self.stop { + if *self.cur.borrow() >= stop { + return Err(new_stop_iteration(vm)); + } + } + + let obj = call_next(vm, &self.iterable)?; + *self.cur.borrow_mut() += 1; + + // TODO is this overflow check required? attempts to copy CPython. + let (next, ovf) = (*self.next.borrow()).overflowing_add(self.step); + *self.next.borrow_mut() = if ovf { self.stop.unwrap() } else { next }; + + Ok(obj) + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let chain = PyItertoolsChain::make_class(ctx); + + let count = ctx.new_class("count", ctx.object()); + PyItertoolsCount::extend_class(ctx, &count); + + let repeat = ctx.new_class("repeat", ctx.object()); + PyItertoolsRepeat::extend_class(ctx, &repeat); + + let starmap = PyItertoolsStarmap::make_class(ctx); + + let takewhile = ctx.new_class("takewhile", ctx.object()); + PyItertoolsTakewhile::extend_class(ctx, &takewhile); + + let islice = PyItertoolsIslice::make_class(ctx); + + py_module!(vm, "itertools", { + "chain" => chain, + "count" => count, + "repeat" => repeat, + "starmap" => starmap, + "takewhile" => takewhile, + "islice" => islice, + }) +} diff --git a/vm/src/stdlib/json.rs b/vm/src/stdlib/json.rs index da373a955c..8b5d21b758 100644 --- a/vm/src/stdlib/json.rs +++ b/vm/src/stdlib/json.rs @@ -1,235 +1,41 @@ -use std::fmt; - -use serde; -use serde::de::{DeserializeSeed, Visitor}; -use serde::ser::{SerializeMap, SerializeSeq}; -use serde_json; - -use crate::function::PyFuncArgs; -use crate::obj::{ - objbool, objdict, objfloat, objint, objsequence, - objstr::{self, PyString}, - objtype, -}; -use crate::pyobject::{ - create_type, DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol, -}; +use crate::obj::objstr::PyStringRef; +use crate::py_serde; +use crate::pyobject::{create_type, ItemProtocol, PyObjectRef, PyResult}; use crate::VirtualMachine; -use num_traits::cast::ToPrimitive; - -// We need to have a VM available to serialise a PyObject based on its subclass, so we implement -// PyObject serialisation via a proxy object which holds a reference to a VM -struct PyObjectSerializer<'s> { - pyobject: &'s PyObjectRef, - vm: &'s VirtualMachine, -} - -impl<'s> PyObjectSerializer<'s> { - fn clone_with_object(&self, pyobject: &'s PyObjectRef) -> PyObjectSerializer { - PyObjectSerializer { - pyobject, - vm: self.vm, - } - } -} - -impl<'s> serde::Serialize for PyObjectSerializer<'s> { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let serialize_seq_elements = - |serializer: S, elements: &Vec| -> Result { - let mut seq = serializer.serialize_seq(Some(elements.len()))?; - for e in elements.iter() { - seq.serialize_element(&self.clone_with_object(e))?; - } - seq.end() - }; - if objtype::isinstance(self.pyobject, &self.vm.ctx.str_type()) { - serializer.serialize_str(&objstr::get_value(&self.pyobject)) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.float_type()) { - serializer.serialize_f64(objfloat::get_value(self.pyobject)) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.bool_type()) { - serializer.serialize_bool(objbool::get_value(self.pyobject)) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.int_type()) { - let v = objint::get_value(self.pyobject); - serializer.serialize_i64(v.to_i64().unwrap()) - // Although this may seem nice, it does not give the right result: - // v.serialize(serializer) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.list_type()) - || objtype::isinstance(self.pyobject, &self.vm.ctx.tuple_type()) - { - let elements = objsequence::get_elements(self.pyobject); - serialize_seq_elements(serializer, &elements) - } else if objtype::isinstance(self.pyobject, &self.vm.ctx.dict_type()) { - let pairs = objdict::get_elements(self.pyobject); - let mut map = serializer.serialize_map(Some(pairs.len()))?; - for (key, e) in pairs.iter() { - map.serialize_entry(&key, &self.clone_with_object(&e.1))?; - } - map.end() - } else if self.pyobject.is(&self.vm.get_none()) { - serializer.serialize_none() - } else { - Err(serde::ser::Error::custom(format!( - "Object of type '{:?}' is not serializable", - self.pyobject.typ() - ))) - } - } -} - -// This object is used as the seed for deserialization so we have access to the PyContext for type -// creation -#[derive(Clone)] -struct PyObjectDeserializer<'c> { - vm: &'c VirtualMachine, -} - -impl<'de> serde::de::DeserializeSeed<'de> for PyObjectDeserializer<'de> { - type Value = PyObjectRef; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_any(self.clone()) - } -} - -impl<'de> Visitor<'de> for PyObjectDeserializer<'de> { - type Value = PyObjectRef; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a type that can deserialise in Python") - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - Ok(self.vm.ctx.new_str(value.to_string())) - } - - fn visit_string(self, value: String) -> Result - where - E: serde::de::Error, - { - Ok(self.vm.ctx.new_str(value)) - } - - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - // The JSON deserializer always uses the i64/u64 deserializers, so we only need to - // implement those for now - Ok(self.vm.ctx.new_int(value)) - } - - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - // The JSON deserializer always uses the i64/u64 deserializers, so we only need to - // implement those for now - Ok(self.vm.ctx.new_int(value)) - } - - fn visit_f64(self, value: f64) -> Result - where - E: serde::de::Error, - { - Ok(self.vm.ctx.new_float(value)) - } - - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - Ok(self.vm.ctx.new_bool(value)) - } - - fn visit_seq(self, mut access: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut seq = Vec::with_capacity(access.size_hint().unwrap_or(0)); - while let Some(value) = access.next_element_seed(self.clone())? { - seq.push(value); - } - Ok(self.vm.ctx.new_list(seq)) - } - - fn visit_map(self, mut access: M) -> Result - where - M: serde::de::MapAccess<'de>, - { - let dict = self.vm.ctx.new_dict(); - // TODO: Given keys must be strings, we can probably do something more efficient - // than wrapping the given object up and then unwrapping it to determine whether or - // not it is a string - while let Some((key_obj, value)) = access.next_entry_seed(self.clone(), self.clone())? { - let key: String = match key_obj.payload::() { - Some(PyString { ref value }) => value.clone(), - _ => unimplemented!("map keys must be strings"), - }; - dict.set_item(&self.vm.ctx, &key, value); - } - Ok(dict) - } - - fn visit_unit(self) -> Result - where - E: serde::de::Error, - { - Ok(self.vm.ctx.none.clone()) - } -} - -pub fn ser_pyobject(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { - let serializer = PyObjectSerializer { pyobject: obj, vm }; - serde_json::to_string(&serializer).map_err(|err| vm.new_type_error(err.to_string())) -} - -pub fn de_pyobject(vm: &VirtualMachine, s: &str) -> PyResult { - let de = PyObjectDeserializer { vm }; - // TODO: Support deserializing string sub-classes - de.deserialize(&mut serde_json::Deserializer::from_str(s)) - .map_err(|err| { - let json_decode_error = vm - .sys_module - .get_item("modules") - .unwrap() - .get_item("json") - .unwrap() - .get_item("JSONDecodeError") - .unwrap(); - let exc = vm.new_exception(json_decode_error, format!("{}", err)); - vm.ctx.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())); - vm.ctx.set_attr(&exc, "colno", vm.ctx.new_int(err.column())); - exc - }) -} +use serde_json; /// Implement json.dumps -fn json_dumps(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - // TODO: Implement non-trivial serialisation case - arg_check!(vm, args, required = [(obj, None)]); - let string = ser_pyobject(vm, obj)?; - Ok(vm.context().new_str(string)) +pub fn json_dumps(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let serializer = py_serde::PyObjectSerializer::new(vm, &obj); + serde_json::to_string(&serializer).map_err(|err| vm.new_type_error(err.to_string())) } /// Implement json.loads -fn json_loads(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { +pub fn json_loads(string: PyStringRef, vm: &VirtualMachine) -> PyResult { // TODO: Implement non-trivial deserialization case - arg_check!(vm, args, required = [(string, Some(vm.ctx.str_type()))]); - - de_pyobject(vm, &objstr::get_value(&string)) + let de_result = + py_serde::deserialize(vm, &mut serde_json::Deserializer::from_str(string.as_str())); + + de_result.map_err(|err| { + let module = vm + .get_attribute(vm.sys_module.clone(), "modules") + .unwrap() + .get_item("json", vm) + .unwrap(); + let json_decode_error = vm.get_attribute(module, "JSONDecodeError").unwrap(); + let json_decode_error = json_decode_error.downcast().unwrap(); + let exc = vm.new_exception(json_decode_error, format!("{}", err)); + vm.set_attr(&exc, "lineno", vm.ctx.new_int(err.line())) + .unwrap(); + vm.set_attr(&exc, "colno", vm.ctx.new_int(err.column())) + .unwrap(); + exc + }) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + // TODO: Make this a proper type with a constructor let json_decode_error = create_type( "JSONDecodeError", @@ -237,7 +43,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { &ctx.exceptions.exception_type, ); - py_module!(ctx, "json", { + py_module!(vm, "json", { "dumps" => ctx.new_rustfunc(json_dumps), "loads" => ctx.new_rustfunc(json_loads), "JSONDecodeError" => json_decode_error diff --git a/vm/src/stdlib/keyword.rs b/vm/src/stdlib/keyword.rs index fe27399c55..062f41df74 100644 --- a/vm/src/stdlib/keyword.rs +++ b/vm/src/stdlib/keyword.rs @@ -6,7 +6,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -18,7 +18,9 @@ fn keyword_iskeyword(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let keyword_kwlist = ctx.new_list( lexer::get_keywords() .keys() @@ -26,7 +28,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { .collect(), ); - py_module!(ctx, "keyword", { + py_module!(vm, "keyword", { "iskeyword" => ctx.new_rustfunc(keyword_iskeyword), "kwlist" => keyword_kwlist }) diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs new file mode 100644 index 0000000000..e7afc86d2a --- /dev/null +++ b/vm/src/stdlib/marshal.rs @@ -0,0 +1,24 @@ +use crate::bytecode; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::obj::objcode::{PyCode, PyCodeRef}; +use crate::pyobject::{IntoPyObject, PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +fn marshal_dumps(co: PyCodeRef, vm: &VirtualMachine) -> PyResult { + PyBytes::new(bincode::serialize(&co.code).unwrap()).into_pyobject(vm) +} + +fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let code = bincode::deserialize::(&code_bytes).unwrap(); + let pycode = PyCode { code }; + pycode.into_pyobject(vm) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "marshal", { + "loads" => ctx.new_rustfunc(marshal_loads), + "dumps" => ctx.new_rustfunc(marshal_dumps) + }) +} diff --git a/vm/src/stdlib/math.rs b/vm/src/stdlib/math.rs index 5af599cb7d..01e3fe6c1d 100644 --- a/vm/src/stdlib/math.rs +++ b/vm/src/stdlib/math.rs @@ -7,8 +7,8 @@ use statrs::function::erf::{erf, erfc}; use statrs::function::gamma::{gamma, ln_gamma}; use crate::function::PyFuncArgs; -use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::obj::{objfloat, objtype}; +use crate::pyobject::{PyObjectRef, PyResult, TypeProtocol}; use crate::vm::VirtualMachine; // Helper macro: @@ -172,8 +172,63 @@ fn math_lgamma(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "math", { +fn try_magic_method(func_name: &str, vm: &VirtualMachine, value: &PyObjectRef) -> PyResult { + let method = vm.get_method_or_type_error(value.clone(), func_name, || { + format!( + "type '{}' doesn't define '{}' method", + value.class().name, + func_name, + ) + })?; + vm.invoke(method, vec![]) +} + +fn math_trunc(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + try_magic_method("__trunc__", vm, value) +} + +fn math_ceil(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + if objtype::isinstance(value, &vm.ctx.float_type) { + let v = objfloat::get_value(value); + Ok(vm.ctx.new_float(v.ceil())) + } else { + try_magic_method("__ceil__", vm, value) + } +} + +fn math_floor(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(value, None)]); + if objtype::isinstance(value, &vm.ctx.float_type) { + let v = objfloat::get_value(value); + Ok(vm.ctx.new_float(v.floor())) + } else { + try_magic_method("__floor__", vm, value) + } +} + +fn math_frexp(value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + objfloat::try_float(&value, vm)?.map_or_else( + || Err(vm.new_type_error(format!("must be real number, not {}", value.class()))), + |value| { + let (m, e) = if value.is_finite() { + let (m, e) = objfloat::ufrexp(value); + (m * value.signum(), e) + } else { + (value, 0) + }; + Ok(vm + .ctx + .new_tuple(vec![vm.ctx.new_float(m), vm.ctx.new_int(e)])) + }, + ) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "math", { // Number theory functions: "fabs" => ctx.new_rustfunc(math_fabs), "isfinite" => ctx.new_rustfunc(math_isfinite), @@ -217,6 +272,13 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { "gamma" => ctx.new_rustfunc(math_gamma), "lgamma" => ctx.new_rustfunc(math_lgamma), + "frexp" => ctx.new_rustfunc(math_frexp), + + // Rounding functions: + "trunc" => ctx.new_rustfunc(math_trunc), + "ceil" => ctx.new_rustfunc(math_ceil), + "floor" => ctx.new_rustfunc(math_floor), + // Constants: "pi" => ctx.new_float(std::f64::consts::PI), // 3.14159... "e" => ctx.new_float(std::f64::consts::E), // 2.71.. diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index b0b61119c3..458a8cfb5e 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,7 +1,14 @@ +#[cfg(feature = "rustpython-parser")] mod ast; +mod binascii; mod dis; -pub(crate) mod json; +mod hashlib; +mod imp; +mod itertools; +mod json; +#[cfg(feature = "rustpython-parser")] mod keyword; +mod marshal; mod math; mod platform; mod pystruct; @@ -9,48 +16,75 @@ mod random; mod re; pub mod socket; mod string; +mod thread; mod time_module; +#[cfg(feature = "rustpython-parser")] mod tokenize; -mod types; +mod unicodedata; +mod warnings; mod weakref; use std::collections::HashMap; +use crate::vm::VirtualMachine; + #[cfg(not(target_arch = "wasm32"))] pub mod io; #[cfg(not(target_arch = "wasm32"))] mod os; +#[cfg(all(unix, not(target_os = "android")))] +mod pwd; -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; -pub type StdlibInitFunc = Box PyObjectRef>; +pub type StdlibInitFunc = Box PyObjectRef>; pub fn get_module_inits() -> HashMap { - let mut modules = HashMap::new(); - modules.insert( - "ast".to_string(), - Box::new(ast::make_module) as StdlibInitFunc, - ); - modules.insert("dis".to_string(), Box::new(dis::make_module)); - modules.insert("json".to_string(), Box::new(json::make_module)); - modules.insert("keyword".to_string(), Box::new(keyword::make_module)); - modules.insert("math".to_string(), Box::new(math::make_module)); - modules.insert("platform".to_string(), Box::new(platform::make_module)); - modules.insert("re".to_string(), Box::new(re::make_module)); - modules.insert("random".to_string(), Box::new(random::make_module)); - modules.insert("string".to_string(), Box::new(string::make_module)); - modules.insert("struct".to_string(), Box::new(pystruct::make_module)); - modules.insert("time".to_string(), Box::new(time_module::make_module)); - modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); - modules.insert("types".to_string(), Box::new(types::make_module)); - modules.insert("_weakref".to_string(), Box::new(weakref::make_module)); + #[allow(unused_mut)] + let mut modules = hashmap! { + "binascii".to_string() => Box::new(binascii::make_module) as StdlibInitFunc, + "dis".to_string() => Box::new(dis::make_module) as StdlibInitFunc, + "hashlib".to_string() => Box::new(hashlib::make_module), + "itertools".to_string() => Box::new(itertools::make_module), + "json".to_string() => Box::new(json::make_module), + "marshal".to_string() => Box::new(marshal::make_module), + "math".to_string() => Box::new(math::make_module), + "platform".to_string() => Box::new(platform::make_module), + "re".to_string() => Box::new(re::make_module), + "random".to_string() => Box::new(random::make_module), + "string".to_string() => Box::new(string::make_module), + "struct".to_string() => Box::new(pystruct::make_module), + "_thread".to_string() => Box::new(thread::make_module), + "time".to_string() => Box::new(time_module::make_module), + "_weakref".to_string() => Box::new(weakref::make_module), + "_imp".to_string() => Box::new(imp::make_module), + "unicodedata".to_string() => Box::new(unicodedata::make_module), + "_warnings".to_string() => Box::new(warnings::make_module), + }; + + // Insert parser related modules: + #[cfg(feature = "rustpython-parser")] + { + modules.insert( + "ast".to_string(), + Box::new(ast::make_module) as StdlibInitFunc, + ); + modules.insert("keyword".to_string(), Box::new(keyword::make_module)); + modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); + } // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] { - modules.insert("io".to_string(), Box::new(io::make_module)); - modules.insert("os".to_string(), Box::new(os::make_module)); + modules.insert("_io".to_string(), Box::new(io::make_module)); + modules.insert("_os".to_string(), Box::new(os::make_module)); modules.insert("socket".to_string(), Box::new(socket::make_module)); } + // Unix-only + #[cfg(all(unix, not(target_os = "android")))] + { + modules.insert("pwd".to_string(), Box::new(pwd::make_module)); + } + modules } diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 42b4160ca8..11f6c16764 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1,13 +1,24 @@ +use std::cell::RefCell; use std::fs::File; use std::fs::OpenOptions; -use std::io::ErrorKind; +use std::io::{self, ErrorKind, Read, Write}; +use std::time::{Duration, SystemTime}; +use std::{env, fs}; +use bitflags::bitflags; use num_traits::cast::ToPrimitive; -use crate::function::PyFuncArgs; -use crate::obj::objint; -use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::function::{IntoPyNativeFunc, PyFuncArgs}; +use crate::obj::objbytes::PyBytesRef; +use crate::obj::objdict::PyDictRef; +use crate::obj::objint::{self, PyInt, PyIntRef}; +use crate::obj::objiter; +use crate::obj::objset::PySet; +use crate::obj::objstr::{self, PyString, PyStringRef}; +use crate::obj::objtype::{self, PyClassRef}; +use crate::pyobject::{ + ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryIntoRef, TypeProtocol, +}; use crate::vm::VirtualMachine; #[cfg(unix)] @@ -50,6 +61,14 @@ pub fn raw_file_number(handle: File) -> i64 { unimplemented!(); } +fn make_path(_vm: &VirtualMachine, path: PyStringRef, dir_fd: &DirFd) -> PyStringRef { + if dir_fd.dir_fd.is_some() { + unimplemented!(); + } else { + path + } +} + pub fn os_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(fileno, Some(vm.ctx.int_type()))]); @@ -63,37 +82,96 @@ pub fn os_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.get_none()) } +bitflags! { + pub struct FileCreationFlags: u32 { + // https://elixir.bootlin.com/linux/v4.8/source/include/uapi/asm-generic/fcntl.h + const O_RDONLY = 0o0000_0000; + const O_WRONLY = 0o0000_0001; + const O_RDWR = 0o0000_0002; + const O_CREAT = 0o0000_0100; + const O_EXCL = 0o0000_0200; + const O_APPEND = 0o0000_2000; + const O_NONBLOCK = 0o0000_4000; + } +} + pub fn os_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, required = [ (name, Some(vm.ctx.str_type())), - (mode, Some(vm.ctx.int_type())) + (flags, Some(vm.ctx.int_type())) + ], + optional = [ + (_mode, Some(vm.ctx.int_type())), + (dir_fd, Some(vm.ctx.int_type())) ] ); - let fname = objstr::get_value(&name); + let name = name.clone().downcast::().unwrap(); + let dir_fd = if let Some(obj) = dir_fd { + DirFd { + dir_fd: Some(obj.clone().downcast::().unwrap()), + } + } else { + DirFd::default() + }; + let fname = &make_path(vm, name, &dir_fd).value; + + let flags = FileCreationFlags::from_bits(objint::get_value(flags).to_u32().unwrap()) + .ok_or(vm.new_value_error("Unsupported flag".to_string()))?; + + let mut options = &mut OpenOptions::new(); + + if flags.contains(FileCreationFlags::O_WRONLY) { + options = options.write(true); + } else if flags.contains(FileCreationFlags::O_RDWR) { + options = options.read(true).write(true); + } else { + options = options.read(true); + } - let handle = match objint::get_value(mode).to_u16().unwrap() { - 0 => OpenOptions::new().read(true).open(&fname), - 1 => OpenOptions::new().write(true).open(&fname), - 512 => OpenOptions::new().write(true).create(true).open(&fname), - _ => OpenOptions::new().read(true).open(&fname), + if flags.contains(FileCreationFlags::O_APPEND) { + options = options.append(true); } - .map_err(|err| match err.kind() { + + if flags.contains(FileCreationFlags::O_CREAT) { + if flags.contains(FileCreationFlags::O_EXCL) { + options = options.create_new(true); + } else { + options = options.create(true); + } + } + + let handle = options + .open(&fname) + .map_err(|err| convert_io_error(vm, err))?; + + Ok(vm.ctx.new_int(raw_file_number(handle))) +} + +fn convert_io_error(vm: &VirtualMachine, err: io::Error) -> PyObjectRef { + let os_error = match err.kind() { ErrorKind::NotFound => { let exc_type = vm.ctx.exceptions.file_not_found_error.clone(); - vm.new_exception(exc_type, format!("No such file or directory: {}", &fname)) + vm.new_exception(exc_type, err.to_string()) } ErrorKind::PermissionDenied => { let exc_type = vm.ctx.exceptions.permission_error.clone(); - vm.new_exception(exc_type, format!("Permission denied: {}", &fname)) + vm.new_exception(exc_type, err.to_string()) } - _ => vm.new_value_error("Unhandled file IO error".to_string()), - })?; - - Ok(vm.ctx.new_int(raw_file_number(handle))) + ErrorKind::AlreadyExists => { + let exc_type = vm.ctx.exceptions.file_exists_error.clone(); + vm.new_exception(exc_type, err.to_string()) + } + _ => vm.new_os_error(err.to_string()), + }; + if let Some(errno) = err.raw_os_error() { + vm.set_attr(&os_error, "errno", vm.ctx.new_int(errno)) + .unwrap(); + } + os_error } fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -113,25 +191,655 @@ fn os_error(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Err(vm.new_os_error(msg)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let py_mod = py_module!(ctx, "os", { - "open" => ctx.new_rustfunc(os_open), +fn os_fsync(fd: PyIntRef, vm: &VirtualMachine) -> PyResult<()> { + let file = rust_file(fd.as_bigint().to_i64().unwrap()); + file.sync_all().map_err(|err| convert_io_error(vm, err))?; + // Avoid closing the fd + raw_file_number(file); + Ok(()) +} + +fn os_read(fd: PyIntRef, n: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; n.as_bigint().to_usize().unwrap()]; + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); + file.read_exact(&mut buffer) + .map_err(|err| convert_io_error(vm, err))?; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_bytes(buffer)) +} + +fn os_write(fd: PyIntRef, data: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let mut file = rust_file(fd.as_bigint().to_i64().unwrap()); + let written = file.write(&data).map_err(|err| convert_io_error(vm, err))?; + + // Avoid closing the fd + raw_file_number(file); + Ok(vm.ctx.new_int(written)) +} + +fn os_remove(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); + fs::remove_file(&path.value).map_err(|err| convert_io_error(vm, err)) +} + +fn os_mkdir(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); + fs::create_dir(&path.value).map_err(|err| convert_io_error(vm, err)) +} + +fn os_mkdirs(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::create_dir_all(&path.value).map_err(|err| convert_io_error(vm, err)) +} + +fn os_rmdir(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult<()> { + let path = make_path(vm, path, &dir_fd); + fs::remove_dir(&path.value).map_err(|err| convert_io_error(vm, err)) +} + +fn os_listdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::read_dir(&path.value) { + Ok(iter) => { + let res: PyResult> = iter + .map(|entry| match entry { + Ok(path) => Ok(vm.ctx.new_str(path.file_name().into_string().unwrap())), + Err(s) => Err(convert_io_error(vm, s)), + }) + .collect(); + Ok(vm.ctx.new_list(res?)) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + } +} + +fn os_putenv(key: PyStringRef, value: PyStringRef, _vm: &VirtualMachine) { + env::set_var(&key.value, &value.value) +} + +fn os_unsetenv(key: PyStringRef, _vm: &VirtualMachine) { + env::remove_var(&key.value) +} + +fn _os_environ(vm: &VirtualMachine) -> PyDictRef { + let environ = vm.ctx.new_dict(); + for (key, value) in env::vars() { + environ.set_item(&key, vm.new_str(value), vm).unwrap(); + } + environ +} + +#[derive(Debug)] +struct DirEntry { + entry: fs::DirEntry, +} + +type DirEntryRef = PyRef; + +impl PyValue for DirEntry { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("_os", "DirEntry") + } +} + +#[derive(FromArgs, Default)] +struct DirFd { + #[pyarg(keyword_only, default = "None")] + dir_fd: Option, +} + +#[derive(FromArgs)] +struct FollowSymlinks { + #[pyarg(keyword_only, default = "true")] + follow_symlinks: bool, +} + +impl DirEntryRef { + fn name(self, _vm: &VirtualMachine) -> String { + self.entry.file_name().into_string().unwrap() + } + + fn path(self, _vm: &VirtualMachine) -> String { + self.entry.path().to_str().unwrap().to_string() + } + + #[allow(clippy::match_bool)] + fn perform_on_metadata( + self, + follow_symlinks: FollowSymlinks, + action: &Fn(fs::Metadata) -> bool, + vm: &VirtualMachine, + ) -> PyResult { + let metadata = match follow_symlinks.follow_symlinks { + true => fs::metadata(self.entry.path()), + false => fs::symlink_metadata(self.entry.path()), + }; + let meta = metadata.map_err(|err| convert_io_error(vm, err))?; + Ok(action(meta)) + } + + fn is_dir(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + self.perform_on_metadata( + follow_symlinks, + &|meta: fs::Metadata| -> bool { meta.is_dir() }, + vm, + ) + } + + fn is_file(self, follow_symlinks: FollowSymlinks, vm: &VirtualMachine) -> PyResult { + self.perform_on_metadata( + follow_symlinks, + &|meta: fs::Metadata| -> bool { meta.is_file() }, + vm, + ) + } + + fn is_symlink(self, vm: &VirtualMachine) -> PyResult { + Ok(self + .entry + .file_type() + .map_err(|err| convert_io_error(vm, err))? + .is_symlink()) + } + + fn stat( + self, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, + ) -> PyResult { + os_stat(self.path(vm).try_into_ref(vm)?, dir_fd, follow_symlinks, vm) + } +} + +#[pyclass] +#[derive(Debug)] +struct ScandirIterator { + entries: RefCell, +} + +impl PyValue for ScandirIterator { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("_os", "ScandirIter") + } +} + +#[pyimpl] +impl ScandirIterator { + #[pymethod(name = "__next__")] + fn next(&self, vm: &VirtualMachine) -> PyResult { + match self.entries.borrow_mut().next() { + Some(entry) => match entry { + Ok(entry) => Ok(DirEntry { entry }.into_ref(vm).into_object()), + Err(s) => Err(convert_io_error(vm, s)), + }, + None => Err(objiter::new_stop_iteration(vm)), + } + } + + #[pymethod(name = "__iter__")] + fn iter(zelf: PyRef, _vm: &VirtualMachine) -> PyRef { + zelf + } +} + +fn os_scandir(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + match fs::read_dir(&path.value) { + Ok(iter) => Ok(ScandirIterator { + entries: RefCell::new(iter), + } + .into_ref(vm) + .into_object()), + Err(s) => Err(convert_io_error(vm, s)), + } +} + +#[derive(Debug)] +struct StatResult { + st_mode: u32, + st_ino: u64, + st_dev: u64, + st_nlink: u64, + st_uid: u32, + st_gid: u32, + st_size: u64, + st_atime: f64, + st_ctime: f64, + st_mtime: f64, +} + +impl PyValue for StatResult { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("_os", "stat_result") + } +} + +type StatResultRef = PyRef; + +impl StatResultRef { + fn st_mode(self, _vm: &VirtualMachine) -> u32 { + self.st_mode + } + + fn st_ino(self, _vm: &VirtualMachine) -> u64 { + self.st_ino + } + + fn st_dev(self, _vm: &VirtualMachine) -> u64 { + self.st_dev + } + + fn st_nlink(self, _vm: &VirtualMachine) -> u64 { + self.st_nlink + } + + fn st_uid(self, _vm: &VirtualMachine) -> u32 { + self.st_uid + } + + fn st_gid(self, _vm: &VirtualMachine) -> u32 { + self.st_gid + } + + fn st_size(self, _vm: &VirtualMachine) -> u64 { + self.st_size + } + + fn st_atime(self, _vm: &VirtualMachine) -> f64 { + self.st_atime + } + + fn st_ctime(self, _vm: &VirtualMachine) -> f64 { + self.st_ctime + } + + fn st_mtime(self, _vm: &VirtualMachine) -> f64 { + self.st_mtime + } +} + +// Copied code from Duration::as_secs_f64 as it's still unstable +fn duration_as_secs_f64(duration: Duration) -> f64 { + (duration.as_secs() as f64) + (duration.subsec_nanos() as f64) / (1_000_000_000 as f64) +} + +fn to_seconds_from_unix_epoch(sys_time: SystemTime) -> f64 { + match sys_time.duration_since(SystemTime::UNIX_EPOCH) { + Ok(duration) => duration_as_secs_f64(duration), + Err(err) => -duration_as_secs_f64(err.duration()), + } +} + +#[cfg(unix)] +fn to_seconds_from_nanos(secs: i64, nanos: i64) -> f64 { + let duration = Duration::new(secs as u64, nanos as u32); + duration_as_secs_f64(duration) +} + +#[cfg(unix)] +macro_rules! os_unix_stat_inner { + ( $path:expr, $follow_symlinks:expr, $vm:expr ) => {{ + #[allow(clippy::match_bool)] + fn get_stats(path: &str, follow_symlinks: bool) -> io::Result { + let meta = match follow_symlinks { + true => fs::metadata(path)?, + false => fs::symlink_metadata(path)?, + }; + + Ok(StatResult { + st_mode: meta.st_mode(), + st_ino: meta.st_ino(), + st_dev: meta.st_dev(), + st_nlink: meta.st_nlink(), + st_uid: meta.st_uid(), + st_gid: meta.st_gid(), + st_size: meta.st_size(), + st_atime: to_seconds_from_unix_epoch(meta.accessed()?), + st_mtime: to_seconds_from_unix_epoch(meta.modified()?), + st_ctime: to_seconds_from_nanos(meta.st_ctime(), meta.st_ctime_nsec()), + }) + } + + get_stats(&$path.value, $follow_symlinks.follow_symlinks) + .map_err(|err| convert_io_error($vm, err)) + }}; +} + +#[cfg(target_os = "linux")] +fn os_stat( + path: PyStringRef, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { + use std::os::linux::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); + os_unix_stat_inner!(path, follow_symlinks, vm) +} + +#[cfg(target_os = "macos")] +fn os_stat( + path: PyStringRef, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { + use std::os::macos::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); + os_unix_stat_inner!(path, follow_symlinks, vm) +} + +#[cfg(target_os = "android")] +fn os_stat( + path: PyStringRef, + dir_fd: DirFd, + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { + use std::os::android::fs::MetadataExt; + let path = make_path(vm, path, &dir_fd); + os_unix_stat_inner!(path, follow_symlinks, vm) +} + +// Copied from CPython fileutils.c +#[cfg(windows)] +fn attributes_to_mode(attr: u32) -> u32 { + const FILE_ATTRIBUTE_DIRECTORY: u32 = 16; + const FILE_ATTRIBUTE_READONLY: u32 = 1; + const S_IFDIR: u32 = 0o040000; + const S_IFREG: u32 = 0o100000; + let mut m: u32 = 0; + if attr & FILE_ATTRIBUTE_DIRECTORY == FILE_ATTRIBUTE_DIRECTORY { + m |= S_IFDIR | 0111; /* IFEXEC for user,group,other */ + } else { + m |= S_IFREG; + } + if attr & FILE_ATTRIBUTE_READONLY == FILE_ATTRIBUTE_READONLY { + m |= 0444; + } else { + m |= 0666; + } + m +} + +#[cfg(windows)] +fn os_stat( + path: PyStringRef, + _dir_fd: DirFd, // TODO: error + follow_symlinks: FollowSymlinks, + vm: &VirtualMachine, +) -> PyResult { + use std::os::windows::fs::MetadataExt; + + fn get_stats(path: &str, follow_symlinks: bool) -> io::Result { + let meta = match follow_symlinks { + true => fs::metadata(path)?, + false => fs::symlink_metadata(path)?, + }; + + Ok(StatResult { + st_mode: attributes_to_mode(meta.file_attributes()), + st_ino: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_dev: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_nlink: 0, // TODO: Not implemented in std::os::windows::fs::MetadataExt. + st_uid: 0, // 0 on windows + st_gid: 0, // 0 on windows + st_size: meta.file_size(), + st_atime: to_seconds_from_unix_epoch(meta.accessed()?), + st_mtime: to_seconds_from_unix_epoch(meta.modified()?), + st_ctime: to_seconds_from_unix_epoch(meta.created()?), + }) + } + + get_stats(&path.value, follow_symlinks.follow_symlinks) + .map_err(|s| vm.new_os_error(s.to_string())) +} + +#[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "android", + windows +)))] +fn os_stat(path: PyStringRef, vm: &VirtualMachine) -> PyResult { + unimplemented!(); +} + +fn os_lstat(path: PyStringRef, dir_fd: DirFd, vm: &VirtualMachine) -> PyResult { + os_stat( + path, + dir_fd, + FollowSymlinks { + follow_symlinks: false, + }, + vm, + ) +} + +#[cfg(unix)] +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { + use std::os::unix::fs as unix_fs; + let dst = make_path(vm, dst, &dir_fd); + unix_fs::symlink(&src.value, &dst.value).map_err(|err| convert_io_error(vm, err)) +} + +#[cfg(windows)] +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + _dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { + use std::os::windows::fs as win_fs; + let ret = match fs::metadata(&dst.value) { + Ok(meta) => { + if meta.is_file() { + win_fs::symlink_file(&src.value, &dst.value) + } else if meta.is_dir() { + win_fs::symlink_dir(&src.value, &dst.value) + } else { + panic!("Uknown file type"); + } + } + Err(_) => win_fs::symlink_file(&src.value, &dst.value), + }; + ret.map_err(|err| convert_io_error(vm, err)) +} + +#[cfg(all(not(unix), not(windows)))] +fn os_symlink( + src: PyStringRef, + dst: PyStringRef, + dir_fd: DirFd, + vm: &VirtualMachine, +) -> PyResult<()> { + unimplemented!(); +} + +fn os_getcwd(vm: &VirtualMachine) -> PyResult { + Ok(env::current_dir() + .map_err(|err| convert_io_error(vm, err))? + .as_path() + .to_str() + .unwrap() + .to_string()) +} + +fn os_chdir(path: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + env::set_current_dir(&path.value).map_err(|err| convert_io_error(vm, err)) +} + +fn os_fspath(path: PyObjectRef, vm: &VirtualMachine) -> PyResult { + if objtype::issubclass(&path.class(), &vm.ctx.str_type()) + || objtype::issubclass(&path.class(), &vm.ctx.bytes_type()) + { + Ok(path) + } else { + Err(vm.new_type_error(format!( + "expected str or bytes object, not {}", + path.class() + ))) + } +} + +fn os_rename(src: PyStringRef, dst: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + fs::rename(&src.value, &dst.value).map_err(|err| convert_io_error(vm, err)) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let os_name = if cfg!(windows) { + "nt".to_string() + } else { + "posix".to_string() + }; + + let environ = _os_environ(vm); + + let scandir_iter = ctx.new_class("ScandirIter", ctx.object()); + ScandirIterator::extend_class(ctx, &scandir_iter); + + let dir_entry = py_class!(ctx, "DirEntry", ctx.object(), { + "name" => ctx.new_property(DirEntryRef::name), + "path" => ctx.new_property(DirEntryRef::path), + "is_dir" => ctx.new_rustfunc(DirEntryRef::is_dir), + "is_file" => ctx.new_rustfunc(DirEntryRef::is_file), + "is_symlink" => ctx.new_rustfunc(DirEntryRef::is_symlink), + "stat" => ctx.new_rustfunc(DirEntryRef::stat), + }); + + let stat_result = py_class!(ctx, "stat_result", ctx.object(), { + "st_mode" => ctx.new_property(StatResultRef::st_mode), + "st_ino" => ctx.new_property(StatResultRef::st_ino), + "st_dev" => ctx.new_property(StatResultRef::st_dev), + "st_nlink" => ctx.new_property(StatResultRef::st_nlink), + "st_uid" => ctx.new_property(StatResultRef::st_uid), + "st_gid" => ctx.new_property(StatResultRef::st_gid), + "st_size" => ctx.new_property(StatResultRef::st_size), + "st_atime" => ctx.new_property(StatResultRef::st_atime), + "st_ctime" => ctx.new_property(StatResultRef::st_ctime), + "st_mtime" => ctx.new_property(StatResultRef::st_mtime), + }); + + struct SupportFunc<'a> { + name: &'a str, + func_obj: PyObjectRef, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + }; + impl<'a> SupportFunc<'a> { + fn new( + vm: &VirtualMachine, + name: &'a str, + func: F, + fd: Option, + dir_fd: Option, + follow_symlinks: Option, + ) -> Self + where + F: IntoPyNativeFunc, + { + let func_obj = vm.ctx.new_rustfunc(func); + Self { + name, + func_obj, + fd, + dir_fd, + follow_symlinks, + } + } + } + let support_funcs = vec![ + SupportFunc::new(vm, "open", os_open, None, Some(false), None), + // access Some Some None + SupportFunc::new(vm, "chdir", os_chdir, Some(false), None, None), + // chflags Some, None Some + // chmod Some Some Some + // chown Some Some Some + // chroot Some None None + SupportFunc::new(vm, "listdir", os_listdir, Some(false), None, None), + SupportFunc::new(vm, "mkdir", os_mkdir, Some(false), Some(false), None), + // mkfifo Some Some None + // mknod Some Some None + // pathconf Some None None + // readlink Some Some None + SupportFunc::new(vm, "remove", os_remove, Some(false), Some(false), None), + SupportFunc::new(vm, "rename", os_rename, Some(false), Some(false), None), + SupportFunc::new(vm, "replace", os_rename, Some(false), Some(false), None), // TODO: Fix replace + SupportFunc::new(vm, "rmdir", os_rmdir, Some(false), Some(false), None), + SupportFunc::new(vm, "scandir", os_scandir, Some(false), None, None), + SupportFunc::new(vm, "stat", os_stat, Some(false), Some(false), Some(false)), + SupportFunc::new(vm, "symlink", os_symlink, None, Some(false), None), + // truncate Some None None + SupportFunc::new(vm, "unlink", os_remove, Some(false), Some(false), None), + // utime Some Some Some + ]; + let supports_fd = PySet::default().into_ref(vm); + let supports_dir_fd = PySet::default().into_ref(vm); + let supports_follow_symlinks = PySet::default().into_ref(vm); + + let module = py_module!(vm, "_os", { "close" => ctx.new_rustfunc(os_close), "error" => ctx.new_rustfunc(os_error), - "O_RDONLY" => ctx.new_int(0), - "O_WRONLY" => ctx.new_int(1), - "O_RDWR" => ctx.new_int(2), - "O_NONBLOCK" => ctx.new_int(4), - "O_APPEND" => ctx.new_int(8), - "O_CREAT" => ctx.new_int(512) + "fsync" => ctx.new_rustfunc(os_fsync), + "read" => ctx.new_rustfunc(os_read), + "write" => ctx.new_rustfunc(os_write), + "mkdirs" => ctx.new_rustfunc(os_mkdirs), + "putenv" => ctx.new_rustfunc(os_putenv), + "unsetenv" => ctx.new_rustfunc(os_unsetenv), + "environ" => environ, + "name" => ctx.new_str(os_name), + "ScandirIter" => scandir_iter, + "DirEntry" => dir_entry, + "stat_result" => stat_result, + "lstat" => ctx.new_rustfunc(os_lstat), + "getcwd" => ctx.new_rustfunc(os_getcwd), + "chdir" => ctx.new_rustfunc(os_chdir), + "fspath" => ctx.new_rustfunc(os_fspath), + "O_RDONLY" => ctx.new_int(FileCreationFlags::O_RDONLY.bits()), + "O_WRONLY" => ctx.new_int(FileCreationFlags::O_WRONLY.bits()), + "O_RDWR" => ctx.new_int(FileCreationFlags::O_RDWR.bits()), + "O_NONBLOCK" => ctx.new_int(FileCreationFlags::O_NONBLOCK.bits()), + "O_APPEND" => ctx.new_int(FileCreationFlags::O_APPEND.bits()), + "O_EXCL" => ctx.new_int(FileCreationFlags::O_EXCL.bits()), + "O_CREAT" => ctx.new_int(FileCreationFlags::O_CREAT.bits()) }); - if cfg!(windows) { - ctx.set_attr(&py_mod, "name", ctx.new_str("nt".to_string())); - } else { - // Assume we're on a POSIX system - ctx.set_attr(&py_mod, "name", ctx.new_str("posix".to_string())); + for support in support_funcs { + if support.fd.unwrap_or(false) { + supports_fd + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + if support.dir_fd.unwrap_or(false) { + supports_dir_fd + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + if support.follow_symlinks.unwrap_or(false) { + supports_follow_symlinks + .clone() + .add(support.func_obj.clone(), vm) + .unwrap(); + } + vm.set_attr(&module, support.name, support.func_obj) + .unwrap(); } - py_mod + extend_module!(vm, module, { + "supports_fd" => supports_fd.into_object(), + "supports_dir_fd" => supports_dir_fd.into_object(), + "supports_follow_symlinks" => supports_follow_symlinks.into_object(), + }); + + module } diff --git a/vm/src/stdlib/platform.rs b/vm/src/stdlib/platform.rs index a87383a723..6d4751a35f 100644 --- a/vm/src/stdlib/platform.rs +++ b/vm/src/stdlib/platform.rs @@ -1,9 +1,10 @@ use crate::function::PyFuncArgs; -use crate::pyobject::{PyContext, PyObjectRef, PyResult}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "platform", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "platform", { "python_compiler" => ctx.new_rustfunc(platform_python_compiler), "python_implementation" => ctx.new_rustfunc(platform_python_implementation), "python_version" => ctx.new_rustfunc(platform_python_version), diff --git a/vm/src/stdlib/pwd.rs b/vm/src/stdlib/pwd.rs new file mode 100644 index 0000000000..85cb2d3eb0 --- /dev/null +++ b/vm/src/stdlib/pwd.rs @@ -0,0 +1,85 @@ +use pwd::Passwd; + +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; +use crate::vm::VirtualMachine; + +impl PyValue for Passwd { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("pwd", "struct_passwd") + } +} + +type PasswdRef = PyRef; + +impl PasswdRef { + fn pw_name(self, _vm: &VirtualMachine) -> String { + self.name.clone() + } + + fn pw_passwd(self, _vm: &VirtualMachine) -> Option { + self.passwd.clone() + } + + fn pw_uid(self, _vm: &VirtualMachine) -> u32 { + self.uid + } + + fn pw_gid(self, _vm: &VirtualMachine) -> u32 { + self.gid + } + + fn pw_gecos(self, _vm: &VirtualMachine) -> Option { + self.gecos.clone() + } + + fn pw_dir(self, _vm: &VirtualMachine) -> String { + self.dir.clone() + } + + fn pw_shell(self, _vm: &VirtualMachine) -> String { + self.shell.clone() + } +} + +fn pwd_getpwnam(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + match Passwd::from_name(&name.value) { + Ok(Some(passwd)) => Ok(passwd), + _ => { + let name_repr = vm.to_repr(name.as_object())?; + let message = vm.new_str(format!("getpwnam(): name not found: {}", name_repr)); + Err(vm.new_key_error(message)) + } + } +} + +fn pwd_getpwuid(uid: u32, vm: &VirtualMachine) -> PyResult { + match Passwd::from_uid(uid) { + Some(passwd) => Ok(passwd), + _ => { + let message = vm.new_str(format!("getpwuid(): uid not found: {}", uid)); + Err(vm.new_key_error(message)) + } + } +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let passwd_type = py_class!(ctx, "struct_passwd", ctx.object(), { + "pw_name" => ctx.new_property(PasswdRef::pw_name), + "pw_passwd" => ctx.new_property(PasswdRef::pw_passwd), + "pw_uid" => ctx.new_property(PasswdRef::pw_uid), + "pw_gid" => ctx.new_property(PasswdRef::pw_gid), + "pw_gecos" => ctx.new_property(PasswdRef::pw_gecos), + "pw_dir" => ctx.new_property(PasswdRef::pw_dir), + "pw_shell" => ctx.new_property(PasswdRef::pw_shell), + }); + + py_module!(vm, "pwd", { + "struct_passwd" => passwd_type, + "getpwnam" => ctx.new_rustfunc(pwd_getpwnam), + "getpwuid" => ctx.new_rustfunc(pwd_getpwuid), + }) +} diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index 9dadc520a9..09f007e607 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -1,6 +1,8 @@ /* * Python struct module. * + * Docs: https://docs.python.org/3/library/struct.html + * * renamed to pystruct since struct is a rust keyword. * * Use this rust module to do byte packing: @@ -8,39 +10,98 @@ */ use std::io::{Cursor, Read, Write}; +use std::iter::Peekable; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use byteorder::{ReadBytesExt, WriteBytesExt}; use num_bigint::BigInt; use num_traits::ToPrimitive; use crate::function::PyFuncArgs; use crate::obj::{objbool, objbytes, objfloat, objint, objstr, objtype}; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::VirtualMachine; +#[derive(Debug)] +struct FormatSpec { + endianness: Endianness, + codes: Vec, +} + +#[derive(Debug)] +enum Endianness { + Native, + Little, + Big, + Network, +} + #[derive(Debug)] struct FormatCode { code: char, - repeat: i32, } -fn parse_format_string(fmt: String) -> Vec { +fn parse_format_string(fmt: String) -> Result { + let mut chars = fmt.chars().peekable(); + // First determine "<", ">","!" or "=" - // TODO + let endianness = parse_endiannes(&mut chars); // Now, analyze struct string furter: + let codes = parse_format_codes(&mut chars)?; + + Ok(FormatSpec { endianness, codes }) +} + +/// Parse endianness +/// See also: https://docs.python.org/3/library/struct.html?highlight=struct#byte-order-size-and-alignment +fn parse_endiannes(chars: &mut Peekable) -> Endianness +where + I: Sized, + I: Iterator, +{ + match chars.peek() { + Some('@') => { + chars.next().unwrap(); + Endianness::Native + } + Some('=') => { + chars.next().unwrap(); + Endianness::Native + } + Some('<') => { + chars.next().unwrap(); + Endianness::Little + } + Some('>') => { + chars.next().unwrap(); + Endianness::Big + } + Some('!') => { + chars.next().unwrap(); + Endianness::Network + } + _ => Endianness::Native, + } +} + +fn parse_format_codes(chars: &mut Peekable) -> Result, String> +where + I: Sized, + I: Iterator, +{ let mut codes = vec![]; - for c in fmt.chars() { + for c in chars { match c { - 'b' | 'B' | 'h' | 'H' | 'i' | 'I' | 'q' | 'Q' | 'f' | 'd' => { - codes.push(FormatCode { code: c, repeat: 1 }) + 'b' | 'B' | 'h' | 'H' | 'i' | 'I' | 'l' | 'L' | 'q' | 'Q' | 'f' | 'd' => { + codes.push(FormatCode { code: c }) } c => { - panic!("Illegal format code {:?}", c); + return Err(format!("Illegal format code {:?}", c)); } } } - codes + + Ok(codes) } fn get_int(vm: &VirtualMachine, arg: &PyObjectRef) -> PyResult { @@ -69,60 +130,112 @@ fn pack_bool(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResu } } -fn pack_i16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_i16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_i16().unwrap(); - data.write_i16::(v).unwrap(); + data.write_i16::(v).unwrap(); Ok(()) } -fn pack_u16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_u16(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_u16().unwrap(); - data.write_u16::(v).unwrap(); + data.write_u16::(v).unwrap(); Ok(()) } -fn pack_i32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_i32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_i32().unwrap(); - data.write_i32::(v).unwrap(); + data.write_i32::(v).unwrap(); Ok(()) } -fn pack_u32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_u32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_u32().unwrap(); - data.write_u32::(v).unwrap(); + data.write_u32::(v).unwrap(); Ok(()) } -fn pack_i64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_i64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_i64().unwrap(); - data.write_i64::(v).unwrap(); + data.write_i64::(v).unwrap(); Ok(()) } -fn pack_u64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_u64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ let v = get_int(vm, arg)?.to_u64().unwrap(); - data.write_u64::(v).unwrap(); + data.write_u64::(v).unwrap(); + Ok(()) +} + +fn pack_f32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ + let v = get_float(vm, arg)? as f32; + data.write_f32::(v).unwrap(); Ok(()) } -fn pack_f32(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { +fn pack_f64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ + let v = get_float(vm, arg)?; + data.write_f64::(v).unwrap(); + Ok(()) +} + +fn get_float(vm: &VirtualMachine, arg: &PyObjectRef) -> PyResult { if objtype::isinstance(&arg, &vm.ctx.float_type()) { - let v = objfloat::get_value(arg) as f32; - data.write_f32::(v).unwrap(); - Ok(()) + Ok(objfloat::get_value(arg)) } else { Err(vm.new_type_error("Expected float".to_string())) } } -fn pack_f64(vm: &VirtualMachine, arg: &PyObjectRef, data: &mut Write) -> PyResult<()> { - if objtype::isinstance(&arg, &vm.ctx.float_type()) { - let v = objfloat::get_value(arg) as f64; - data.write_f64::(v).unwrap(); - Ok(()) - } else { - Err(vm.new_type_error("Expected float".to_string())) +fn pack_item( + vm: &VirtualMachine, + code: &FormatCode, + arg: &PyObjectRef, + data: &mut Write, +) -> PyResult<()> +where + Endianness: byteorder::ByteOrder, +{ + match code.code { + 'b' => pack_i8(vm, arg, data)?, + 'B' => pack_u8(vm, arg, data)?, + '?' => pack_bool(vm, arg, data)?, + 'h' => pack_i16::(vm, arg, data)?, + 'H' => pack_u16::(vm, arg, data)?, + 'i' | 'l' => pack_i32::(vm, arg, data)?, + 'I' | 'L' => pack_u32::(vm, arg, data)?, + 'q' => pack_i64::(vm, arg, data)?, + 'Q' => pack_u64::(vm, arg, data)?, + 'f' => pack_f32::(vm, arg, data)?, + 'd' => pack_f64::(vm, arg, data)?, + c => { + panic!("Unsupported format code {:?}", c); + } } + Ok(()) } fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -136,30 +249,26 @@ fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if objtype::isinstance(&fmt_arg, &vm.ctx.str_type()) { let fmt_str = objstr::get_value(&fmt_arg); - let codes = parse_format_string(fmt_str); + let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?;; - if codes.len() + 1 == args.args.len() { + if format_spec.codes.len() + 1 == args.args.len() { // Create data vector: let mut data = Vec::::new(); // Loop over all opcodes: - for (code, arg) in codes.iter().zip(args.args.iter().skip(1)) { + for (code, arg) in format_spec.codes.iter().zip(args.args.iter().skip(1)) { debug!("code: {:?}", code); - match code.code { - 'b' => pack_i8(vm, arg, &mut data)?, - 'B' => pack_u8(vm, arg, &mut data)?, - '?' => pack_bool(vm, arg, &mut data)?, - 'h' => pack_i16(vm, arg, &mut data)?, - 'H' => pack_u16(vm, arg, &mut data)?, - 'i' => pack_i32(vm, arg, &mut data)?, - 'I' => pack_u32(vm, arg, &mut data)?, - 'l' => pack_i32(vm, arg, &mut data)?, - 'L' => pack_u32(vm, arg, &mut data)?, - 'q' => pack_i64(vm, arg, &mut data)?, - 'Q' => pack_u64(vm, arg, &mut data)?, - 'f' => pack_f32(vm, arg, &mut data)?, - 'd' => pack_f64(vm, arg, &mut data)?, - c => { - panic!("Unsupported format code {:?}", c); + match format_spec.endianness { + Endianness::Little => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Big => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Network => { + pack_item::(vm, code, arg, &mut data)? + } + Endianness::Native => { + pack_item::(vm, code, arg, &mut data)? } } } @@ -168,7 +277,7 @@ fn struct_pack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { } else { Err(vm.new_type_error(format!( "Expected {} arguments (got: {})", - codes.len() + 1, + format_spec.codes.len() + 1, args.args.len() ))) } @@ -199,57 +308,81 @@ fn unpack_bool(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { } } -fn unpack_i16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_i16::() { +fn unpack_i16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_i16::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_u16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_u16::() { +fn unpack_u16(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_u16::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_i32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_i32::() { +fn unpack_i32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_i32::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_u32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_u32::() { +fn unpack_u32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_u32::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_i64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_i64::() { +fn unpack_i64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_i64::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_u64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_u64::() { +fn unpack_u64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_u64::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_int(v)), } } -fn unpack_f32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_f32::() { +fn unpack_f32(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_f32::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_float(f64::from(v))), } } -fn unpack_f64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult { - match rdr.read_f64::() { +fn unpack_f64(vm: &VirtualMachine, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match rdr.read_f64::() { Err(err) => panic!("Error in reading {:?}", err), Ok(v) => Ok(vm.ctx.new_float(v)), } @@ -267,39 +400,55 @@ fn struct_unpack(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { let fmt_str = objstr::get_value(&fmt); - let codes = parse_format_string(fmt_str); + let format_spec = parse_format_string(fmt_str).map_err(|e| vm.new_value_error(e))?; let data = objbytes::get_value(buffer).to_vec(); let mut rdr = Cursor::new(data); let mut items = vec![]; - for code in codes { + for code in format_spec.codes { debug!("unpack code: {:?}", code); - match code.code { - 'b' => items.push(unpack_i8(vm, &mut rdr)?), - 'B' => items.push(unpack_u8(vm, &mut rdr)?), - '?' => items.push(unpack_bool(vm, &mut rdr)?), - 'h' => items.push(unpack_i16(vm, &mut rdr)?), - 'H' => items.push(unpack_u16(vm, &mut rdr)?), - 'i' => items.push(unpack_i32(vm, &mut rdr)?), - 'I' => items.push(unpack_u32(vm, &mut rdr)?), - 'l' => items.push(unpack_i32(vm, &mut rdr)?), - 'L' => items.push(unpack_u32(vm, &mut rdr)?), - 'q' => items.push(unpack_i64(vm, &mut rdr)?), - 'Q' => items.push(unpack_u64(vm, &mut rdr)?), - 'f' => items.push(unpack_f32(vm, &mut rdr)?), - 'd' => items.push(unpack_f64(vm, &mut rdr)?), - c => { - panic!("Unsupported format code {:?}", c); - } - } + let item = match format_spec.endianness { + Endianness::Little => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Big => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Network => unpack_code::(vm, &code, &mut rdr)?, + Endianness::Native => unpack_code::(vm, &code, &mut rdr)?, + }; + items.push(item); } Ok(vm.ctx.new_tuple(items)) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "struct", { +fn unpack_code(vm: &VirtualMachine, code: &FormatCode, rdr: &mut Read) -> PyResult +where + Endianness: byteorder::ByteOrder, +{ + match code.code { + 'b' => unpack_i8(vm, rdr), + 'B' => unpack_u8(vm, rdr), + '?' => unpack_bool(vm, rdr), + 'h' => unpack_i16::(vm, rdr), + 'H' => unpack_u16::(vm, rdr), + 'i' | 'l' => unpack_i32::(vm, rdr), + 'I' | 'L' => unpack_u32::(vm, rdr), + 'q' => unpack_i64::(vm, rdr), + 'Q' => unpack_u64::(vm, rdr), + 'f' => unpack_f32::(vm, rdr), + 'd' => unpack_f64::(vm, rdr), + c => { + panic!("Unsupported format code {:?}", c); + } + } +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let struct_error = ctx.new_class("struct.error", ctx.object()); + + py_module!(vm, "struct", { "pack" => ctx.new_rustfunc(struct_pack), - "unpack" => ctx.new_rustfunc(struct_unpack) + "unpack" => ctx.new_rustfunc(struct_unpack), + "error" => struct_error, }) } diff --git a/vm/src/stdlib/random.rs b/vm/src/stdlib/random.rs index 82c1f410c3..cf3c0ea183 100644 --- a/vm/src/stdlib/random.rs +++ b/vm/src/stdlib/random.rs @@ -4,11 +4,13 @@ use rand::distributions::{Distribution, Normal}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "random", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "random", { "guass" => ctx.new_rustfunc(random_gauss), "normalvariate" => ctx.new_rustfunc(random_normalvariate), "random" => ctx.new_rustfunc(random_random), diff --git a/vm/src/stdlib/re.rs b/vm/src/stdlib/re.rs index 863e24c406..f427389802 100644 --- a/vm/src/stdlib/re.rs +++ b/vm/src/stdlib/re.rs @@ -4,206 +4,123 @@ * This module fits the python re interface onto the rust regular expression * system. */ - -use std::path::PathBuf; - use regex::{Match, Regex}; -use crate::function::PyFuncArgs; -use crate::import; -use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtype::PyClassRef; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue}; use crate::vm::VirtualMachine; impl PyValue for Regex { - fn class(vm: &VirtualMachine) -> PyObjectRef { + fn class(vm: &VirtualMachine) -> PyClassRef { vm.class("re", "Pattern") } } -/// Create the python `re` module with all its members. -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let match_type = py_class!(ctx, "Match", ctx.object(), { - "start" => ctx.new_rustfunc(match_start), - "end" => ctx.new_rustfunc(match_end) - }); - - let pattern_type = py_class!(ctx, "Pattern", ctx.object(), { - "match" => ctx.new_rustfunc(pattern_match), - "search" => ctx.new_rustfunc(pattern_search) - }); +/// Inner data for a match object. +#[derive(Debug)] +struct PyMatch { + start: usize, + end: usize, +} - py_module!(ctx, "re", { - "compile" => ctx.new_rustfunc(re_compile), - "Match" => match_type, - "match" => ctx.new_rustfunc(re_match), - "Pattern" => pattern_type, - "search" => ctx.new_rustfunc(re_search) - }) +impl PyValue for PyMatch { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("re", "Match") + } } -/// Implement re.match -/// See also: -/// https://docs.python.org/3/library/re.html#re.match -fn re_match(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (pattern, Some(vm.ctx.str_type())), - (string, Some(vm.ctx.str_type())) - ] - ); - let regex = make_regex(vm, pattern)?; - let search_text = objstr::get_value(string); - - do_match(vm, ®ex, search_text) +type PyRegexRef = PyRef; +type PyMatchRef = PyRef; + +fn re_match(pattern: PyStringRef, string: PyStringRef, vm: &VirtualMachine) -> PyResult { + let regex = make_regex(vm, &pattern.value)?; + do_match(vm, ®ex, &string.value) } -/// Implement re.search -/// See also: -/// https://docs.python.org/3/library/re.html#re.search -fn re_search(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (pattern, Some(vm.ctx.str_type())), - (string, Some(vm.ctx.str_type())) - ] - ); - - // let pattern_str = objstr::get_value(&pattern); - let regex = make_regex(vm, pattern)?; - let search_text = objstr::get_value(string); - - do_search(vm, ®ex, search_text) +fn re_search(pattern: PyStringRef, string: PyStringRef, vm: &VirtualMachine) -> PyResult { + let regex = make_regex(vm, &pattern.value)?; + do_search(vm, ®ex, &string.value) } -fn do_match(vm: &VirtualMachine, regex: &Regex, search_text: String) -> PyResult { +fn do_match(vm: &VirtualMachine, regex: &Regex, search_text: &str) -> PyResult { // TODO: implement match! do_search(vm, regex, search_text) } -fn do_search(vm: &VirtualMachine, regex: &Regex, search_text: String) -> PyResult { - match regex.find(&search_text) { +fn do_search(vm: &VirtualMachine, regex: &Regex, search_text: &str) -> PyResult { + match regex.find(search_text) { None => Ok(vm.get_none()), Some(result) => create_match(vm, &result), } } -fn make_regex(vm: &VirtualMachine, pattern: &PyObjectRef) -> PyResult { - let pattern_str = objstr::get_value(pattern); - - match Regex::new(&pattern_str) { +fn make_regex(vm: &VirtualMachine, pattern: &str) -> PyResult { + match Regex::new(pattern) { Ok(regex) => Ok(regex), Err(err) => Err(vm.new_value_error(format!("Error in regex: {:?}", err))), } } -/// Inner data for a match object. -#[derive(Debug)] -struct PyMatch { - start: usize, - end: usize, -} - -impl PyValue for PyMatch { - fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.class("re", "Match") - } -} - /// Take a found regular expression and convert it to proper match object. fn create_match(vm: &VirtualMachine, match_value: &Match) -> PyResult { - // Return match object: - // TODO: implement match object - // TODO: how to refer to match object defined in this - let module = import::import_module(vm, PathBuf::default(), "re").unwrap(); - let match_class = vm.ctx.get_attr(&module, "Match").unwrap(); - // let mo = vm.invoke(match_class, PyFuncArgs::default())?; // let txt = vm.ctx.new_str(result.as_str().to_string()); // vm.ctx.set_attr(&mo, "str", txt); - let match_value = PyMatch { + Ok(PyMatch { start: match_value.start(), end: match_value.end(), - }; - - Ok(PyObject::new(match_value, match_class.clone())) + } + .into_ref(vm) + .into_object()) } -/// Compile a regular expression into a Pattern object. -/// See also: -/// https://docs.python.org/3/library/re.html#re.compile -fn re_compile(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(pattern, Some(vm.ctx.str_type()))] // TODO: flags=0 - ); - - let regex = make_regex(vm, pattern)?; - // TODO: retrieval of this module is akward: - let module = import::import_module(vm, PathBuf::default(), "re").unwrap(); - let pattern_class = vm.ctx.get_attr(&module, "Pattern").unwrap(); - - Ok(PyObject::new(regex, pattern_class.clone())) +fn re_compile(pattern: PyStringRef, vm: &VirtualMachine) -> PyResult { + make_regex(vm, &pattern.value) } -fn pattern_match(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (text, Some(vm.ctx.str_type()))] - ); - - let regex = get_regex(zelf); - let search_text = objstr::get_value(text); - do_match(vm, ®ex, search_text) +fn re_escape(pattern: PyStringRef, _vm: &VirtualMachine) -> String { + regex::escape(&pattern.value) } -fn pattern_search(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (text, Some(vm.ctx.str_type()))] - ); - - let regex = get_regex(zelf); - let search_text = objstr::get_value(text); - do_search(vm, ®ex, search_text) +impl PyRegexRef { + fn match_(self, text: PyStringRef, vm: &VirtualMachine) -> PyResult { + do_match(vm, &self, &text.value) + } + fn search(self, text: PyStringRef, vm: &VirtualMachine) -> PyResult { + do_search(vm, &self, &text.value) + } } -/// Returns start of match -/// see: https://docs.python.org/3/library/re.html#re.Match.start -fn match_start(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - // TODO: implement groups - let m = get_match(zelf); - Ok(vm.new_int(m.start)) +impl PyMatchRef { + fn start(self, _vm: &VirtualMachine) -> usize { + self.start + } + fn end(self, _vm: &VirtualMachine) -> usize { + self.end + } } -fn match_end(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - // TODO: implement groups - let m = get_match(zelf); - Ok(vm.new_int(m.end)) -} +/// Create the python `re` module with all its members. +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; -/// Retrieve inner rust regex from python object: -fn get_regex(obj: &PyObjectRef) -> &Regex { - // TODO: Regex shouldn't be stored in payload directly, create newtype wrapper - if let Some(regex) = obj.payload::() { - return regex; - } - panic!("Inner error getting regex {:?}", obj); -} + let match_type = py_class!(ctx, "Match", ctx.object(), { + "start" => ctx.new_rustfunc(PyMatchRef::start), + "end" => ctx.new_rustfunc(PyMatchRef::end) + }); -/// Retrieve inner rust match from python object: -fn get_match(obj: &PyObjectRef) -> &PyMatch { - if let Some(value) = obj.payload::() { - return value; - } - panic!("Inner error getting match {:?}", obj); + let pattern_type = py_class!(ctx, "Pattern", ctx.object(), { + "match" => ctx.new_rustfunc(PyRegexRef::match_), + "search" => ctx.new_rustfunc(PyRegexRef::search) + }); + + py_module!(vm, "re", { + "compile" => ctx.new_rustfunc(re_compile), + "escape" => ctx.new_rustfunc(re_escape), + "Match" => match_type, + "match" => ctx.new_rustfunc(re_match), + "Pattern" => pattern_type, + "search" => ctx.new_rustfunc(re_search) + }) } diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 4e75cfa417..68934bd016 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -3,16 +3,15 @@ use std::io; use std::io::Read; use std::io::Write; use std::net::{SocketAddr, TcpListener, TcpStream, ToSocketAddrs, UdpSocket}; -use std::ops::Deref; - -use crate::function::PyFuncArgs; -use crate::obj::objbytes; -use crate::obj::objint; -use crate::obj::objsequence::get_elements; -use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol}; + +use crate::obj::objbytes::PyBytesRef; +use crate::obj::objint::PyIntRef; +use crate::obj::objstr::PyStringRef; +use crate::obj::objtuple::PyTupleRef; +use crate::pyobject::{PyObjectRef, PyRef, PyResult, PyValue, TryFromObject}; use crate::vm::VirtualMachine; +use crate::obj::objtype::PyClassRef; use num_traits::ToPrimitive; #[derive(Debug, Copy, Clone)] @@ -22,13 +21,13 @@ enum AddressFamily { Inet6 = 3, } -impl AddressFamily { - fn from_i32(vm: &VirtualMachine, value: i32) -> Result { - match value { +impl TryFromObject for AddressFamily { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match i32::try_from_object(vm, obj)? { 1 => Ok(AddressFamily::Unix), 2 => Ok(AddressFamily::Inet), 3 => Ok(AddressFamily::Inet6), - _ => Err(vm.new_os_error(format!("Unknown address family value: {}", value))), + value => Err(vm.new_os_error(format!("Unknown address family value: {}", value))), } } } @@ -39,12 +38,12 @@ enum SocketKind { Dgram = 2, } -impl SocketKind { - fn from_i32(vm: &VirtualMachine, value: i32) -> Result { - match value { +impl TryFromObject for SocketKind { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + match i32::try_from_object(vm, obj)? { 1 => Ok(SocketKind::Stream), 2 => Ok(SocketKind::Dgram), - _ => Err(vm.new_os_error(format!("Unknown socket kind value: {}", value))), + value => Err(vm.new_os_error(format!("Unknown socket kind value: {}", value))), } } } @@ -85,6 +84,33 @@ impl Connection { _ => Err(io::Error::new(io::ErrorKind::Other, "oh no!")), } } + + #[cfg(unix)] + fn fileno(&self) -> i64 { + use std::os::unix::io::AsRawFd; + let raw_fd = match self { + Connection::TcpListener(con) => con.as_raw_fd(), + Connection::UdpSocket(con) => con.as_raw_fd(), + Connection::TcpStream(con) => con.as_raw_fd(), + }; + raw_fd as i64 + } + + #[cfg(windows)] + fn fileno(&self) -> i64 { + use std::os::windows::io::AsRawSocket; + let raw_fd = match self { + Connection::TcpListener(con) => con.as_raw_socket(), + Connection::UdpSocket(con) => con.as_raw_socket(), + Connection::TcpStream(con) => con.as_raw_socket(), + }; + raw_fd as i64 + } + + #[cfg(all(not(unix), not(windows)))] + fn fileno(&self) -> i64 { + unimplemented!(); + } } impl Read for Connection { @@ -118,9 +144,8 @@ pub struct Socket { } impl PyValue for Socket { - fn class(_vm: &VirtualMachine) -> PyObjectRef { - // TODO - unimplemented!() + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("socket", "socket") } } @@ -134,282 +159,209 @@ impl Socket { } } -fn get_socket<'a>(obj: &'a PyObjectRef) -> impl Deref + 'a { - obj.payload::().unwrap() -} +type SocketRef = PyRef; -fn socket_new(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (cls, None), - (family_int, Some(vm.ctx.int_type())), - (kind_int, Some(vm.ctx.int_type())) - ] - ); - - let address_family = - AddressFamily::from_i32(vm, objint::get_value(family_int).to_i32().unwrap())?; - let kind = SocketKind::from_i32(vm, objint::get_value(kind_int).to_i32().unwrap())?; - - Ok(PyObject::new( - Socket::new(address_family, kind), - cls.clone(), - )) -} +impl SocketRef { + fn new( + cls: PyClassRef, + family: AddressFamily, + kind: SocketKind, + vm: &VirtualMachine, + ) -> PyResult { + Socket::new(family, kind).into_ref_with_type(vm, cls) + } -fn socket_connect(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (address, Some(vm.ctx.tuple_type()))] - ); - - let address_string = get_address_string(vm, address)?; - - let socket = get_socket(zelf); - - match socket.socket_kind { - SocketKind::Stream => match TcpStream::connect(address_string) { - Ok(stream) => { - socket - .con - .borrow_mut() - .replace(Connection::TcpStream(stream)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - SocketKind::Dgram => { - if let Some(Connection::UdpSocket(con)) = socket.con.borrow().as_ref() { - match con.connect(address_string) { - Ok(_) => Ok(vm.get_none()), - Err(s) => Err(vm.new_os_error(s.to_string())), + fn connect(self, address: Address, vm: &VirtualMachine) -> PyResult<()> { + let address_string = address.get_address_string(); + + match self.socket_kind { + SocketKind::Stream => match TcpStream::connect(address_string) { + Ok(stream) => { + self.con.borrow_mut().replace(Connection::TcpStream(stream)); + Ok(()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + SocketKind::Dgram => { + if let Some(Connection::UdpSocket(con)) = self.con.borrow().as_ref() { + match con.connect(address_string) { + Ok(_) => Ok(()), + Err(s) => Err(vm.new_os_error(s.to_string())), + } + } else { + Err(vm.new_type_error("".to_string())) } - } else { - Err(vm.new_type_error("".to_string())) } } } -} -fn socket_bind(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (address, Some(vm.ctx.tuple_type()))] - ); - - let address_string = get_address_string(vm, address)?; - - let socket = get_socket(zelf); - - match socket.socket_kind { - SocketKind::Stream => match TcpListener::bind(address_string) { - Ok(stream) => { - socket - .con - .borrow_mut() - .replace(Connection::TcpListener(stream)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - SocketKind::Dgram => match UdpSocket::bind(address_string) { - Ok(dgram) => { - socket - .con - .borrow_mut() - .replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) - } - Err(s) => Err(vm.new_os_error(s.to_string())), - }, - } -} - -fn get_address_string(vm: &VirtualMachine, address: &PyObjectRef) -> Result { - let args = PyFuncArgs { - args: get_elements(address).to_vec(), - kwargs: vec![], - }; - arg_check!( - vm, - args, - required = [ - (host, Some(vm.ctx.str_type())), - (port, Some(vm.ctx.int_type())) - ] - ); - - Ok(format!( - "{}:{}", - objstr::get_value(host), - objint::get_value(port).to_string() - )) -} + fn bind(self, address: Address, vm: &VirtualMachine) -> PyResult<()> { + let address_string = address.get_address_string(); -fn socket_listen(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(_zelf, None), (_num, Some(vm.ctx.int_type()))] - ); - Ok(vm.get_none()) -} + match self.socket_kind { + SocketKind::Stream => match TcpListener::bind(address_string) { + Ok(stream) => { + self.con + .borrow_mut() + .replace(Connection::TcpListener(stream)); + Ok(()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + SocketKind::Dgram => match UdpSocket::bind(address_string) { + Ok(dgram) => { + self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); + Ok(()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, + } + } -fn socket_accept(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); + fn listen(self, _num: PyIntRef, _vm: &VirtualMachine) {} - let socket = get_socket(zelf); + fn accept(self, vm: &VirtualMachine) -> PyResult { + let ret = match self.con.borrow_mut().as_mut() { + Some(v) => v.accept(), + None => return Err(vm.new_type_error("".to_string())), + }; - let ret = match socket.con.borrow_mut().as_mut() { - Some(v) => v.accept(), - None => return Err(vm.new_type_error("".to_string())), - }; + let (tcp_stream, addr) = match ret { + Ok((socket, addr)) => (socket, addr), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }; - let (tcp_stream, addr) = match ret { - Ok((socket, addr)) => (socket, addr), - Err(s) => return Err(vm.new_os_error(s.to_string())), - }; + let socket = Socket { + address_family: self.address_family, + socket_kind: self.socket_kind, + con: RefCell::new(Some(Connection::TcpStream(tcp_stream))), + } + .into_ref(vm); - let socket = Socket { - address_family: socket.address_family, - socket_kind: socket.socket_kind, - con: RefCell::new(Some(Connection::TcpStream(tcp_stream))), - }; + let addr_tuple = get_addr_tuple(vm, addr)?; - let sock_obj = PyObject::new(socket, zelf.typ()); + Ok(vm.ctx.new_tuple(vec![socket.into_object(), addr_tuple])) + } - let addr_tuple = get_addr_tuple(vm, addr)?; + fn recv(self, bufsize: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; bufsize.as_bigint().to_usize().unwrap()]; + match self.con.borrow_mut().as_mut() { + Some(v) => match v.read_exact(&mut buffer) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }, + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.ctx.new_bytes(buffer)) + } - Ok(vm.ctx.new_tuple(vec![sock_obj, addr_tuple])) -} + fn recvfrom(self, bufsize: PyIntRef, vm: &VirtualMachine) -> PyResult { + let mut buffer = vec![0u8; bufsize.as_bigint().to_usize().unwrap()]; + let ret = match self.con.borrow().as_ref() { + Some(v) => v.recv_from(&mut buffer), + None => return Err(vm.new_type_error("".to_string())), + }; -fn socket_recv(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (bufsize, Some(vm.ctx.int_type()))] - ); - let socket = get_socket(zelf); - - let mut buffer = vec![0u8; objint::get_value(bufsize).to_usize().unwrap()]; - match socket.con.borrow_mut().as_mut() { - Some(v) => match v.read_exact(&mut buffer) { - Ok(_) => (), + let addr = match ret { + Ok((_size, addr)) => addr, Err(s) => return Err(vm.new_os_error(s.to_string())), - }, - None => return Err(vm.new_type_error("".to_string())), - }; - Ok(vm.ctx.new_bytes(buffer)) -} + }; -fn socket_recvfrom(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (bufsize, Some(vm.ctx.int_type()))] - ); + let addr_tuple = get_addr_tuple(vm, addr)?; - let socket = get_socket(zelf); - - let mut buffer = vec![0u8; objint::get_value(bufsize).to_usize().unwrap()]; - let ret = match socket.con.borrow().as_ref() { - Some(v) => v.recv_from(&mut buffer), - None => return Err(vm.new_type_error("".to_string())), - }; - - let addr = match ret { - Ok((_size, addr)) => addr, - Err(s) => return Err(vm.new_os_error(s.to_string())), - }; - - let addr_tuple = get_addr_tuple(vm, addr)?; + Ok(vm.ctx.new_tuple(vec![vm.ctx.new_bytes(buffer), addr_tuple])) + } - Ok(vm.ctx.new_tuple(vec![vm.ctx.new_bytes(buffer), addr_tuple])) -} + fn send(self, bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<()> { + match self.con.borrow_mut().as_mut() { + Some(v) => match v.write(&bytes) { + Ok(_) => (), + Err(s) => return Err(vm.new_os_error(s.to_string())), + }, + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(()) + } -fn socket_send(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, None), (bytes, Some(vm.ctx.bytes_type()))] - ); - let socket = get_socket(zelf); - - match socket.con.borrow_mut().as_mut() { - Some(v) => match v.write(&objbytes::get_value(&bytes)) { - Ok(_) => (), - Err(s) => return Err(vm.new_os_error(s.to_string())), - }, - None => return Err(vm.new_type_error("".to_string())), - }; - Ok(vm.get_none()) -} + fn sendto(self, bytes: PyBytesRef, address: Address, vm: &VirtualMachine) -> PyResult<()> { + let address_string = address.get_address_string(); -fn socket_sendto(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [ - (zelf, None), - (bytes, Some(vm.ctx.bytes_type())), - (address, Some(vm.ctx.tuple_type())) - ] - ); - let address_string = get_address_string(vm, address)?; - - let socket = get_socket(zelf); - - match socket.socket_kind { - SocketKind::Dgram => { - if let Some(v) = socket.con.borrow().as_ref() { - return match v.send_to(&objbytes::get_value(&bytes), address_string) { - Ok(_) => Ok(vm.get_none()), - Err(s) => Err(vm.new_os_error(s.to_string())), - }; - } - // Doing implicit bind - match UdpSocket::bind("0.0.0.0:0") { - Ok(dgram) => match dgram.send_to(&objbytes::get_value(&bytes), address_string) { - Ok(_) => { - socket - .con - .borrow_mut() - .replace(Connection::UdpSocket(dgram)); - Ok(vm.get_none()) - } + match self.socket_kind { + SocketKind::Dgram => { + if let Some(v) = self.con.borrow().as_ref() { + return match v.send_to(&bytes, address_string) { + Ok(_) => Ok(()), + Err(s) => Err(vm.new_os_error(s.to_string())), + }; + } + // Doing implicit bind + match UdpSocket::bind("0.0.0.0:0") { + Ok(dgram) => match dgram.send_to(&bytes, address_string) { + Ok(_) => { + self.con.borrow_mut().replace(Connection::UdpSocket(dgram)); + Ok(()) + } + Err(s) => Err(vm.new_os_error(s.to_string())), + }, Err(s) => Err(vm.new_os_error(s.to_string())), - }, - Err(s) => Err(vm.new_os_error(s.to_string())), + } } + _ => Err(vm.new_not_implemented_error("".to_string())), } - _ => Err(vm.new_not_implemented_error("".to_string())), } -} -fn socket_close(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); + fn close(self, _vm: &VirtualMachine) { + self.con.borrow_mut().take(); + } + + fn fileno(self, vm: &VirtualMachine) -> PyResult { + let fileno = match self.con.borrow_mut().as_mut() { + Some(v) => v.fileno(), + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.ctx.new_int(fileno)) + } + + fn getsockname(self, vm: &VirtualMachine) -> PyResult { + let addr = match self.con.borrow().as_ref() { + Some(v) => v.local_addr(), + None => return Err(vm.new_type_error("".to_string())), + }; - let socket = get_socket(zelf); - socket.con.borrow_mut().take(); - Ok(vm.get_none()) + match addr { + Ok(addr) => get_addr_tuple(vm, addr), + Err(s) => Err(vm.new_os_error(s.to_string())), + } + } } -fn socket_getsockname(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(zelf, None)]); - let socket = get_socket(zelf); +struct Address { + host: String, + port: usize, +} - let addr = match socket.con.borrow().as_ref() { - Some(v) => v.local_addr(), - None => return Err(vm.new_type_error("".to_string())), - }; +impl Address { + fn get_address_string(self) -> String { + format!("{}:{}", self.host, self.port.to_string()) + } +} - match addr { - Ok(addr) => get_addr_tuple(vm, addr), - Err(s) => Err(vm.new_os_error(s.to_string())), +impl TryFromObject for Address { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + let tuple = PyTupleRef::try_from_object(vm, obj)?; + if tuple.elements.len() != 2 { + Err(vm.new_type_error("Address tuple should have only 2 values".to_string())) + } else { + Ok(Address { + host: PyStringRef::try_from_object(vm, tuple.elements[0].clone())? + .value + .to_string(), + port: PyIntRef::try_from_object(vm, tuple.elements[1].clone())? + .as_bigint() + .to_usize() + .unwrap(), + }) + } } } @@ -420,25 +372,28 @@ fn get_addr_tuple(vm: &VirtualMachine, addr: SocketAddr) -> PyResult { Ok(vm.ctx.new_tuple(vec![ip, port])) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let socket = py_class!(ctx, "socket", ctx.object(), { - "__new__" => ctx.new_rustfunc(socket_new), - "connect" => ctx.new_rustfunc(socket_connect), - "recv" => ctx.new_rustfunc(socket_recv), - "send" => ctx.new_rustfunc(socket_send), - "bind" => ctx.new_rustfunc(socket_bind), - "accept" => ctx.new_rustfunc(socket_accept), - "listen" => ctx.new_rustfunc(socket_listen), - "close" => ctx.new_rustfunc(socket_close), - "getsockname" => ctx.new_rustfunc(socket_getsockname), - "sendto" => ctx.new_rustfunc(socket_sendto), - "recvfrom" => ctx.new_rustfunc(socket_recvfrom), + "__new__" => ctx.new_rustfunc(SocketRef::new), + "connect" => ctx.new_rustfunc(SocketRef::connect), + "recv" => ctx.new_rustfunc(SocketRef::recv), + "send" => ctx.new_rustfunc(SocketRef::send), + "bind" => ctx.new_rustfunc(SocketRef::bind), + "accept" => ctx.new_rustfunc(SocketRef::accept), + "listen" => ctx.new_rustfunc(SocketRef::listen), + "close" => ctx.new_rustfunc(SocketRef::close), + "getsockname" => ctx.new_rustfunc(SocketRef::getsockname), + "sendto" => ctx.new_rustfunc(SocketRef::sendto), + "recvfrom" => ctx.new_rustfunc(SocketRef::recvfrom), + "fileno" => ctx.new_rustfunc(SocketRef::fileno), }); - py_module!(ctx, "socket", { + py_module!(vm, "socket", { "AF_INET" => ctx.new_int(AddressFamily::Inet as i32), "SOCK_STREAM" => ctx.new_int(SocketKind::Stream as i32), "SOCK_DGRAM" => ctx.new_int(SocketKind::Dgram as i32), - "socket" => socket.clone(), + "socket" => socket, }) } diff --git a/vm/src/stdlib/string.rs b/vm/src/stdlib/string.rs index 157dae035f..c255ffc935 100644 --- a/vm/src/stdlib/string.rs +++ b/vm/src/stdlib/string.rs @@ -3,9 +3,12 @@ * */ -use crate::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { let ascii_lowercase = "abcdefghijklmnopqrstuvwxyz".to_string(); let ascii_uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".to_string(); let ascii_letters = format!("{}{}", ascii_lowercase, ascii_uppercase); @@ -19,7 +22,7 @@ pub fn make_module(ctx: &PyContext) -> PyObjectRef { */ // Constants: - py_module!(ctx, "string", { + py_module!(vm, "string", { "ascii_letters" => ctx.new_str(ascii_letters), "ascii_lowercase" => ctx.new_str(ascii_lowercase), "ascii_uppercase" => ctx.new_str(ascii_uppercase), diff --git a/vm/src/stdlib/thread.rs b/vm/src/stdlib/thread.rs new file mode 100644 index 0000000000..414e7d72e7 --- /dev/null +++ b/vm/src/stdlib/thread.rs @@ -0,0 +1,58 @@ +/// Implementation of the _thread module, currently noop implementation as RustPython doesn't yet +/// support threading +use super::super::pyobject::PyObjectRef; +use crate::function::PyFuncArgs; +use crate::pyobject::PyResult; +use crate::vm::VirtualMachine; + +fn rlock_acquire(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) +} + +fn rlock_release(_zelf: PyObjectRef, _vm: &VirtualMachine) {} + +fn rlock_enter(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(instance, None)]); + Ok(instance.clone()) +} + +fn rlock_exit(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + // The context manager protocol requires these, but we don't use them + required = [ + (_instance, None), + (_exception_type, None), + (_exception_value, None), + (_traceback, None) + ] + ); + Ok(vm.get_none()) +} + +fn get_ident(_vm: &VirtualMachine) -> u32 { + 1 +} + +fn allocate_lock(vm: &VirtualMachine) -> PyResult { + let lock_class = vm.class("_thread", "RLock"); + vm.invoke(lock_class.into_object(), vec![]) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let rlock_type = py_class!(ctx, "_thread.RLock", ctx.object(), { + "acquire" => ctx.new_rustfunc(rlock_acquire), + "release" => ctx.new_rustfunc(rlock_release), + "__enter__" => ctx.new_rustfunc(rlock_enter), + "__exit__" => ctx.new_rustfunc(rlock_exit), + }); + + py_module!(vm, "_thread", { + "RLock" => rlock_type, + "get_ident" => ctx.new_rustfunc(get_ident), + "allocate_lock" => ctx.new_rustfunc(allocate_lock), + }) +} diff --git a/vm/src/stdlib/time_module.rs b/vm/src/stdlib/time_module.rs index dfe3ab8e35..6c818689e0 100644 --- a/vm/src/stdlib/time_module.rs +++ b/vm/src/stdlib/time_module.rs @@ -5,7 +5,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::function::PyFuncArgs; use crate::obj::objfloat; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn time_sleep(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -32,8 +32,10 @@ fn time_time(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(value) } -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "time", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "time", { "sleep" => ctx.new_rustfunc(time_sleep), "time" => ctx.new_rustfunc(time_time) }) diff --git a/vm/src/stdlib/tokenize.rs b/vm/src/stdlib/tokenize.rs index 0aa5df3d2a..000dd5e1a3 100644 --- a/vm/src/stdlib/tokenize.rs +++ b/vm/src/stdlib/tokenize.rs @@ -8,7 +8,7 @@ use rustpython_parser::lexer; use crate::function::PyFuncArgs; use crate::obj::objstr; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::pyobject::{PyObjectRef, PyResult}; use crate::vm::VirtualMachine; fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { @@ -25,8 +25,10 @@ fn tokenize_tokenize(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // TODO: create main function when called with -m -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "tokenize", { +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "tokenize", { "tokenize" => ctx.new_rustfunc(tokenize_tokenize) }) } diff --git a/vm/src/stdlib/types.rs b/vm/src/stdlib/types.rs deleted file mode 100644 index 279cdbdf9e..0000000000 --- a/vm/src/stdlib/types.rs +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Dynamic type creation and names for built in types. - */ - -use crate::function::PyFuncArgs; -use crate::obj::objtype; -use crate::pyobject::{PyContext, PyObjectRef, PyResult, TypeProtocol}; -use crate::VirtualMachine; - -fn types_new_class(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(name, Some(vm.ctx.str_type()))], - optional = [(bases, None), (_kwds, None), (_exec_body, None)] - ); - - let bases: PyObjectRef = match bases { - Some(bases) => bases.clone(), - None => vm.ctx.new_tuple(vec![]), - }; - let dict = vm.ctx.new_dict(); - objtype::type_new_class(vm, &vm.ctx.type_type(), name, &bases, &dict) -} - -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "types", { - "new_class" => ctx.new_rustfunc(types_new_class), - "FunctionType" => ctx.function_type(), - "LambdaType" => ctx.function_type(), - "CodeType" => ctx.code_type(), - "FrameType" => ctx.frame_type() - }) -} diff --git a/vm/src/stdlib/unicodedata.rs b/vm/src/stdlib/unicodedata.rs new file mode 100644 index 0000000000..7c9c865da7 --- /dev/null +++ b/vm/src/stdlib/unicodedata.rs @@ -0,0 +1,94 @@ +/* Access to the unicode database. + See also: https://docs.python.org/3/library/unicodedata.html +*/ + +use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{PyObjectRef, PyResult}; +use crate::vm::VirtualMachine; + +use unic::char::property::EnumeratedCharProperty; +use unic::ucd::category::GeneralCategory; +use unic::ucd::Name; +use unicode_names2; + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + let unidata_version = unic::UNICODE_VERSION.to_string(); + + py_module!(vm, "unicodedata", { + "bidirectional" => ctx.new_rustfunc(bidirectional), + "category" => ctx.new_rustfunc(category), + "name" => ctx.new_rustfunc(name), + "lookup" => ctx.new_rustfunc(lookup), + "normalize" => ctx.new_rustfunc(normalize), + "unidata_version" => ctx.new_str(unidata_version), + }) +} + +fn category(character: PyStringRef, vm: &VirtualMachine) -> PyResult { + let my_char = extract_char(character, vm)?; + let category = GeneralCategory::of(my_char); + Ok(vm.new_str(category.abbr_name().to_string())) +} + +fn lookup(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + // TODO: we might want to use unic_ucd instead of unicode_names2 for this too, if possible: + if let Some(character) = unicode_names2::character(&name.value) { + Ok(vm.new_str(character.to_string())) + } else { + Err(vm.new_key_error(vm.new_str(format!("undefined character name '{}'", name)))) + } +} + +fn name( + character: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, +) -> PyResult { + let my_char = extract_char(character, vm)?; + + if let Some(name) = Name::of(my_char) { + Ok(vm.new_str(name.to_string())) + } else { + match default { + OptionalArg::Present(obj) => Ok(obj), + OptionalArg::Missing => { + Err(vm.new_value_error("character name not found!".to_string())) + } + } + } +} + +fn bidirectional(character: PyStringRef, vm: &VirtualMachine) -> PyResult { + use unic::bidi::BidiClass; + let my_char = extract_char(character, vm)?; + let cls = BidiClass::of(my_char); + Ok(vm.new_str(cls.abbr_name().to_string())) +} + +fn normalize(form: PyStringRef, unistr: PyStringRef, vm: &VirtualMachine) -> PyResult { + use unic::normal::StrNormalForm; + let text = &unistr.value; + let normalized_text = match form.value.as_ref() { + "NFC" => text.nfc().collect::(), + "NFKC" => text.nfkc().collect::(), + "NFD" => text.nfd().collect::(), + "NFKD" => text.nfkd().collect::(), + _ => { + return Err(vm.new_value_error("unistr must be one of NFC, NFD".to_string())); + } + }; + + Ok(vm.new_str(normalized_text)) +} + +fn extract_char(character: PyStringRef, vm: &VirtualMachine) -> PyResult { + if character.value.len() != 1 { + return Err(vm.new_type_error("argument must be an unicode character, not str".to_string())); + } + + let my_char: char = character.value.chars().next().unwrap(); + Ok(my_char) +} diff --git a/vm/src/stdlib/warnings.rs b/vm/src/stdlib/warnings.rs new file mode 100644 index 0000000000..cb97ead480 --- /dev/null +++ b/vm/src/stdlib/warnings.rs @@ -0,0 +1,37 @@ +use crate::function::OptionalArg; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; + +#[derive(FromArgs)] +struct WarnArgs { + #[pyarg(positional_only, optional = false)] + message: PyStringRef, + #[pyarg(positional_or_keyword, optional = true)] + category: OptionalArg, + #[pyarg(positional_or_keyword, optional = true)] + stacklevel: OptionalArg, +} + +fn warnings_warn(args: WarnArgs, _vm: &VirtualMachine) { + // TODO: Implement correctly + let level = match args.stacklevel { + OptionalArg::Present(l) => l, + OptionalArg::Missing => 1, + }; + eprintln!( + "Warning: {} , category: {:?}, level: {}", + args.message.as_str(), + args.category, + level + ) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + let module = py_module!(vm, "_warnings", { + "warn" => ctx.new_rustfunc(warnings_warn), + }); + + module +} diff --git a/vm/src/stdlib/weakref.rs b/vm/src/stdlib/weakref.rs index 5616c3ddfe..58a23704d6 100644 --- a/vm/src/stdlib/weakref.rs +++ b/vm/src/stdlib/weakref.rs @@ -5,10 +5,34 @@ //! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html) //! -use super::super::pyobject::{PyContext, PyObjectRef}; +use crate::pyobject::PyObjectRef; +use crate::vm::VirtualMachine; +use std::rc::Rc; -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - py_module!(ctx, "_weakref", { - "ref" => ctx.weakref_type() +fn weakref_getweakrefcount(obj: PyObjectRef, _vm: &VirtualMachine) -> usize { + Rc::weak_count(&obj) +} + +fn weakref_getweakrefs(_obj: PyObjectRef, vm: &VirtualMachine) -> PyObjectRef { + // TODO: implement this, may require a different gc + vm.ctx.new_list(vec![]) +} + +fn weakref_remove_dead_weakref(_obj: PyObjectRef, _key: PyObjectRef, _vm: &VirtualMachine) { + // TODO +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "_weakref", { + "ref" => ctx.weakref_type(), + "proxy" => ctx.weakproxy_type(), + "getweakrefcount" => ctx.new_rustfunc(weakref_getweakrefcount), + "getweakrefs" => ctx.new_rustfunc(weakref_getweakrefs), + "ReferenceType" => ctx.weakref_type(), + "ProxyType" => ctx.weakproxy_type(), + "CallableProxyType" => ctx.weakproxy_type(), + "_remove_dead_weakref" => ctx.new_rustfunc(weakref_remove_dead_weakref), }) } diff --git a/vm/src/sysmodule.rs b/vm/src/sysmodule.rs index 8a45bc1c7e..c20f7b1677 100644 --- a/vm/src/sysmodule.rs +++ b/vm/src/sysmodule.rs @@ -1,11 +1,10 @@ use std::rc::Rc; use std::{env, mem}; -use num_traits::ToPrimitive; - -use crate::function::PyFuncArgs; -use crate::obj::objint; -use crate::pyobject::{DictProtocol, PyContext, PyObjectRef, PyResult, TypeProtocol}; +use crate::frame::FrameRef; +use crate::function::{OptionalArg, PyFuncArgs}; +use crate::obj::objstr::PyStringRef; +use crate::pyobject::{IntoPyObject, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyResult}; use crate::vm::VirtualMachine; /* @@ -18,32 +17,54 @@ fn argv(ctx: &PyContext) -> PyObjectRef { ctx.new_list(argv) } -fn frame_idx(vm: &VirtualMachine, offset: Option<&PyObjectRef>) -> Result { - if let Some(int) = offset { - if let Some(offset) = objint::get_value(&int).to_usize() { - if offset > vm.frames.borrow().len() - 1 { - return Err(vm.new_value_error("call stack is not deep enough".to_string())); - } - return Ok(offset); - } +fn getframe(offset: OptionalArg, vm: &VirtualMachine) -> PyResult { + let offset = offset.into_option().unwrap_or(0); + if offset > vm.frames.borrow().len() - 1 { + return Err(vm.new_value_error("call stack is not deep enough".to_string())); } - Ok(0) -} - -fn getframe(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [], - optional = [(offset, Some(vm.ctx.int_type()))] - ); - - let idx = frame_idx(vm, offset)?; - let idx = vm.frames.borrow().len() - idx - 1; + let idx = vm.frames.borrow().len() - offset - 1; let frame = &vm.frames.borrow()[idx]; Ok(frame.clone()) } +/// sys.flags +/// +/// Flags provided through command line arguments or environment vars. +#[pystruct_sequence(name = "flags")] +#[derive(Default, Debug)] +struct SysFlags { + /// -d + debug: bool, + /// -i + inspect: bool, + /// -i + interactive: bool, + /// -O or -OO + optimize: u8, + /// -B + dont_write_bytecode: bool, + /// -s + no_user_site: bool, + /// -S + no_site: bool, + /// -E + ignore_environment: bool, + /// -v + verbose: bool, + /// -b + bytes_warning: bool, + /// -q + quiet: bool, + /// -R + hash_randomization: bool, + /// -I + isolated: bool, + /// -X dev + dev_mode: bool, + /// -X utf8 + utf8_mode: bool, +} + fn sys_getrefcount(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(object, None)]); let size = Rc::strong_count(&object); @@ -57,21 +78,88 @@ fn sys_getsizeof(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_int(size)) } -pub fn make_module(ctx: &PyContext, builtins: PyObjectRef) -> PyObjectRef { - let path_list = match env::var_os("PYTHONPATH") { - Some(paths) => env::split_paths(&paths) - .map(|path| { - ctx.new_str( - path.to_str() - .expect("PYTHONPATH isn't valid unicode") - .to_string(), - ) - }) - .collect(), - None => vec![], +fn sys_getfilesystemencoding(_vm: &VirtualMachine) -> String { + // TODO: implmement non-utf-8 mode. + "utf-8".to_string() +} + +#[cfg(not(windows))] +fn sys_getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { + "surrogateescape".to_string() +} + +#[cfg(windows)] +fn sys_getfilesystemencodeerrors(_vm: &VirtualMachine) -> String { + "surrogatepass".to_string() +} + +// TODO implement string interning, this will be key for performance +fn sys_intern(value: PyStringRef, _vm: &VirtualMachine) -> PyStringRef { + value +} + +pub fn make_module(vm: &VirtualMachine, module: PyObjectRef, builtins: PyObjectRef) { + let ctx = &vm.ctx; + + let flags_type = SysFlags::make_class(ctx); + // TODO parse command line arguments and environment variables to populate SysFlags + let flags = SysFlags::default() + .into_struct_sequence(vm, flags_type) + .unwrap(); + + // TODO Add crate version to this namespace + let implementation = py_namespace!(vm, { + "name" => ctx.new_str("RustPython".to_string()), + "cache_tag" => ctx.new_str("rustpython-01".to_string()), + }); + + let path_list = if cfg!(target_arch = "wasm32") { + vec![] + } else { + fn get_paths(env_variable_name: &str) -> Vec { + let paths = env::var_os(env_variable_name); + match paths { + Some(paths) => env::split_paths(&paths) + .map(|path| { + path.into_os_string() + .into_string() + .expect(&format!("{} isn't valid unicode", env_variable_name)) + }) + .collect(), + None => vec![], + } + } + + get_paths("RUSTPYTHONPATH") + .into_iter() + .chain(get_paths("PYTHONPATH").into_iter()) + .map(|path| ctx.new_str(path)) + .collect() }; let path = ctx.new_list(path_list); + let platform = if cfg!(target_os = "linux") { + "linux".to_string() + } else if cfg!(target_os = "macos") { + "darwin".to_string() + } else if cfg!(target_os = "windows") { + "win32".to_string() + } else if cfg!(target_os = "android") { + // Linux as well. see https://bugs.python.org/issue32637 + "linux".to_string() + } else { + "unknown".to_string() + }; + + // https://doc.rust-lang.org/reference/conditional-compilation.html#target_endian + let bytorder = if cfg!(target_endian = "little") { + "little".to_string() + } else if cfg!(target_endian = "big") { + "big".to_string() + } else { + "unknown".to_string() + }; + let sys_doc = "This module provides access to some objects used or maintained by the interpreter and to functions that interact strongly with the interpreter. @@ -141,23 +229,39 @@ setprofile() -- set the global profiling function setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; + let mut module_names: Vec<_> = vm.stdlib_inits.borrow().keys().cloned().collect(); + module_names.push("sys".to_string()); + module_names.push("builtins".to_string()); + module_names.sort(); let modules = ctx.new_dict(); - let sys_name = "sys"; - let sys_mod = py_module!(ctx, sys_name, { + extend_module!(vm, module, { + "__name__" => ctx.new_str(String::from("sys")), "argv" => argv(ctx), + "builtin_module_names" => ctx.new_tuple(module_names.iter().map(|v| v.into_pyobject(vm).unwrap()).collect()), + "byteorder" => ctx.new_str(bytorder), + "flags" => flags, "getrefcount" => ctx.new_rustfunc(sys_getrefcount), "getsizeof" => ctx.new_rustfunc(sys_getsizeof), + "implementation" => implementation, + "getfilesystemencoding" => ctx.new_rustfunc(sys_getfilesystemencoding), + "getfilesystemencodeerrors" => ctx.new_rustfunc(sys_getfilesystemencodeerrors), + "intern" => ctx.new_rustfunc(sys_intern), "maxsize" => ctx.new_int(std::usize::MAX), "path" => path, "ps1" => ctx.new_str(">>>>> ".to_string()), "ps2" => ctx.new_str("..... ".to_string()), "__doc__" => ctx.new_str(sys_doc.to_string()), "_getframe" => ctx.new_rustfunc(getframe), + "modules" => modules.clone(), + "warnoptions" => ctx.new_list(vec![]), + "platform" => ctx.new_str(platform), + "meta_path" => ctx.new_list(vec![]), + "path_hooks" => ctx.new_list(vec![]), + "path_importer_cache" => ctx.new_dict(), + "pycache_prefix" => vm.get_none(), + "dont_write_bytecode" => vm.new_bool(true), }); - modules.set_item(&ctx, sys_name, sys_mod.clone()); - modules.set_item(&ctx, "builtins", builtins); - ctx.set_attr(&sys_mod, "modules", modules); - - sys_mod + modules.set_item("sys", module.clone(), vm).unwrap(); + modules.set_item("builtins", builtins.clone(), vm).unwrap(); } diff --git a/vm/src/vm.rs b/vm/src/vm.rs index d8bdda0203..f117b83f31 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -4,8 +4,6 @@ //! https://github.com/ProgVal/pythonvm-rust/blob/master/src/processor/mod.rs //! -extern crate rustpython_parser; - use std::cell::{Ref, RefCell}; use std::collections::hash_map::HashMap; use std::collections::hash_set::HashSet; @@ -14,27 +12,33 @@ use std::sync::{Mutex, MutexGuard}; use crate::builtins; use crate::bytecode; -use crate::frame::{ExecutionResult, Frame, Scope}; +use crate::frame::{ExecutionResult, Frame, FrameRef, Scope}; +use crate::frozen; use crate::function::PyFuncArgs; +use crate::import; use crate::obj::objbool; use crate::obj::objbuiltinfunc::PyBuiltinFunction; -use crate::obj::objcode; -use crate::obj::objframe; +use crate::obj::objcode::{PyCode, PyCodeRef}; +use crate::obj::objdict::PyDictRef; use crate::obj::objfunction::{PyFunction, PyMethod}; -use crate::obj::objgenerator; +use crate::obj::objgenerator::PyGenerator; +use crate::obj::objint::PyInt; use crate::obj::objiter; -use crate::obj::objlist::PyList; use crate::obj::objsequence; use crate::obj::objstr::{PyString, PyStringRef}; -use crate::obj::objtuple::PyTuple; +use crate::obj::objtuple::PyTupleRef; use crate::obj::objtype; +use crate::obj::objtype::PyClassRef; +use crate::pyhash; use crate::pyobject::{ - AttributeProtocol, DictProtocol, IdProtocol, PyContext, PyObjectRef, PyResult, TryFromObject, - TryIntoRef, TypeProtocol, + IdProtocol, ItemProtocol, PyContext, PyObjectRef, PyResult, PyValue, TryFromObject, TryIntoRef, + TypeProtocol, }; use crate::stdlib; use crate::sysmodule; use num_bigint::BigInt; +#[cfg(feature = "rustpython-compiler")] +use rustpython_compiler::{compile, error::CompileError}; // use objects::objects; @@ -47,8 +51,11 @@ pub struct VirtualMachine { pub sys_module: PyObjectRef, pub stdlib_inits: RefCell>, pub ctx: PyContext, - pub frames: RefCell>, + pub frames: RefCell>, pub wasm_id: Option, + pub exceptions: RefCell>, + pub frozen: RefCell>, + pub import_func: RefCell, } impl VirtualMachine { @@ -57,59 +64,93 @@ impl VirtualMachine { let ctx = PyContext::new(); // Hard-core modules: - let builtins = builtins::make_module(&ctx); - let sysmod = sysmodule::make_module(&ctx, builtins.clone()); + let builtins = ctx.new_module("builtins", ctx.new_dict()); + let sysmod = ctx.new_module("sys", ctx.new_dict()); let stdlib_inits = RefCell::new(stdlib::get_module_inits()); - VirtualMachine { - builtins, - sys_module: sysmod, + let frozen = RefCell::new(frozen::get_module_inits()); + let import_func = RefCell::new(ctx.none()); + let vm = VirtualMachine { + builtins: builtins.clone(), + sys_module: sysmod.clone(), stdlib_inits, ctx, frames: RefCell::new(vec![]), wasm_id: None, - } + exceptions: RefCell::new(vec![]), + frozen, + import_func, + }; + + builtins::make_module(&vm, builtins.clone()); + sysmodule::make_module(&vm, sysmod, builtins); + vm } - pub fn run_code_obj(&self, code: PyObjectRef, scope: Scope) -> PyResult { - let frame = self.ctx.new_frame(code, scope); + pub fn run_code_obj(&self, code: PyCodeRef, scope: Scope) -> PyResult { + let frame = Frame::new(code, scope).into_ref(self); self.run_frame_full(frame) } - pub fn run_frame_full(&self, frame: PyObjectRef) -> PyResult { + pub fn run_frame_full(&self, frame: FrameRef) -> PyResult { match self.run_frame(frame)? { ExecutionResult::Return(value) => Ok(value), _ => panic!("Got unexpected result from function"), } } - pub fn run_frame(&self, frame: PyObjectRef) -> PyResult { + pub fn run_frame(&self, frame: FrameRef) -> PyResult { self.frames.borrow_mut().push(frame.clone()); - let frame = objframe::get_value(&frame); let result = frame.run(self); self.frames.borrow_mut().pop(); result } - pub fn current_frame(&self) -> Ref { - Ref::map(self.frames.borrow(), |frames| { - let index = frames.len() - 1; - let current_frame = &frames[index]; - objframe::get_value(current_frame) - }) + pub fn frame_throw( + &self, + frame: FrameRef, + exception: PyObjectRef, + ) -> PyResult { + self.frames.borrow_mut().push(frame.clone()); + let result = frame.throw(self, exception); + self.frames.borrow_mut().pop(); + result + } + + pub fn current_frame(&self) -> Option> { + let frames = self.frames.borrow(); + if frames.is_empty() { + None + } else { + Some(Ref::map(self.frames.borrow(), |frames| { + frames.last().unwrap() + })) + } } pub fn current_scope(&self) -> Ref { - let frame = self.current_frame(); + let frame = self + .current_frame() + .expect("called current_scope but no frames on the stack"); Ref::map(frame, |f| &f.scope) } - pub fn class(&self, module: &str, class: &str) -> PyObjectRef { + pub fn try_class(&self, module: &str, class: &str) -> PyResult { + let class = self + .get_attribute(self.import(module, &self.ctx.new_tuple(vec![]), 0)?, class)? + .downcast() + .expect("not a class"); + Ok(class) + } + + pub fn class(&self, module: &str, class: &str) -> PyClassRef { let module = self - .import(module) + .import(module, &self.ctx.new_tuple(vec![]), 0) .unwrap_or_else(|_| panic!("unable to import {}", module)); - self.get_attribute(module.clone(), class) - .unwrap_or_else(|_| panic!("module {} has no class {}", module, class)) + let class = self + .get_attribute(module.clone(), class) + .unwrap_or_else(|_| panic!("module {} has no class {}", module, class)); + class.downcast().expect("not a class") } /// Create a new python string object. @@ -127,25 +168,20 @@ impl VirtualMachine { self.ctx.new_bool(b) } - pub fn new_dict(&self) -> PyObjectRef { - self.ctx.new_dict() + fn new_exception_obj(&self, exc_type: PyClassRef, args: Vec) -> PyResult { + // TODO: add repr of args into logging? + info!("New exception created: {}", exc_type.name); + self.invoke(exc_type.into_object(), args) } - pub fn new_empty_exception(&self, exc_type: PyObjectRef) -> PyResult { - info!("New exception created: no msg"); - let args = PyFuncArgs::default(); - self.invoke(exc_type, args) + pub fn new_empty_exception(&self, exc_type: PyClassRef) -> PyResult { + self.new_exception_obj(exc_type, vec![]) } - pub fn new_exception(&self, exc_type: PyObjectRef, msg: String) -> PyObjectRef { - // TODO: exc_type may be user-defined exception, so we should return PyResult - // TODO: maybe there is a clearer way to create an instance: - info!("New exception created: {}", msg); - let pymsg = self.new_str(msg); - let args: Vec = vec![pymsg]; - - // Call function: - self.invoke(exc_type, args).unwrap() + /// Create Python instance of `exc_type` with message as first element of `args` tuple + pub fn new_exception(&self, exc_type: PyClassRef, msg: String) -> PyObjectRef { + let pystr_msg = self.new_str(msg); + self.new_exception_obj(exc_type, vec![pystr_msg]).unwrap() } pub fn new_attribute_error(&self, msg: String) -> PyObjectRef { @@ -158,17 +194,22 @@ impl VirtualMachine { self.new_exception(type_error, msg) } + pub fn new_name_error(&self, msg: String) -> PyObjectRef { + let name_error = self.ctx.exceptions.name_error.clone(); + self.new_exception(name_error, msg) + } + pub fn new_unsupported_operand_error( &self, a: PyObjectRef, b: PyObjectRef, op: &str, ) -> PyObjectRef { - let a_type_name = objtype::get_type_name(&a.typ()); - let b_type_name = objtype::get_type_name(&b.typ()); self.new_type_error(format!( "Unsupported operand types for '{}': '{}' and '{}'", - op, a_type_name, b_type_name + op, + a.class().name, + b.class().name )) } @@ -184,9 +225,9 @@ impl VirtualMachine { self.new_exception(value_error, msg) } - pub fn new_key_error(&self, msg: String) -> PyObjectRef { + pub fn new_key_error(&self, obj: PyObjectRef) -> PyObjectRef { let key_error = self.ctx.exceptions.key_error.clone(); - self.new_exception(key_error, msg) + self.new_exception_obj(key_error, vec![obj]).unwrap() } pub fn new_index_error(&self, msg: String) -> PyObjectRef { @@ -209,19 +250,37 @@ impl VirtualMachine { self.new_exception(overflow_error, msg) } + #[cfg(feature = "rustpython-compiler")] + pub fn new_syntax_error(&self, error: &CompileError) -> PyObjectRef { + let syntax_error_type = self.ctx.exceptions.syntax_error.clone(); + let syntax_error = self.new_exception(syntax_error_type, error.to_string()); + let lineno = self.new_int(error.location.row()); + self.set_attr(&syntax_error, "lineno", lineno).unwrap(); + syntax_error + } + + pub fn new_import_error(&self, msg: String) -> PyObjectRef { + let import_error = self.ctx.exceptions.import_error.clone(); + self.new_exception(import_error, msg) + } + + pub fn new_scope_with_builtins(&self) -> Scope { + Scope::with_builtins(None, self.ctx.new_dict(), self) + } + pub fn get_none(&self) -> PyObjectRef { self.ctx.none() } - pub fn get_type(&self) -> PyObjectRef { + pub fn get_type(&self) -> PyClassRef { self.ctx.type_type() } - pub fn get_object(&self) -> PyObjectRef { + pub fn get_object(&self) -> PyClassRef { self.ctx.object() } - pub fn get_locals(&self) -> PyObjectRef { + pub fn get_locals(&self) -> PyDictRef { self.current_scope().get_locals().clone() } @@ -235,8 +294,8 @@ impl VirtualMachine { TryFromObject::try_from_object(self, str) } - pub fn to_pystr(&self, obj: &PyObjectRef) -> Result { - let py_str_obj = self.to_str(obj)?; + pub fn to_pystr<'a, T: Into<&'a PyObjectRef>>(&'a self, obj: T) -> Result { + let py_str_obj = self.to_str(obj.into())?; Ok(py_str_obj.value.clone()) } @@ -245,42 +304,66 @@ impl VirtualMachine { TryFromObject::try_from_object(self, repr) } - pub fn import(&self, module: &str) -> PyResult { - let builtins_import = self.builtins.get_item("__import__"); - match builtins_import { - Some(func) => self.invoke(func, vec![self.ctx.new_str(module.to_string())]), - None => Err(self.new_exception( - self.ctx.exceptions.import_error.clone(), - "__import__ not found".to_string(), - )), - } + pub fn import(&self, module: &str, from_list: &PyObjectRef, level: usize) -> PyResult { + let sys_modules = self.get_attribute(self.sys_module.clone(), "modules")?; + sys_modules + .get_item(module.to_string(), self) + .or_else(|_| { + let import_func = self + .get_attribute(self.builtins.clone(), "__import__") + .map_err(|_| self.new_import_error("__import__ not found".to_string()))?; + + let (locals, globals) = if let Some(frame) = self.current_frame() { + ( + frame.scope.get_locals().into_object(), + frame.scope.globals.clone().into_object(), + ) + } else { + (self.get_none(), self.get_none()) + }; + self.invoke( + import_func, + vec![ + self.ctx.new_str(module.to_string()), + globals, + locals, + from_list.clone(), + self.ctx.new_int(level), + ], + ) + }) + .map_err(|exc| import::remove_importlib_frames(self, &exc)) } /// Determines if `obj` is an instance of `cls`, either directly, indirectly or virtually via /// the __instancecheck__ magic method. - pub fn isinstance(&self, obj: &PyObjectRef, cls: &PyObjectRef) -> PyResult { + pub fn isinstance(&self, obj: &PyObjectRef, cls: &PyClassRef) -> PyResult { // cpython first does an exact check on the type, although documentation doesn't state that // https://github.com/python/cpython/blob/a24107b04c1277e3c1105f98aff5bfa3a98b33a0/Objects/abstract.c#L2408 - if Rc::ptr_eq(&obj.typ(), cls) { + if Rc::ptr_eq(&obj.class().into_object(), cls.as_object()) { Ok(true) } else { - let ret = self.call_method(cls, "__instancecheck__", vec![obj.clone()])?; + let ret = self.call_method(cls.as_object(), "__instancecheck__", vec![obj.clone()])?; objbool::boolval(self, ret) } } /// Determines if `subclass` is a subclass of `cls`, either directly, indirectly or virtually /// via the __subclasscheck__ magic method. - pub fn issubclass(&self, subclass: &PyObjectRef, cls: &PyObjectRef) -> PyResult { - let ret = self.call_method(cls, "__subclasscheck__", vec![subclass.clone()])?; + pub fn issubclass(&self, subclass: &PyClassRef, cls: &PyClassRef) -> PyResult { + let ret = self.call_method( + cls.as_object(), + "__subclasscheck__", + vec![subclass.clone().into_object()], + )?; objbool::boolval(self, ret) } pub fn call_get_descriptor(&self, attr: PyObjectRef, obj: PyObjectRef) -> PyResult { - let attr_class = attr.typ(); - if let Some(descriptor) = attr_class.get_attr("__get__") { - let cls = obj.typ(); - self.invoke(descriptor, vec![attr, obj.clone(), cls]) + let attr_class = attr.class(); + if let Some(descriptor) = objtype::class_get_attr(&attr_class, "__get__") { + let cls = obj.class(); + self.invoke(descriptor, vec![attr, obj.clone(), cls.into_object()]) } else { Ok(attr) } @@ -291,8 +374,8 @@ impl VirtualMachine { T: Into, { // This is only used in the vm for magic methods, which use a greatly simplified attribute lookup. - let cls = obj.typ(); - match cls.get_attr(method_name) { + let cls = obj.class(); + match objtype::class_get_attr(&cls, method_name) { Some(func) => { trace!( "vm.call_method {:?} {:?} {:?} -> {:?}", @@ -308,53 +391,63 @@ impl VirtualMachine { } } - pub fn invoke(&self, func_ref: PyObjectRef, args: T) -> PyResult - where - T: Into, - { - let args = args.into(); + fn _invoke(&self, func_ref: PyObjectRef, args: PyFuncArgs) -> PyResult { trace!("Invoke: {:?} {:?}", func_ref, args); if let Some(PyFunction { ref code, ref scope, ref defaults, + ref kw_only_defaults, }) = func_ref.payload() { - return self.invoke_python_function(code, scope, defaults, args); - } - if let Some(PyMethod { + self.invoke_python_function(code, scope, defaults, kw_only_defaults, args) + } else if let Some(PyMethod { ref function, ref object, }) = func_ref.payload() { - return self.invoke(function.clone(), args.insert(object.clone())); - } - if let Some(PyBuiltinFunction { ref value }) = func_ref.payload() { - return value(self, args); + self.invoke(function.clone(), args.insert(object.clone())) + } else if let Some(PyBuiltinFunction { ref value }) = func_ref.payload() { + value(self, args) + } else { + // TODO: is it safe to just invoke __call__ otherwise? + trace!("invoke __call__ for: {:?}", &func_ref.payload); + self.call_method(&func_ref, "__call__", args) } + } - // TODO: is it safe to just invoke __call__ otherwise? - trace!("invoke __call__ for: {:?}", func_ref.payload); - self.call_method(&func_ref, "__call__", args) + // TODO: make func_ref an &PyObjectRef + #[inline] + pub fn invoke(&self, func_ref: PyObjectRef, args: T) -> PyResult + where + T: Into, + { + self._invoke(func_ref, args.into()) } fn invoke_python_function( &self, - code: &PyObjectRef, + code: &PyCodeRef, scope: &Scope, - defaults: &PyObjectRef, - args: PyFuncArgs, + defaults: &Option, + kw_only_defaults: &Option, + func_args: PyFuncArgs, ) -> PyResult { - let code_object = objcode::get_value(code); - let scope = scope.child_scope(&self.ctx); - self.fill_locals_from_args(&code_object, &scope.get_locals(), args, defaults)?; + let scope = scope.new_child_scope(&self.ctx); + self.fill_locals_from_args( + &code.code, + &scope.get_locals(), + func_args, + defaults, + kw_only_defaults, + )?; // Construct frame: - let frame = self.ctx.new_frame(code.clone(), scope); + let frame = Frame::new(code.clone(), scope).into_ref(self); // If we have a generator, create a new generator - if code_object.is_generator { - Ok(objgenerator::new_generator(frame, self).into_object()) + if code.code.is_generator { + Ok(PyGenerator::new(frame, self).into_object()) } else { self.run_frame_full(frame) } @@ -363,19 +456,14 @@ impl VirtualMachine { pub fn invoke_with_locals( &self, function: PyObjectRef, - cells: PyObjectRef, - locals: PyObjectRef, + cells: PyDictRef, + locals: PyDictRef, ) -> PyResult { - if let Some(PyFunction { - code, - scope, - defaults: _, - }) = &function.payload() - { + if let Some(PyFunction { code, scope, .. }) = &function.payload() { let scope = scope - .child_scope_with_locals(cells) - .child_scope_with_locals(locals); - let frame = self.ctx.new_frame(code.clone(), scope); + .new_child_scope_with_locals(cells) + .new_child_scope_with_locals(locals); + let frame = Frame::new(code.clone(), scope).into_ref(self); return self.run_frame_full(frame); } panic!( @@ -387,11 +475,12 @@ impl VirtualMachine { fn fill_locals_from_args( &self, code_object: &bytecode::CodeObject, - locals: &PyObjectRef, - args: PyFuncArgs, - defaults: &PyObjectRef, + locals: &PyDictRef, + func_args: PyFuncArgs, + defaults: &Option, + kw_only_defaults: &Option, ) -> PyResult<()> { - let nargs = args.args.len(); + let nargs = func_args.args.len(); let nexpected_args = code_object.arg_names.len(); // This parses the arguments from args and kwargs into @@ -409,8 +498,8 @@ impl VirtualMachine { // Copy positional arguments into local variables for i in 0..n { let arg_name = &code_object.arg_names[i]; - let arg = &args.args[i]; - locals.set_item(&self.ctx, arg_name, arg.clone()); + let arg = &func_args.args[i]; + locals.set_item(arg_name, arg.clone(), self)?; } // Pack other positional arguments in to *args: @@ -418,17 +507,14 @@ impl VirtualMachine { bytecode::Varargs::Named(ref vararg_name) => { let mut last_args = vec![]; for i in n..nargs { - let arg = &args.args[i]; + let arg = &func_args.args[i]; last_args.push(arg.clone()); } let vararg_value = self.ctx.new_tuple(last_args); - locals.set_item(&self.ctx, vararg_name, vararg_value); - } - bytecode::Varargs::Unnamed => { - // just ignore the rest of the args + locals.set_item(vararg_name, vararg_value, self)?; } - bytecode::Varargs::None => { + bytecode::Varargs::Unnamed | bytecode::Varargs::None => { // Check the number of positional arguments if nargs > nexpected_args { return Err(self.new_type_error(format!( @@ -442,28 +528,28 @@ impl VirtualMachine { // Do we support `**kwargs` ? let kwargs = match code_object.varkeywords { bytecode::Varargs::Named(ref kwargs_name) => { - let d = self.new_dict(); - locals.set_item(&self.ctx, kwargs_name, d.clone()); + let d = self.ctx.new_dict(); + locals.set_item(kwargs_name, d.as_object().clone(), self)?; Some(d) } - bytecode::Varargs::Unnamed => Some(self.new_dict()), + bytecode::Varargs::Unnamed => Some(self.ctx.new_dict()), bytecode::Varargs::None => None, }; // Handle keyword arguments - for (name, value) in args.kwargs { + for (name, value) in func_args.kwargs { // Check if we have a parameter with this name: if code_object.arg_names.contains(&name) || code_object.kwonlyarg_names.contains(&name) { - if locals.contains_key(&name) { + if locals.contains_key(&name, self) { return Err( self.new_type_error(format!("Got multiple values for argument '{}'", name)) ); } - locals.set_item(&self.ctx, &name, value); + locals.set_item(&name, value, self)?; } else if let Some(d) = &kwargs { - d.set_item(&self.ctx, &name, value); + d.set_item(&name, value, self)?; } else { return Err( self.new_type_error(format!("Got an unexpected keyword argument '{}'", name)) @@ -474,23 +560,15 @@ impl VirtualMachine { // Add missing positional arguments, if we have fewer positional arguments than the // function definition calls for if nargs < nexpected_args { - let available_defaults = if defaults.is(&self.get_none()) { - vec![] - } else if let Some(list) = defaults.payload::() { - list.elements.borrow().clone() - } else if let Some(tuple) = defaults.payload::() { - tuple.elements.borrow().clone() - } else { - panic!("function defaults not tuple or None"); - }; + let num_defaults_available = defaults.as_ref().map_or(0, |d| d.elements.len()); // Given the number of defaults available, check all the arguments for which we // _don't_ have defaults; if any are missing, raise an exception - let required_args = nexpected_args - available_defaults.len(); + let required_args = nexpected_args - num_defaults_available; let mut missing = vec![]; for i in 0..required_args { let variable_name = &code_object.arg_names[i]; - if !locals.contains_key(variable_name) { + if !locals.contains_key(variable_name, self) { missing.push(variable_name) } } @@ -501,35 +579,32 @@ impl VirtualMachine { missing ))); } - - // We have sufficient defaults, so iterate over the corresponding names and use - // the default if we don't already have a value - for (default_index, i) in (required_args..nexpected_args).enumerate() { - let arg_name = &code_object.arg_names[i]; - if !locals.contains_key(arg_name) { - locals.set_item( - &self.ctx, - arg_name, - available_defaults[default_index].clone(), - ); + if let Some(defaults) = defaults { + let defaults = &defaults.elements; + // We have sufficient defaults, so iterate over the corresponding names and use + // the default if we don't already have a value + for (default_index, i) in (required_args..nexpected_args).enumerate() { + let arg_name = &code_object.arg_names[i]; + if !locals.contains_key(arg_name, self) { + locals.set_item(arg_name, defaults[default_index].clone(), self)?; + } } } }; // Check if kw only arguments are all present: - let kwdefs: HashMap = HashMap::new(); for arg_name in &code_object.kwonlyarg_names { - if !locals.contains_key(arg_name) { - if kwdefs.contains_key(arg_name) { - // If not yet specified, take the default value - unimplemented!(); - } else { - // No default value and not specified. - return Err(self.new_type_error(format!( - "Missing required kw only argument: '{}'", - arg_name - ))); + if !locals.contains_key(arg_name, self) { + if let Some(kw_only_defaults) = kw_only_defaults { + if let Some(default) = kw_only_defaults.get_item_option(arg_name, self)? { + locals.set_item(arg_name, default, self)?; + continue; + } } + + // No default value and not specified. + return Err(self + .new_type_error(format!("Missing required kw only argument: '{}'", arg_name))); } } @@ -538,10 +613,10 @@ impl VirtualMachine { pub fn extract_elements(&self, value: &PyObjectRef) -> PyResult> { // Extract elements from item, if possible: - let elements = if objtype::isinstance(value, &self.ctx.tuple_type()) - || objtype::isinstance(value, &self.ctx.list_type()) - { - objsequence::get_elements(value).to_vec() + let elements = if objtype::isinstance(value, &self.ctx.tuple_type()) { + objsequence::get_elements_tuple(value).to_vec() + } else if objtype::isinstance(value, &self.ctx.list_type()) { + objsequence::get_elements_list(value).to_vec() } else { let iter = objiter::get_iter(self, value)?; objiter::get_all(self, &iter)? @@ -559,29 +634,49 @@ impl VirtualMachine { self.call_method(&obj, "__getattribute__", vec![attr_name.into_object()]) } - pub fn set_attr( - &self, - obj: &PyObjectRef, - attr_name: PyObjectRef, - attr_value: PyObjectRef, - ) -> PyResult { - self.call_method(&obj, "__setattr__", vec![attr_name, attr_value]) + pub fn set_attr(&self, obj: &PyObjectRef, attr_name: K, attr_value: V) -> PyResult + where + K: TryIntoRef, + V: Into, + { + let attr_name = attr_name.try_into_ref(self)?; + self.call_method( + obj, + "__setattr__", + vec![attr_name.into_object(), attr_value.into()], + ) } - pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult { - self.call_method(&obj, "__delattr__", vec![attr_name]) + pub fn del_attr(&self, obj: &PyObjectRef, attr_name: PyObjectRef) -> PyResult<()> { + self.call_method(&obj, "__delattr__", vec![attr_name])?; + Ok(()) } // get_method should be used for internal access to magic methods (by-passing // the full getattribute look-up. - pub fn get_method(&self, obj: PyObjectRef, method_name: &str) -> PyResult { - let cls = obj.typ(); - match cls.get_attr(method_name) { + pub fn get_method_or_type_error( + &self, + obj: PyObjectRef, + method_name: &str, + err_msg: F, + ) -> PyResult + where + F: FnOnce() -> String, + { + let cls = obj.class(); + match objtype::class_get_attr(&cls, method_name) { Some(method) => self.call_get_descriptor(method, obj.clone()), - None => Err(self.new_type_error(format!("{} has no method {:?}", obj, method_name))), + None => Err(self.new_type_error(err_msg())), } } + /// May return exception, if `__get__` descriptor raises one + pub fn get_method(&self, obj: PyObjectRef, method_name: &str) -> Option { + let cls = obj.class(); + let method = objtype::class_get_attr(&cls, method_name)?; + Some(self.call_get_descriptor(method, obj.clone())) + } + /// Calls a method on `obj` passing `arg`, if the method exists. /// /// Otherwise, or if the result is the special `NotImplemented` built-in constant, @@ -596,13 +691,13 @@ impl VirtualMachine { where F: Fn(&VirtualMachine, PyObjectRef, PyObjectRef) -> PyResult, { - if let Ok(method) = self.get_method(obj.clone(), method) { + if let Some(method_or_err) = self.get_method(obj.clone(), method) { + let method = method_or_err?; let result = self.invoke(method, vec![arg.clone()])?; if !result.is(&self.ctx.not_implemented()) { return Ok(result); } } - unsupported(self, obj, arg) } @@ -631,12 +726,24 @@ impl VirtualMachine { }) } - pub fn serialize(&self, obj: &PyObjectRef) -> PyResult { - crate::stdlib::json::ser_pyobject(self, obj) + pub fn is_callable(&self, obj: &PyObjectRef) -> bool { + match_class!(obj, + PyFunction => true, + PyMethod => true, + PyBuiltinFunction => true, + obj => objtype::class_has_attr(&obj.class(), "__call__"), + ) } - pub fn deserialize(&self, s: &str) -> PyResult { - crate::stdlib::json::de_pyobject(self, s) + #[cfg(feature = "rustpython-compiler")] + pub fn compile( + &self, + source: &str, + mode: &compile::Mode, + source_path: String, + ) -> Result { + compile::compile(source, mode, source_path) + .map(|codeobj| PyCode::new(codeobj).into_ref(self)) } pub fn _sub(&self, a: PyObjectRef, b: PyObjectRef) -> PyResult { @@ -857,6 +964,49 @@ impl VirtualMachine { Err(vm.new_unsupported_operand_error(a, b, ">=")) }) } + + pub fn _hash(&self, obj: &PyObjectRef) -> PyResult { + let hash_obj = self.call_method(obj, "__hash__", vec![])?; + if objtype::isinstance(&hash_obj, &self.ctx.int_type()) { + Ok(hash_obj.payload::().unwrap().hash(self)) + } else { + Err(self.new_type_error("__hash__ method should return an integer".to_string())) + } + } + + // https://docs.python.org/3/reference/expressions.html#membership-test-operations + fn _membership_iter_search(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + let iter = objiter::get_iter(self, &haystack)?; + loop { + if let Some(element) = objiter::get_next_object(self, &iter)? { + let equal = self._eq(needle.clone(), element.clone())?; + if objbool::get_value(&equal) { + return Ok(self.new_bool(true)); + } else { + continue; + } + } else { + return Ok(self.new_bool(false)); + } + } + } + + pub fn _membership(&self, haystack: PyObjectRef, needle: PyObjectRef) -> PyResult { + if let Some(method_or_err) = self.get_method(haystack.clone(), "__contains__") { + let method = method_or_err?; + self.invoke(method, vec![needle]) + } else { + self._membership_iter_search(haystack, needle) + } + } + + pub fn push_exception(&self, exc: PyObjectRef) { + self.exceptions.borrow_mut().push(exc) + } + + pub fn pop_exception(&self) -> Option { + self.exceptions.borrow_mut().pop() + } } impl Default for VirtualMachine { @@ -909,7 +1059,7 @@ mod tests { #[test] fn test_add_py_integers() { - let mut vm = VirtualMachine::new(); + let vm = VirtualMachine::new(); let a = vm.ctx.new_int(33_i32); let b = vm.ctx.new_int(12_i32); let res = vm._add(a, b).unwrap(); @@ -919,7 +1069,7 @@ mod tests { #[test] fn test_multiply_str() { - let mut vm = VirtualMachine::new(); + let vm = VirtualMachine::new(); let a = vm.ctx.new_str(String::from("Hello ")); let b = vm.ctx.new_int(4_i32); let res = vm._mul(a, b).unwrap(); diff --git a/wasm/demo/package.json b/wasm/demo/package.json index a03c80aced..6f66b120d6 100644 --- a/wasm/demo/package.json +++ b/wasm/demo/package.json @@ -5,6 +5,7 @@ "main": "index.js", "dependencies": { "codemirror": "^5.42.0", + "serve": "^11.0.2", "xterm": "^3.8.0" }, "devDependencies": { @@ -24,7 +25,7 @@ "build": "webpack", "dist": "webpack --mode production", "test": "cd ../tests; pipenv run pytest", - "ci": "start-server-and-test dev http-get://localhost:8080 test" + "ci": "start-server-and-test 'npm run build && serve dist/ -n -p 8080' :8080 test" }, "repository": { "type": "git", diff --git a/wasm/demo/snippets/mandelbrot.py b/wasm/demo/snippets/mandelbrot.py index 2d664fdedf..b4010c7539 100644 --- a/wasm/demo/snippets/mandelbrot.py +++ b/wasm/demo/snippets/mandelbrot.py @@ -2,8 +2,7 @@ h = 50.0 def mandel(): - """Print a mandelbrot fractal to the console, yielding after each character - is printed""" + """Print a mandelbrot fractal to the console, yielding after each character is printed""" y = 0.0 while y < h: x = 0.0 @@ -32,12 +31,14 @@ def mandel(): y += 1 yield +# run the mandelbrot + try: from browser import request_animation_frame except: request_animation_frame = None gen = mandel() def gen_cb(_time=None): - gen.__next__() + for _ in range(4): gen.__next__() request_animation_frame(gen_cb) if request_animation_frame: gen_cb() -else: list(gen) +else: any(gen) diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index 9b58afe398..602e252417 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -1,6 +1,6 @@ import './style.css'; import 'codemirror/lib/codemirror.css'; -import 'xterm/dist/xterm.css'; +import 'xterm/lib/xterm.css'; // A dependency graph that contains any wasm must all be imported // asynchronously. This `index.js` file does the single async import, so diff --git a/wasm/demo/src/main.js b/wasm/demo/src/main.js index f5034f62a1..9b0fca8ef8 100644 --- a/wasm/demo/src/main.js +++ b/wasm/demo/src/main.js @@ -82,12 +82,6 @@ snippets.addEventListener('change', updateSnippet); // option selected for the `select`, but the textarea won't be updated) updateSnippet(); -const prompt = '>>>>> '; - -const term = new Terminal(); -term.open(document.getElementById('terminal')); -term.write(prompt); - function removeNonAscii(str) { if (str === null || str === '') return false; else str = str.toString(); @@ -99,38 +93,80 @@ function printToConsole(data) { term.write(removeNonAscii(data) + '\r\n'); } +const term = new Terminal(); +term.open(document.getElementById('terminal')); + const terminalVM = rp.vmStore.init('term_vm'); terminalVM.setStdout(printToConsole); -var input = ''; +function getPrompt(name = 'ps1') { + terminalVM.exec(` +try: + import sys as __sys + __prompt = __sys.${name} +except: + __prompt = '' +finally: + del __sys +`); + return String(terminalVM.eval('__prompt')); +} + +term.write(getPrompt()); + +function resetInput() { + continuedInput = []; + input = ''; + continuing = false; +} + +let continuedInput, input, continuing; +resetInput(); + +let ps2; + term.on('data', data => { const code = data.charCodeAt(0); if (code == 13) { // CR - if (input[input.length - 1] == ':') { - input += data; - term.write('\r\n.....'); - } else { - term.write('\r\n'); - try { - terminalVM.execSingle(input); - } catch (err) { - if (err instanceof WebAssembly.RuntimeError) { - err = window.__RUSTPYTHON_ERROR || err; - } - printToConsole(err); + term.write('\r\n'); + continuedInput.push(input); + if (continuing) { + if (input === '') { + continuing = false; + } else { + input = ''; + term.write(ps2); + return; + } + } + try { + terminalVM.execSingle(continuedInput.join('\n')); + } catch (err) { + if (err instanceof SyntaxError && err.message.includes('EOF')) { + ps2 = getPrompt('ps2'); + term.write(ps2); + continuing = true; + input = ''; + return; + } else if (err instanceof WebAssembly.RuntimeError) { + err = window.__RUSTPYTHON_ERROR || err; } - term.write(prompt); - input = ''; + printToConsole(err); } - } else if (code == 127) { + resetInput(); + term.write(getPrompt()); + } else if (code == 127 || code == 8) { + // Backspace if (input.length > 0) { term.write('\b \b'); input = input.slice(0, -1); } - } else if (code < 32 || code == 127) { + } else if (code < 32) { // Control - return; + term.write('\r\n' + getPrompt()); + input = ''; + continuedInput = []; } else { // Visible term.write(data); diff --git a/wasm/demo/src/style.css b/wasm/demo/src/style.css index a8a5de5004..8fc18ecce6 100644 --- a/wasm/demo/src/style.css +++ b/wasm/demo/src/style.css @@ -1,6 +1,6 @@ textarea { font-family: monospace; - resize: none; + resize: vertical; } #code, diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 769455aea1..9c78a02904 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustpython_wasm" -version = "0.1.0-pre-alpha.1" -authors = ["Ryan Liddle "] +version = "0.1.0-pre-alpha.2" +authors = ["RustPython Team"] license = "MIT" description = "A Python-3 (CPython >= 3.5.0) Interpreter written in Rust, compiled to WASM" repository = "https://github.com/RustPython/RustPython/tree/master/wasm/lib" @@ -11,11 +11,14 @@ edition = "2018" crate-type = ["cdylib", "rlib"] [dependencies] -rustpython_parser = { path = "../../parser" } -rustpython_vm = { path = "../../vm" } +rustpython-compiler = { path = "../../compiler" } +rustpython-parser = { path = "../../parser" } +rustpython-vm = { path = "../../vm" } cfg-if = "0.1.2" wasm-bindgen = "0.2" wasm-bindgen-futures = "0.3" +serde-wasm-bindgen = "0.1" +serde = "1.0" js-sys = "0.3" futures = "0.1" num-traits = "0.2" diff --git a/wasm/lib/src/browser.py b/wasm/lib/src/browser.py new file mode 100644 index 0000000000..b146f696ac --- /dev/null +++ b/wasm/lib/src/browser.py @@ -0,0 +1,37 @@ +from _browser import * + +from _js import JsValue +from _window import window + + +jsstr = window.new_from_str + + +_alert = window.get_prop("alert") + + +def alert(msg): + if type(msg) != str: + raise TypeError("msg must be a string") + _alert.call(jsstr(msg)) + + +_confirm = window.get_prop("confirm") + + +def confirm(msg): + if type(msg) != str: + raise TypeError("msg must be a string") + return _confirm.call(jsstr(msg)).as_bool() + + +_prompt = window.get_prop("prompt") + + +def prompt(msg, default_val=None): + if type(msg) != str: + raise TypeError("msg must be a string") + if default_val is not None and type(default_val) != str: + raise TypeError("default_val must be a string") + + return _prompt.call(*(jsstr(arg) for arg in [msg, default_val] if arg)).as_str() diff --git a/wasm/lib/src/browser_module.rs b/wasm/lib/src/browser_module.rs index 7fc2a2476e..364fa614bb 100644 --- a/wasm/lib/src/browser_module.rs +++ b/wasm/lib/src/browser_module.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - use futures::Future; use js_sys::Promise; use num_traits::cast::ToPrimitive; @@ -7,15 +5,17 @@ use wasm_bindgen::prelude::*; use wasm_bindgen::JsCast; use wasm_bindgen_futures::{future_to_promise, JsFuture}; -use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::import::import_module; -use rustpython_vm::obj::{objint, objstr}; +use rustpython_vm::function::{OptionalArg, PyFuncArgs}; +use rustpython_vm::import::import_file; +use rustpython_vm::obj::{ + objdict::PyDictRef, objint::PyIntRef, objstr::PyStringRef, objtype::PyClassRef, +}; use rustpython_vm::pyobject::{ - AttributeProtocol, PyContext, PyObject, PyObjectRef, PyResult, PyValue, TypeProtocol, + PyCallable, PyClassImpl, PyObject, PyObjectRef, PyRef, PyResult, PyValue, }; use rustpython_vm::VirtualMachine; -use crate::{convert, vm_class::AccessibleVM, wasm_builtins::window}; +use crate::{convert, vm_class::weak_vm, wasm_builtins::window}; enum FetchResponseFormat { Json, @@ -41,27 +41,38 @@ impl FetchResponseFormat { } } -fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(url, Some(vm.ctx.str_type()))]); - - let promise_type = import_promise_type(vm)?; +#[derive(FromArgs)] +struct FetchArgs { + #[pyarg(keyword_only, default = "None")] + response_format: Option, + #[pyarg(keyword_only, default = "None")] + method: Option, + #[pyarg(keyword_only, default = "None")] + headers: Option, + #[pyarg(keyword_only, default = "None")] + body: Option, + #[pyarg(keyword_only, default = "None")] + content_type: Option, +} - let response_format = - args.get_optional_kwarg_with_type("response_format", vm.ctx.str_type(), vm)?; - let method = args.get_optional_kwarg_with_type("method", vm.ctx.str_type(), vm)?; - let headers = args.get_optional_kwarg_with_type("headers", vm.ctx.dict_type(), vm)?; - let body = args.get_optional_kwarg("body"); - let content_type = args.get_optional_kwarg_with_type("content_type", vm.ctx.str_type(), vm)?; +fn browser_fetch(url: PyStringRef, args: FetchArgs, vm: &VirtualMachine) -> PyResult { + let FetchArgs { + response_format, + method, + headers, + body, + content_type, + } = args; let response_format = match response_format { - Some(s) => FetchResponseFormat::from_str(vm, &objstr::get_value(&s))?, + Some(s) => FetchResponseFormat::from_str(vm, s.as_str())?, None => FetchResponseFormat::Text, }; let mut opts = web_sys::RequestInit::new(); match method { - Some(s) => opts.method(&objstr::get_value(&s)), + Some(s) => opts.method(s.as_str()), None => opts.method("GET"), }; @@ -69,15 +80,15 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { opts.body(Some(&convert::py_to_js(vm, body))); } - let request = web_sys::Request::new_with_str_and_init(&objstr::get_value(url), &opts) + let request = web_sys::Request::new_with_str_and_init(url.as_str(), &opts) .map_err(|err| convert::js_py_typeerror(vm, err))?; if let Some(headers) = headers { let h = request.headers(); - for (key, value) in rustpython_vm::obj::objdict::get_key_value_pairs(&headers) { - let key = &vm.to_str(&key)?.value; - let value = &vm.to_str(&value)?.value; - h.set(key, value) + for (key, value) in headers { + let key = vm.to_str(&key)?; + let value = vm.to_str(&value)?; + h.set(key.as_str(), value.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } } @@ -85,7 +96,7 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { if let Some(content_type) = content_type { request .headers() - .set("Content-Type", &objstr::get_value(&content_type)) + .set("Content-Type", content_type.as_str()) .map_err(|err| convert::js_py_typeerror(vm, err))?; } @@ -101,12 +112,10 @@ fn browser_fetch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { }) .and_then(JsFuture::from); - Ok(PyPromise::new_obj(promise_type, future_to_promise(future))) + Ok(PyPromise::from_future(future).into_ref(vm).into_object()) } -fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(func, Some(vm.ctx.function_type()))]); - +fn browser_request_animation_frame(func: PyCallable, vm: &VirtualMachine) -> PyResult { use std::{cell::RefCell, rc::Rc}; // this basic setup for request_animation_frame taken from: @@ -115,18 +124,16 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR let f = Rc::new(RefCell::new(None)); let g = f.clone(); - let func = func.clone(); - - let acc_vm = AccessibleVM::from(vm); + let weak_vm = weak_vm(vm); *g.borrow_mut() = Some(Closure::wrap(Box::new(move |time: f64| { - let stored_vm = acc_vm + let stored_vm = weak_vm .upgrade() .expect("that the vm is valid from inside of request_animation_frame"); let vm = &stored_vm.vm; let func = func.clone(); let args = vec![vm.ctx.new_float(time)]; - let _ = vm.invoke(func, args); + let _ = vm.invoke(func.into_object(), args); let closure = f.borrow_mut().take(); drop(closure); @@ -141,10 +148,8 @@ fn browser_request_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyR Ok(vm.ctx.new_int(id)) } -fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(id, Some(vm.ctx.int_type()))]); - - let id = objint::get_value(id).to_i32().ok_or_else(|| { +fn browser_cancel_animation_frame(id: PyIntRef, vm: &VirtualMachine) -> PyResult { + let id = id.as_bigint().to_i32().ok_or_else(|| { vm.new_exception( vm.ctx.exceptions.value_error.clone(), "Integer too large to convert to i32 for animationFrame id".into(), @@ -158,184 +163,226 @@ fn browser_cancel_animation_frame(vm: &VirtualMachine, args: PyFuncArgs) -> PyRe Ok(vm.get_none()) } +#[pyclass(name = "Promise")] #[derive(Debug)] pub struct PyPromise { value: Promise, } +pub type PyPromiseRef = PyRef; impl PyValue for PyPromise { - fn class(vm: &VirtualMachine) -> PyObjectRef { - vm.class(BROWSER_NAME, "Promise") + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("browser", "Promise") } } +#[pyimpl] impl PyPromise { - pub fn new_obj(promise_type: PyObjectRef, value: Promise) -> PyObjectRef { - PyObject::new(PyPromise { value }, promise_type) + pub fn new(value: Promise) -> PyPromise { + PyPromise { value } + } + pub fn from_future(future: F) -> PyPromise + where + F: Future + 'static, + { + PyPromise::new(future_to_promise(future)) + } + pub fn value(&self) -> Promise { + self.value.clone() } -} -pub fn get_promise_value(obj: &PyObjectRef) -> Promise { - if let Some(promise) = obj.payload::() { - return promise.value.clone(); + #[pymethod] + fn then( + &self, + on_fulfill: PyCallable, + on_reject: OptionalArg, + vm: &VirtualMachine, + ) -> PyPromiseRef { + let weak_vm = weak_vm(vm); + + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { + let stored_vm = &weak_vm + .upgrade() + .expect("that the vm is valid when the promise resolves"); + let vm = &stored_vm.vm; + let ret = match res { + Ok(val) => { + let args = if val.is_null() { + vec![] + } else { + vec![convert::js_to_py(vm, val)] + }; + vm.invoke(on_fulfill.into_object(), PyFuncArgs::new(args, vec![])) + } + Err(err) => { + if let OptionalArg::Present(on_reject) = on_reject { + let err = convert::js_to_py(vm, err); + vm.invoke(on_reject.into_object(), PyFuncArgs::new(vec![err], vec![])) + } else { + return Err(err); + } + } + }; + convert::pyresult_to_jsresult(vm, ret) + }); + + PyPromise::from_future(ret_future).into_ref(vm) } - panic!("Inner error getting promise") -} -pub fn import_promise_type(vm: &VirtualMachine) -> PyResult { - match import_module(vm, PathBuf::default(), BROWSER_NAME)?.get_attr("Promise") { - Some(promise) => Ok(promise), - None => Err(vm.new_not_implemented_error("No Promise".to_string())), + #[pymethod] + fn catch(&self, on_reject: PyCallable, vm: &VirtualMachine) -> PyPromiseRef { + let weak_vm = weak_vm(vm); + + let ret_future = JsFuture::from(self.value.clone()).then(move |res| { + res.or_else(|err| { + let stored_vm = weak_vm + .upgrade() + .expect("that the vm is valid when the promise resolves"); + let vm = &stored_vm.vm; + let err = convert::js_to_py(vm, err); + let res = vm.invoke(on_reject.into_object(), PyFuncArgs::new(vec![err], vec![])); + convert::pyresult_to_jsresult(vm, res) + }) + }); + + PyPromise::from_future(ret_future).into_ref(vm) } } -fn promise_then(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let promise_type = import_promise_type(vm)?; - arg_check!( - vm, - args, - required = [ - (zelf, Some(promise_type.clone())), - (on_fulfill, Some(vm.ctx.function_type())) - ], - optional = [(on_reject, Some(vm.ctx.function_type()))] - ); +#[pyclass] +#[derive(Debug)] +struct Document { + doc: web_sys::Document, +} - let on_fulfill = on_fulfill.clone(); - let on_reject = on_reject.cloned(); +impl PyValue for Document { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("browser", "Document") + } +} - let acc_vm = AccessibleVM::from(vm); +#[pyimpl] +impl Document { + #[pymethod] + fn query(&self, query: PyStringRef, vm: &VirtualMachine) -> PyResult { + let elem = self + .doc + .query_selector(&query.value) + .map_err(|err| convert::js_py_typeerror(vm, err))?; + let elem = match elem { + Some(elem) => Element { elem }.into_ref(vm).into_object(), + None => vm.get_none(), + }; + Ok(elem) + } +} - let promise = get_promise_value(zelf); +#[pyclass] +#[derive(Debug)] +struct Element { + elem: web_sys::Element, +} - let ret_future = JsFuture::from(promise).then(move |res| { - let stored_vm = &acc_vm - .upgrade() - .expect("that the vm is valid when the promise resolves"); - let vm = &stored_vm.vm; - let ret = match res { - Ok(val) => { - let val = convert::js_to_py(vm, val); - vm.invoke(on_fulfill, PyFuncArgs::new(vec![val], vec![])) - } - Err(err) => { - if let Some(on_reject) = on_reject { - let err = convert::js_to_py(vm, err); - vm.invoke(on_reject, PyFuncArgs::new(vec![err], vec![])) - } else { - return Err(err); - } - } - }; - convert::pyresult_to_jsresult(vm, ret) - }); +impl PyValue for Element { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("browser", "Element") + } +} - let ret_promise = future_to_promise(ret_future); +#[pyimpl] +impl Element { + #[pymethod] + fn get_attr( + &self, + attr: PyStringRef, + default: OptionalArg, + vm: &VirtualMachine, + ) -> PyObjectRef { + match self.elem.get_attribute(&attr.value) { + Some(s) => vm.new_str(s), + None => default.into_option().unwrap_or_else(|| vm.get_none()), + } + } - Ok(PyPromise::new_obj(promise_type, ret_promise)) + #[pymethod] + fn set_attr(&self, attr: PyStringRef, value: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { + self.elem + .set_attribute(&attr.value, &value.value) + .map_err(|err| convert::js_to_py(vm, err)) + } } -fn promise_catch(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - let promise_type = import_promise_type(vm)?; - arg_check!( - vm, - args, - required = [ - (zelf, Some(promise_type.clone())), - (on_reject, Some(vm.ctx.function_type())) - ] - ); +fn browser_load_module(module: PyStringRef, path: PyStringRef, vm: &VirtualMachine) -> PyResult { + let weak_vm = weak_vm(vm); - let on_reject = on_reject.clone(); + let mut opts = web_sys::RequestInit::new(); + opts.method("GET"); - let acc_vm = AccessibleVM::from(vm); + let request = web_sys::Request::new_with_str_and_init(path.as_str(), &opts) + .map_err(|err| convert::js_py_typeerror(vm, err))?; - let promise = get_promise_value(zelf); + let window = window(); + let request_prom = window.fetch_with_request(&request); - let ret_future = JsFuture::from(promise).then(move |res| match res { - Ok(val) => Ok(val), - Err(err) => { - let stored_vm = acc_vm + let future = JsFuture::from(request_prom) + .and_then(move |val| { + let response = val + .dyn_into::() + .expect("val to be of type Response"); + response.text() + }) + .and_then(JsFuture::from) + .and_then(move |text| { + let stored_vm = &weak_vm .upgrade() .expect("that the vm is valid when the promise resolves"); let vm = &stored_vm.vm; - let err = convert::js_to_py(vm, err); - let res = vm.invoke(on_reject, PyFuncArgs::new(vec![err], vec![])); - convert::pyresult_to_jsresult(vm, res) - } - }); - - let ret_promise = future_to_promise(ret_future); - - Ok(PyPromise::new_obj(promise_type, ret_promise)) -} - -fn browser_alert(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); - - window() - .alert_with_message(&objstr::get_value(message)) - .expect("alert() not to fail"); + let resp_text = text.as_string().unwrap(); + let res = import_file(vm, module.as_str(), "WEB".to_string(), resp_text); + match res { + Ok(_) => Ok(JsValue::null()), + Err(err) => Err(convert::py_err_to_js_err(vm, &err)), + } + }); - Ok(vm.get_none()) + Ok(PyPromise::from_future(future).into_ref(vm).into_object()) } -fn browser_confirm(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!(vm, args, required = [(message, Some(vm.ctx.str_type()))]); +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; - let result = window() - .confirm_with_message(&objstr::get_value(message)) - .expect("confirm() not to fail"); + let promise = PyPromise::make_class(ctx); - Ok(vm.new_bool(result)) -} + let document_class = Document::make_class(ctx); -fn browser_prompt(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(message, Some(vm.ctx.str_type()))], - optional = [(default, Some(vm.ctx.str_type()))] + let document = PyObject::new( + Document { + doc: window().document().expect("Document missing from window"), + }, + document_class.clone(), + None, ); - let result = if let Some(default) = default { - window().prompt_with_message_and_default( - &objstr::get_value(message), - &objstr::get_value(default), - ) - } else { - window().prompt_with_message(&objstr::get_value(message)) - }; - - let result = match result.expect("prompt() not to fail") { - Some(result) => vm.new_str(result), - None => vm.get_none(), - }; - - Ok(result) -} - -const BROWSER_NAME: &str = "browser"; + let element = Element::make_class(ctx); -pub fn make_module(ctx: &PyContext) -> PyObjectRef { - let promise = py_class!(ctx, "Promise", ctx.object(), { - "then" => ctx.new_rustfunc(promise_then), - "catch" => ctx.new_rustfunc(promise_catch) - }); - - py_module!(ctx, BROWSER_NAME, { + py_module!(vm, "browser", { "fetch" => ctx.new_rustfunc(browser_fetch), "request_animation_frame" => ctx.new_rustfunc(browser_request_animation_frame), "cancel_animation_frame" => ctx.new_rustfunc(browser_cancel_animation_frame), "Promise" => promise, - "alert" => ctx.new_rustfunc(browser_alert), - "confirm" => ctx.new_rustfunc(browser_confirm), - "prompt" => ctx.new_rustfunc(browser_prompt), + "Document" => document_class, + "document" => document, + "Element" => element, + "load_module" => ctx.new_rustfunc(browser_load_module), }) } pub fn setup_browser_module(vm: &VirtualMachine) { vm.stdlib_inits .borrow_mut() - .insert(BROWSER_NAME.to_string(), Box::new(make_module)); + .insert("_browser".to_string(), Box::new(make_module)); + vm.frozen.borrow_mut().insert( + "browser".to_string(), + py_compile_bytecode!(file = "src/browser.py"), + ); } diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 122ff1763e..6de4a5987a 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -1,14 +1,16 @@ use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, Uint8Array}; use num_traits::cast::ToPrimitive; +use serde_wasm_bindgen; use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::obj::{objbytes, objint, objsequence, objtype}; -use rustpython_vm::pyobject::{AttributeProtocol, DictProtocol, PyObjectRef, PyResult}; +use rustpython_vm::py_serde; +use rustpython_vm::pyobject::{ItemProtocol, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module; -use crate::vm_class::{AccessibleVM, WASMVirtualMachine}; +use crate::vm_class::{stored_vm_from_wasm, WASMVirtualMachine}; pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { macro_rules! map_exceptions { @@ -20,8 +22,9 @@ pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { } }; } - let msg = match py_err - .get_attr("msg") + let msg = match vm + .get_attribute(py_err.clone(), "msg") + .ok() .and_then(|msg| vm.to_pystr(&msg).ok()) { Some(msg) => msg, @@ -37,12 +40,12 @@ pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { &vm.ctx.exceptions.name_error => js_sys::ReferenceError::new, &vm.ctx.exceptions.syntax_error => js_sys::SyntaxError::new, }); - if let Some(tb) = py_err.get_attr("__traceback__") { + if let Ok(tb) = vm.get_attribute(py_err.clone(), "__traceback__") { if objtype::isinstance(&tb, &vm.ctx.list_type()) { - let elements = objsequence::get_elements(&tb).to_vec(); + let elements = objsequence::get_elements_list(&tb).to_vec(); if let Some(top) = elements.get(0) { if objtype::isinstance(&top, &vm.ctx.tuple_type()) { - let element = objsequence::get_elements(&top); + let element = objsequence::get_elements_tuple(&top); if let Some(lineno) = objint::to_int(vm, &element[1], 10) .ok() @@ -80,11 +83,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { return Err(err); } }; - let acc_vm = AccessibleVM::from(wasm_vm.clone()); - let stored_vm = acc_vm - .upgrade() - .expect("acc. VM to be invalid when WASM vm is valid"); - let vm = &stored_vm.vm; + let vm = &stored_vm_from_wasm(&wasm_vm).vm; let mut py_func_args = PyFuncArgs::default(); if let Some(ref args) = args { for arg in args.values() { @@ -96,7 +95,7 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { let (key, val) = pair?; py_func_args .kwargs - .push((js_sys::JsString::from(key).into(), js_to_py(vm, val))); + .insert(js_sys::JsString::from(key).into(), js_to_py(vm, val)); } } let result = vm.invoke(py_obj.clone(), py_func_args); @@ -114,9 +113,9 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { } } // the browser module might not be injected - if let Ok(promise_type) = browser_module::import_promise_type(vm) { - if objtype::isinstance(&py_obj, &promise_type) { - return browser_module::get_promise_value(&py_obj).into(); + if vm.try_class("browser", "Promise").is_ok() { + if let Some(py_prom) = py_obj.payload::() { + return py_prom.value().into(); } } @@ -124,17 +123,17 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { || objtype::isinstance(&py_obj, &vm.ctx.bytearray_type()) { let bytes = objbytes::get_value(&py_obj); - let arr = Uint8Array::new_with_length(bytes.len() as u32); - for (i, byte) in bytes.iter().enumerate() { - Reflect::set(&arr, &(i as u32).into(), &(*byte).into()) - .expect("setting Uint8Array value failed"); + unsafe { + // `Uint8Array::view` is an `unsafe fn` because it provides + // a direct view into the WASM linear memory; if you were to allocate + // something with Rust that view would probably become invalid. It's safe + // because we then copy the array using `Uint8Array::slice`. + let view = Uint8Array::view(&bytes); + view.slice(0, bytes.len() as u32).into() } - arr.into() } else { - match vm.serialize(&py_obj) { - Ok(json) => js_sys::JSON::parse(&json).unwrap_or(JsValue::UNDEFINED), - Err(_) => JsValue::UNDEFINED, - } + py_serde::serialize(vm, &py_obj, &serde_wasm_bindgen::Serializer::new()) + .unwrap_or(JsValue::UNDEFINED) } } @@ -158,8 +157,10 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { if js_val.is_object() { if let Some(promise) = js_val.dyn_ref::() { // the browser module might not be injected - if let Ok(promise_type) = browser_module::import_promise_type(vm) { - return browser_module::PyPromise::new_obj(promise_type, promise.clone()); + if vm.try_class("browser", "Promise").is_ok() { + return browser_module::PyPromise::new(promise.clone()) + .into_ref(vm) + .into_object(); } } if Array::is_array(&js_val) { @@ -179,18 +180,18 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { .cloned() .unwrap_or_else(|| js_val.unchecked_ref::().buffer()), ); - let mut vec = Vec::with_capacity(u8_array.length() as usize); - // TODO: use Uint8Array::copy_to once updating js_sys doesn't break everything - u8_array.for_each(&mut |byte, _, _| vec.push(byte)); + let mut vec = vec![0; u8_array.length() as usize]; + u8_array.copy_to(&mut vec); vm.ctx.new_bytes(vec) } else { - let dict = vm.new_dict(); + let dict = vm.ctx.new_dict(); for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item(&vm.ctx, &String::from(js_sys::JsString::from(key)), py_val); + dict.set_item(&String::from(js_sys::JsString::from(key)), py_val, vm) + .unwrap(); } - dict + dict.into_object() } } else if js_val.is_function() { let func = js_sys::Function::from(js_val); @@ -223,10 +224,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { // Because `JSON.stringify(undefined)` returns undefined vm.get_none() } else { - let json = match js_sys::JSON::stringify(&js_val) { - Ok(json) => String::from(json), - Err(_) => return vm.get_none(), - }; - vm.deserialize(&json).unwrap_or_else(|_| vm.get_none()) + py_serde::deserialize(vm, serde_wasm_bindgen::Deserializer::from(js_val)) + .unwrap_or_else(|_| vm.get_none()) } } diff --git a/wasm/lib/src/js_module.rs b/wasm/lib/src/js_module.rs new file mode 100644 index 0000000000..6db733d66e --- /dev/null +++ b/wasm/lib/src/js_module.rs @@ -0,0 +1,255 @@ +use js_sys::{Array, Object, Reflect}; +use rustpython_vm::function::Args; +use rustpython_vm::obj::{objfloat::PyFloatRef, objstr::PyStringRef, objtype::PyClassRef}; +use rustpython_vm::pyobject::{ + create_type, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, +}; +use rustpython_vm::VirtualMachine; +use wasm_bindgen::{prelude::*, JsCast}; + +#[wasm_bindgen(inline_js = " +export function has_prop(target, prop) { return prop in Object(target); } +export function get_prop(target, prop) { return target[prop]; } +export function set_prop(target, prop, value) { target[prop] = value; } +export function type_of(a) { return typeof a; } +export function instance_of(lhs, rhs) { return lhs instanceof rhs; } +")] +extern "C" { + #[wasm_bindgen(catch)] + fn has_prop(target: &JsValue, prop: &JsValue) -> Result; + #[wasm_bindgen(catch)] + fn get_prop(target: &JsValue, prop: &JsValue) -> Result; + #[wasm_bindgen(catch)] + fn set_prop(target: &JsValue, prop: &JsValue, value: &JsValue) -> Result<(), JsValue>; + #[wasm_bindgen] + fn type_of(a: &JsValue) -> String; + #[wasm_bindgen(catch)] + fn instance_of(lhs: &JsValue, rhs: &JsValue) -> Result; +} + +#[pyclass(name = "JsValue")] +#[derive(Debug)] +pub struct PyJsValue { + value: JsValue, +} +type PyJsValueRef = PyRef; + +impl PyValue for PyJsValue { + fn class(vm: &VirtualMachine) -> PyClassRef { + vm.class("_js", "JsValue") + } +} + +enum JsProperty { + Str(PyStringRef), + Js(PyJsValueRef), +} + +impl TryFromObject for JsProperty { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + PyStringRef::try_from_object(vm, obj.clone()) + .map(JsProperty::Str) + .or_else(|_| PyJsValueRef::try_from_object(vm, obj).map(JsProperty::Js)) + } +} + +impl JsProperty { + fn to_jsvalue(self) -> JsValue { + match self { + JsProperty::Str(s) => s.as_str().into(), + JsProperty::Js(value) => value.value.clone(), + } + } +} + +#[pyimpl] +impl PyJsValue { + #[inline] + pub fn new(value: impl Into) -> PyJsValue { + PyJsValue { + value: value.into(), + } + } + + #[pymethod] + fn null(&self, _vm: &VirtualMachine) -> PyJsValue { + PyJsValue::new(JsValue::NULL) + } + + #[pymethod] + fn undefined(&self, _vm: &VirtualMachine) -> PyJsValue { + PyJsValue::new(JsValue::UNDEFINED) + } + + #[pymethod] + fn new_from_str(&self, s: PyStringRef, _vm: &VirtualMachine) -> PyJsValue { + PyJsValue::new(s.as_str()) + } + + #[pymethod] + fn new_from_float(&self, n: PyFloatRef, _vm: &VirtualMachine) -> PyJsValue { + PyJsValue::new(n.to_f64()) + } + + #[pymethod] + fn new_object(&self, opts: NewObjectOptions, vm: &VirtualMachine) -> PyResult { + let value = if let Some(proto) = opts.prototype { + if let Some(proto) = proto.value.dyn_ref::() { + Object::create(proto) + } else if proto.value.is_null() { + Object::create(proto.value.unchecked_ref()) + } else { + return Err(vm.new_value_error(format!("prototype must be an Object or null"))); + } + } else { + Object::new() + }; + Ok(PyJsValue::new(value)) + } + + #[pymethod] + fn has_prop(&self, name: JsProperty, vm: &VirtualMachine) -> PyResult { + has_prop(&self.value, &name.to_jsvalue()).map_err(|err| new_js_error(vm, err)) + } + + #[pymethod] + fn get_prop(&self, name: JsProperty, vm: &VirtualMachine) -> PyResult { + let name = &name.to_jsvalue(); + if has_prop(&self.value, name).map_err(|err| new_js_error(vm, err))? { + get_prop(&self.value, name) + .map(PyJsValue::new) + .map_err(|err| new_js_error(vm, err)) + } else { + Err(vm.new_attribute_error(format!("No attribute {:?} on JS value", name))) + } + } + + #[pymethod] + fn set_prop(&self, name: JsProperty, value: PyJsValueRef, vm: &VirtualMachine) -> PyResult<()> { + set_prop(&self.value, &name.to_jsvalue(), &value.value).map_err(|err| new_js_error(vm, err)) + } + + #[pymethod] + fn call( + &self, + args: Args, + opts: CallOptions, + vm: &VirtualMachine, + ) -> PyResult { + let func = self + .value + .dyn_ref::() + .ok_or_else(|| vm.new_type_error("JS value is not callable".to_string()))?; + let this = opts + .this + .map(|this| this.value.clone()) + .unwrap_or(JsValue::UNDEFINED); + let js_args = Array::new(); + for arg in args { + js_args.push(&arg.value); + } + Reflect::apply(func, &this, &js_args) + .map(PyJsValue::new) + .map_err(|err| new_js_error(vm, err)) + } + + #[pymethod] + fn construct( + &self, + args: Args, + opts: NewObjectOptions, + vm: &VirtualMachine, + ) -> PyResult { + let ctor = self + .value + .dyn_ref::() + .ok_or_else(|| vm.new_type_error("JS value is not callable".to_string()))?; + let proto = opts + .prototype + .as_ref() + .and_then(|proto| proto.value.dyn_ref::().clone()); + let js_args = Array::new(); + for arg in args { + js_args.push(&arg.value); + } + let constructed_result = if let Some(proto) = proto { + Reflect::construct_with_new_target(ctor, &js_args, &proto) + } else { + Reflect::construct(ctor, &js_args) + }; + + constructed_result + .map(PyJsValue::new) + .map_err(|err| new_js_error(vm, err)) + } + + #[pymethod] + fn as_str(&self, _vm: &VirtualMachine) -> Option { + self.value.as_string() + } + + #[pymethod] + fn as_float(&self, _vm: &VirtualMachine) -> Option { + self.value.as_f64() + } + + #[pymethod] + fn as_bool(&self, _vm: &VirtualMachine) -> Option { + self.value.as_bool() + } + + #[pymethod(name = "typeof")] + fn type_of(&self, _vm: &VirtualMachine) -> String { + type_of(&self.value) + } + + #[pymethod] + /// Checks that `typeof self == "object" && self !== null`. Use instead + /// of `value.typeof() == "object"` + fn is_object(&self, _vm: &VirtualMachine) -> bool { + self.value.is_object() + } + + #[pymethod] + fn instanceof(&self, rhs: PyJsValueRef, vm: &VirtualMachine) -> PyResult { + instance_of(&self.value, &rhs.value).map_err(|err| new_js_error(vm, err)) + } + + #[pymethod(name = "__repr__")] + fn repr(&self, _vm: &VirtualMachine) -> String { + format!("{:?}", self.value) + } +} + +#[derive(FromArgs)] +struct CallOptions { + #[pyarg(keyword_only, default = "None")] + this: Option, +} + +#[derive(FromArgs)] +struct NewObjectOptions { + #[pyarg(keyword_only, default = "None")] + prototype: Option, +} + +fn new_js_error(vm: &VirtualMachine, err: JsValue) -> PyObjectRef { + let exc = vm.new_exception(vm.class("_js", "JsError"), format!("{:?}", err)); + vm.set_attr(&exc, "js_value", PyJsValue::new(err).into_ref(vm)) + .unwrap(); + exc +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + py_module!(vm, "_js", { + "JsError" => create_type("JsError", &ctx.type_type, &ctx.exceptions.exception_type), + "JsValue" => PyJsValue::make_class(ctx), + }) +} + +pub fn setup_js_module(vm: &VirtualMachine) { + vm.stdlib_inits + .borrow_mut() + .insert("_js".to_string(), Box::new(make_module)); +} diff --git a/wasm/lib/src/lib.rs b/wasm/lib/src/lib.rs index 35d4595199..52049ee076 100644 --- a/wasm/lib/src/lib.rs +++ b/wasm/lib/src/lib.rs @@ -1,5 +1,6 @@ pub mod browser_module; pub mod convert; +pub mod js_module; pub mod vm_class; pub mod wasm_builtins; diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 9d80854f73..1437af4668 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -5,40 +5,51 @@ use std::rc::{Rc, Weak}; use js_sys::{Object, Reflect, SyntaxError, TypeError}; use wasm_bindgen::prelude::*; -use rustpython_vm::compile; +use rustpython_compiler::{compile, error::CompileErrorType}; use rustpython_vm::frame::{NameProtocol, Scope}; use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::pyobject::{PyContext, PyObjectRef, PyResult}; +use rustpython_vm::import; +use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue}; use rustpython_vm::VirtualMachine; use crate::browser_module::setup_browser_module; use crate::convert; +use crate::js_module; use crate::wasm_builtins; -pub trait HeldRcInner {} - -impl HeldRcInner for T {} - pub(crate) struct StoredVirtualMachine { pub vm: VirtualMachine, pub scope: RefCell, /// you can put a Rc in here, keep it as a Weak, and it'll be held only for /// as long as the StoredVM is alive - held_rcs: RefCell>>, + held_objects: RefCell>, } impl StoredVirtualMachine { fn new(id: String, inject_browser_module: bool) -> StoredVirtualMachine { let mut vm = VirtualMachine::new(); - let scope = vm.ctx.new_scope(); + vm.wasm_id = Some(id); + let scope = vm.new_scope_with_builtins(); + + js_module::setup_js_module(&vm); if inject_browser_module { + vm.stdlib_inits.borrow_mut().insert( + "_window".to_string(), + Box::new(|vm| { + py_module!(vm, "_window", { + "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(vm), + }) + }), + ); setup_browser_module(&vm); } - vm.wasm_id = Some(id); + + import::init_importlib(&vm, false); + StoredVirtualMachine { vm, scope: RefCell::new(scope), - held_rcs: RefCell::new(Vec::new()), + held_objects: RefCell::new(Vec::new()), } } } @@ -47,9 +58,26 @@ impl StoredVirtualMachine { // probably gets compiled down to a normal-ish static varible, like Atomic* types do: // https://rustwasm.github.io/2018/10/24/multithreading-rust-and-wasm.html#atomic-instructions thread_local! { - static STORED_VMS: RefCell>> = - RefCell::default(); - static ACTIVE_VMS: RefCell> = RefCell::default(); + static STORED_VMS: RefCell>> = RefCell::default(); +} + +pub fn get_vm_id(vm: &VirtualMachine) -> &str { + vm.wasm_id + .as_ref() + .expect("VirtualMachine inside of WASM crate should have wasm_id set") +} +pub(crate) fn stored_vm_from_wasm(wasm_vm: &WASMVirtualMachine) -> Rc { + STORED_VMS.with(|cell| { + cell.borrow() + .get(&wasm_vm.id) + .expect("VirtualMachine is not valid") + .clone() + }) +} +pub(crate) fn weak_vm(vm: &VirtualMachine) -> Weak { + let id = get_vm_id(vm); + STORED_VMS + .with(|cell| Rc::downgrade(cell.borrow().get(id).expect("VirtualMachine is not valid"))) } #[wasm_bindgen(js_name = vmStore)] @@ -109,44 +137,6 @@ impl VMStore { } } -#[derive(Clone)] -pub(crate) struct AccessibleVM { - weak: Weak, - id: String, -} - -impl AccessibleVM { - pub fn from_id(id: String) -> AccessibleVM { - let weak = STORED_VMS - .with(|cell| Rc::downgrade(cell.borrow().get(&id).expect("WASM VM to be valid"))); - AccessibleVM { weak, id } - } - - pub fn upgrade(&self) -> Option> { - self.weak.upgrade() - } -} - -impl From for AccessibleVM { - fn from(vm: WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id) - } -} -impl From<&WASMVirtualMachine> for AccessibleVM { - fn from(vm: &WASMVirtualMachine) -> AccessibleVM { - AccessibleVM::from_id(vm.id.clone()) - } -} -impl From<&VirtualMachine> for AccessibleVM { - fn from(vm: &VirtualMachine) -> AccessibleVM { - AccessibleVM::from_id( - vm.wasm_id - .clone() - .expect("VM passed to from::() to have wasm_id be Some()"), - ) - } -} - #[wasm_bindgen(js_name = VirtualMachine)] #[derive(Clone)] pub struct WASMVirtualMachine { @@ -178,13 +168,13 @@ impl WASMVirtualMachine { STORED_VMS.with(|cell| cell.borrow().contains_key(&self.id)) } - pub(crate) fn push_held_rc( + pub(crate) fn push_held_rc( &self, - rc: Rc, - ) -> Result, JsValue> { + obj: PyObjectRef, + ) -> Result>, JsValue> { self.with(|stored_vm| { - let weak = Rc::downgrade(&rc); - stored_vm.held_rcs.borrow_mut().push(rc); + let weak = Rc::downgrade(&obj); + stored_vm.held_objects.borrow_mut().push(obj); weak }) } @@ -224,15 +214,15 @@ impl WASMVirtualMachine { fn error() -> JsValue { TypeError::new("Unknown stdout option, please pass a function or 'console'").into() } - let print_fn: Box PyResult> = - if let Some(s) = stdout.as_string() { - match s.as_str() { - "console" => Box::new(wasm_builtins::builtin_print_console), - _ => return Err(error()), - } - } else if stdout.is_function() { - let func = js_sys::Function::from(stdout); - Box::new(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { + let print_fn: PyObjectRef = if let Some(s) = stdout.as_string() { + match s.as_str() { + "console" => vm.ctx.new_rustfunc(wasm_builtins::builtin_print_console), + _ => return Err(error()), + } + } else if stdout.is_function() { + let func = js_sys::Function::from(stdout); + vm.ctx + .new_rustfunc(move |vm: &VirtualMachine, args: PyFuncArgs| -> PyResult { func.call1( &JsValue::UNDEFINED, &wasm_builtins::format_print_args(vm, args)?.into(), @@ -240,16 +230,15 @@ impl WASMVirtualMachine { .map_err(|err| convert::js_to_py(vm, err))?; Ok(vm.get_none()) }) - } else if stdout.is_undefined() || stdout.is_null() { - fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { - Ok(vm.get_none()) - } - Box::new(noop) - } else { - return Err(error()); - }; - let rustfunc = vm.ctx.new_rustfunc(print_fn); - vm.ctx.set_attr(&vm.builtins, "print", rustfunc); + } else if stdout.is_undefined() || stdout.is_null() { + fn noop(vm: &VirtualMachine, _args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) + } + vm.ctx.new_rustfunc(noop) + } else { + return Err(error()); + }; + vm.set_attr(&vm.builtins, "print", print_fn).unwrap(); Ok(()) })? } @@ -266,12 +255,12 @@ impl WASMVirtualMachine { let mod_name = name.clone(); - let stdlib_init_fn = move |ctx: &PyContext| { - let py_mod = ctx.new_module(&name, ctx.new_dict()); + let stdlib_init_fn = move |vm: &VirtualMachine| { + let module = vm.ctx.new_module(&name, vm.ctx.new_dict()); for (key, value) in module_items.clone() { - ctx.set_attr(&py_mod, &key, value); + vm.set_attr(&module, key, value).unwrap(); } - py_mod + module }; vm.stdlib_inits @@ -289,40 +278,30 @@ impl WASMVirtualMachine { ref vm, ref scope, .. }| { source.push('\n'); - let code = - compile::compile(&source, &mode, "".to_string(), vm.ctx.code_type()); + let code = vm.compile(&source, &mode, "".to_string()); let code = code.map_err(|err| { let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err)); - if let rustpython_vm::error::CompileError::Parse(ref parse_error) = err { + if let CompileErrorType::Parse(ref parse_error) = err.error { use rustpython_parser::error::ParseError; if let ParseError::EOF(Some(ref loc)) | ParseError::ExtraToken((ref loc, ..)) | ParseError::InvalidToken(ref loc) | ParseError::UnrecognizedToken((ref loc, ..), _) = parse_error { - let _ = Reflect::set( - &js_err, - &"row".into(), - &(loc.get_row() as u32).into(), - ); - let _ = Reflect::set( - &js_err, - &"col".into(), - &(loc.get_column() as u32).into(), - ); + let _ = + Reflect::set(&js_err, &"row".into(), &(loc.row() as u32).into()); + let _ = + Reflect::set(&js_err, &"col".into(), &(loc.column() as u32).into()); } if let ParseError::ExtraToken((_, _, ref loc)) | ParseError::UnrecognizedToken((_, _, ref loc), _) = parse_error { - let _ = Reflect::set( - &js_err, - &"endrow".into(), - &(loc.get_row() as u32).into(), - ); + let _ = + Reflect::set(&js_err, &"endrow".into(), &(loc.row() as u32).into()); let _ = Reflect::set( &js_err, &"endcol".into(), - &(loc.get_column() as u32).into(), + &(loc.column() as u32).into(), ); } } diff --git a/wasm/lib/src/wasm_builtins.rs b/wasm/lib/src/wasm_builtins.rs index d5bc830dfe..e2fed62c2b 100644 --- a/wasm/lib/src/wasm_builtins.rs +++ b/wasm/lib/src/wasm_builtins.rs @@ -7,9 +7,12 @@ use js_sys::{self, Array}; use web_sys::{self, console}; -use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::obj::{objstr, objtype}; -use rustpython_vm::pyobject::{IdProtocol, PyObjectRef, PyResult, TypeProtocol}; +use rustpython_vm::function::{Args, KwArgs, PyFuncArgs}; +use rustpython_vm::obj::{ + objstr::{self, PyStringRef}, + objtype, +}; +use rustpython_vm::pyobject::{IdProtocol, ItemProtocol, PyObjectRef, PyResult, TypeProtocol}; use rustpython_vm::VirtualMachine; pub(crate) fn window() -> web_sys::Window { @@ -25,7 +28,7 @@ pub fn format_print_args(vm: &VirtualMachine, args: PyFuncArgs) -> Result Result '2.7'", - "version": "==6.0.0" + "version": "==7.0.0" }, "pluggy": { "hashes": [ @@ -54,11 +54,11 @@ }, "pytest": { "hashes": [ - "sha256:592eaa2c33fae68c7d75aacf042efc9f77b27c08a6224a4f59beab8d9a420523", - "sha256:ad3ad5c450284819ecde191a654c09b0ec72257a2c711b9633d677c71c9850c4" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.3.1" + "version": "==4.4.1" }, "selenium": { "hashes": [ @@ -77,10 +77,10 @@ }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:4c291ca23bbb55c76518905869ef34bdd5f0e46af7afe6861e8375643ffee1a0", + "sha256:9a247273df709c4fedb38c711e44292304f73f39ab01beda9f6b9fc375669ac3" ], - "version": "==1.24.1" + "version": "==1.24.2" } }, "develop": {} diff --git a/wasm/tests/test_demo.py b/wasm/tests/test_demo.py index 9dce406811..a9694f0716 100644 --- a/wasm/tests/test_demo.py +++ b/wasm/tests/test_demo.py @@ -1,4 +1,5 @@ import time +import sys from selenium import webdriver from selenium.webdriver.firefox.options import Options @@ -16,12 +17,23 @@ return output; """ +def print_stack(driver): + stack = driver.execute_script( + "return window.__RUSTPYTHON_ERROR_MSG + '\\n' + window.__RUSTPYTHON_ERROR_STACK" + ) + print(f"RustPython error stack:\n{stack}", file=sys.stderr) + + @pytest.fixture(scope="module") def driver(request): options = Options() options.add_argument('-headless') driver = webdriver.Firefox(options=options) - driver.get("http://localhost:8080") + try: + driver.get("http://localhost:8080") + except Exception as e: + print_stack(driver) + raise time.sleep(5) yield driver driver.close() @@ -35,4 +47,9 @@ def driver(request): ) def test_demo(driver, script, output): script = RUN_CODE_TEMPLATE.format(script) - assert driver.execute_script(script).strip() == output + try: + script_output = driver.execute_script(script) + except Exception as e: + print_stack(driver) + raise + assert script_output.strip() == output diff --git a/whats_left.sh b/whats_left.sh new file mode 100755 index 0000000000..84ae3a03ca --- /dev/null +++ b/whats_left.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +ALL_SECTIONS=(methods modules) + +GREEN='' +BOLD='' +NC='(B' + +h() { + # uppercase input + header_name=$(echo "$@" | tr "[:lower:]" "[:upper:]") + echo "$GREEN$BOLD===== $header_name =====$NC" +} + +cd "$(dirname "$0")" + +( + cd tests + # -I means isolate from environment; we don't want any pip packages to be listed + python3 -I not_impl_gen.py +) + +# show the building first, so people aren't confused why it's taking so long to +# run whats_left_to_implement +cargo build --release + +if [ $# -eq 0 ]; then + sections=(${ALL_SECTIONS[@]}) +else + sections=($@) +fi + +for section in "${sections[@]}"; do + section=$(echo "$section" | tr "[:upper:]" "[:lower:]") + snippet=tests/snippets/whats_left_$section.py + if ! [[ -f $snippet ]]; then + echo "Invalid section $section" >&2 + continue + fi + h "$section" >&2 + cargo run --release -q -- "$snippet" +done