diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fa6d58bbb7..24993639945 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,7 +153,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - name: Start anvil - run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --port 3021 & + run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --timestamp 1743944919 --port 3021 & - name: Install graph CLI run: curl -sSL http://cli.thegraph.com/install.sh | sudo bash diff --git a/Cargo.lock b/Cargo.lock index b2ae24e0169..33815c70807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ dependencies = [ "cfg-if 1.0.0", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449" +checksum = "bfff2b17d272a5e3e201feda444e2c24b011fa722951268d1bd8b9b5bc6dc449" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -203,7 +203,6 @@ dependencies = [ "mime", "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", @@ -211,33 +210,31 @@ dependencies = [ "serde_urlencoded", "static_assertions_next", "tempfile", - "thiserror", - "uuid", + "thiserror 1.0.61", ] [[package]] name = "async-graphql-axum" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aa80e171205c6d562057fd5a49167c8fbe61f7db2bed6540f6d4f2234d7ff2" +checksum = "6bf2882c816094fef6e39d381b8e9b710e5943e7bdef5198496441d5083164fa" dependencies = [ "async-graphql", - "async-trait", - "axum 0.7.5", + "axum 0.8.1", "bytes", "futures-util", "serde_json", "tokio", "tokio-stream", "tokio-util 0.7.11", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", ] [[package]] name = "async-graphql-derive" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f" +checksum = "d8e5d0c6697def2f79ccbd972fb106b633173a6066e430b480e1ff9376a7561a" dependencies = [ "Inflector", "async-graphql-parser", @@ -246,15 +243,15 @@ dependencies = [ "proc-macro2", "quote", "strum", - "syn 2.0.69", - "thiserror", + "syn 2.0.87", + "thiserror 1.0.61", ] [[package]] name = "async-graphql-parser" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad" +checksum = "8531ee6d292c26df31c18c565ff22371e7bdfffe7f5e62b69537db0b8fd554dc" dependencies = [ "async-graphql-value", "pest", @@ -264,9 +261,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" +checksum = "741110dda927420a28fbc1c310543d3416f789a6ba96859c2c265843a0a96887" dependencies = [ "bytes", "indexmap 2.2.6", @@ -282,14 +279,14 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -298,13 +295,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -315,7 +312,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -338,50 +335,49 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.6.20" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", + "axum-core 0.4.3", "bytes", "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.29", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", "pin-project-lite", "rustversion", "serde", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", ] [[package]] name = "axum" -version = "0.7.5" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" dependencies = [ - "async-trait", - "axum-core 0.4.3", - "base64 0.21.7", + "axum-core 0.5.0", + "base64 0.22.1", "bytes", + "form_urlencoded", "futures-util", "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -394,37 +390,39 @@ dependencies = [ "sha1", "sync_wrapper 1.0.1", "tokio", - "tokio-tungstenite 0.21.0", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tungstenite", + "tower 0.5.2", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] [[package]] name = "axum-core" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.12", - "http-body 0.4.6", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", "mime", + "pin-project-lite", "rustversion", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sync_wrapper 0.1.2", + "tower-layer 0.3.3", + "tower-service 0.3.3", ] [[package]] name = "axum-core" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" dependencies = [ - "async-trait", "bytes", "futures-util", "http 1.1.0", @@ -433,9 +431,9 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sync_wrapper 1.0.1", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] @@ -569,15 +567,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "675f87afced0413c9bb02843499dbbd3882a237645883f71a2b59644a6d2f753" dependencies = [ "arrayref", "arrayvec 0.7.4", "cc", "cfg-if 1.0.0", - "constant_time_eq 0.3.0", + "constant_time_eq 0.3.1", ] [[package]] @@ -643,22 +641,22 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] [[package]] name = "cc" -version = "1.0.105" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ "jobserver", "libc", - "once_cell", + "shlex", ] [[package]] @@ -731,7 +729,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -783,9 +781,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "convert_case" @@ -803,11 +801,21 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -1078,7 +1086,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1089,7 +1097,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1182,14 +1190,14 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] name = "diesel" -version = "2.2.4" +version = "2.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe8e2e68695bd615d7e4f3227c0727b151330d3e253b525086c348d055d5e" +checksum = "04001f23ba8843dc315804fa324000376084dfb1c30794ff68dd279e6e5696d5" dependencies = [ "bigdecimal 0.3.1", "bitflags 2.6.0", @@ -1203,7 +1211,6 @@ dependencies = [ "pq-sys", "r2d2", "serde_json", - "uuid", ] [[package]] @@ -1215,29 +1222,29 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] name = "diesel-dynamic-schema" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71eda9b13a55533594231b0763c36bc21058ccb82ed17eaeb41b3cbb897c1bb1" +checksum = "061bbe2d02508364c50153226524b7fc224f56031a5e927b0bc5f1f2b48de6a6" dependencies = [ "diesel", ] [[package]] name = "diesel_derives" -version = "2.2.1" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59de76a222c2b8059f789cbe07afbfd8deb8c31dd0bc2a21f85e256c1def8259" +checksum = "e7f2c3de51e2ba6bf2a648285696137aaf0f5f487bcbea93972fe8a364e131a4" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1257,7 +1264,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1344,6 +1351,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "dsl_auto_type" version = "0.1.1" @@ -1355,7 +1373,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1451,7 +1469,7 @@ dependencies = [ "serde", "serde_json", "sha3", - "thiserror", + "thiserror 1.0.61", "uint 0.9.5", ] @@ -1528,7 +1546,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ "byteorder", - "rand", + "rand 0.8.5", "rustc-hex", "static_assertions", ] @@ -1647,7 +1665,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1727,7 +1745,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", ] [[package]] @@ -1765,7 +1795,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "time", ] @@ -1815,7 +1845,7 @@ dependencies = [ "http 1.1.0", "http-body-util", "humantime", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "isatty", "itertools 0.13.0", @@ -1830,9 +1860,9 @@ dependencies = [ "petgraph", "priority-queue", "prometheus", - "prost 0.12.6", - "prost-types 0.12.6", - "rand", + "prost", + "prost-types", + "rand 0.8.5", "regex", "reqwest", "semver", @@ -1842,6 +1872,7 @@ dependencies = [ "serde_plain", "serde_regex", "serde_yaml", + "sha2", "slog", "slog-async", "slog-envlogger", @@ -1850,7 +1881,7 @@ dependencies = [ "stable-hash 0.3.4", "stable-hash 0.4.4", "strum_macros", - "thiserror", + "thiserror 1.0.61", "tiny-keccak 1.5.0", "tokio", "tokio-retry", @@ -1873,8 +1904,8 @@ dependencies = [ "graph", "graph-runtime-derive", "graph-runtime-wasm", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "serde", "sha2", "tonic-build", @@ -1886,26 +1917,10 @@ version = "0.36.0" dependencies = [ "anyhow", "heck 0.5.0", - "protobuf 3.5.0", + "protobuf 3.7.1", "protobuf-parse", ] -[[package]] -name = "graph-chain-cosmos" -version = "0.36.0" -dependencies = [ - "anyhow", - "graph", - "graph-chain-common", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost 0.12.6", - "prost-types 0.12.6", - "semver", - "serde", - "tonic-build", -] - [[package]] name = "graph-chain-ethereum" version = "0.36.0" @@ -1919,13 +1934,12 @@ dependencies = [ "hex", "itertools 0.13.0", "jsonrpc-core", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "semver", "serde", "tiny-keccak 1.5.0", "tonic-build", - "uuid", ] [[package]] @@ -1937,8 +1951,8 @@ dependencies = [ "graph", "graph-runtime-derive", "graph-runtime-wasm", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "serde", "tonic-build", "trigger-filters", @@ -1954,8 +1968,8 @@ dependencies = [ "graph-runtime-wasm", "hex", "lazy_static", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "semver", "serde", "tokio", @@ -1973,7 +1987,6 @@ dependencies = [ "cid", "graph", "graph-chain-arweave", - "graph-chain-cosmos", "graph-chain-ethereum", "graph-chain-near", "graph-chain-substreams", @@ -1981,7 +1994,6 @@ dependencies = [ "serde_yaml", "tower 0.4.13 (git+https://github.com/tower-rs/tower.git)", "tower-test", - "uuid", "wiremock", ] @@ -2011,7 +2023,6 @@ dependencies = [ "git-testament", "graph", "graph-chain-arweave", - "graph-chain-cosmos", "graph-chain-ethereum", "graph-chain-near", "graph-chain-substreams", @@ -2021,7 +2032,6 @@ dependencies = [ "graph-server-index-node", "graph-server-json-rpc", "graph-server-metrics", - "graph-server-websocket", "graph-store-postgres", "graphman", "graphman-server", @@ -2042,7 +2052,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -2053,7 +2063,7 @@ dependencies = [ "graph-chain-ethereum", "graph-runtime-derive", "graph-runtime-wasm", - "rand", + "rand 0.8.5", "semver", "test-store", "wasmtime", @@ -2074,7 +2084,6 @@ dependencies = [ "parity-wasm", "semver", "serde_yaml", - "uuid", "wasm-instrument", "wasmtime", ] @@ -2093,11 +2102,10 @@ dependencies = [ name = "graph-server-index-node" version = "0.36.0" dependencies = [ - "blake3 1.5.1", + "blake3 1.6.1", "git-testament", "graph", "graph-chain-arweave", - "graph-chain-cosmos", "graph-chain-ethereum", "graph-chain-near", "graph-chain-substreams", @@ -2120,17 +2128,6 @@ dependencies = [ "graph", ] -[[package]] -name = "graph-server-websocket" -version = "0.36.0" -dependencies = [ - "graph", - "serde", - "serde_derive", - "tokio-tungstenite 0.23.1", - "uuid", -] - [[package]] name = "graph-store-postgres" version = "0.36.0" @@ -2138,7 +2135,7 @@ dependencies = [ "Inflector", "anyhow", "async-trait", - "blake3 1.5.1", + "blake3 1.6.1", "chrono", "clap", "derive_more", @@ -2161,11 +2158,10 @@ dependencies = [ "postgres", "postgres-openssl", "pretty_assertions", - "rand", + "rand 0.8.5", "serde", "serde_json", "stable-hash 0.3.4", - "uuid", ] [[package]] @@ -2200,7 +2196,7 @@ dependencies = [ "proc-macro-utils", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -2213,7 +2209,7 @@ dependencies = [ "graph-store-postgres", "graphman-store", "itertools 0.13.0", - "thiserror", + "thiserror 1.0.61", "tokio", ] @@ -2224,7 +2220,7 @@ dependencies = [ "anyhow", "async-graphql", "async-graphql-axum", - "axum 0.7.5", + "axum 0.8.1", "chrono", "diesel", "graph", @@ -2237,7 +2233,7 @@ dependencies = [ "serde_json", "slog", "test-store", - "thiserror", + "thiserror 1.0.61", "tokio", "tower-http", ] @@ -2259,7 +2255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" dependencies = [ "combine", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -2324,7 +2320,7 @@ dependencies = [ "pest_derive", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -2532,16 +2528,16 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", "tracing", "want", ] [[package]] name = "hyper" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -2566,26 +2562,27 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", - "rustls 0.23.10", - "rustls-native-certs", + "rustls", + "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls", + "tower-service 0.3.3", ] [[package]] name = "hyper-timeout" -version = "0.4.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 0.14.29", + "hyper 1.6.0", + "hyper-util", "pin-project-lite", "tokio", - "tokio-io-timeout", + "tower-service 0.3.3", ] [[package]] @@ -2596,31 +2593,30 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "native-tls", "tokio", "tokio-native-tls", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", ] [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", "tracing", ] @@ -2655,10 +2651,128 @@ checksum = "d1fcc7f316b2c079dde77564a1360639c1a956a23fa96122732e416cb10717bb" dependencies = [ "cfg-if 1.0.0", "num-traits", - "rand", + "rand 0.8.5", "static_assertions", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "id-arena" version = "2.2.1" @@ -2684,12 +2798,23 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -2905,11 +3030,11 @@ dependencies = [ "jsonrpsee-types", "lazy_static", "parking_lot", - "rand", + "rand 0.8.5", "rustc-hash 1.1.0", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tokio", "tracing", "unicase", @@ -2943,7 +3068,7 @@ dependencies = [ "beef", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tracing", ] @@ -2990,6 +3115,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + [[package]] name = "lock_api" version = "0.4.12" @@ -3039,6 +3170,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "maybe-owned" version = "0.3.4" @@ -3132,7 +3269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3174,12 +3311,6 @@ dependencies = [ "unsigned-varint 0.7.2", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "multimap" version = "0.10.0" @@ -3198,7 +3329,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.0", "security-framework-sys", "tempfile", ] @@ -3298,12 +3429,12 @@ dependencies = [ "chrono", "futures 0.3.30", "humantime", - "hyper 1.4.0", + "hyper 1.6.0", "itertools 0.13.0", "parking_lot", "percent-encoding", "quick-xml", - "rand", + "rand 0.8.5", "reqwest", "ring", "rustls-pemfile", @@ -3330,9 +3461,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.66" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.6.0", "cfg-if 1.0.0", @@ -3351,7 +3482,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3362,9 +3493,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", @@ -3461,7 +3592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", - "thiserror", + "thiserror 1.0.61", "ucd-trie", ] @@ -3485,7 +3616,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3544,7 +3675,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3605,7 +3736,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand", + "rand 0.8.5", "sha2", "stringprep", ] @@ -3652,16 +3783,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.20" @@ -3669,7 +3790,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3739,114 +3860,59 @@ dependencies = [ "parking_lot", "protobuf 2.28.0", "reqwest", - "thiserror", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", + "thiserror 1.0.61", ] [[package]] name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive 0.12.6", -] - -[[package]] -name = "prost-build" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.10.5", - "lazy_static", - "log", - "multimap 0.8.3", - "petgraph", - "prettyplease 0.1.25", - "prost 0.11.9", - "prost-types 0.11.9", - "regex", - "syn 1.0.109", - "tempfile", - "which", + "prost-derive", ] [[package]] name = "prost-build" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "bytes", "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.13.0", "log", - "multimap 0.10.0", + "multimap", "once_cell", "petgraph", - "prettyplease 0.2.20", - "prost 0.12.6", - "prost-types 0.12.6", + "prettyplease", + "prost", + "prost-types", "regex", - "syn 2.0.69", + "syn 2.0.87", "tempfile", ] [[package]] name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-derive" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.69", -] - -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", + "syn 2.0.87", ] [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost 0.12.6", + "prost", ] [[package]] @@ -3857,38 +3923,38 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "protobuf" -version = "3.5.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df67496db1a89596beaced1579212e9b7c53c22dca1d9745de00ead76573d514" +checksum = "a3a7c64d9bf75b1b8d981124c14c179074e8caa7dfe7b6a12e6222ddcd0c8f72" dependencies = [ "once_cell", "protobuf-support", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "protobuf-parse" -version = "3.5.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a16027030d4ec33e423385f73bb559821827e9ec18c50e7874e4d6de5a4e96f" +checksum = "322330e133eab455718444b4e033ebfac7c6528972c784fcde28d2cc783c6257" dependencies = [ "anyhow", "indexmap 2.2.6", "log", - "protobuf 3.5.0", + "protobuf 3.7.1", "protobuf-support", "tempfile", - "thiserror", + "thiserror 1.0.61", "which", ] [[package]] name = "protobuf-support" -version = "3.5.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e2d30ab1878b2e72d1e2fc23ff5517799c9929e2cf81a8516f9f4dcf2b9cf3" +checksum = "b088fd20b938a875ea00843b6faf48579462630015c3788d397ad6a786663252" dependencies = [ - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3921,8 +3987,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 1.1.0", - "rustls 0.23.10", - "thiserror", + "rustls", + "thiserror 1.0.61", "tokio", "tracing", ] @@ -3934,12 +4000,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring", "rustc-hash 2.0.0", - "rustls 0.23.10", + "rustls", "slab", - "thiserror", + "thiserror 1.0.61", "tinyvec", "tracing", ] @@ -3990,18 +4056,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", ] [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "rand" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.21", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -4010,7 +4097,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.1", ] [[package]] @@ -4063,9 +4159,9 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -4126,7 +4222,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-rustls", "hyper-tls", "hyper-util", @@ -4140,8 +4236,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.10", - "rustls-native-certs", + "rustls", + "rustls-native-certs 0.7.1", "rustls-pemfile", "rustls-pki-types", "serde", @@ -4151,9 +4247,9 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.0", + "tokio-rustls", "tokio-util 0.7.11", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -4164,15 +4260,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if 1.0.0", - "getrandom", + "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -4235,11 +4330,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" dependencies = [ "log", + "once_cell", "ring", "rustls-pki-types", "rustls-webpki", @@ -4248,30 +4344,28 @@ dependencies = [ ] [[package]] -name = "rustls" -version = "0.23.10" +name = "rustls-native-certs" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ - "once_cell", - "ring", + "openssl-probe", + "rustls-pemfile", "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", + "schannel", + "security-framework 2.11.0", ] [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.2.0", ] [[package]] @@ -4286,9 +4380,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] name = "rustls-webpki" @@ -4371,7 +4465,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -4379,9 +4486,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -4413,7 +4520,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4502,7 +4609,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4572,6 +4679,12 @@ dependencies = [ "dirs", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -4694,7 +4807,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4718,7 +4831,7 @@ dependencies = [ "futures 0.3.30", "httparse", "log", - "rand", + "rand 0.8.5", "sha-1", ] @@ -4825,14 +4938,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] name = "substreams" -version = "0.5.20" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392f77309a4e36d7839d0552a38557b53894200aba239f3d0725ec167ebf4297" +checksum = "5bb63116b90d4c174114fb237a8916dd995c939874f7576333990a44d78b642a" dependencies = [ "anyhow", "bigdecimal 0.3.1", @@ -4844,22 +4957,22 @@ dependencies = [ "pad", "pest", "pest_derive", - "prost 0.11.9", - "prost-build 0.11.9", - "prost-types 0.11.9", + "prost", + "prost-build", + "prost-types", "substreams-macro", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "substreams-entity-change" -version = "1.3.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2c7fca123abff659d15ed30da5b605fa954a29e912c94260c488d0d18f9107d" +checksum = "0587b8d5dd7bffb0415d544c31e742c4cabdb81bbe9a3abfffff125185e4e9e8" dependencies = [ "base64 0.13.1", - "prost 0.11.9", - "prost-types 0.11.9", + "prost", + "prost-types", "substreams", ] @@ -4869,26 +4982,28 @@ version = "0.36.0" [[package]] name = "substreams-macro" -version = "0.5.20" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc7137347f05d26c7007dced97b4caef67a13b3d422789d969fe6e4cd8cc4a" +checksum = "f36f36e9da94db29f49daf3ab6b47b529b57c43fc5d58bc35b160aaad1a7233f" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "substreams-near-core" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9922f437e6cb86b62cfd8bdede93937def710616ac2825ffff06b8770bbd06df" +checksum = "01ef8a763c5a5604b16f4898ab75d39494ef785c457aaca1fd7761b299f40fbf" dependencies = [ "bs58 0.4.0", - "prost 0.11.9", - "prost-build 0.11.9", - "prost-types 0.11.9", + "getrandom 0.2.15", + "hex", + "prost", + "prost-build", + "prost-types", ] [[package]] @@ -4896,7 +5011,7 @@ name = "substreams-trigger-filter" version = "0.36.0" dependencies = [ "hex", - "prost 0.11.9", + "prost", "substreams", "substreams-entity-change", "substreams-near-core", @@ -4923,9 +5038,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.69" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -4944,6 +5059,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -4951,7 +5077,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -5039,7 +5165,7 @@ dependencies = [ "hex-literal 0.4.1", "lazy_static", "pretty_assertions", - "prost-types 0.12.6", + "prost-types", ] [[package]] @@ -5048,7 +5174,16 @@ version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.61", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -5059,7 +5194,18 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] @@ -5121,6 +5267,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.7.0" @@ -5155,16 +5311,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.3.0" @@ -5173,7 +5319,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5217,7 +5363,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand", + "rand 0.8.5", "socket2", "tokio", "tokio-util 0.7.11", @@ -5231,18 +5377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" dependencies = [ "pin-project", - "rand", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", + "rand 0.8.5", "tokio", ] @@ -5252,16 +5387,16 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls", "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -5284,26 +5419,14 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.21.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.21.0", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.23.0", + "tungstenite", ] [[package]] @@ -5391,47 +5514,50 @@ dependencies = [ [[package]] name = "tonic" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum 0.6.20", - "base64 0.21.7", + "axum 0.7.5", + "base64 0.22.1", "bytes", "flate2", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.29", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.6.0", "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", - "prost 0.12.6", - "rustls-native-certs", + "prost", + "rustls-native-certs 0.8.1", "rustls-pemfile", - "rustls-pki-types", + "socket2", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls", "tokio-stream", "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] [[package]] name = "tonic-build" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ - "prettyplease 0.2.20", + "prettyplease", "proc-macro2", - "prost-build 0.12.6", + "prost-build", + "prost-types", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5445,12 +5571,12 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util 0.7.11", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] @@ -5468,8 +5594,24 @@ dependencies = [ "sync_wrapper 0.1.2", "tokio", "tokio-util 0.7.11", - "tower-layer 0.3.2 (git+https://github.com/tower-rs/tower.git)", - "tower-service 0.3.2 (git+https://github.com/tower-rs/tower.git)", + "tower-layer 0.3.2", + "tower-service 0.3.2", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.1", + "tokio", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] @@ -5485,31 +5627,31 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "pin-project-lite", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", ] [[package]] name = "tower-layer" version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" [[package]] name = "tower-layer" -version = "0.3.2" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" [[package]] name = "tower-service" -version = "0.3.2" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-test" @@ -5520,8 +5662,8 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-test", - "tower-layer 0.3.2 (git+https://github.com/tower-rs/tower.git)", - "tower-service 0.3.2 (git+https://github.com/tower-rs/tower.git)", + "tower-layer 0.3.2", + "tower-service 0.3.2", ] [[package]] @@ -5544,7 +5686,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5581,38 +5723,18 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.1.0", - "httparse", - "log", - "rand", - "sha1", - "thiserror", - "url", - "utf-8", -] - -[[package]] -name = "tungstenite" -version = "0.23.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ - "byteorder", "bytes", "data-encoding", "http 1.1.0", "httparse", "log", - "rand", + "rand 0.9.0", "sha1", - "thiserror", + "thiserror 2.0.12", "utf-8", ] @@ -5735,12 +5857,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.3", "percent-encoding", ] @@ -5750,6 +5872,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -5758,13 +5892,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.9.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" -dependencies = [ - "getrandom", - "serde", -] +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" [[package]] name = "vcpkg" @@ -5809,6 +5939,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -5836,7 +5975,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -5870,7 +6009,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6016,7 +6155,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -6046,7 +6185,7 @@ dependencies = [ "log", "object 0.32.2", "target-lexicon", - "thiserror", + "thiserror 1.0.61", "wasmparser 0.116.1", "wasmtime-cranelift-shared", "wasmtime-environ", @@ -6084,7 +6223,7 @@ dependencies = [ "serde", "serde_derive", "target-lexicon", - "thiserror", + "thiserror 1.0.61", "wasmparser 0.116.1", "wasmtime-types", ] @@ -6170,7 +6309,7 @@ dependencies = [ "memfd", "memoffset", "paste", - "rand", + "rand 0.8.5", "rustix", "sptr", "wasm-encoder 0.36.2", @@ -6192,7 +6331,7 @@ dependencies = [ "cranelift-entity", "serde", "serde_derive", - "thiserror", + "thiserror 1.0.61", "wasmparser 0.116.1", ] @@ -6204,7 +6343,7 @@ checksum = "f50f51f8d79bfd2aa8e9d9a0ae7c2d02b45fe412e62ff1b87c0c81b07c738231" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -6299,7 +6438,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" dependencies = [ "native-tls", - "thiserror", + "thiserror 1.0.61", "tokio", "url", ] @@ -6547,7 +6686,7 @@ dependencies = [ "futures 0.3.30", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "log", "once_cell", @@ -6558,6 +6697,15 @@ dependencies = [ "url", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "wit-parser" version = "0.13.2" @@ -6575,6 +6723,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -6596,13 +6756,46 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf01143b2dd5d134f11f545cf9f1431b13b749695cb33bcce051e7568f99478" +dependencies = [ + "zerocopy-derive 0.8.21", ] [[package]] @@ -6613,7 +6806,39 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712c8386f4f4299382c9abee219bee7084f78fb939d88b6840fcc1320d5f6da2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -6622,6 +6847,28 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/Cargo.toml b/Cargo.toml index 751a19d6213..b938992bc30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,13 +4,26 @@ members = [ "core", "core/graphman", "core/graphman_store", - "chain/*", + "chain/arweave", + "chain/common", + "chain/ethereum", + "chain/near", + "chain/substreams", "graphql", "node", - "runtime/*", - "server/*", - "store/*", - "substreams/*", + "runtime/derive", + "runtime/test", + "runtime/wasm", + "server/graphman", + "server/http", + "server/index-node", + "server/json-rpc", + "server/metrics", + "store/postgres", + "store/test-store", + "substreams/substreams-head-tracker", + "substreams/substreams-trigger-filter", + "substreams/trigger-filters", "graph", "tests", "graph/derive", @@ -27,17 +40,24 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] anyhow = "1.0" -async-graphql = { version = "7.0.11", features = ["chrono", "uuid"] } -async-graphql-axum = "7.0.11" -axum = "0.7.5" -bs58 = "0.5.1" +async-graphql = { version = "7.0.15", features = ["chrono"] } +async-graphql-axum = "7.0.15" +axum = "0.8.1" chrono = "0.4.38" +bs58 = "0.5.1" clap = { version = "4.5.4", features = ["derive", "env"] } derivative = "2.2.0" -diesel = { version = "2.2.4", features = ["postgres", "serde_json", "numeric", "r2d2", "chrono", "uuid", "i-implement-a-third-party-backend-and-opt-into-breaking-changes"] } +diesel = { version = "2.2.7", features = [ + "postgres", + "serde_json", + "numeric", + "r2d2", + "chrono", + "i-implement-a-third-party-backend-and-opt-into-breaking-changes", +] } diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } -diesel-dynamic-schema = { version = "0.2.1", features = ["postgres"] } -diesel_derives = "2.1.4" +diesel-dynamic-schema = { version = "0.2.3", features = ["postgres"] } +diesel_derives = "2.2.3" diesel_migrations = "2.1.0" graph = { path = "./graph" } graph-core = { path = "./core" } @@ -47,8 +67,8 @@ graphman = { path = "./core/graphman" } graphman-store = { path = "./core/graphman_store" } itertools = "0.13.0" lazy_static = "1.5.0" -prost = "0.12.6" -prost-types = "0.12.6" +prost = "0.13" +prost-types = "0.13" regex = "1.5.4" reqwest = "0.12.5" serde = { version = "1.0.126", features = ["rc"] } @@ -56,18 +76,24 @@ serde_derive = "1.0.125" serde_json = { version = "1.0", features = ["arbitrary_precision"] } serde_regex = "1.1.0" serde_yaml = "0.9.21" -slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] } +slog = { version = "2.7.0", features = [ + "release_max_level_trace", + "max_level_trace", +] } sqlparser = "0.46.0" strum = { version = "0.26", features = ["derive"] } -syn = { version = "2.0.66", features = ["full"] } +syn = { version = "2.0.87", features = ["full"] } test-store = { path = "./store/test-store" } thiserror = "1.0.25" tokio = { version = "1.38.0", features = ["full"] } -tonic = { version = "0.11.0", features = ["tls-roots", "gzip"] } -tonic-build = { version = "0.11.0", features = ["prost"] } +tonic = { version = "0.12.3", features = ["tls-roots", "gzip"] } +tonic-build = { version = "0.12.3", features = ["prost"] } tower-http = { version = "0.5.2", features = ["cors"] } wasmparser = "0.118.1" wasmtime = "15.0.1" +substreams = "=0.6.0" +substreams-entity-change = "2" +substreams-near-core = "=0.10.2" # Incremental compilation on Rust 1.58 causes an ICE on build. As soon as graph node builds again, these can be removed. [profile.test] diff --git a/NEWS.md b/NEWS.md index a8843422fde..719d2f12e49 100644 --- a/NEWS.md +++ b/NEWS.md @@ -446,8 +446,8 @@ Not Relevant @@ -1155,7 +1155,7 @@ storage](./docs/config.md) and spread subgraph deployments, and the load coming from indexing and querying them across multiple independent Postgres databases. -**This feature is considered experimenatal. We encourage users to try this +**This feature is considered experimental. We encourage users to try this out in a test environment, but do not recommend it yet for production use** In particular, the details of how sharding is configured may change in backwards-incompatible ways in the future. diff --git a/README.md b/README.md index ff31fdad758..b856653fd95 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ To build and run this project you need to have the following installed on your s - Note that `rustfmt`, which is part of the default Rust installation, is a build-time requirement. - PostgreSQL – [PostgreSQL Downloads](https://www.postgresql.org/download/) - IPFS – [Installing IPFS](https://docs.ipfs.io/install/) -- Profobuf Compiler - [Installing Protobuf](https://grpc.io/docs/protoc-installation/) +- Protobuf Compiler - [Installing Protobuf](https://grpc.io/docs/protoc-installation/) For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. diff --git a/chain/arweave/build.rs b/chain/arweave/build.rs index ea8153e7bd1..c0abc5fb8c9 100644 --- a/chain/arweave/build.rs +++ b/chain/arweave/build.rs @@ -2,6 +2,6 @@ fn main() { println!("cargo:rerun-if-changed=proto"); tonic_build::configure() .out_dir("src/protobuf") - .compile(&["proto/arweave.proto"], &["proto"]) + .compile_protos(&["proto/arweave.proto"], &["proto"]) .expect("Failed to compile Firehose Arweave proto(s)"); } diff --git a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs index 39f83444cae..17f44498491 100644 --- a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs +++ b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs @@ -1,11 +1,9 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { /// Firehose block version (unrelated to Arweave block version) @@ -75,7 +73,6 @@ pub struct Block { pub poa: ::core::option::Option, } /// A succinct proof of access to a recall byte found in a TX -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProofOfAccess { /// The recall byte option chosen; global offset of index byte @@ -94,7 +91,6 @@ pub struct ProofOfAccess { #[prost(bytes = "vec", tag = "4")] pub chunk: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Transaction { /// 1 or 2 for v1 or v2 transactions. More allowable in the future @@ -137,7 +133,6 @@ pub struct Transaction { #[prost(message, optional, tag = "12")] pub reward: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Tag { #[prost(bytes = "vec", tag = "1")] diff --git a/chain/common/Cargo.toml b/chain/common/Cargo.toml index 2715c18a845..6c1cfd9dc03 100644 --- a/chain/common/Cargo.toml +++ b/chain/common/Cargo.toml @@ -7,6 +7,6 @@ edition.workspace = true [dependencies] protobuf = "3.0.2" -protobuf-parse = "3.5.0" +protobuf-parse = "3.7.1" anyhow = "1" heck = "0.5" diff --git a/chain/cosmos/Cargo.toml b/chain/cosmos/Cargo.toml deleted file mode 100644 index 4d3b598d046..00000000000 --- a/chain/cosmos/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "graph-chain-cosmos" -version.workspace = true -edition = "2018" - -[build-dependencies] -tonic-build = { workspace = true } -graph-chain-common = { path = "../common" } - -[dependencies] -graph = { path = "../../graph" } -prost = { workspace = true } -prost-types = { workspace = true } -serde = { workspace = true } -anyhow = "1.0" -semver = "1.0.23" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/cosmos/build.rs b/chain/cosmos/build.rs deleted file mode 100644 index c618d3b466d..00000000000 --- a/chain/cosmos/build.rs +++ /dev/null @@ -1,54 +0,0 @@ -const PROTO_FILE: &str = "proto/cosmos.proto"; - -fn main() { - println!("cargo:rerun-if-changed=proto"); - - let types = - graph_chain_common::parse_proto_file(PROTO_FILE).expect("Unable to parse proto file!"); - - let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) - .filter(|t| t.is_array) - .map(|t| t.type_name.clone()) - .collect::>(); - - let mut builder = tonic_build::configure().out_dir("src/protobuf"); - - for (name, ptype) in types { - //generate Asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_asc_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - //generate data index id - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_network_type_id(Cosmos)]", - ); - - //generate conversion from rust type to asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_from_rust_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - if array_types.contains(&ptype.name) { - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_array_type(Cosmos)]", - ); - } - } - - builder - .compile(&[PROTO_FILE], &["proto"]) - .expect("Failed to compile Firehose Cosmos proto(s)"); -} diff --git a/chain/cosmos/proto/cosmos.proto b/chain/cosmos/proto/cosmos.proto deleted file mode 100644 index c32502da1e9..00000000000 --- a/chain/cosmos/proto/cosmos.proto +++ /dev/null @@ -1,368 +0,0 @@ -syntax = "proto3"; - -package sf.cosmos.type.v1; - -option go_package = "github.com/figment-networks/proto-cosmos/pb/sf/cosmos/type/v1;pbcosmos"; - -import "google/protobuf/descriptor.proto"; -import "google/protobuf/any.proto"; -import "gogoproto/gogo.proto"; -import "cosmos_proto/cosmos.proto"; -import "firehose/annotations.proto"; - -message Block { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; - EvidenceList evidence = 2 [(gogoproto.nullable) = false]; - Commit last_commit = 3; - ResponseBeginBlock result_begin_block = 4 [(firehose.required) = true]; - ResponseEndBlock result_end_block = 5 [(firehose.required) = true]; - repeated TxResult transactions = 7; - repeated Validator validator_updates = 8; -} - -// HeaderOnlyBlock is a standard [Block] structure where all other fields are -// removed so that hydrating that object from a [Block] bytes payload will -// drastically reduce allocated memory required to hold the full block. -// -// This can be used to unpack a [Block] when only the [Header] information -// is required and greatly reduce required memory. -message HeaderOnlyBlock { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; -} - -message EventData { - Event event = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3; -} - -message TransactionData { - TxResult tx = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; -} - -message MessageData { - google.protobuf.Any message = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3 [(firehose.required) = true]; -} - -message TransactionContext { - bytes hash = 1; - uint32 index = 2; - uint32 code = 3; - int64 gas_wanted = 4; - int64 gas_used = 5; -} - -message Header { - Consensus version = 1 [(gogoproto.nullable) = false]; - string chain_id = 2 [(gogoproto.customname) = "ChainID"]; - uint64 height = 3; - Timestamp time = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - BlockID last_block_id = 5 [(firehose.required) = true, (gogoproto.nullable) = false]; - bytes last_commit_hash = 6; - bytes data_hash = 7; - bytes validators_hash = 8; - bytes next_validators_hash = 9; - bytes consensus_hash = 10; - bytes app_hash = 11; - bytes last_results_hash = 12; - bytes evidence_hash = 13; - bytes proposer_address = 14; - bytes hash = 15; -} - -message Consensus { - option (gogoproto.equal) = true; - - uint64 block = 1; - uint64 app = 2; -} - -message Timestamp { - int64 seconds = 1; - int32 nanos = 2; -} - -message BlockID { - bytes hash = 1; - PartSetHeader part_set_header = 2 [(gogoproto.nullable) = false]; -} - -message PartSetHeader { - uint32 total = 1; - bytes hash = 2; -} - -message EvidenceList { - repeated Evidence evidence = 1 [(gogoproto.nullable) = false]; -} - -message Evidence { - oneof sum { - DuplicateVoteEvidence duplicate_vote_evidence = 1; - LightClientAttackEvidence light_client_attack_evidence = 2; - } -} - -message DuplicateVoteEvidence { - EventVote vote_a = 1; - EventVote vote_b = 2; - int64 total_voting_power = 3; - int64 validator_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message EventVote { - SignedMsgType event_vote_type = 1 [json_name = "type"]; - uint64 height = 2; - int32 round = 3; - BlockID block_id = 4 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes validator_address = 6; - int32 validator_index = 7; - bytes signature = 8; -} - -enum SignedMsgType { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - SIGNED_MSG_TYPE_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "UnknownType"]; - SIGNED_MSG_TYPE_PREVOTE = 1 [(gogoproto.enumvalue_customname) = "PrevoteType"]; - SIGNED_MSG_TYPE_PRECOMMIT = 2 [(gogoproto.enumvalue_customname) = "PrecommitType"]; - SIGNED_MSG_TYPE_PROPOSAL = 32 [(gogoproto.enumvalue_customname) = "ProposalType"]; -} - -message LightClientAttackEvidence { - LightBlock conflicting_block = 1; - int64 common_height = 2; - repeated Validator byzantine_validators = 3; - int64 total_voting_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message LightBlock { - SignedHeader signed_header = 1; - ValidatorSet validator_set = 2; -} - -message SignedHeader { - Header header = 1; - Commit commit = 2; -} - -message Commit { - int64 height = 1; - int32 round = 2; - BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; -} - -message CommitSig { - BlockIDFlag block_id_flag = 1; - bytes validator_address = 2; - Timestamp timestamp = 3 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes signature = 4; -} - -enum BlockIDFlag { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - BLOCK_ID_FLAG_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "BlockIDFlagUnknown"]; - BLOCK_ID_FLAG_ABSENT = 1 [(gogoproto.enumvalue_customname) = "BlockIDFlagAbsent"]; - BLOCK_ID_FLAG_COMMIT = 2 [(gogoproto.enumvalue_customname) = "BlockIDFlagCommit"]; - BLOCK_ID_FLAG_NIL = 3 [(gogoproto.enumvalue_customname) = "BlockIDFlagNil"]; -} - -message ValidatorSet { - repeated Validator validators = 1; - Validator proposer = 2; - int64 total_voting_power = 3; -} - -message Validator { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 voting_power = 3; - int64 proposer_priority = 4; -} - -message PublicKey { - option (gogoproto.compare) = true; - option (gogoproto.equal) = true; - - oneof sum { - bytes ed25519 = 1; - bytes secp256k1 = 2; - } -} - -message ResponseBeginBlock { - repeated Event events = 1 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; -} - -message Event { - string event_type = 1 [json_name = "type"]; - repeated EventAttribute attributes = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"]; -} - -message EventAttribute { - string key = 1; - string value = 2; - bool index = 3; -} - -message ResponseEndBlock { - repeated ValidatorUpdate validator_updates = 1; - ConsensusParams consensus_param_updates = 2; - repeated Event events = 3; -} - -message ValidatorUpdate { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 power = 3; -} - -message ConsensusParams { - BlockParams block = 1 [(gogoproto.nullable) = false]; - EvidenceParams evidence = 2 [(gogoproto.nullable) = false]; - ValidatorParams validator = 3 [(gogoproto.nullable) = false]; - VersionParams version = 4 [(gogoproto.nullable) = false]; -} - -message BlockParams { - int64 max_bytes = 1; - int64 max_gas = 2; -} - -message EvidenceParams { - int64 max_age_num_blocks = 1; - Duration max_age_duration = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; - int64 max_bytes = 3; -} - -message Duration { - int64 seconds = 1; - int32 nanos = 2; -} - -message ValidatorParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - repeated string pub_key_types = 1; -} - -message VersionParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - uint64 app_version = 1; -} - -message TxResult { - uint64 height = 1; - uint32 index = 2; - Tx tx = 3 [(firehose.required) = true]; - ResponseDeliverTx result = 4 [(firehose.required) = true]; - bytes hash = 5; -} - -message Tx { - TxBody body = 1 [(firehose.required) = true]; - AuthInfo auth_info = 2; - repeated bytes signatures = 3; -} - -message TxBody { - repeated google.protobuf.Any messages = 1; - string memo = 2; - uint64 timeout_height = 3; - repeated google.protobuf.Any extension_options = 1023; - repeated google.protobuf.Any non_critical_extension_options = 2047; -} - -message Any { - string type_url = 1; - bytes value = 2; -} - -message AuthInfo { - repeated SignerInfo signer_infos = 1; - Fee fee = 2; - Tip tip = 3; -} - -message SignerInfo { - google.protobuf.Any public_key = 1; - ModeInfo mode_info = 2; - uint64 sequence = 3; -} - -message ModeInfo { - oneof sum { - ModeInfoSingle single = 1; - ModeInfoMulti multi = 2; - } -} - -message ModeInfoSingle { - SignMode mode = 1; -} - -enum SignMode { - SIGN_MODE_UNSPECIFIED = 0; - SIGN_MODE_DIRECT = 1; - SIGN_MODE_TEXTUAL = 2; - SIGN_MODE_LEGACY_AMINO_JSON = 127; -} - -message ModeInfoMulti { - CompactBitArray bitarray = 1; - repeated ModeInfo mode_infos = 2; -} - -message CompactBitArray { - option (gogoproto.goproto_stringer) = false; - - uint32 extra_bits_stored = 1; - bytes elems = 2; -} - -message Fee { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - uint64 gas_limit = 2; - string payer = 3 [(cosmos_proto.scalar) = "cosmos.AddressString"]; - string granter = 4 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message Coin { - option (gogoproto.equal) = true; - - string denom = 1; - string amount = 2 [(gogoproto.customtype) = "Int", (gogoproto.nullable) = false]; -} - -message Tip { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - string tipper = 2 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message ResponseDeliverTx { - uint32 code = 1; - bytes data = 2; - string log = 3; - string info = 4; - int64 gas_wanted = 5; - int64 gas_used = 6; - repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; - string codespace = 8; -} - -message ValidatorSetUpdates { - repeated Validator validator_updates = 1; -} diff --git a/chain/cosmos/proto/cosmos_proto/cosmos.proto b/chain/cosmos/proto/cosmos_proto/cosmos.proto deleted file mode 100644 index 5c63b86f063..00000000000 --- a/chain/cosmos/proto/cosmos_proto/cosmos.proto +++ /dev/null @@ -1,97 +0,0 @@ -syntax = "proto3"; -package cosmos_proto; - -import "google/protobuf/descriptor.proto"; - -option go_package = "github.com/cosmos/cosmos-proto;cosmos_proto"; - -extend google.protobuf.MessageOptions { - - // implements_interface is used to indicate the type name of the interface - // that a message implements so that it can be used in google.protobuf.Any - // fields that accept that interface. A message can implement multiple - // interfaces. Interfaces should be declared using a declare_interface - // file option. - repeated string implements_interface = 93001; -} - -extend google.protobuf.FieldOptions { - - // accepts_interface is used to annotate that a google.protobuf.Any - // field accepts messages that implement the specified interface. - // Interfaces should be declared using a declare_interface file option. - string accepts_interface = 93001; - - // scalar is used to indicate that this field follows the formatting defined - // by the named scalar which should be declared with declare_scalar. Code - // generators may choose to use this information to map this field to a - // language-specific type representing the scalar. - string scalar = 93002; -} - -extend google.protobuf.FileOptions { - - // declare_interface declares an interface type to be used with - // accepts_interface and implements_interface. Interface names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given interface type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/interfaces.proto in the file descriptor set. - repeated InterfaceDescriptor declare_interface = 793021; - - // declare_scalar declares a scalar type to be used with - // the scalar field option. Scalar names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given scalar type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/scalars.proto in the file descriptor set. - repeated ScalarDescriptor declare_scalar = 793022; -} - -// InterfaceDescriptor describes an interface type to be used with -// accepts_interface and implements_interface and declared by declare_interface. -message InterfaceDescriptor { - - // name is the name of the interface. It should be a short-name (without - // a period) such that the fully qualified name of the interface will be - // package.name, ex. for the package a.b and interface named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the interface and its - // purpose. - string description = 2; -} - -// ScalarDescriptor describes an scalar type to be used with -// the scalar field option and declared by declare_scalar. -// Scalars extend simple protobuf built-in types with additional -// syntax and semantics, for instance to represent big integers. -// Scalars should ideally define an encoding such that there is only one -// valid syntactical representation for a given semantic meaning, -// i.e. the encoding should be deterministic. -message ScalarDescriptor { - - // name is the name of the scalar. It should be a short-name (without - // a period) such that the fully qualified name of the scalar will be - // package.name, ex. for the package a.b and scalar named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the scalar and its - // encoding format. For instance a big integer or decimal scalar should - // specify precisely the expected encoding format. - string description = 2; - - // field_type is the type of field with which this scalar can be used. - // Scalars can be used with one and only one type of field so that - // encoding standards and simple and clear. Currently only string and - // bytes fields are supported for scalars. - repeated ScalarType field_type = 3; -} - -enum ScalarType { - SCALAR_TYPE_UNSPECIFIED = 0; - SCALAR_TYPE_STRING = 1; - SCALAR_TYPE_BYTES = 2; -} diff --git a/chain/cosmos/proto/firehose/annotations.proto b/chain/cosmos/proto/firehose/annotations.proto deleted file mode 100644 index 1476c1ab08d..00000000000 --- a/chain/cosmos/proto/firehose/annotations.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package firehose; - -option go_package = "github.com/streamingfast/pbgo/sf/firehose/v1;pbfirehose"; - -import "google/protobuf/descriptor.proto"; - -extend google.protobuf.FieldOptions { - optional bool required = 77001; -} diff --git a/chain/cosmos/proto/gogoproto/gogo.proto b/chain/cosmos/proto/gogoproto/gogo.proto deleted file mode 100644 index 49e78f99fe5..00000000000 --- a/chain/cosmos/proto/gogoproto/gogo.proto +++ /dev/null @@ -1,145 +0,0 @@ -// Protocol Buffers for Go with Gadgets -// -// Copyright (c) 2013, The GoGo Authors. All rights reserved. -// http://github.com/gogo/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -package gogoproto; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "GoGoProtos"; -option go_package = "github.com/gogo/protobuf/gogoproto"; - -extend google.protobuf.EnumOptions { - optional bool goproto_enum_prefix = 62001; - optional bool goproto_enum_stringer = 62021; - optional bool enum_stringer = 62022; - optional string enum_customname = 62023; - optional bool enumdecl = 62024; -} - -extend google.protobuf.EnumValueOptions { - optional string enumvalue_customname = 66001; -} - -extend google.protobuf.FileOptions { - optional bool goproto_getters_all = 63001; - optional bool goproto_enum_prefix_all = 63002; - optional bool goproto_stringer_all = 63003; - optional bool verbose_equal_all = 63004; - optional bool face_all = 63005; - optional bool gostring_all = 63006; - optional bool populate_all = 63007; - optional bool stringer_all = 63008; - optional bool onlyone_all = 63009; - - optional bool equal_all = 63013; - optional bool description_all = 63014; - optional bool testgen_all = 63015; - optional bool benchgen_all = 63016; - optional bool marshaler_all = 63017; - optional bool unmarshaler_all = 63018; - optional bool stable_marshaler_all = 63019; - - optional bool sizer_all = 63020; - - optional bool goproto_enum_stringer_all = 63021; - optional bool enum_stringer_all = 63022; - - optional bool unsafe_marshaler_all = 63023; - optional bool unsafe_unmarshaler_all = 63024; - - optional bool goproto_extensions_map_all = 63025; - optional bool goproto_unrecognized_all = 63026; - optional bool gogoproto_import = 63027; - optional bool protosizer_all = 63028; - optional bool compare_all = 63029; - optional bool typedecl_all = 63030; - optional bool enumdecl_all = 63031; - - optional bool goproto_registration = 63032; - optional bool messagename_all = 63033; - - optional bool goproto_sizecache_all = 63034; - optional bool goproto_unkeyed_all = 63035; -} - -extend google.protobuf.MessageOptions { - optional bool goproto_getters = 64001; - optional bool goproto_stringer = 64003; - optional bool verbose_equal = 64004; - optional bool face = 64005; - optional bool gostring = 64006; - optional bool populate = 64007; - optional bool stringer = 67008; - optional bool onlyone = 64009; - - optional bool equal = 64013; - optional bool description = 64014; - optional bool testgen = 64015; - optional bool benchgen = 64016; - optional bool marshaler = 64017; - optional bool unmarshaler = 64018; - optional bool stable_marshaler = 64019; - - optional bool sizer = 64020; - - optional bool unsafe_marshaler = 64023; - optional bool unsafe_unmarshaler = 64024; - - optional bool goproto_extensions_map = 64025; - optional bool goproto_unrecognized = 64026; - - optional bool protosizer = 64028; - optional bool compare = 64029; - - optional bool typedecl = 64030; - - optional bool messagename = 64033; - - optional bool goproto_sizecache = 64034; - optional bool goproto_unkeyed = 64035; -} - -extend google.protobuf.FieldOptions { - optional bool nullable = 65001; - optional bool embed = 65002; - optional string customtype = 65003; - optional string customname = 65004; - optional string jsontag = 65005; - optional string moretags = 65006; - optional string casttype = 65007; - optional string castkey = 65008; - optional string castvalue = 65009; - - optional bool stdtime = 65010; - optional bool stdduration = 65011; - optional bool wktpointer = 65012; - - optional string castrepeated = 65013; -} diff --git a/chain/cosmos/src/adapter.rs b/chain/cosmos/src/adapter.rs deleted file mode 100644 index 746c91e2e07..00000000000 --- a/chain/cosmos/src/adapter.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::collections::HashSet; - -use prost::Message; -use prost_types::Any; - -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::firehose::EventTypeFilter; -use graph::prelude::*; - -const EVENT_TYPE_FILTER_TYPE_URL: &str = - "type.googleapis.com/sf.cosmos.transform.v1.EventTypeFilter"; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter { - pub(crate) event_type_filter: CosmosEventTypeFilter, - pub(crate) block_filter: CosmosBlockFilter, -} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { - self.event_type_filter - .extend_from_data_sources(data_sources.clone()); - self.block_filter.extend_from_data_sources(data_sources); - } - - fn node_capabilities(&self) -> bc::EmptyNodeCapabilities { - bc::EmptyNodeCapabilities::default() - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - if self.block_filter.trigger_every_block { - return vec![]; - } - - if self.event_type_filter.event_types.is_empty() { - return vec![]; - } - - let filter = EventTypeFilter { - event_types: Vec::from_iter(self.event_type_filter.event_types), - }; - - vec![Any { - type_url: EVENT_TYPE_FILTER_TYPE_URL.to_string(), - value: filter.encode_to_vec(), - }] - } -} - -pub type EventType = String; - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosEventTypeFilter { - pub event_types: HashSet, -} - -impl CosmosEventTypeFilter { - pub(crate) fn matches(&self, event_type: &EventType) -> bool { - self.event_types.contains(event_type) - } - - fn extend_from_data_sources<'a>(&mut self, data_sources: impl Iterator) { - self.event_types.extend( - data_sources.flat_map(|data_source| data_source.events().map(ToString::to_string)), - ); - } -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosBlockFilter { - pub trigger_every_block: bool, -} - -impl CosmosBlockFilter { - fn extend_from_data_sources<'a>( - &mut self, - mut data_sources: impl Iterator, - ) { - if !self.trigger_every_block { - self.trigger_every_block = data_sources.any(DataSource::has_block_handler); - } - } -} - -#[cfg(test)] -mod test { - use graph::blockchain::TriggerFilter as _; - - use super::*; - - #[test] - fn test_trigger_filters() { - let cases = [ - (TriggerFilter::test_new(false, &[]), None), - (TriggerFilter::test_new(true, &[]), None), - (TriggerFilter::test_new(true, &["event_1", "event_2"]), None), - ( - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - Some(event_type_filter_with(&["event_1", "event_2", "event_3"])), - ), - ]; - - for (trigger_filter, expected_filter) in cases { - let firehose_filter = trigger_filter.to_firehose_filter(); - let decoded_filter = decode_filter(firehose_filter); - - assert_eq!(decoded_filter.is_some(), expected_filter.is_some()); - - if let (Some(mut expected_filter), Some(mut decoded_filter)) = - (expected_filter, decoded_filter) - { - // event types may be in different order - expected_filter.event_types.sort(); - decoded_filter.event_types.sort(); - - assert_eq!(decoded_filter, expected_filter); - } - } - } - - impl TriggerFilter { - pub(crate) fn test_new(trigger_every_block: bool, event_types: &[&str]) -> TriggerFilter { - TriggerFilter { - event_type_filter: CosmosEventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - }, - block_filter: CosmosBlockFilter { - trigger_every_block, - }, - } - } - } - - fn event_type_filter_with(event_types: &[&str]) -> EventTypeFilter { - EventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - } - } - - fn decode_filter(proto_filters: Vec) -> Option { - assert!(proto_filters.len() <= 1); - - let proto_filter = proto_filters.get(0)?; - - assert_eq!(proto_filter.type_url, EVENT_TYPE_FILTER_TYPE_URL); - - let firehose_filter = EventTypeFilter::decode(&*proto_filter.value) - .expect("Could not decode EventTypeFilter from protobuf Any"); - - Some(firehose_filter) - } -} diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs deleted file mode 100644 index 353b1e4dbbe..00000000000 --- a/chain/cosmos/src/chain.rs +++ /dev/null @@ -1,715 +0,0 @@ -use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; -use graph::blockchain::{BlockIngestor, NoopDecoderHook, TriggerFilterWrapper}; -use graph::components::network_provider::ChainName; -use graph::env::EnvVars; -use graph::prelude::MetricsRegistry; -use graph::substreams::Clock; -use std::collections::BTreeSet; -use std::convert::TryFrom; -use std::sync::Arc; - -use graph::blockchain::block_stream::{BlockStreamError, BlockStreamMapper, FirehoseCursor}; -use graph::blockchain::client::ChainClient; -use graph::blockchain::{BasicBlockchainBuilder, BlockchainBuilder, NoopRuntimeAdapter}; -use graph::cheap_clone::CheapClone; -use graph::components::store::{DeploymentCursorTracker, SourceableStore}; -use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::{ - blockchain::{ - block_stream::{ - BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseError, - FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, - }, - firehose_block_stream::FirehoseBlockStream, - Block as _, BlockHash, BlockPtr, Blockchain, BlockchainKind, EmptyNodeCapabilities, - IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - firehose::{self, FirehoseEndpoint, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, -}; -use prost::Message; - -use crate::data_source::{ - DataSource, DataSourceTemplate, EventOrigin, UnresolvedDataSource, UnresolvedDataSourceTemplate, -}; -use crate::trigger::CosmosTrigger; -use crate::{codec, Block, TriggerFilter}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: ChainName, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: cosmos") - } -} - -#[async_trait] -impl BlockchainBuilder for BasicBlockchainBuilder { - async fn build(self, _config: &Arc) -> Chain { - Chain { - logger_factory: self.logger_factory, - name: self.name, - client: Arc::new(ChainClient::new_firehose(self.firehose_endpoints)), - chain_store: self.chain_store, - metrics_registry: self.metrics_registry, - } - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Cosmos; - - type Client = (); - type Block = codec::Block; - - type DataSource = DataSource; - - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = CosmosTrigger; - - type MappingTrigger = CosmosTrigger; - - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - type DecoderHook = NoopDecoderHook; - - fn is_refetch_block_required(&self) -> bool { - false - } - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - fn triggers_adapter( - &self, - _loc: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = TriggersAdapter {}; - Ok(Arc::new(adapter)) - } - - async fn new_block_stream( - &self, - deployment: DeploymentLocator, - store: impl DeploymentCursorTracker, - start_blocks: Vec, - _source_subgraph_stores: Vec>, - filter: Arc>, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = self - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); - - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper { - adapter, - filter: filter.chain_filter.clone(), - }); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - self.chain_client(), - store.block_ptr(), - store.firehose_cursor(), - firehose_mapper, - start_blocks, - logger, - self.metrics_registry.clone(), - ))) - } - - fn chain_store(&self) -> Arc { - self.chain_store.cheap_clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint().await?; - - firehose_endpoint - .block_ptr_for_number::(logger, number) - .await - .map_err(Into::into) - } - - fn runtime(&self) -> anyhow::Result<(Arc>, Self::DecoderHook)> { - Ok((Arc::new(NoopRuntimeAdapter::default()), NoopDecoderHook)) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } - - async fn block_ingestor(&self) -> anyhow::Result> { - let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), - self.chain_client(), - self.logger_factory - .component_logger("CosmosFirehoseBlockIngestor", None), - self.name.clone(), - ); - Ok(Box::new(ingestor)) - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - _root: Option, - ) -> Result, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - async fn load_block_ptrs_by_numbers( - &self, - _logger: Logger, - _block_numbers: BTreeSet, - ) -> Result, Error> { - todo!() - } - async fn chain_head_ptr(&self) -> Result, Error> { - unimplemented!() - } - - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result<(Vec>, BlockNumber), Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &TriggerFilter, - ) -> Result, Error> { - let shared_block = Arc::new(block.clone()); - - let header_only_block = codec::HeaderOnlyBlock::from(&block); - - let mut triggers: Vec<_> = shared_block - .begin_block_events()? - .cloned() - // FIXME (Cosmos): Optimize. Should use an Arc instead of cloning the - // block. This is not currently possible because EventData is automatically - // generated. - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::BeginBlock, - ) - }) - .chain(shared_block.transactions().flat_map(|tx| { - tx.result - .as_ref() - .unwrap() - .events - .iter() - .filter_map(|e| { - filter_event_trigger( - filter, - e.clone(), - &header_only_block, - Some(build_tx_context(tx)), - EventOrigin::DeliverTx, - ) - }) - .collect::>() - })) - .chain( - shared_block - .end_block_events()? - .cloned() - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::EndBlock, - ) - }), - ) - .collect(); - - triggers.extend(shared_block.transactions().cloned().flat_map(|tx_result| { - let mut triggers: Vec<_> = Vec::new(); - if let Some(tx) = tx_result.tx.clone() { - if let Some(tx_body) = tx.body { - triggers.extend(tx_body.messages.into_iter().map(|message| { - CosmosTrigger::with_message( - message, - header_only_block.clone(), - build_tx_context(&tx_result), - ) - })); - } - } - triggers.push(CosmosTrigger::with_transaction( - tx_result, - header_only_block.clone(), - )); - triggers - })); - - if filter.block_filter.trigger_every_block { - triggers.push(CosmosTrigger::Block(shared_block.cheap_clone())); - } - - Ok(BlockWithTriggers::new(block, triggers, logger)) - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - /// Panics if `block` is genesis. - /// But that's ok since this is only called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -/// Returns a new event trigger only if the given event matches the event filter. -fn filter_event_trigger( - filter: &TriggerFilter, - event: codec::Event, - block: &codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, -) -> Option { - if filter.event_type_filter.matches(&event.event_type) { - Some(CosmosTrigger::with_event( - event, - block.clone(), - tx_context, - origin, - )) - } else { - None - } -} - -fn build_tx_context(tx: &codec::TxResult) -> codec::TransactionContext { - codec::TransactionContext { - hash: tx.hash.clone(), - index: tx.index, - code: tx.result.as_ref().unwrap().code, - gas_wanted: tx.result.as_ref().unwrap().gas_wanted, - gas_used: tx.result.as_ref().unwrap().gas_used, - } -} - -pub struct FirehoseMapper { - adapter: Arc>, - filter: Arc, -} - -#[async_trait] -impl BlockStreamMapper for FirehoseMapper { - fn decode_block( - &self, - output: Option<&[u8]>, - ) -> Result, BlockStreamError> { - let block = match output { - Some(block) => crate::Block::decode(block)?, - None => Err(anyhow::anyhow!( - "cosmos mapper is expected to always have a block" - ))?, - }; - - Ok(Some(block)) - } - - async fn block_with_triggers( - &self, - logger: &Logger, - block: crate::Block, - ) -> Result, BlockStreamError> { - self.adapter - .triggers_in_block(logger, block, self.filter.as_ref()) - .await - .map_err(BlockStreamError::from) - } - - async fn handle_substreams_block( - &self, - _logger: &Logger, - _clock: Clock, - _cursor: FirehoseCursor, - _block: Vec, - ) -> Result, BlockStreamError> { - unimplemented!() - } -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - fn trigger_filter(&self) -> &TriggerFilter { - self.filter.as_ref() - } - - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - ) -> Result, FirehoseError> { - let step = ForkStep::try_from(response.step).unwrap_or_else(|_| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down struct that would decode only a few fields and ignore all the rest. - // unwrap: Input cannot be None so output will be error or block. - let block = self - .decode_block(Some(any_block.value.as_ref())) - .map_err(Error::from)? - .unwrap(); - - match step { - ForkStep::StepNew => Ok(BlockStreamEvent::ProcessBlock( - self.block_with_triggers(logger, block) - .await - .map_err(Error::from)?, - FirehoseCursor::from(response.cursor.clone()), - )), - - ForkStep::StepUndo => { - let parent_ptr = block - .parent_ptr() - .map_err(FirehoseError::from)? - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - ForkStep::StepFinal => { - panic!( - "final step is not handled and should not be requested in the Firehose request" - ) - } - - ForkStep::StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - async fn final_block_ptr_for( - &self, - logger: &Logger, - endpoint: &Arc, - block: &codec::Block, - ) -> Result { - // Cosmos provides instant block finality. - self.block_ptr_for_number(logger, endpoint, block.number()) - .await - } -} - -#[cfg(test)] -mod test { - use graph::{ - blockchain::Trigger, - prelude::{ - slog::{o, Discard, Logger}, - tokio, - }, - }; - - use super::*; - - use codec::{ - Block, Event, Header, HeaderOnlyBlock, ResponseBeginBlock, ResponseDeliverTx, - ResponseEndBlock, TxResult, - }; - - #[tokio::test] - async fn test_trigger_filters() { - let adapter = TriggersAdapter {}; - let logger = Logger::root(Discard, o!()); - - let block_with_events = Block::test_with_event_types( - vec!["begin_event_1", "begin_event_2", "begin_event_3"], - vec!["tx_event_1", "tx_event_2", "tx_event_3"], - vec!["end_event_1", "end_event_2", "end_event_3"], - ); - - let header_only_block = HeaderOnlyBlock::from(&block_with_events); - - let cases = [ - ( - Block::test_new(), - TriggerFilter::test_new(false, &[]), - vec![], - ), - ( - Block::test_new(), - TriggerFilter::test_new(true, &[]), - vec![CosmosTrigger::Block(Arc::new(Block::test_new()))], - ), - ( - Block::test_new(), - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - vec![], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(false, &["begin_event_3", "tx_event_3", "end_event_3"]), - vec![ - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_3"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[2])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_3"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(true, &["begin_event_3", "tx_event_2", "end_event_1"]), - vec![ - CosmosTrigger::Block(Arc::new(block_with_events.clone())), - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_2"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[1])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_1"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ]; - - for (block, trigger_filter, expected_triggers) in cases { - let triggers = adapter - .triggers_in_block(&logger, block, &trigger_filter) - .await - .expect("failed to get triggers in block"); - - assert_eq!( - triggers.trigger_data.len(), - expected_triggers.len(), - "Expected trigger list to contain exactly {:?}, but it didn't: {:?}", - expected_triggers, - triggers.trigger_data - ); - - // they may not be in the same order - for trigger in expected_triggers { - assert!( - triggers.trigger_data.iter().any(|t| match t { - Trigger::Chain(t) => t == &trigger, - _ => false, - }), - "Expected trigger list to contain {:?}, but it only contains: {:?}", - trigger, - triggers.trigger_data - ); - } - } - } - - impl Block { - fn test_new() -> Block { - Block::test_with_event_types(vec![], vec![], vec![]) - } - - fn test_with_event_types( - begin_event_types: Vec<&str>, - tx_event_types: Vec<&str>, - end_event_types: Vec<&str>, - ) -> Block { - Block { - header: Some(Header { - version: None, - chain_id: "test".to_string(), - height: 1, - time: None, - last_block_id: None, - last_commit_hash: vec![], - data_hash: vec![], - validators_hash: vec![], - next_validators_hash: vec![], - consensus_hash: vec![], - app_hash: vec![], - last_results_hash: vec![], - evidence_hash: vec![], - proposer_address: vec![], - hash: vec![], - }), - evidence: None, - last_commit: None, - result_begin_block: Some(ResponseBeginBlock { - events: begin_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - result_end_block: Some(ResponseEndBlock { - validator_updates: vec![], - consensus_param_updates: None, - events: end_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - transactions: tx_event_types - .into_iter() - .map(TxResult::test_with_event_type) - .collect(), - validator_updates: vec![], - } - } - } - - impl Event { - fn test_with_type(event_type: &str) -> Event { - Event { - event_type: event_type.to_string(), - attributes: vec![], - } - } - } - - impl TxResult { - fn test_with_event_type(event_type: &str) -> TxResult { - TxResult { - height: 1, - index: 1, - tx: None, - result: Some(ResponseDeliverTx { - code: 1, - data: vec![], - log: "".to_string(), - info: "".to_string(), - gas_wanted: 1, - gas_used: 1, - codespace: "".to_string(), - events: vec![Event::test_with_type(event_type)], - }), - hash: vec![], - } - } - } -} diff --git a/chain/cosmos/src/codec.rs b/chain/cosmos/src/codec.rs deleted file mode 100644 index bdc05c1b820..00000000000 --- a/chain/cosmos/src/codec.rs +++ /dev/null @@ -1,198 +0,0 @@ -pub(crate) use crate::protobuf::pbcodec::*; - -use graph::blockchain::{Block as BlockchainBlock, BlockTime}; -use graph::{ - blockchain::BlockPtr, - prelude::{anyhow::anyhow, BlockNumber, Error}, -}; - -use std::convert::TryFrom; - -impl Block { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn begin_block_events(&self) -> Result, Error> { - let events = self - .result_begin_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_begin_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn end_block_events(&self) -> Result, Error> { - let events = self - .result_end_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_end_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn transactions(&self) -> impl Iterator { - self.transactions.iter() - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: Block) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a Block> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a Block) -> Result { - let header = b.header()?; - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } - - fn timestamp(&self) -> BlockTime { - let time = self.header().unwrap().time.as_ref().unwrap(); - BlockTime::since_epoch(time.seconds, time.nanos as u32) - } -} - -impl HeaderOnlyBlock { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl From<&Block> for HeaderOnlyBlock { - fn from(b: &Block) -> HeaderOnlyBlock { - HeaderOnlyBlock { - header: b.header.clone(), - } - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: HeaderOnlyBlock) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a HeaderOnlyBlock> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a HeaderOnlyBlock) -> Result { - let header = b.header()?; - - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for HeaderOnlyBlock { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } - - fn timestamp(&self) -> BlockTime { - let time = self.header().unwrap().time.as_ref().unwrap(); - BlockTime::since_epoch(time.seconds, time.nanos as u32) - } -} - -impl EventData { - pub fn event(&self) -> Result<&Event, Error> { - self.event - .as_ref() - .ok_or_else(|| anyhow!("event data missing event field")) - } - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("event data missing block field")) - } -} - -impl TransactionData { - pub fn tx_result(&self) -> Result<&TxResult, Error> { - self.tx - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing tx field")) - } - - pub fn response_deliver_tx(&self) -> Result<&ResponseDeliverTx, Error> { - self.tx_result()? - .result - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing result field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing block field")) - } -} - -impl MessageData { - pub fn message(&self) -> Result<&prost_types::Any, Error> { - self.message - .as_ref() - .ok_or_else(|| anyhow!("message data missing message field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("message data missing block field")) - } -} diff --git a/chain/cosmos/src/data_source.rs b/chain/cosmos/src/data_source.rs deleted file mode 100644 index f09448ecbe8..00000000000 --- a/chain/cosmos/src/data_source.rs +++ /dev/null @@ -1,726 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{Context, Error, Result}; - -use graph::components::subgraph::InstanceDSTemplateInfo; -use graph::{ - blockchain::{self, Block, Blockchain, TriggerWithHandler}, - components::store::StoredDynamicDataSource, - data::subgraph::DataSourceContext, - derive::CheapClone, - prelude::{ - anyhow, async_trait, BlockNumber, CheapClone, Deserialize, Link, LinkResolver, Logger, - }, -}; - -use crate::chain::Chain; -use crate::codec; -use crate::trigger::CosmosTrigger; - -pub const COSMOS_KIND: &str = "cosmos"; -const BLOCK_HANDLER_KIND: &str = "block"; -const EVENT_HANDLER_KIND: &str = "event"; -const TRANSACTION_HANDLER_KIND: &str = "transaction"; -const MESSAGE_HANDLER_KIND: &str = "message"; - -const DYNAMIC_DATA_SOURCE_ERROR: &str = "Cosmos subgraphs do not support dynamic data sources"; -const TEMPLATE_ERROR: &str = "Cosmos subgraphs do not support templates"; - -/// Runtime representation of a data source. -// Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens -// of thousands of data sources in memory at once. -#[derive(Clone, Debug)] -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info( - _info: InstanceDSTemplateInfo, - _template: &graph::data_source::DataSourceTemplate, - ) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } - - fn address(&self) -> Option<&[u8]> { - None - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn handler_kinds(&self) -> HashSet<&str> { - let mut kinds = HashSet::new(); - - let Mapping { - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - .. - } = &self.mapping; - - if !block_handlers.is_empty() { - kinds.insert(BLOCK_HANDLER_KIND); - } - - if !event_handlers.is_empty() { - kinds.insert(EVENT_HANDLER_KIND); - } - - if !transaction_handlers.is_empty() { - kinds.insert(TRANSACTION_HANDLER_KIND); - } - - if !message_handlers.is_empty() { - kinds.insert(MESSAGE_HANDLER_KIND); - } - - kinds - } - - fn end_block(&self) -> Option { - self.source.end_block - } - - fn match_and_decode( - &self, - trigger: &::TriggerData, - block: &Arc<::Block>, - _logger: &Logger, - ) -> Result>> { - if self.source.start_block > block.number() { - return Ok(None); - } - - let handler = match trigger { - CosmosTrigger::Block(_) => match self.handler_for_block() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Event { event_data, origin } => { - match self.handler_for_event(event_data.event()?, *origin) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - - CosmosTrigger::Transaction(_) => match self.handler_for_transaction() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Message(message_data) => { - match self.handler_for_message(message_data.message()?) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.cheap_clone(), - handler, - block.ptr(), - block.timestamp(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - self.creation_block - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - context, - - // The creation block is ignored for detection duplicate data sources. - // Contract ABI equality is implicit in `source` and `mapping.abis` equality. - creation_block: _, - } = self; - - // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs - // used at runtime but not needed to define uniqueness; each runtime host should be for a - // unique data source. - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.block_handlers == other.mapping.block_handlers - && mapping.event_handlers == other.mapping.event_handlers - && mapping.transaction_handlers == other.mapping.transaction_handlers - && mapping.message_handlers == other.mapping.message_handlers - && context == &other.context - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR); - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) - } - - fn validate(&self, _: &semver::Version) -> Vec { - let mut errors = Vec::new(); - - if self.kind != COSMOS_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - COSMOS_KIND, - self.kind - )) - } - - // Ensure there is only one block handler - if self.mapping.block_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated block handlers")); - } - - // Ensure there is only one transaction handler - if self.mapping.transaction_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated transaction handlers")); - } - - // Ensure that each event type + origin filter combination has only one handler - - // group handler origin filters by event type - let mut event_types = HashMap::with_capacity(self.mapping.event_handlers.len()); - for event_handler in self.mapping.event_handlers.iter() { - let origins = event_types - .entry(&event_handler.event) - // 3 is the maximum number of valid handlers for an event type (1 for each origin) - .or_insert(HashSet::with_capacity(3)); - - // insert returns false if value was already in the set - if !origins.insert(event_handler.origin) { - errors.push(multiple_origin_err( - &event_handler.event, - event_handler.origin, - )) - } - } - - // Ensure each event type either has: - // 1 handler with no origin filter - // OR - // 1 or more handlers with origin filter - for (event_type, origins) in event_types.iter() { - if origins.len() > 1 && !origins.iter().all(Option::is_some) { - errors.push(combined_origins_err(event_type)) - } - } - - // Ensure each message handlers is unique - let mut message_type_urls = HashSet::with_capacity(self.mapping.message_handlers.len()); - for message_handler in self.mapping.message_handlers.iter() { - if !message_type_urls.insert(message_handler.message.clone()) { - errors.push(duplicate_url_type(&message_handler.message)) - } - } - - errors - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } -} - -impl DataSource { - fn from_manifest( - kind: String, - network: Option, - name: String, - source: Source, - mapping: Mapping, - context: Option, - ) -> Result { - // Data sources in the manifest are created "before genesis" so they have no creation block. - let creation_block = None; - - Ok(DataSource { - kind, - network, - name, - source, - mapping, - context: Arc::new(context), - creation_block, - }) - } - - fn handler_for_block(&self) -> Option { - self.mapping.block_handlers.first().cloned() - } - - fn handler_for_transaction(&self) -> Option { - self.mapping.transaction_handlers.first().cloned() - } - - fn handler_for_message(&self, message: &::prost_types::Any) -> Option { - self.mapping - .message_handlers - .iter() - .find(|handler| handler.message == message.type_url) - .cloned() - } - - fn handler_for_event( - &self, - event: &codec::Event, - event_origin: EventOrigin, - ) -> Option { - self.mapping - .event_handlers - .iter() - .find(|handler| { - let event_type_matches = event.event_type == handler.event; - - if let Some(handler_origin) = handler.origin { - event_type_matches && event_origin == handler_origin - } else { - event_type_matches - } - }) - .cloned() - } - - pub(crate) fn has_block_handler(&self) -> bool { - !self.mapping.block_handlers.is_empty() - } - - /// Return an iterator over all event types from event handlers. - pub(crate) fn events(&self) -> impl Iterator { - self.mapping - .event_handlers - .iter() - .map(|handler| handler.event.as_str()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSource { - kind, - network, - name, - source, - mapping, - context, - } = self; - - let mapping = mapping.resolve(resolver, logger).await.with_context(|| { - format!( - "failed to resolve data source {} with source {}", - name, source.start_block - ) - })?; - - DataSource::from_manifest(kind, network, name, source, mapping, context) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - async fn resolve( - self, - _resolver: &Arc, - _logger: &Logger, - _manifest_idx: u32, - ) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn name(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn api_version(&self) -> semver::Version { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn runtime(&self) -> Option>> { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn manifest_idx(&self) -> u32 { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn kind(&self) -> &str { - &self.kind - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - pub api_version: String, - pub language: String, - pub entities: Vec, - #[serde(default)] - pub block_handlers: Vec, - #[serde(default)] - pub event_handlers: Vec, - #[serde(default)] - pub transaction_handlers: Vec, - #[serde(default)] - pub message_handlers: Vec, - pub file: Link, -} - -impl UnresolvedMapping { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let UnresolvedMapping { - api_version, - language, - entities, - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - file: link, - } = self; - - let api_version = semver::Version::parse(&api_version)?; - - let module_bytes = resolver - .cat(logger, &link) - .await - .with_context(|| format!("failed to resolve mapping {}", link.link))?; - - Ok(Mapping { - api_version, - language, - entities, - block_handlers: block_handlers.clone(), - event_handlers: event_handlers.clone(), - transaction_handlers: transaction_handlers.clone(), - message_handlers: message_handlers.clone(), - runtime: Arc::new(module_bytes), - link, - }) - } -} - -#[derive(Clone, Debug)] -pub struct Mapping { - pub api_version: semver::Version, - pub language: String, - pub entities: Vec, - pub block_handlers: Vec, - pub event_handlers: Vec, - pub transaction_handlers: Vec, - pub message_handlers: Vec, - pub runtime: Arc>, - pub link: Link, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingEventHandler { - pub event: String, - pub origin: Option, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingTransactionHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingMessageHandler { - pub message: String, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Source { - #[serde(default)] - pub start_block: BlockNumber, - pub(crate) end_block: Option, -} - -#[derive(Clone, Copy, CheapClone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub enum EventOrigin { - BeginBlock, - DeliverTx, - EndBlock, -} - -fn multiple_origin_err(event_type: &str, origin: Option) -> Error { - let origin_err_name = match origin { - Some(origin) => format!("{:?}", origin), - None => "no".to_string(), - }; - - anyhow!( - "data source has multiple {} event handlers with {} origin", - event_type, - origin_err_name, - ) -} - -fn combined_origins_err(event_type: &str) -> Error { - anyhow!( - "data source has combined origin and no-origin {} event handlers", - event_type - ) -} - -fn duplicate_url_type(message: &str) -> Error { - anyhow!( - "data source has more than one message handler for message {} ", - message - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - use graph::{blockchain::DataSource as _, data::subgraph::LATEST_VERSION}; - - #[test] - fn test_event_handlers_origin_validation() { - let cases = [ - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_3", None), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::EndBlock)), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - ]), - vec![ - multiple_origin_err("event_1", None), - combined_origins_err("event_2"), - multiple_origin_err("event_3", Some(EventOrigin::EndBlock)), - ], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(&LATEST_VERSION); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - #[test] - fn test_message_handlers_duplicate() { - let cases = [ - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_1".to_string(), - }, - ]), - vec![], - ), - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - ]), - vec![duplicate_url_type("message_0")], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(&LATEST_VERSION); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - impl DataSource { - fn with_event_handlers(event_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { - start_block: 1, - end_block: None, - }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers, - transaction_handlers: vec![], - message_handlers: vec![], - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - - fn with_message_handlers(message_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { - start_block: 1, - end_block: None, - }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers: vec![], - transaction_handlers: vec![], - message_handlers, - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - } - - impl MappingEventHandler { - fn with_origin(event_type: &str, origin: Option) -> MappingEventHandler { - MappingEventHandler { - event: event_type.to_string(), - origin, - handler: "handler".to_string(), - } - } - } -} diff --git a/chain/cosmos/src/lib.rs b/chain/cosmos/src/lib.rs deleted file mode 100644 index 5d0bf16d050..00000000000 --- a/chain/cosmos/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -mod adapter; -pub mod chain; -pub mod codec; -mod data_source; -mod protobuf; -pub mod runtime; -mod trigger; - -// ETHDEP: These concrete types should probably not be exposed. -pub use data_source::{DataSource, DataSourceTemplate}; - -pub use crate::adapter::TriggerFilter; -pub use crate::chain::Chain; - -pub use protobuf::pbcodec; -pub use protobuf::pbcodec::Block; diff --git a/chain/cosmos/src/protobuf/.gitignore b/chain/cosmos/src/protobuf/.gitignore deleted file mode 100644 index 96786948080..00000000000 --- a/chain/cosmos/src/protobuf/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/google.protobuf.rs -/gogoproto.rs -/cosmos_proto.rs -/firehose.rs diff --git a/chain/cosmos/src/protobuf/mod.rs b/chain/cosmos/src/protobuf/mod.rs deleted file mode 100644 index f12336b2f4f..00000000000 --- a/chain/cosmos/src/protobuf/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[rustfmt::skip] -#[path = "sf.cosmos.r#type.v1.rs"] -pub mod pbcodec; - -pub use graph_runtime_wasm::asc_abi::class::{Array, Uint8Array}; - -pub use crate::runtime::abi::*; -pub use pbcodec::*; diff --git a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs b/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs deleted file mode 100644 index a3938e2c9c1..00000000000 --- a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs +++ /dev/null @@ -1,839 +0,0 @@ -// This file is @generated by prost-build. -#[graph_runtime_derive::generate_asc_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub last_commit: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result_begin_block: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub result_end_block: ::core::option::Option, - #[prost(message, repeated, tag = "7")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "8")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are -/// removed so that hydrating that object from a \[Block\] bytes payload will -/// drastically reduce allocated memory required to hold the full block. -/// -/// This can be used to unpack a \[Block\] when only the \[Header\] information -/// is required and greatly reduce required memory. -#[graph_runtime_derive::generate_asc_type(__required__{header:Header})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{header:Header})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, -} -#[graph_runtime_derive::generate_asc_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventData { - #[prost(message, optional, tag = "1")] - pub event: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionData { - #[prost(message, optional, tag = "1")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MessageData { - #[prost(message, optional, tag = "1")] - pub message: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionContext { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(uint32, tag = "3")] - pub code: u32, - #[prost(int64, tag = "4")] - pub gas_wanted: i64, - #[prost(int64, tag = "5")] - pub gas_used: i64, -} -#[graph_runtime_derive::generate_asc_type(__required__{last_block_id:BlockID})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{last_block_id:BlockID})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Header { - #[prost(message, optional, tag = "1")] - pub version: ::core::option::Option, - #[prost(string, tag = "2")] - pub chain_id: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub height: u64, - #[prost(message, optional, tag = "4")] - pub time: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub last_block_id: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub last_commit_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "7")] - pub data_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "8")] - pub validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "9")] - pub next_validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "10")] - pub consensus_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "11")] - pub app_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "12")] - pub last_results_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "13")] - pub evidence_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "14")] - pub proposer_address: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "15")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Consensus { - #[prost(uint64, tag = "1")] - pub block: u64, - #[prost(uint64, tag = "2")] - pub app: u64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Timestamp { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockId { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub part_set_header: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PartSetHeader { - #[prost(uint32, tag = "1")] - pub total: u32, - #[prost(bytes = "vec", tag = "2")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceList { - #[prost(message, repeated, tag = "1")] - pub evidence: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Evidence { - #[prost(oneof = "evidence::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `Evidence`. -pub mod evidence { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - DuplicateVoteEvidence(super::DuplicateVoteEvidence), - #[prost(message, tag = "2")] - LightClientAttackEvidence(super::LightClientAttackEvidence), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DuplicateVoteEvidence { - #[prost(message, optional, tag = "1")] - pub vote_a: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub vote_b: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, - #[prost(int64, tag = "4")] - pub validator_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventVote { - #[prost(enumeration = "SignedMsgType", tag = "1")] - pub event_vote_type: i32, - #[prost(uint64, tag = "2")] - pub height: u64, - #[prost(int32, tag = "3")] - pub round: i32, - #[prost(message, optional, tag = "4")] - pub block_id: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(int32, tag = "7")] - pub validator_index: i32, - #[prost(bytes = "vec", tag = "8")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightClientAttackEvidence { - #[prost(message, optional, tag = "1")] - pub conflicting_block: ::core::option::Option, - #[prost(int64, tag = "2")] - pub common_height: i64, - #[prost(message, repeated, tag = "3")] - pub byzantine_validators: ::prost::alloc::vec::Vec, - #[prost(int64, tag = "4")] - pub total_voting_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightBlock { - #[prost(message, optional, tag = "1")] - pub signed_header: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub validator_set: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedHeader { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub commit: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Commit { - #[prost(int64, tag = "1")] - pub height: i64, - #[prost(int32, tag = "2")] - pub round: i32, - #[prost(message, optional, tag = "3")] - pub block_id: ::core::option::Option, - #[prost(message, repeated, tag = "4")] - pub signatures: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CommitSig { - #[prost(enumeration = "BlockIdFlag", tag = "1")] - pub block_id_flag: i32, - #[prost(bytes = "vec", tag = "2")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "3")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "4")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSet { - #[prost(message, repeated, tag = "1")] - pub validators: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub proposer: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Validator { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub voting_power: i64, - #[prost(int64, tag = "4")] - pub proposer_priority: i64, -} -#[graph_runtime_derive::generate_asc_type(sum{ed25519:Vec, secp256k1:Vec})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(sum{ed25519:Vec, secp256k1:Vec})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PublicKey { - #[prost(oneof = "public_key::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `PublicKey`. -pub mod public_key { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(bytes, tag = "1")] - Ed25519(::prost::alloc::vec::Vec), - #[prost(bytes, tag = "2")] - Secp256k1(::prost::alloc::vec::Vec), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseBeginBlock { - #[prost(message, repeated, tag = "1")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Event { - #[prost(string, tag = "1")] - pub event_type: ::prost::alloc::string::String, - #[prost(message, repeated, tag = "2")] - pub attributes: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventAttribute { - #[prost(string, tag = "1")] - pub key: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub value: ::prost::alloc::string::String, - #[prost(bool, tag = "3")] - pub index: bool, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseEndBlock { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub consensus_param_updates: ::core::option::Option, - #[prost(message, repeated, tag = "3")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorUpdate { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ConsensusParams { - #[prost(message, optional, tag = "1")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub validator: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub version: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockParams { - #[prost(int64, tag = "1")] - pub max_bytes: i64, - #[prost(int64, tag = "2")] - pub max_gas: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceParams { - #[prost(int64, tag = "1")] - pub max_age_num_blocks: i64, - #[prost(message, optional, tag = "2")] - pub max_age_duration: ::core::option::Option, - #[prost(int64, tag = "3")] - pub max_bytes: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Duration { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorParams { - #[prost(string, repeated, tag = "1")] - pub pub_key_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct VersionParams { - #[prost(uint64, tag = "1")] - pub app_version: u64, -} -#[graph_runtime_derive::generate_asc_type(__required__{tx:Tx, result:ResponseDeliverTx})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:Tx, - result:ResponseDeliverTx} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxResult { - #[prost(uint64, tag = "1")] - pub height: u64, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result: ::core::option::Option, - #[prost(bytes = "vec", tag = "5")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type(__required__{body:TxBody})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{body:TxBody})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tx { - #[prost(message, optional, tag = "1")] - pub body: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub auth_info: ::core::option::Option, - #[prost(bytes = "vec", repeated, tag = "3")] - pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxBody { - #[prost(message, repeated, tag = "1")] - pub messages: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(string, tag = "2")] - pub memo: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub timeout_height: u64, - #[prost(message, repeated, tag = "1023")] - pub extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(message, repeated, tag = "2047")] - pub non_critical_extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Any { - #[prost(string, tag = "1")] - pub type_url: ::prost::alloc::string::String, - #[prost(bytes = "vec", tag = "2")] - pub value: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AuthInfo { - #[prost(message, repeated, tag = "1")] - pub signer_infos: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub fee: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tip: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignerInfo { - #[prost(message, optional, tag = "1")] - pub public_key: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub mode_info: ::core::option::Option, - #[prost(uint64, tag = "3")] - pub sequence: u64, -} -#[graph_runtime_derive::generate_asc_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfo { - #[prost(oneof = "mode_info::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `ModeInfo`. -pub mod mode_info { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - Single(super::ModeInfoSingle), - #[prost(message, tag = "2")] - Multi(super::ModeInfoMulti), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoSingle { - #[prost(enumeration = "SignMode", tag = "1")] - pub mode: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoMulti { - #[prost(message, optional, tag = "1")] - pub bitarray: ::core::option::Option, - #[prost(message, repeated, tag = "2")] - pub mode_infos: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CompactBitArray { - #[prost(uint32, tag = "1")] - pub extra_bits_stored: u32, - #[prost(bytes = "vec", tag = "2")] - pub elems: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Fee { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "2")] - pub gas_limit: u64, - #[prost(string, tag = "3")] - pub payer: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub granter: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Coin { - #[prost(string, tag = "1")] - pub denom: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub amount: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tip { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(string, tag = "2")] - pub tipper: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseDeliverTx { - #[prost(uint32, tag = "1")] - pub code: u32, - #[prost(bytes = "vec", tag = "2")] - pub data: ::prost::alloc::vec::Vec, - #[prost(string, tag = "3")] - pub log: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub info: ::prost::alloc::string::String, - #[prost(int64, tag = "5")] - pub gas_wanted: i64, - #[prost(int64, tag = "6")] - pub gas_used: i64, - #[prost(message, repeated, tag = "7")] - pub events: ::prost::alloc::vec::Vec, - #[prost(string, tag = "8")] - pub codespace: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSetUpdates { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignedMsgType { - Unknown = 0, - Prevote = 1, - Precommit = 2, - Proposal = 32, -} -impl SignedMsgType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignedMsgType::Unknown => "SIGNED_MSG_TYPE_UNKNOWN", - SignedMsgType::Prevote => "SIGNED_MSG_TYPE_PREVOTE", - SignedMsgType::Precommit => "SIGNED_MSG_TYPE_PRECOMMIT", - SignedMsgType::Proposal => "SIGNED_MSG_TYPE_PROPOSAL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGNED_MSG_TYPE_UNKNOWN" => Some(Self::Unknown), - "SIGNED_MSG_TYPE_PREVOTE" => Some(Self::Prevote), - "SIGNED_MSG_TYPE_PRECOMMIT" => Some(Self::Precommit), - "SIGNED_MSG_TYPE_PROPOSAL" => Some(Self::Proposal), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum BlockIdFlag { - Unknown = 0, - Absent = 1, - Commit = 2, - Nil = 3, -} -impl BlockIdFlag { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - BlockIdFlag::Unknown => "BLOCK_ID_FLAG_UNKNOWN", - BlockIdFlag::Absent => "BLOCK_ID_FLAG_ABSENT", - BlockIdFlag::Commit => "BLOCK_ID_FLAG_COMMIT", - BlockIdFlag::Nil => "BLOCK_ID_FLAG_NIL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "BLOCK_ID_FLAG_UNKNOWN" => Some(Self::Unknown), - "BLOCK_ID_FLAG_ABSENT" => Some(Self::Absent), - "BLOCK_ID_FLAG_COMMIT" => Some(Self::Commit), - "BLOCK_ID_FLAG_NIL" => Some(Self::Nil), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignMode { - Unspecified = 0, - Direct = 1, - Textual = 2, - LegacyAminoJson = 127, -} -impl SignMode { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignMode::Unspecified => "SIGN_MODE_UNSPECIFIED", - SignMode::Direct => "SIGN_MODE_DIRECT", - SignMode::Textual => "SIGN_MODE_TEXTUAL", - SignMode::LegacyAminoJson => "SIGN_MODE_LEGACY_AMINO_JSON", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGN_MODE_UNSPECIFIED" => Some(Self::Unspecified), - "SIGN_MODE_DIRECT" => Some(Self::Direct), - "SIGN_MODE_TEXTUAL" => Some(Self::Textual), - "SIGN_MODE_LEGACY_AMINO_JSON" => Some(Self::LegacyAminoJson), - _ => None, - } - } -} diff --git a/chain/cosmos/src/runtime/abi.rs b/chain/cosmos/src/runtime/abi.rs deleted file mode 100644 index af9260b63be..00000000000 --- a/chain/cosmos/src/runtime/abi.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::protobuf::*; -use graph::runtime::HostExportError; -pub use graph::semver::Version; - -pub use graph::runtime::{ - asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, - DeterministicHostError, IndexForAscTypeId, ToAscObj, -}; -/* -TODO: AscBytesArray seem to be generic to all chains, but AscIndexId pins it to Cosmos -****************** this can be moved to runtime graph/runtime/src/asc_heap.rs, but IndexForAscTypeId::CosmosBytesArray ****** -*/ -pub struct AscBytesArray(pub Array>); - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self - .iter() - .map(|x| asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(x), gas)) - .collect(); - - Ok(AscBytesArray(Array::new(&content?, heap, gas)?)) - } -} - -//this can be moved to runtime -impl AscType for AscBytesArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -//we will have to keep this chain specific (Inner/Outer) -impl AscIndexId for AscBytesArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::CosmosBytesArray; -} - -/************************************************************************** */ -// this can be moved to runtime - prost_types::Any -impl ToAscObj for prost_types::Any { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscAny { - type_url: asc_new(heap, &self.type_url, gas)?, - value: asc_new( - heap, - &graph_runtime_wasm::asc_abi::class::Bytes(&self.value), - gas, - )?, - ..Default::default() - }) - } -} - -//this can be moved to runtime - prost_types::Any -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - - Ok(AscAnyArray(Array::new(&content?, heap, gas)?)) - } -} diff --git a/chain/cosmos/src/runtime/mod.rs b/chain/cosmos/src/runtime/mod.rs deleted file mode 100644 index 78d10d4d7d3..00000000000 --- a/chain/cosmos/src/runtime/mod.rs +++ /dev/null @@ -1,348 +0,0 @@ -pub mod abi; - -#[cfg(test)] -mod test { - use crate::protobuf::*; - - use graph::semver::Version; - - /// A macro that takes an ASC struct value definition and calls AscBytes methods to check that - /// memory layout is padded properly. - macro_rules! assert_asc_bytes { - ($struct_name:ident { - $($field:ident : $field_value:expr),+ - $(,)? // trailing - }) => { - let value = $struct_name { - $($field: $field_value),+ - }; - - // just call the function. it will panic on misalignments - let asc_bytes = value.to_asc_bytes().unwrap(); - - let value_004 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 4)).unwrap(); - let value_005 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 5)).unwrap(); - - // turn the values into bytes again to verify that they are the same as the original - // because these types usually don't implement PartialEq - assert_eq!( - asc_bytes, - value_004.to_asc_bytes().unwrap(), - "Expected {} v0.0.4 asc bytes to be the same", - stringify!($struct_name) - ); - assert_eq!( - asc_bytes, - value_005.to_asc_bytes().unwrap(), - "Expected {} v0.0.5 asc bytes to be the same", - stringify!($struct_name) - ); - }; - } - - #[test] - fn test_asc_type_alignment() { - // TODO: automatically generate these tests for each struct in derive(AscType) macro - - assert_asc_bytes!(AscBlock { - header: new_asc_ptr(), - evidence: new_asc_ptr(), - last_commit: new_asc_ptr(), - result_begin_block: new_asc_ptr(), - result_end_block: new_asc_ptr(), - transactions: new_asc_ptr(), - validator_updates: new_asc_ptr(), - }); - - assert_asc_bytes!(AscHeaderOnlyBlock { - header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventData { - event: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionData { - tx: new_asc_ptr(), - block: new_asc_ptr(), - }); - - assert_asc_bytes!(AscMessageData { - message: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionContext { - hash: new_asc_ptr(), - index: 20, - code: 20, - gas_wanted: 20, - gas_used: 20, - }); - - assert_asc_bytes!(AscHeader { - version: new_asc_ptr(), - chain_id: new_asc_ptr(), - height: 20, - time: new_asc_ptr(), - last_block_id: new_asc_ptr(), - last_commit_hash: new_asc_ptr(), - data_hash: new_asc_ptr(), - validators_hash: new_asc_ptr(), - next_validators_hash: new_asc_ptr(), - consensus_hash: new_asc_ptr(), - app_hash: new_asc_ptr(), - last_results_hash: new_asc_ptr(), - evidence_hash: new_asc_ptr(), - proposer_address: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscConsensus { block: 0, app: 0 }); - - assert_asc_bytes!(AscTimestamp { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscBlockId { - hash: new_asc_ptr(), - part_set_header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscPartSetHeader { - total: 20, - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidenceList { - evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidence { - duplicate_vote_evidence: new_asc_ptr(), - light_client_attack_evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscDuplicateVoteEvidence { - vote_a: new_asc_ptr(), - vote_b: new_asc_ptr(), - total_voting_power: 20, - validator_power: 20, - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventVote { - event_vote_type: 20, - height: 20, - round: 20, - block_id: new_asc_ptr(), - timestamp: new_asc_ptr(), - validator_address: new_asc_ptr(), - validator_index: 20, - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightClientAttackEvidence { - conflicting_block: new_asc_ptr(), - common_height: 20, - total_voting_power: 20, - byzantine_validators: new_asc_ptr(), - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightBlock { - signed_header: new_asc_ptr(), - validator_set: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignedHeader { - header: new_asc_ptr(), - commit: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommit { - height: 20, - round: 20, - block_id: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommitSig { - block_id_flag: 20, - validator_address: new_asc_ptr(), - timestamp: new_asc_ptr(), - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSet { - validators: new_asc_ptr(), - proposer: new_asc_ptr(), - total_voting_power: 20, - }); - - assert_asc_bytes!(AscValidator { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - voting_power: 20, - proposer_priority: 20, - }); - - assert_asc_bytes!(AscPublicKey { - ed25519: new_asc_ptr(), - secp256k1: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseBeginBlock { - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvent { - event_type: new_asc_ptr(), - attributes: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventAttribute { - key: new_asc_ptr(), - value: new_asc_ptr(), - index: true, - }); - - assert_asc_bytes!(AscResponseEndBlock { - validator_updates: new_asc_ptr(), - consensus_param_updates: new_asc_ptr(), - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorUpdate { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - power: 20, - }); - - assert_asc_bytes!(AscConsensusParams { - block: new_asc_ptr(), - evidence: new_asc_ptr(), - validator: new_asc_ptr(), - version: new_asc_ptr(), - }); - - assert_asc_bytes!(AscBlockParams { - max_bytes: 20, - max_gas: 20, - }); - - assert_asc_bytes!(AscEvidenceParams { - max_age_num_blocks: 20, - max_age_duration: new_asc_ptr(), - max_bytes: 20, - }); - - assert_asc_bytes!(AscDuration { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscValidatorParams { - pub_key_types: new_asc_ptr(), - }); - - assert_asc_bytes!(AscVersionParams { app_version: 20 }); - - assert_asc_bytes!(AscTxResult { - height: 20, - index: 20, - tx: new_asc_ptr(), - result: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTx { - body: new_asc_ptr(), - auth_info: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTxBody { - messages: new_asc_ptr(), - memo: new_asc_ptr(), - timeout_height: 20, - extension_options: new_asc_ptr(), - non_critical_extension_options: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAny { - type_url: new_asc_ptr(), - value: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAuthInfo { - signer_infos: new_asc_ptr(), - fee: new_asc_ptr(), - tip: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignerInfo { - public_key: new_asc_ptr(), - mode_info: new_asc_ptr(), - sequence: 20, - }); - - assert_asc_bytes!(AscModeInfo { - single: new_asc_ptr(), - multi: new_asc_ptr(), - }); - - assert_asc_bytes!(AscModeInfoSingle { mode: 20 }); - - assert_asc_bytes!(AscModeInfoMulti { - bitarray: new_asc_ptr(), - mode_infos: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCompactBitArray { - extra_bits_stored: 20, - elems: new_asc_ptr(), - }); - - assert_asc_bytes!(AscFee { - amount: new_asc_ptr(), - gas_limit: 20, - payer: new_asc_ptr(), - granter: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCoin { - denom: new_asc_ptr(), - amount: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTip { - amount: new_asc_ptr(), - tipper: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseDeliverTx { - code: 20, - data: new_asc_ptr(), - log: new_asc_ptr(), - info: new_asc_ptr(), - gas_wanted: 20, - gas_used: 20, - events: new_asc_ptr(), - codespace: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSetUpdates { - validator_updates: new_asc_ptr(), - }); - } - - // non-null AscPtr - fn new_asc_ptr() -> AscPtr { - AscPtr::new(12) - } -} diff --git a/chain/cosmos/src/trigger.rs b/chain/cosmos/src/trigger.rs deleted file mode 100644 index 9700a75bf76..00000000000 --- a/chain/cosmos/src/trigger.rs +++ /dev/null @@ -1,364 +0,0 @@ -use std::{cmp::Ordering, sync::Arc}; - -use graph::blockchain::{Block, BlockHash, MappingTriggerTrait, TriggerData}; -use graph::derive::CheapClone; -use graph::prelude::{BlockNumber, Error}; -use graph::runtime::HostExportError; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr}; -use graph_runtime_wasm::module::ToAscPtr; - -use crate::codec; -use crate::data_source::EventOrigin; - -// Logging the block is too verbose, so this strips the block from the trigger for Debug. -impl std::fmt::Debug for CosmosTrigger { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[allow(unused)] - #[derive(Debug)] - pub enum MappingTriggerWithoutBlock<'e> { - Block, - Event { - event_type: &'e str, - origin: EventOrigin, - }, - Transaction, - Message, - } - - let trigger_without_block = match self { - CosmosTrigger::Block(_) => MappingTriggerWithoutBlock::Block, - CosmosTrigger::Event { event_data, origin } => MappingTriggerWithoutBlock::Event { - event_type: &event_data.event().map_err(|_| std::fmt::Error)?.event_type, - origin: *origin, - }, - CosmosTrigger::Transaction(_) => MappingTriggerWithoutBlock::Transaction, - CosmosTrigger::Message(_) => MappingTriggerWithoutBlock::Message, - }; - - write!(f, "{:?}", trigger_without_block) - } -} - -impl ToAscPtr for CosmosTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - Ok(match self { - CosmosTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - CosmosTrigger::Event { event_data, .. } => { - asc_new(heap, event_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Transaction(transaction_data) => { - asc_new(heap, transaction_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Message(message_data) => { - asc_new(heap, message_data.as_ref(), gas)?.erase() - } - }) - } -} - -#[derive(Clone, CheapClone)] -pub enum CosmosTrigger { - Block(Arc), - Event { - event_data: Arc, - origin: EventOrigin, - }, - Transaction(Arc), - Message(Arc), -} - -impl PartialEq for CosmosTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, - ( - Self::Event { - event_data: a_event_data, - origin: a_origin, - }, - Self::Event { - event_data: b_event_data, - origin: b_origin, - }, - ) => { - if let (Ok(a_event), Ok(b_event)) = (a_event_data.event(), b_event_data.event()) { - let mut attributes_a = a_event.attributes.clone(); - attributes_a.sort_by(|a, b| a.key.cmp(&b.key)); - - let mut attributes_b = b_event.attributes.clone(); - attributes_b.sort_by(|a, b| a.key.cmp(&b.key)); - - a_event.event_type == b_event.event_type - && a_origin == b_origin - && attributes_a == attributes_b - } else { - false - } - } - (Self::Transaction(a_ptr), Self::Transaction(b_ptr)) => a_ptr == b_ptr, - (Self::Message(a_ptr), Self::Message(b_ptr)) => a_ptr == b_ptr, - _ => false, - } - } -} - -impl Eq for CosmosTrigger {} - -impl CosmosTrigger { - pub(crate) fn with_event( - event: codec::Event, - block: codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, - ) -> CosmosTrigger { - CosmosTrigger::Event { - event_data: Arc::new(codec::EventData { - event: Some(event), - block: Some(block), - tx: tx_context, - }), - origin, - } - } - - pub(crate) fn with_transaction( - tx_result: codec::TxResult, - block: codec::HeaderOnlyBlock, - ) -> CosmosTrigger { - CosmosTrigger::Transaction(Arc::new(codec::TransactionData { - tx: Some(tx_result), - block: Some(block), - })) - } - - pub(crate) fn with_message( - message: ::prost_types::Any, - block: codec::HeaderOnlyBlock, - tx_context: codec::TransactionContext, - ) -> CosmosTrigger { - CosmosTrigger::Message(Arc::new(codec::MessageData { - message: Some(message), - block: Some(block), - tx: Some(tx_context), - })) - } - - pub fn block_number(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.number()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.number()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.number()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.number()), - } - } - - pub fn block_hash(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.hash()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.hash()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.hash()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.hash()), - } - } - - fn error_context(&self) -> std::string::String { - match self { - CosmosTrigger::Block(..) => { - if let (Ok(block_number), Ok(block_hash)) = (self.block_number(), self.block_hash()) - { - format!("block #{block_number}, hash {block_hash}") - } else { - "block".to_string() - } - } - CosmosTrigger::Event { event_data, origin } => { - if let (Ok(event), Ok(block_number), Ok(block_hash)) = - (event_data.event(), self.block_number(), self.block_hash()) - { - format!( - "event type {}, origin: {:?}, block #{block_number}, hash {block_hash}", - event.event_type, origin, - ) - } else { - "event".to_string() - } - } - CosmosTrigger::Transaction(transaction_data) => { - if let (Ok(block_number), Ok(block_hash), Ok(response_deliver_tx)) = ( - self.block_number(), - self.block_hash(), - transaction_data.response_deliver_tx(), - ) { - format!( - "block #{block_number}, hash {block_hash}, transaction log: {}", - response_deliver_tx.log - ) - } else { - "transaction".to_string() - } - } - CosmosTrigger::Message(message_data) => { - if let (Ok(message), Ok(block_number), Ok(block_hash)) = ( - message_data.message(), - self.block_number(), - self.block_hash(), - ) { - format!( - "message type {}, block #{block_number}, hash {block_hash}", - message.type_url, - ) - } else { - "message".to_string() - } - } - } - } -} - -impl Ord for CosmosTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - // Events have no intrinsic ordering information, so we keep the order in - // which they are included in the `events` field - (Self::Event { .. }, Self::Event { .. }) => Ordering::Equal, - - // Keep the order when comparing two message triggers - (Self::Message(..), Self::Message(..)) => Ordering::Equal, - - // Transactions are ordered by their index inside the block - (Self::Transaction(a), Self::Transaction(b)) => { - if let (Ok(a_tx_result), Ok(b_tx_result)) = (a.tx_result(), b.tx_result()) { - a_tx_result.index.cmp(&b_tx_result.index) - } else { - Ordering::Equal - } - } - - // Keep the order when comparing two block triggers - (Self::Block(..), Self::Block(..)) => Ordering::Equal, - - // Event triggers always come first - (Self::Event { .. }, _) => Ordering::Greater, - (_, Self::Event { .. }) => Ordering::Less, - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Less, - (_, Self::Block(..)) => Ordering::Greater, - - // Message triggers before Transaction triggers - (Self::Message(..), Self::Transaction(..)) => Ordering::Greater, - (Self::Transaction(..), Self::Message(..)) => Ordering::Less, - } - } -} - -impl PartialOrd for CosmosTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl TriggerData for CosmosTrigger { - fn error_context(&self) -> String { - self.error_context() - } - - fn address_match(&self) -> Option<&[u8]> { - None - } -} - -impl MappingTriggerTrait for CosmosTrigger { - fn error_context(&self) -> String { - self.error_context() - } -} -#[cfg(test)] -mod tests { - use crate::codec::TxResult; - - use super::*; - - #[test] - fn test_cosmos_trigger_ordering() { - let event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let other_event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let other_message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let transaction_trigger = CosmosTrigger::Transaction(Arc::::new( - codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 1, - ..Default::default() - }), - }, - )); - let other_transaction_trigger = CosmosTrigger::Transaction( - Arc::::new(codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 2, - ..Default::default() - }), - }), - ); - let block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - let other_block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - - assert_eq!(event_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&message_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&other_event_trigger), Ordering::Equal); - - assert_eq!(message_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&other_message_trigger), Ordering::Equal); - assert_eq!(message_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(transaction_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!( - transaction_trigger.cmp(&other_transaction_trigger), - Ordering::Less - ); - assert_eq!( - other_transaction_trigger.cmp(&transaction_trigger), - Ordering::Greater - ); - assert_eq!(transaction_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(transaction_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(block_trigger.cmp(&other_block_trigger), Ordering::Equal); - assert_eq!(block_trigger.cmp(&transaction_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&event_trigger), Ordering::Less); - } -} diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml index 61ae59ab4af..43d1afb9bd3 100644 --- a/chain/ethereum/Cargo.toml +++ b/chain/ethereum/Cargo.toml @@ -22,7 +22,6 @@ graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] base64 = "0" -uuid = { version = "1.9.1", features = ["v4"] } [build-dependencies] tonic-build = { workspace = true } diff --git a/chain/ethereum/build.rs b/chain/ethereum/build.rs index 8ccae67aa92..227a50914a6 100644 --- a/chain/ethereum/build.rs +++ b/chain/ethereum/build.rs @@ -3,6 +3,6 @@ fn main() { tonic_build::configure() .out_dir("src/protobuf") - .compile(&["proto/ethereum.proto"], &["proto"]) + .compile_protos(&["proto/ethereum.proto"], &["proto"]) .expect("Failed to compile Firehose Ethereum proto(s)"); } diff --git a/chain/ethereum/examples/firehose.rs b/chain/ethereum/examples/firehose.rs index e5f85964fe1..5a70794dfe2 100644 --- a/chain/ethereum/examples/firehose.rs +++ b/chain/ethereum/examples/firehose.rs @@ -38,6 +38,7 @@ async fn main() -> Result<(), Error> { false, SubgraphLimit::Unlimited, metrics, + false, )); loop { diff --git a/chain/ethereum/proto/ethereum.proto b/chain/ethereum/proto/ethereum.proto index 3c9f7378c7d..42adbd0ffa6 100644 --- a/chain/ethereum/proto/ethereum.proto +++ b/chain/ethereum/proto/ethereum.proto @@ -13,7 +13,7 @@ message Block { uint64 size = 4; BlockHeader header = 5; - // Uncles represents block produced with a valid solution but were not actually choosen + // Uncles represents block produced with a valid solution but were not actually chosen // as the canonical block for the given height so they are mostly "forked" blocks. // // If the Block has been produced using the Proof of Stake consensus algorithm, this @@ -285,7 +285,7 @@ message Log { bytes data = 3; // Index is the index of the log relative to the transaction. This index - // is always populated regardless of the state revertion of the the call + // is always populated regardless of the state reversion of the call // that emitted this log. uint32 index = 4; @@ -294,7 +294,7 @@ message Log { // An **important** notice is that this field will be 0 when the call // that emitted the log has been reverted by the chain. // - // Currently, there is two locations where a Log can be obtained: + // Currently, there are two locations where a Log can be obtained: // - block.transaction_traces[].receipt.logs[] // - block.transaction_traces[].calls[].logs[] // @@ -341,7 +341,7 @@ message Call { reserved 29; // In Ethereum, a call can be either: - // - Successfull, execution passes without any problem encountered + // - Successful, execution passes without any problem encountered // - Failed, execution failed, and remaining gas should be consumed // - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded // @@ -355,7 +355,7 @@ message Call { // see above for details about those flags. string failure_reason = 11; - // This field represents wheter or not the state changes performed + // This field represents whether or not the state changes performed // by this call were correctly recorded by the blockchain. // // On Ethereum, a transaction can record state changes even if some @@ -412,7 +412,7 @@ message BalanceChange { BigInt new_value = 3; Reason reason = 4; - // Obtain all balanche change reasons under deep mind repository: + // Obtain all balance change reasons under deep mind repository: // // ```shell // ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq @@ -466,7 +466,7 @@ message CodeChange { // The gas is computed per actual op codes. Doing them completely might prove // overwhelming in most cases. // -// Hence, we only index some of them, those that are costy like all the calls +// Hence, we only index some of them, those that are costly like all the calls // one, log events, return data, etc. message GasChange { uint64 old_value = 1; @@ -505,4 +505,4 @@ message GasChange { } uint64 ordinal = 4; -} \ No newline at end of file +} diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index f632ee36d93..911d4d3ebfe 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -17,7 +17,7 @@ use graph::prelude::{ EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, }; use graph::schema::InputSchema; -use graph::slog::{debug, error, trace}; +use graph::slog::{debug, error, trace, warn}; use graph::substreams::Clock; use graph::{ blockchain::{ @@ -257,6 +257,7 @@ pub struct EthereumAdapterSelector { client: Arc>, registry: Arc, chain_store: Arc, + eth_adapters: Arc, } impl EthereumAdapterSelector { @@ -265,12 +266,14 @@ impl EthereumAdapterSelector { client: Arc>, registry: Arc, chain_store: Arc, + eth_adapters: Arc, ) -> Self { Self { logger_factory, client, registry, chain_store, + eth_adapters, } } } @@ -296,6 +299,7 @@ impl TriggersAdapterSelector for EthereumAdapterSelector { chain_store: self.chain_store.cheap_clone(), unified_api_version, capabilities: *capabilities, + eth_adapters: self.eth_adapters.cheap_clone(), }; Ok(Arc::new(adapter)) } @@ -610,7 +614,7 @@ impl Blockchain for Chain { // present in the DB. Box::new(PollingBlockIngestor::new( logger, - graph::env::ENV_VARS.reorg_threshold, + graph::env::ENV_VARS.reorg_threshold(), self.chain_client(), self.chain_store().cheap_clone(), self.polling_ingestor_interval, @@ -739,6 +743,7 @@ pub struct TriggersAdapter { chain_client: Arc>, capabilities: NodeCapabilities, unified_api_version: UnifiedMappingApiVersion, + eth_adapters: Arc, } /// Fetches blocks from the cache based on block numbers, excluding duplicates @@ -784,12 +789,34 @@ async fn fetch_unique_blocks_from_cache( "Loading {} block(s) not in the block cache", missing_blocks.len() ); - debug!(logger, "Missing blocks {:?}", missing_blocks); + trace!(logger, "Missing blocks {:?}", missing_blocks.len()); } (blocks, missing_blocks) } +// This is used to load blocks from the RPC. +async fn load_blocks_with_rpc( + logger: &Logger, + adapter: Arc, + chain_store: Arc, + block_numbers: BTreeSet, +) -> Result> { + let logger_clone = logger.clone(); + load_blocks( + logger, + chain_store, + block_numbers, + |missing_numbers| async move { + adapter + .load_block_ptrs_by_numbers_rpc(logger_clone, missing_numbers) + .try_collect() + .await + }, + ) + .await +} + /// Fetches blocks by their numbers, first attempting to load from cache. /// Missing blocks are retrieved from an external source, with all blocks sorted and converted to `BlockFinality` format. async fn load_blocks( @@ -847,6 +874,37 @@ impl TriggersAdapterTrait for TriggersAdapter { ) -> Result> { match &*self.chain_client { ChainClient::Firehose(endpoints) => { + // If the force_rpc_for_block_ptrs flag is set, we will use the RPC to load the blocks + // even if the firehose is available. If no adapter is available, we will log an error. + // And then fallback to the firehose. + if ENV_VARS.force_rpc_for_block_ptrs { + trace!( + logger, + "Loading blocks from RPC (force_rpc_for_block_ptrs is set)"; + "block_numbers" => format!("{:?}", block_numbers) + ); + match self.eth_adapters.cheapest_with(&self.capabilities).await { + Ok(adapter) => { + match load_blocks_with_rpc( + &logger, + adapter, + self.chain_store.clone(), + block_numbers.clone(), + ) + .await + { + Ok(blocks) => return Ok(blocks), + Err(e) => { + warn!(logger, "Error loading blocks from RPC: {}", e); + } + } + } + Err(e) => { + warn!(logger, "Error getting cheapest adapter: {}", e); + } + } + } + trace!( logger, "Loading blocks from firehose"; @@ -884,29 +942,16 @@ impl TriggersAdapterTrait for TriggersAdapter { .await } - ChainClient::Rpc(client) => { + ChainClient::Rpc(eth_adapters) => { trace!( logger, "Loading blocks from RPC"; "block_numbers" => format!("{:?}", block_numbers) ); - let adapter = client.cheapest_with(&self.capabilities).await?; - let chain_store = self.chain_store.clone(); - let logger_clone = logger.clone(); - - load_blocks( - &logger, - chain_store, - block_numbers, - |missing_numbers| async move { - adapter - .load_block_ptrs_by_numbers_rpc(logger_clone, missing_numbers) - .try_collect() - .await - }, - ) - .await + let adapter = eth_adapters.cheapest_with(&self.capabilities).await?; + load_blocks_with_rpc(&logger, adapter, self.chain_store.clone(), block_numbers) + .await } } } @@ -973,10 +1018,12 @@ impl TriggersAdapterTrait for TriggersAdapter { ChainClient::Firehose(endpoints) => { let endpoint = endpoints.endpoint().await?; let block = endpoint - .get_block_by_number::(ptr.number as u64, &self.logger) + .get_block_by_number_with_retry::(ptr.number as u64, &self.logger) .await - .map_err(|e| anyhow!("Failed to fetch block from firehose: {}", e))?; - + .context(format!( + "Failed to fetch block {} from firehose", + ptr.number + ))?; Ok(block.hash() == ptr.hash) } ChainClient::Rpc(adapter) => { diff --git a/chain/ethereum/src/env.rs b/chain/ethereum/src/env.rs index bc7223dbc07..027a26b623f 100644 --- a/chain/ethereum/src/env.rs +++ b/chain/ethereum/src/env.rs @@ -91,6 +91,10 @@ pub struct EnvVars { /// This is a comma separated list of chain ids for which the gas field will not be set /// when calling `eth_call`. pub eth_call_no_gas: Vec, + /// Set by the flag `GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS`. On by default. + /// When enabled, forces the use of RPC instead of Firehose for loading block pointers by numbers. + /// This is used in composable subgraphs. Firehose can be slow for loading block pointers by numbers. + pub force_rpc_for_block_ptrs: bool, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -141,6 +145,7 @@ impl From for EnvVars { .filter(|s| !s.is_empty()) .map(str::to_string) .collect(), + force_rpc_for_block_ptrs: x.force_rpc_for_block_ptrs.0, } } } @@ -192,4 +197,6 @@ struct Inner { genesis_block_number: u64, #[envconfig(from = "GRAPH_ETH_CALL_NO_GAS", default = "421613,421614")] eth_call_no_gas: String, + #[envconfig(from = "GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS", default = "true")] + force_rpc_for_block_ptrs: EnvVarBoolean, } diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 7173c069c65..e0714c24f02 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -147,6 +147,7 @@ impl EthereumAdapter { let retry_log_message = format!("trace_filter RPC call for block range: [{}..{}]", from, to); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -295,6 +296,7 @@ impl EthereumAdapter { let eth_adapter = self.clone(); let retry_log_message = format!("eth_getLogs RPC call for block range: [{}..{}]", from, to); retry(retry_log_message, &logger) + .redact_log_urls(true) .when(move |res: &Result<_, web3::error::Error>| match res { Ok(_) => false, Err(e) => !too_many_logs_fingerprints @@ -511,6 +513,7 @@ impl EthereumAdapter { let retry_log_message = format!("eth_getCode RPC call for block {}", block_ptr); retry(retry_log_message, &logger) + .redact_log_urls(true) .when(|result| match result { Ok(_) => false, Err(_) => true, @@ -546,6 +549,7 @@ impl EthereumAdapter { let retry_log_message = format!("eth_getBalance RPC call for block {}", block_ptr); retry(retry_log_message, &logger) + .redact_log_urls(true) .when(|result| match result { Ok(_) => false, Err(_) => true, @@ -586,6 +590,7 @@ impl EthereumAdapter { let block_id = self.block_ptr_to_id(&block_ptr); let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -765,6 +770,7 @@ impl EthereumAdapter { stream::iter_ok::<_, Error>(ids.into_iter().map(move |hash| { let web3 = web3.clone(); retry(format!("load block {}", hash), &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -799,6 +805,7 @@ impl EthereumAdapter { async move { retry(format!("load block {}", number), &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -856,6 +863,7 @@ impl EthereumAdapter { stream::iter_ok::<_, Error>(block_nums.into_iter().map(move |block_num| { let web3 = web3.clone(); retry(format!("load block ptr {}", block_num), &logger) + .redact_log_urls(true) .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) @@ -1140,6 +1148,7 @@ impl EthereumAdapter { let web3 = self.web3.clone(); u64::try_from( retry("chain_id RPC call", &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1175,6 +1184,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let metrics = self.metrics.clone(); let provider = self.provider().to_string(); let net_version_future = retry("net_version RPC call", &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(20) .run(move || { @@ -1203,6 +1213,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ENV_VARS.genesis_block_number ); let gen_block_hash_future = retry(retry_log_message, &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(30) .run(move || { @@ -1254,6 +1265,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let web3 = self.web3.clone(); Box::new( retry("eth_getBlockByNumber(latest) no txs RPC call", logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1288,6 +1300,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let web3 = self.web3.clone(); Box::new( retry("eth_getBlockByNumber(latest) with txs RPC call", logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1345,6 +1358,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ); Box::new( retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1376,6 +1390,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ); Box::new( retry(retry_log_message, &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1458,6 +1473,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ); Box::new( retry(retry_log_message, logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1525,6 +1541,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let web3 = self.web3.clone(); let logger = logger.clone(); let res = retry(retry_log_message, &logger) + .redact_log_urls(true) .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) @@ -2279,6 +2296,7 @@ async fn fetch_transaction_receipts_in_batch_with_retry( block_hash ); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .no_logging() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) @@ -2406,6 +2424,7 @@ async fn fetch_block_receipts_with_retry( // Perform the retry operation let receipts_option = retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || web3.eth().block_receipts(BlockId::Hash(block_hash)).boxed()) @@ -2450,6 +2469,7 @@ async fn fetch_transaction_receipt_with_retry( transaction_hash ); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || web3.eth().transaction_receipt(transaction_hash).boxed()) diff --git a/chain/ethereum/src/network.rs b/chain/ethereum/src/network.rs index 43b5a04b63a..d654db71276 100644 --- a/chain/ethereum/src/network.rs +++ b/chain/ethereum/src/network.rs @@ -325,7 +325,6 @@ mod tests { url::Url, }; use std::sync::Arc; - use uuid::Uuid; use crate::{EthereumAdapter, EthereumAdapterTrait, ProviderEthRpcMetrics, Transport}; @@ -679,18 +678,14 @@ mod tests { #[tokio::test] async fn eth_adapter_selection_multiple_adapters() { let logger = Logger::root(Discard, o!()); - let unavailable_provider = Uuid::new_v4().to_string(); - let error_provider = Uuid::new_v4().to_string(); - let no_error_provider = Uuid::new_v4().to_string(); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; let mock_registry = Arc::new(MetricsRegistry::mock()); let metrics = Arc::new(EndpointMetrics::new( logger, - &[ - unavailable_provider.clone(), - error_provider.clone(), - no_error_provider.clone(), - ], + &[unavailable_provider, error_provider, no_error_provider], mock_registry.clone(), )); let logger = graph::log::logger(true); @@ -718,7 +713,7 @@ mod tests { ]; // Set errors - metrics.report_for_test(&ProviderName::from(error_provider.clone()), false); + metrics.report_for_test(&ProviderName::from(error_provider), false); let mut no_retest_adapters = vec![]; let mut always_retest_adapters = vec![]; @@ -796,18 +791,14 @@ mod tests { #[tokio::test] async fn eth_adapter_selection_single_adapter() { let logger = Logger::root(Discard, o!()); - let unavailable_provider = Uuid::new_v4().to_string(); - let error_provider = Uuid::new_v4().to_string(); - let no_error_provider = Uuid::new_v4().to_string(); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; let mock_registry = Arc::new(MetricsRegistry::mock()); let metrics = Arc::new(EndpointMetrics::new( logger, - &[ - unavailable_provider, - error_provider.clone(), - no_error_provider.clone(), - ], + &[unavailable_provider, error_provider, no_error_provider], mock_registry.clone(), )); let chain_id: Word = "chain_id".into(); @@ -815,7 +806,7 @@ mod tests { let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); // Set errors - metrics.report_for_test(&ProviderName::from(error_provider.clone()), false); + metrics.report_for_test(&ProviderName::from(error_provider), false); let mut no_retest_adapters = vec![]; no_retest_adapters.push(EthereumNetworkAdapter { diff --git a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs index 6d13e187d14..4ab8d0a1324 100644 --- a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs +++ b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { #[prost(int32, tag = "1")] @@ -12,7 +11,7 @@ pub struct Block { pub size: u64, #[prost(message, optional, tag = "5")] pub header: ::core::option::Option, - /// Uncles represents block produced with a valid solution but were not actually choosen + /// Uncles represents block produced with a valid solution but were not actually chosen /// as the canonical block for the given height so they are mostly "forked" blocks. /// /// If the Block has been produced using the Proof of Stake consensus algorithm, this @@ -32,7 +31,6 @@ pub struct Block { /// /// WARN: this is a client-side optimization pattern and should be moved in the /// consuming code. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct HeaderOnlyBlock { #[prost(message, optional, tag = "5")] @@ -41,7 +39,6 @@ pub struct HeaderOnlyBlock { /// BlockWithRefs is a lightweight block, with traces and transactions /// purged from the `block` within, and only. It is used in transports /// to pass block data around. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockWithRefs { #[prost(string, tag = "1")] @@ -53,19 +50,16 @@ pub struct BlockWithRefs { #[prost(bool, tag = "4")] pub irreversible: bool, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionRefs { #[prost(bytes = "vec", repeated, tag = "1")] pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct UnclesHeaders { #[prost(message, repeated, tag = "1")] pub uncles: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockRef { #[prost(bytes = "vec", tag = "1")] @@ -73,7 +67,6 @@ pub struct BlockRef { #[prost(uint64, tag = "2")] pub number: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockHeader { #[prost(bytes = "vec", tag = "1")] @@ -163,13 +156,11 @@ pub struct BlockHeader { #[prost(message, optional, tag = "18")] pub base_fee_per_gas: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionTrace { /// consensus @@ -290,9 +281,9 @@ pub mod transaction_trace { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Type::TrxTypeLegacy => "TRX_TYPE_LEGACY", - Type::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", - Type::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", + Self::TrxTypeLegacy => "TRX_TYPE_LEGACY", + Self::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", + Self::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -308,7 +299,6 @@ pub mod transaction_trace { } /// AccessTuple represents a list of storage keys for a given contract's address and is used /// for AccessList construction. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccessTuple { #[prost(bytes = "vec", tag = "1")] @@ -317,7 +307,6 @@ pub struct AccessTuple { pub storage_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } /// TransactionTraceWithBlockRef -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionTraceWithBlockRef { #[prost(message, optional, tag = "1")] @@ -325,7 +314,6 @@ pub struct TransactionTraceWithBlockRef { #[prost(message, optional, tag = "2")] pub block_ref: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionReceipt { /// State root is an intermediate state_root hash, computed in-between transactions to make @@ -350,7 +338,6 @@ pub struct TransactionReceipt { #[prost(message, repeated, tag = "4")] pub logs: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Log { #[prost(bytes = "vec", tag = "1")] @@ -360,7 +347,7 @@ pub struct Log { #[prost(bytes = "vec", tag = "3")] pub data: ::prost::alloc::vec::Vec, /// Index is the index of the log relative to the transaction. This index - /// is always populated regardless of the state revertion of the the call + /// is always populated regardless of the state reversion of the call /// that emitted this log. #[prost(uint32, tag = "4")] pub index: u32, @@ -369,7 +356,7 @@ pub struct Log { /// An **important** notice is that this field will be 0 when the call /// that emitted the log has been reverted by the chain. /// - /// Currently, there is two locations where a Log can be obtained: + /// Currently, there are two locations where a Log can be obtained: /// - block.transaction_traces\[\].receipt.logs\[\] /// - block.transaction_traces\[\].calls\[\].logs\[\] /// @@ -384,7 +371,6 @@ pub struct Log { #[prost(uint64, tag = "7")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Call { #[prost(uint32, tag = "1")] @@ -432,7 +418,7 @@ pub struct Call { #[prost(message, repeated, tag = "28")] pub gas_changes: ::prost::alloc::vec::Vec, /// In Ethereum, a call can be either: - /// - Successfull, execution passes without any problem encountered + /// - Successful, execution passes without any problem encountered /// - Failed, execution failed, and remaining gas should be consumed /// - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded /// @@ -447,7 +433,7 @@ pub struct Call { /// see above for details about those flags. #[prost(string, tag = "11")] pub failure_reason: ::prost::alloc::string::String, - /// This field represents wheter or not the state changes performed + /// This field represents whether or not the state changes performed /// by this call were correctly recorded by the blockchain. /// /// On Ethereum, a transaction can record state changes even if some @@ -477,7 +463,6 @@ pub struct Call { #[prost(message, repeated, tag = "33")] pub account_creations: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct StorageChange { #[prost(bytes = "vec", tag = "1")] @@ -491,7 +476,6 @@ pub struct StorageChange { #[prost(uint64, tag = "5")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BalanceChange { #[prost(bytes = "vec", tag = "1")] @@ -507,7 +491,7 @@ pub struct BalanceChange { } /// Nested message and enum types in `BalanceChange`. pub mod balance_change { - /// Obtain all balanche change reasons under deep mind repository: + /// Obtain all balance change reasons under deep mind repository: /// /// ```shell /// ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq @@ -550,22 +534,22 @@ pub mod balance_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", - Reason::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", - Reason::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", - Reason::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", - Reason::Transfer => "REASON_TRANSFER", - Reason::GenesisBalance => "REASON_GENESIS_BALANCE", - Reason::GasBuy => "REASON_GAS_BUY", - Reason::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", - Reason::RewardFeeReset => "REASON_REWARD_FEE_RESET", - Reason::GasRefund => "REASON_GAS_REFUND", - Reason::TouchAccount => "REASON_TOUCH_ACCOUNT", - Reason::SuicideRefund => "REASON_SUICIDE_REFUND", - Reason::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", - Reason::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", - Reason::Burn => "REASON_BURN", + Self::Unknown => "REASON_UNKNOWN", + Self::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", + Self::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", + Self::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", + Self::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", + Self::Transfer => "REASON_TRANSFER", + Self::GenesisBalance => "REASON_GENESIS_BALANCE", + Self::GasBuy => "REASON_GAS_BUY", + Self::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", + Self::RewardFeeReset => "REASON_REWARD_FEE_RESET", + Self::GasRefund => "REASON_GAS_REFUND", + Self::TouchAccount => "REASON_TOUCH_ACCOUNT", + Self::SuicideRefund => "REASON_SUICIDE_REFUND", + Self::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", + Self::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", + Self::Burn => "REASON_BURN", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -592,7 +576,6 @@ pub mod balance_change { } } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct NonceChange { #[prost(bytes = "vec", tag = "1")] @@ -604,7 +587,6 @@ pub struct NonceChange { #[prost(uint64, tag = "4")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountCreation { #[prost(bytes = "vec", tag = "1")] @@ -612,7 +594,6 @@ pub struct AccountCreation { #[prost(uint64, tag = "2")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CodeChange { #[prost(bytes = "vec", tag = "1")] @@ -632,10 +613,9 @@ pub struct CodeChange { /// The gas is computed per actual op codes. Doing them completely might prove /// overwhelming in most cases. /// -/// Hence, we only index some of them, those that are costy like all the calls +/// Hence, we only index some of them, those that are costly like all the calls /// one, log events, return data, etc. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct GasChange { #[prost(uint64, tag = "1")] pub old_value: u64, @@ -696,27 +676,27 @@ pub mod gas_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::Call => "REASON_CALL", - Reason::CallCode => "REASON_CALL_CODE", - Reason::CallDataCopy => "REASON_CALL_DATA_COPY", - Reason::CodeCopy => "REASON_CODE_COPY", - Reason::CodeStorage => "REASON_CODE_STORAGE", - Reason::ContractCreation => "REASON_CONTRACT_CREATION", - Reason::ContractCreation2 => "REASON_CONTRACT_CREATION2", - Reason::DelegateCall => "REASON_DELEGATE_CALL", - Reason::EventLog => "REASON_EVENT_LOG", - Reason::ExtCodeCopy => "REASON_EXT_CODE_COPY", - Reason::FailedExecution => "REASON_FAILED_EXECUTION", - Reason::IntrinsicGas => "REASON_INTRINSIC_GAS", - Reason::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", - Reason::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", - Reason::Return => "REASON_RETURN", - Reason::ReturnDataCopy => "REASON_RETURN_DATA_COPY", - Reason::Revert => "REASON_REVERT", - Reason::SelfDestruct => "REASON_SELF_DESTRUCT", - Reason::StaticCall => "REASON_STATIC_CALL", - Reason::StateColdAccess => "REASON_STATE_COLD_ACCESS", + Self::Unknown => "REASON_UNKNOWN", + Self::Call => "REASON_CALL", + Self::CallCode => "REASON_CALL_CODE", + Self::CallDataCopy => "REASON_CALL_DATA_COPY", + Self::CodeCopy => "REASON_CODE_COPY", + Self::CodeStorage => "REASON_CODE_STORAGE", + Self::ContractCreation => "REASON_CONTRACT_CREATION", + Self::ContractCreation2 => "REASON_CONTRACT_CREATION2", + Self::DelegateCall => "REASON_DELEGATE_CALL", + Self::EventLog => "REASON_EVENT_LOG", + Self::ExtCodeCopy => "REASON_EXT_CODE_COPY", + Self::FailedExecution => "REASON_FAILED_EXECUTION", + Self::IntrinsicGas => "REASON_INTRINSIC_GAS", + Self::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", + Self::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", + Self::Return => "REASON_RETURN", + Self::ReturnDataCopy => "REASON_RETURN_DATA_COPY", + Self::Revert => "REASON_REVERT", + Self::SelfDestruct => "REASON_SELF_DESTRUCT", + Self::StaticCall => "REASON_STATIC_CALL", + Self::StateColdAccess => "REASON_STATE_COLD_ACCESS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -763,10 +743,10 @@ impl TransactionTraceStatus { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - TransactionTraceStatus::Unknown => "UNKNOWN", - TransactionTraceStatus::Succeeded => "SUCCEEDED", - TransactionTraceStatus::Failed => "FAILED", - TransactionTraceStatus::Reverted => "REVERTED", + Self::Unknown => "UNKNOWN", + Self::Succeeded => "SUCCEEDED", + Self::Failed => "FAILED", + Self::Reverted => "REVERTED", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -799,12 +779,12 @@ impl CallType { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - CallType::Unspecified => "UNSPECIFIED", - CallType::Call => "CALL", - CallType::Callcode => "CALLCODE", - CallType::Delegate => "DELEGATE", - CallType::Static => "STATIC", - CallType::Create => "CREATE", + Self::Unspecified => "UNSPECIFIED", + Self::Call => "CALL", + Self::Callcode => "CALLCODE", + Self::Delegate => "DELEGATE", + Self::Static => "STATIC", + Self::Create => "CREATE", } } /// Creates an enum from field names used in the ProtoBuf definition. diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index d88bf2b22d7..1a4af0663cb 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -37,7 +37,7 @@ impl AscType for AscLogParamArray { } } -impl ToAscObj for Vec { +impl ToAscObj for &[ethabi::LogParam] { fn to_asc_obj( &self, heap: &mut H, @@ -121,6 +121,7 @@ impl AscIndexId for AscLogArray { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub struct AscUnresolvedContractCall_0_0_4 { pub contract_name: AscPtr, pub contract_address: AscPtr, @@ -201,6 +202,7 @@ impl AscIndexId for AscEthereumBlock { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumBlock_0_0_6 { pub hash: AscPtr, pub parent_hash: AscPtr, @@ -225,6 +227,7 @@ impl AscIndexId for AscEthereumBlock_0_0_6 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_1 { pub hash: AscPtr, pub index: AscPtr, @@ -241,6 +244,7 @@ impl AscIndexId for AscEthereumTransaction_0_0_1 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_2 { pub hash: AscPtr, pub index: AscPtr, @@ -258,6 +262,7 @@ impl AscIndexId for AscEthereumTransaction_0_0_2 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_6 { pub hash: AscPtr, pub index: AscPtr, @@ -346,6 +351,7 @@ impl AscIndexId for AscEthereumTransactionReceipt { /// `receipt` field. #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumEvent_0_0_7 where T: AscType, @@ -392,6 +398,7 @@ impl AscIndexId for AscEthereumCall { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumCall_0_0_3 where T: AscType, @@ -413,146 +420,146 @@ where const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumCall; } -impl ToAscObj for EthereumBlockData { +impl<'a> ToAscObj for EthereumBlockData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumBlock { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas)?, + parent_hash: asc_new(heap, self.parent_hash(), gas)?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas)?, + author: asc_new(heap, self.author(), gas)?, + state_root: asc_new(heap, self.state_root(), gas)?, + transactions_root: asc_new(heap, self.transactions_root(), gas)?, + receipts_root: asc_new(heap, self.receipts_root(), gas)?, + number: asc_new(heap, &BigInt::from(self.number()), gas)?, + gas_used: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_used()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + timestamp: asc_new(heap, &BigInt::from_unsigned_u256(self.timestamp()), gas)?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas)?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, )?, size: self - .size + .size() .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) .unwrap_or(Ok(AscPtr::null()))?, }) } } -impl ToAscObj for EthereumBlockData { +impl<'a> ToAscObj for EthereumBlockData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumBlock_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas)?, + parent_hash: asc_new(heap, self.parent_hash(), gas)?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas)?, + author: asc_new(heap, self.author(), gas)?, + state_root: asc_new(heap, self.state_root(), gas)?, + transactions_root: asc_new(heap, self.transactions_root(), gas)?, + receipts_root: asc_new(heap, self.receipts_root(), gas)?, + number: asc_new(heap, &BigInt::from(self.number()), gas)?, + gas_used: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_used()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + timestamp: asc_new(heap, &BigInt::from_unsigned_u256(self.timestamp()), gas)?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas)?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, )?, size: self - .size + .size() .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) .unwrap_or(Ok(AscPtr::null()))?, base_fee_per_block: self - .base_fee_per_gas + .base_fee_per_gas() .map(|base_fee| asc_new(heap, &BigInt::from_unsigned_u256(&base_fee), gas)) .unwrap_or(Ok(AscPtr::null()))?, }) } } -impl ToAscObj for EthereumTransactionData { +impl<'a> ToAscObj for EthereumTransactionData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_1 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, + hash: asc_new(heap, self.hash(), gas)?, + index: asc_new(heap, &BigInt::from_unsigned_u128(self.index()), gas)?, + from: asc_new(heap, self.from(), gas)?, to: self - .to + .to() .map(|to| asc_new(heap, &to, gas)) .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + value: asc_new(heap, &BigInt::from_unsigned_u256(self.value()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_price()), gas)?, }) } } -impl ToAscObj for EthereumTransactionData { +impl<'a> ToAscObj for EthereumTransactionData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_2 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, + hash: asc_new(heap, self.hash(), gas)?, + index: asc_new(heap, &BigInt::from_unsigned_u128(self.index()), gas)?, + from: asc_new(heap, self.from(), gas)?, to: self - .to + .to() .map(|to| asc_new(heap, &to, gas)) .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, + value: asc_new(heap, &BigInt::from_unsigned_u256(self.value()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_price()), gas)?, + input: asc_new(heap, self.input(), gas)?, }) } } -impl ToAscObj for EthereumTransactionData { +impl<'a> ToAscObj for EthereumTransactionData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, + hash: asc_new(heap, self.hash(), gas)?, + index: asc_new(heap, &BigInt::from_unsigned_u128(self.index()), gas)?, + from: asc_new(heap, self.from(), gas)?, to: self - .to + .to() .map(|to| asc_new(heap, &to, gas)) .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, - nonce: asc_new(heap, &BigInt::from_unsigned_u256(&self.nonce), gas)?, + value: asc_new(heap, &BigInt::from_unsigned_u256(self.value()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_price()), gas)?, + input: asc_new(heap, self.input(), gas)?, + nonce: asc_new(heap, &BigInt::from_unsigned_u256(self.nonce()), gas)?, }) } } -impl ToAscObj> for EthereumEventData +impl<'a, T, B> ToAscObj> for EthereumEventData<'a> where T: AscType + AscIndexId, B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + EthereumTransactionData<'a>: ToAscObj, + EthereumBlockData<'a>: ToAscObj, { fn to_asc_obj( &self, @@ -560,17 +567,17 @@ where gas: &GasCounter, ) -> Result, HostExportError> { Ok(AscEthereumEvent { - address: asc_new(heap, &self.address, gas)?, - log_index: asc_new(heap, &BigInt::from_unsigned_u256(&self.log_index), gas)?, + address: asc_new(heap, self.address(), gas)?, + log_index: asc_new(heap, &BigInt::from_unsigned_u256(self.log_index()), gas)?, transaction_log_index: asc_new( heap, - &BigInt::from_unsigned_u256(&self.transaction_log_index), + &BigInt::from_unsigned_u256(self.transaction_log_index()), gas, )?, log_type: self - .log_type - .clone() - .map(|log_type| asc_new(heap, &log_type, gas)) + .log_type() + .as_ref() + .map(|log_type| asc_new(heap, log_type, gas)) .unwrap_or(Ok(AscPtr::null()))?, block: asc_new::(heap, &self.block, gas)?, transaction: asc_new::(heap, &self.transaction, gas)?, @@ -579,13 +586,13 @@ where } } -impl ToAscObj> - for (EthereumEventData, Option<&TransactionReceipt>) +impl<'a, T, B> ToAscObj> + for (EthereumEventData<'a>, Option<&TransactionReceipt>) where T: AscType + AscIndexId, B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + EthereumTransactionData<'a>: ToAscObj, + EthereumBlockData<'a>: ToAscObj, { fn to_asc_obj( &self, @@ -711,14 +718,14 @@ impl ToAscObj for &TransactionReceipt { } } -impl ToAscObj for EthereumCallData { +impl<'a> ToAscObj for EthereumCallData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumCall { - address: asc_new(heap, &self.to, gas)?, + address: asc_new(heap, self.to(), gas)?, block: asc_new(heap, &self.block, gas)?, transaction: asc_new(heap, &self.transaction, gas)?, inputs: asc_new(heap, &self.inputs, gas)?, @@ -727,8 +734,8 @@ impl ToAscObj for EthereumCallData { } } -impl ToAscObj> - for EthereumCallData +impl<'a> ToAscObj> + for EthereumCallData<'a> { fn to_asc_obj( &self, @@ -739,8 +746,8 @@ impl ToAscObj { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, + to: asc_new(heap, self.to(), gas)?, + from: asc_new(heap, self.from(), gas)?, block: asc_new(heap, &self.block, gas)?, transaction: asc_new(heap, &self.transaction, gas)?, inputs: asc_new(heap, &self.inputs, gas)?, @@ -749,8 +756,8 @@ impl ToAscObj> - for EthereumCallData +impl<'a> ToAscObj> + for EthereumCallData<'a> { fn to_asc_obj( &self, @@ -761,8 +768,8 @@ impl ToAscObj { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, + to: asc_new(heap, self.to(), gas)?, + from: asc_new(heap, self.from(), gas)?, block: asc_new(heap, &self.block, gas)?, transaction: asc_new(heap, &self.transaction, gas)?, inputs: asc_new(heap, &self.inputs, gas)?, diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index a5d83690b4b..f9f4ced2978 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -10,7 +10,6 @@ use graph::prelude::ethabi::ethereum_types::U128; use graph::prelude::ethabi::ethereum_types::U256; use graph::prelude::ethabi::ethereum_types::U64; use graph::prelude::ethabi::Address; -use graph::prelude::ethabi::Bytes; use graph::prelude::ethabi::LogParam; use graph::prelude::web3::types::Block; use graph::prelude::web3::types::Log; @@ -26,7 +25,6 @@ use graph::runtime::AscPtr; use graph::runtime::HostExportError; use graph::semver::Version; use graph_runtime_wasm::module::ToAscPtr; -use std::ops::Deref; use std::{cmp::Ordering, sync::Arc}; use crate::runtime::abi::AscEthereumBlock; @@ -42,6 +40,8 @@ use crate::runtime::abi::AscEthereumTransaction_0_0_6; // ETHDEP: This should be defined in only one place. type LightEthereumBlock = Block; +static U256_DEFAULT: U256 = U256::zero(); + pub enum MappingTrigger { Log { block: Arc, @@ -145,15 +145,12 @@ impl ToAscPtr for MappingTrigger { calls: _, } => { let api_version = heap.api_version(); - let ethereum_event_data = EthereumEventData { - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), - address: log.address, - log_index: log.log_index.unwrap_or(U256::zero()), - transaction_log_index: log.log_index.unwrap_or(U256::zero()), - log_type: log.log_type.clone(), - params, - }; + let ethereum_event_data = EthereumEventData::new( + block.as_ref(), + transaction.as_ref(), + log.as_ref(), + ¶ms, + ); if api_version >= API_VERSION_0_0_7 { asc_new::< AscEthereumEvent_0_0_7< @@ -194,14 +191,7 @@ impl ToAscPtr for MappingTrigger { inputs, outputs, } => { - let call = EthereumCallData { - to: call.to, - from: call.from, - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), - inputs, - outputs, - }; + let call = EthereumCallData::new(&block, &transaction, &call, &inputs, &outputs); if heap.api_version() >= Version::new(0, 0, 6) { asc_new::< AscEthereumCall_0_0_3, @@ -420,99 +410,211 @@ impl TriggerData for EthereumTrigger { } /// Ethereum block data. -#[derive(Clone, Debug, Default)] -pub struct EthereumBlockData { - pub hash: H256, - pub parent_hash: H256, - pub uncles_hash: H256, - pub author: H160, - pub state_root: H256, - pub transactions_root: H256, - pub receipts_root: H256, - pub number: U64, - pub gas_used: U256, - pub gas_limit: U256, - pub timestamp: U256, - pub difficulty: U256, - pub total_difficulty: U256, - pub size: Option, - pub base_fee_per_gas: Option, +#[derive(Clone, Debug)] +pub struct EthereumBlockData<'a> { + block: &'a Block, } -impl<'a, T> From<&'a Block> for EthereumBlockData { - fn from(block: &'a Block) -> EthereumBlockData { - EthereumBlockData { - hash: block.hash.unwrap(), - parent_hash: block.parent_hash, - uncles_hash: block.uncles_hash, - author: block.author, - state_root: block.state_root, - transactions_root: block.transactions_root, - receipts_root: block.receipts_root, - number: block.number.unwrap(), - gas_used: block.gas_used, - gas_limit: block.gas_limit, - timestamp: block.timestamp, - difficulty: block.difficulty, - total_difficulty: block.total_difficulty.unwrap_or_default(), - size: block.size, - base_fee_per_gas: block.base_fee_per_gas, - } +impl<'a> From<&'a Block> for EthereumBlockData<'a> { + fn from(block: &'a Block) -> EthereumBlockData<'a> { + EthereumBlockData { block } + } +} + +impl<'a> EthereumBlockData<'a> { + pub fn hash(&self) -> &H256 { + self.block.hash.as_ref().unwrap() + } + + pub fn parent_hash(&self) -> &H256 { + &self.block.parent_hash + } + + pub fn uncles_hash(&self) -> &H256 { + &self.block.uncles_hash + } + + pub fn author(&self) -> &H160 { + &self.block.author + } + + pub fn state_root(&self) -> &H256 { + &self.block.state_root + } + + pub fn transactions_root(&self) -> &H256 { + &self.block.transactions_root + } + + pub fn receipts_root(&self) -> &H256 { + &self.block.receipts_root + } + + pub fn number(&self) -> U64 { + self.block.number.unwrap() + } + + pub fn gas_used(&self) -> &U256 { + &self.block.gas_used + } + + pub fn gas_limit(&self) -> &U256 { + &self.block.gas_limit + } + + pub fn timestamp(&self) -> &U256 { + &self.block.timestamp + } + + pub fn difficulty(&self) -> &U256 { + &self.block.difficulty + } + + pub fn total_difficulty(&self) -> &U256 { + self.block + .total_difficulty + .as_ref() + .unwrap_or(&U256_DEFAULT) + } + + pub fn size(&self) -> &Option { + &self.block.size + } + + pub fn base_fee_per_gas(&self) -> &Option { + &self.block.base_fee_per_gas } } /// Ethereum transaction data. #[derive(Clone, Debug)] -pub struct EthereumTransactionData { - pub hash: H256, - pub index: U128, - pub from: H160, - pub to: Option, - pub value: U256, - pub gas_limit: U256, - pub gas_price: U256, - pub input: Bytes, - pub nonce: U256, +pub struct EthereumTransactionData<'a> { + tx: &'a Transaction, } -impl From<&'_ Transaction> for EthereumTransactionData { - fn from(tx: &Transaction) -> EthereumTransactionData { +impl<'a> EthereumTransactionData<'a> { + // We don't implement `From` because it causes confusion with the `from` + // accessor method + fn new(tx: &'a Transaction) -> EthereumTransactionData<'a> { + EthereumTransactionData { tx } + } + + pub fn hash(&self) -> &H256 { + &self.tx.hash + } + + pub fn index(&self) -> U128 { + self.tx.transaction_index.unwrap().as_u64().into() + } + + pub fn from(&self) -> &H160 { // unwrap: this is always `Some` for txns that have been mined // (see https://github.com/tomusdrw/rust-web3/pull/407) - let from = tx.from.unwrap(); - EthereumTransactionData { - hash: tx.hash, - index: tx.transaction_index.unwrap().as_u64().into(), - from, - to: tx.to, - value: tx.value, - gas_limit: tx.gas, - gas_price: tx.gas_price.unwrap_or(U256::zero()), // EIP-1559 made this optional. - input: tx.input.0.clone(), - nonce: tx.nonce, - } + self.tx.from.as_ref().unwrap() + } + + pub fn to(&self) -> &Option { + &self.tx.to + } + + pub fn value(&self) -> &U256 { + &self.tx.value + } + + pub fn gas_limit(&self) -> &U256 { + &self.tx.gas + } + + pub fn gas_price(&self) -> &U256 { + // EIP-1559 made this optional. + self.tx.gas_price.as_ref().unwrap_or(&U256_DEFAULT) + } + + pub fn input(&self) -> &[u8] { + &self.tx.input.0 + } + + pub fn nonce(&self) -> &U256 { + &self.tx.nonce } } /// An Ethereum event logged from a specific contract address and block. #[derive(Debug, Clone)] -pub struct EthereumEventData { - pub address: Address, - pub log_index: U256, - pub transaction_log_index: U256, - pub log_type: Option, - pub block: EthereumBlockData, - pub transaction: EthereumTransactionData, - pub params: Vec, +pub struct EthereumEventData<'a> { + pub block: EthereumBlockData<'a>, + pub transaction: EthereumTransactionData<'a>, + pub params: &'a [LogParam], + log: &'a Log, +} + +impl<'a> EthereumEventData<'a> { + pub fn new( + block: &'a Block, + tx: &'a Transaction, + log: &'a Log, + params: &'a [LogParam], + ) -> Self { + EthereumEventData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(tx), + log, + params, + } + } + + pub fn address(&self) -> &Address { + &self.log.address + } + + pub fn log_index(&self) -> &U256 { + self.log.log_index.as_ref().unwrap_or(&U256_DEFAULT) + } + + pub fn transaction_log_index(&self) -> &U256 { + self.log + .transaction_log_index + .as_ref() + .unwrap_or(&U256_DEFAULT) + } + + pub fn log_type(&self) -> &Option { + &self.log.log_type + } } /// An Ethereum call executed within a transaction within a block to a contract address. #[derive(Debug, Clone)] -pub struct EthereumCallData { - pub from: Address, - pub to: Address, - pub block: EthereumBlockData, - pub transaction: EthereumTransactionData, - pub inputs: Vec, - pub outputs: Vec, +pub struct EthereumCallData<'a> { + pub block: EthereumBlockData<'a>, + pub transaction: EthereumTransactionData<'a>, + pub inputs: &'a [LogParam], + pub outputs: &'a [LogParam], + call: &'a EthereumCall, +} + +impl<'a> EthereumCallData<'a> { + fn new( + block: &'a Block, + transaction: &'a Transaction, + call: &'a EthereumCall, + inputs: &'a [LogParam], + outputs: &'a [LogParam], + ) -> EthereumCallData<'a> { + EthereumCallData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(transaction), + inputs, + outputs, + call, + } + } + + pub fn from(&self) -> &Address { + &self.call.from + } + + pub fn to(&self) -> &Address { + &self.call.to + } } diff --git a/chain/near/build.rs b/chain/near/build.rs index 611f861baf2..0bb50d10b27 100644 --- a/chain/near/build.rs +++ b/chain/near/build.rs @@ -3,7 +3,7 @@ fn main() { tonic_build::configure() .out_dir("src/protobuf") .extern_path(".sf.near.codec.v1", "crate::codec::pbcodec") - .compile( + .compile_protos( &["proto/near.proto", "proto/substreams-triggers.proto"], &["proto"], ) diff --git a/chain/near/src/protobuf/receipts.v1.rs b/chain/near/src/protobuf/receipts.v1.rs index 00d3e2fe004..2b844103e9a 100644 --- a/chain/near/src/protobuf/receipts.v1.rs +++ b/chain/near/src/protobuf/receipts.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockAndReceipts { #[prost(message, optional, tag = "1")] diff --git a/chain/substreams/build.rs b/chain/substreams/build.rs index 8cccc11fe3a..330a01a8c68 100644 --- a/chain/substreams/build.rs +++ b/chain/substreams/build.rs @@ -3,6 +3,6 @@ fn main() { tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) + .compile_protos(&["proto/codec.proto"], &["proto"]) .expect("Failed to compile Substreams entity proto(s)"); } diff --git a/chain/substreams/examples/substreams.rs b/chain/substreams/examples/substreams.rs index eac9397b893..a5af2bbe25c 100644 --- a/chain/substreams/examples/substreams.rs +++ b/chain/substreams/examples/substreams.rs @@ -57,6 +57,7 @@ async fn main() -> Result<(), Error> { false, SubgraphLimit::Unlimited, Arc::new(endpoint_metrics), + true, )); let client = Arc::new(ChainClient::new_firehose(FirehoseEndpoints::for_testing( diff --git a/chain/substreams/src/protobuf/substreams.entity.v1.rs b/chain/substreams/src/protobuf/substreams.entity.v1.rs index 372748908d7..4077f281ad7 100644 --- a/chain/substreams/src/protobuf/substreams.entity.v1.rs +++ b/chain/substreams/src/protobuf/substreams.entity.v1.rs @@ -1,11 +1,9 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct EntityChanges { #[prost(message, repeated, tag = "5")] pub entity_changes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct EntityChange { #[prost(string, tag = "1")] @@ -47,10 +45,10 @@ pub mod entity_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Operation::Unset => "UNSET", - Operation::Create => "CREATE", - Operation::Update => "UPDATE", - Operation::Delete => "DELETE", + Self::Unset => "UNSET", + Self::Create => "CREATE", + Self::Update => "UPDATE", + Self::Delete => "DELETE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -65,7 +63,6 @@ pub mod entity_change { } } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Value { #[prost(oneof = "value::Typed", tags = "1, 2, 3, 4, 5, 6, 7, 10")] @@ -73,7 +70,6 @@ pub struct Value { } /// Nested message and enum types in `Value`. pub mod value { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Typed { #[prost(int32, tag = "1")] @@ -95,13 +91,11 @@ pub mod value { Array(super::Array), } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Array { #[prost(message, repeated, tag = "1")] pub value: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Field { #[prost(string, tag = "1")] diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs index 13a5bed1a1d..405b6f8a116 100644 --- a/chain/substreams/src/trigger.rs +++ b/chain/substreams/src/trigger.rs @@ -15,7 +15,6 @@ use graph::{ substreams::Modules, }; use graph_runtime_wasm::module::ToAscPtr; -use lazy_static::__Deref; use std::{collections::BTreeSet, sync::Arc}; use crate::{Block, Chain, NoopDataSourceTemplate, ParsedChanges}; @@ -179,18 +178,6 @@ impl blockchain::TriggersAdapter for TriggersAdapter { } } -fn write_poi_event( - proof_of_indexing: &SharedProofOfIndexing, - poi_event: &ProofOfIndexingEvent, - causality_region: &str, - logger: &Logger, -) { - if let Some(proof_of_indexing) = proof_of_indexing { - let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); - proof_of_indexing.write(logger, causality_region, poi_event); - } -} - pub struct TriggerProcessor { pub locator: DeploymentLocator, } @@ -226,8 +213,7 @@ where return Err(MappingError::Unknown(anyhow!("Detected UNSET entity operation, either a server error or there's a new type of operation and we're running an outdated protobuf"))); } ParsedChanges::Upsert { key, entity } => { - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::SetEntity { entity_type: key.entity_type.typename(), id: &key.entity_id.to_string(), @@ -249,8 +235,7 @@ where let id = entity_key.entity_id.clone(); state.entity_cache.remove(entity_key); - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::RemoveEntity { entity_type: entity_type.typename(), id: &id.to_string(), diff --git a/core/Cargo.toml b/core/Cargo.toml index 4533b5d8880..4140b37ffbe 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -13,7 +13,6 @@ graph = { path = "../graph" } graph-chain-arweave = { path = "../chain/arweave" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-near = { path = "../chain/near" } -graph-chain-cosmos = { path = "../chain/cosmos" } graph-chain-substreams = { path = "../chain/substreams" } graph-runtime-wasm = { path = "../runtime/wasm" } serde_yaml = { workspace = true } @@ -24,5 +23,4 @@ anyhow = "1.0" [dev-dependencies] tower-test = { git = "https://github.com/tower-rs/tower.git" } -uuid = { version = "1.9.1", features = ["v4"] } wiremock = "0.6.1" diff --git a/core/graphman/src/commands/deployment/reassign.rs b/core/graphman/src/commands/deployment/reassign.rs index 2e5916a7aae..5d3d633e082 100644 --- a/core/graphman/src/commands/deployment/reassign.rs +++ b/core/graphman/src/commands/deployment/reassign.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::anyhow; use graph::components::store::DeploymentLocator; use graph::components::store::StoreEvent; -use graph::prelude::EntityChange; +use graph::prelude::AssignmentChange; use graph::prelude::NodeId; use graph_store_postgres::command_support::catalog; use graph_store_postgres::command_support::catalog::Site; @@ -74,7 +74,7 @@ pub fn reassign_deployment( let primary_conn = primary_pool.get().map_err(GraphmanError::from)?; let mut catalog_conn = catalog::Connection::new(primary_conn); - let changes: Vec = match catalog_conn + let changes: Vec = match catalog_conn .assigned_node(&deployment.site) .map_err(GraphmanError::from)? { diff --git a/core/src/polling_monitor/ipfs_service.rs b/core/src/polling_monitor/ipfs_service.rs index f8c68976216..86a5feef0ab 100644 --- a/core/src/polling_monitor/ipfs_service.rs +++ b/core/src/polling_monitor/ipfs_service.rs @@ -104,7 +104,6 @@ mod test { use graph::log::discard; use graph::tokio; use tower::ServiceExt; - use uuid::Uuid; use wiremock::matchers as m; use wiremock::Mock; use wiremock::MockServer; @@ -114,7 +113,11 @@ mod test { #[tokio::test] async fn cat_file_in_folder() { - let random_bytes = Uuid::new_v4().as_bytes().to_vec(); + let random_bytes = "One morning, when Gregor Samsa woke \ + from troubled dreams, he found himself transformed in his bed \ + into a horrible vermin" + .as_bytes() + .to_vec(); let ipfs_file = ("dir/file.txt", random_bytes.clone()); let add_resp = add_files_to_local_ipfs_node_for_testing([ipfs_file]) diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index ef265978ede..3f35d570a7d 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -126,11 +126,7 @@ impl> IndexingContext { ) -> Result { let error_count = state.deterministic_errors.len(); - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } + proof_of_indexing.start_handler(causality_region); let start = Instant::now(); @@ -156,16 +152,12 @@ impl> IndexingContext { let elapsed = start.elapsed().as_secs_f64(); subgraph_metrics.observe_trigger_processing_duration(elapsed); - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing.write_deterministic_error(logger, causality_region); } Ok(state) diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 2d54a90417c..8c2b76e5b6c 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -124,21 +124,6 @@ impl SubgraphInstanceManagerTrait for SubgraphInstanceManager< self.start_subgraph_inner(logger, loc, runner).await } - BlockchainKind::Cosmos => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - deployment_status_metric, - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } BlockchainKind::Substreams => { let runner = instance_manager .build_subgraph_runner::( diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index 258f4bbcd15..3a712b6daa9 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -154,25 +154,11 @@ where let logger = self.logger.clone(); self.subscription_manager - .subscribe(FromIterator::from_iter([SubscriptionFilter::Assignment])) + .subscribe() .map_err(|()| anyhow!("Entity change stream failed")) .map(|event| { - // We're only interested in the SubgraphDeploymentAssignment change; we - // know that there is at least one, as that is what we subscribed to - let filter = SubscriptionFilter::Assignment; - let assignments = event - .changes - .iter() - .filter(|change| filter.matches(change)) - .map(|change| match change { - EntityChange::Data { .. } => unreachable!(), - EntityChange::Assignment { - deployment, - operation, - } => (deployment.clone(), operation.clone()), - }) - .collect::>(); - stream::iter_ok(assignments) + let changes: Vec<_> = event.changes.iter().cloned().map(AssignmentChange::into_parts).collect(); + stream::iter_ok(changes) }) .flatten() .and_then( @@ -183,7 +169,7 @@ where ); match operation { - EntityChangeOperation::Set => { + AssignmentOperation::Set => { store .assignment_status(&deployment) .map_err(|e| { @@ -220,7 +206,7 @@ where } }) } - EntityChangeOperation::Removed => { + AssignmentOperation::Removed => { // Send remove event without checking node ID. // If node ID does not match, then this is a no-op when handled in // assignment provider. @@ -381,24 +367,6 @@ where ) .await? } - BlockchainKind::Cosmos => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - history_blocks, - ) - .await? - } BlockchainKind::Substreams => { create_subgraph_version::( &logger, diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 1e0fa2d4c8e..922c7a4003c 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -3,7 +3,6 @@ use crate::subgraph::error::BlockProcessingError; use crate::subgraph::inputs::IndexingInputs; use crate::subgraph::state::IndexingState; use crate::subgraph::stream::new_block_stream; -use atomic_refcell::AtomicRefCell; use graph::blockchain::block_stream::{ BlockStreamError, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, }; @@ -367,14 +366,8 @@ where debug!(logger, "Start processing block"; "triggers" => triggers.len()); - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; + let proof_of_indexing = + SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); @@ -633,8 +626,7 @@ where return Err(BlockProcessingError::Canceled); } - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); + if let Some(proof_of_indexing) = proof_of_indexing.into_inner() { update_proof_of_indexing( proof_of_indexing, block.timestamp(), @@ -1160,7 +1152,7 @@ where // PoI ignores offchain events. // See also: poi-ignores-offchain - let proof_of_indexing = None; + let proof_of_indexing = SharedProofOfIndexing::ignored(); let causality_region = ""; let trigger = TriggerData::Offchain(trigger); @@ -1318,14 +1310,8 @@ where .deployment_head .set(block_ptr.number as f64); - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; + let proof_of_indexing = + SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); @@ -1380,8 +1366,7 @@ where return Err(BlockProcessingError::Canceled.into()); } - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); + if let Some(proof_of_indexing) = proof_of_indexing.into_inner() { update_proof_of_indexing( proof_of_indexing, block_time, diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs index 0057e9e1354..c3123e87268 100644 --- a/core/src/subgraph/trigger_processor.rs +++ b/core/src/subgraph/trigger_processor.rs @@ -39,11 +39,7 @@ where return Ok(state); } - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } + proof_of_indexing.start_handler(causality_region); for HostedTrigger { host, @@ -73,16 +69,12 @@ where } } - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing.write_deterministic_error(logger, causality_region); } Ok(state) diff --git a/docs/aggregations.md b/docs/aggregations.md index 301bb70c659..fafbd4d3305 100644 --- a/docs/aggregations.md +++ b/docs/aggregations.md @@ -1,10 +1,5 @@ # Timeseries and aggregations -**This feature is experimental. We very much encourage users to try this -out, but we might still need to make material changes to what's described -here in a manner that is not backwards compatible. That might require -deleting and redeploying any subgraph that uses the features here.** - _This feature is available from spec version 1.1.0 onwards_ ## Overview @@ -188,8 +183,9 @@ accepts the following arguments: - Optional `timestamp_{gte|gt|lt|lte|eq|in}` filters to restrict the range of timestamps to return. The timestamp to filter by must be a string containing microseconds since the epoch. The value `"1704164640000000"` - corresponds to `2024-01-02T03:04Z`. -- Timeseries are always sorted by `timestamp` and `id` in descending order + corresponds to `2024-01-02T03:04Z` +- Timeseries are sorted by `timestamp` and `id` in descending order by + default ```graphql token_stats(interval: "hour", diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 5d2b501f2a6..46903185ccf 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -112,18 +112,10 @@ those. result is checked while the response is being constructed, so that execution does not take more memory than what is configured. The default value for both is unlimited. -- `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`: maximum number of GraphQL - operations per WebSocket connection. Any operation created after the limit - will return an error to the client. Default: 1000. - `GRAPH_GRAPHQL_HTTP_PORT` : Port for the GraphQL HTTP server -- `GRAPH_GRAPHQL_WS_PORT` : Port for the GraphQL WebSocket server - `GRAPH_SQL_STATEMENT_TIMEOUT`: the maximum number of seconds an individual SQL query is allowed to take during GraphQL execution. Default: unlimited -- `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`: disables the internal - mechanism that is used to trigger updates on GraphQL subscriptions. When - this variable is set to any value, `graph-node` will still accept GraphQL - subscriptions, but they won't receive any updates. - `ENABLE_GRAPHQL_VALIDATIONS`: enables GraphQL validations, based on the GraphQL specification. This will validate and ensure every query executes follows the execution rules. Default: `false` @@ -185,11 +177,10 @@ those. query, and the `query_id` of the GraphQL query that caused the SQL query. These SQL queries are marked with `component: GraphQlRunner` There are additional SQL queries that get logged when `sql` is given. These are - queries caused by mappings when processing blocks for a subgraph, and - queries caused by subscriptions. If `cache` is present in addition to - `gql`, also logs information for each toplevel GraphQL query field - whether that could be retrieved from cache or not. Defaults to no - logging. + queries caused by mappings when processing blocks for a subgraph. If + `cache` is present in addition to `gql`, also logs information for each + toplevel GraphQL query field whether that could be retrieved from cache + or not. Defaults to no logging. - `GRAPH_LOG_TIME_FORMAT`: Custom log time format.Default value is `%b %d %H:%M:%S%.3f`. More information [here](https://docs.rs/chrono/latest/chrono/#formatting-and-parsing). - `STORE_CONNECTION_POOL_SIZE`: How many simultaneous connections to allow to the store. Due to implementation details, this value may not be strictly adhered to. Defaults to 10. @@ -232,6 +223,18 @@ those. copying or grafting should take. This limits how long transactions for such long running operations will be, and therefore helps control bloat in other tables. Value is in seconds and defaults to 180s. +- `GRAPH_STORE_BATCH_TIMEOUT`: How long a batch operation during copying or + grafting is allowed to take at most. This is meant to guard against + batches that are catastrophically big and should be set to a small + multiple of `GRAPH_STORE_BATCH_TARGET_DURATION`, like 10 times that + value, and needs to be at least 2 times that value when set. If this + timeout is hit, the batch size is reset to 1 so we can be sure that + batches stay below `GRAPH_STORE_BATCH_TARGET_DURATION` and the smaller + batch is retried. Value is in seconds and defaults to unlimited. +- `GRAPH_STORE_BATCH_WORKERS`: The number of workers to use for batch + operations. If there are idle connectiosn, each subgraph copy operation + will use up to this many workers to copy tables in parallel. Defaults + to 1 and must be at least 1 - `GRAPH_START_BLOCK`: block hash:block number where the forked subgraph will start indexing at. - `GRAPH_FORK_BASE`: api url for where the graph node will fork from, use `https://api.thegraph.com/subgraphs/id/` for the hosted service. diff --git a/docs/implementation/add-chain.md b/docs/implementation/add-chain.md index f4af3371f25..718c89bf24a 100644 --- a/docs/implementation/add-chain.md +++ b/docs/implementation/add-chain.md @@ -45,7 +45,7 @@ fn main() { println!("cargo:rerun-if-changed=proto"); tonic_build::configure() .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) + .compile_protos(&["proto/codec.proto"], &["proto"]) .expect("Failed to compile Firehose CoolChain proto(s)"); } ``` diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 3ea0c0bf349..dc4bd6e42e9 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -48,6 +48,7 @@ serde_derive = { workspace = true } serde_json = { workspace = true } serde_regex = { workspace = true } serde_yaml = { workspace = true } +sha2 = "0.10.8" slog = { version = "2.7.0", features = [ "release_max_level_trace", "max_level_trace", @@ -75,7 +76,7 @@ tokio = { version = "1.38.0", features = [ tokio-stream = { version = "0.1.15", features = ["sync"] } tokio-retry = "0.3.0" toml = "0.8.8" -url = "2.5.2" +url = "2.5.4" prometheus = "0.13.4" priority-queue = "2.0.3" tonic = { workspace = true } diff --git a/graph/build.rs b/graph/build.rs index 3cc00c0dc07..d67e110edf4 100644 --- a/graph/build.rs +++ b/graph/build.rs @@ -2,12 +2,11 @@ fn main() { println!("cargo:rerun-if-changed=proto"); tonic_build::configure() .out_dir("src/firehose") - .compile( + .compile_protos( &[ "proto/firehose.proto", "proto/ethereum/transforms.proto", "proto/near/transforms.proto", - "proto/cosmos/transforms.proto", ], &["proto"], ) @@ -16,13 +15,14 @@ fn main() { tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .out_dir("src/substreams") - .compile(&["proto/substreams.proto"], &["proto"]) + .compile_protos(&["proto/substreams.proto"], &["proto"]) .expect("Failed to compile Substreams proto(s)"); tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .extern_path(".sf.substreams.v1", "crate::substreams") + .extern_path(".sf.firehose.v2", "crate::firehose") .out_dir("src/substreams_rpc") - .compile(&["proto/substreams-rpc.proto"], &["proto"]) + .compile_protos(&["proto/substreams-rpc.proto"], &["proto"]) .expect("Failed to compile Substreams RPC proto(s)"); } diff --git a/graph/proto/substreams-rpc.proto b/graph/proto/substreams-rpc.proto index a0ba1b72037..28298458480 100644 --- a/graph/proto/substreams-rpc.proto +++ b/graph/proto/substreams-rpc.proto @@ -4,39 +4,46 @@ package sf.substreams.rpc.v2; import "google/protobuf/any.proto"; import "substreams.proto"; +import "firehose.proto"; -service Stream { - rpc Blocks(Request) returns (stream Response); +service EndpointInfo { + rpc Info(sf.firehose.v2.InfoRequest) returns (sf.firehose.v2.InfoResponse); } +service Stream { rpc Blocks(Request) returns (stream Response); } + message Request { int64 start_block_num = 1; string start_cursor = 2; uint64 stop_block_num = 3; // With final_block_only, you only receive blocks that are irreversible: - // 'final_block_height' will be equal to current block and no 'undo_signal' will ever be sent + // 'final_block_height' will be equal to current block and no 'undo_signal' + // will ever be sent bool final_blocks_only = 4; - // Substreams has two mode when executing your module(s) either development mode or production - // mode. Development and production modes impact the execution of Substreams, important aspects - // of execution include: + // Substreams has two mode when executing your module(s) either development + // mode or production mode. Development and production modes impact the + // execution of Substreams, important aspects of execution include: // * The time required to reach the first byte. // * The speed that large ranges get executed. // * The module logs and outputs sent back to the client. // - // By default, the engine runs in developer mode, with richer and deeper output. Differences - // between production and development modes include: - // * Forward parallel execution is enabled in production mode and disabled in development mode - // * The time required to reach the first byte in development mode is faster than in production mode. + // By default, the engine runs in developer mode, with richer and deeper + // output. Differences between production and development modes include: + // * Forward parallel execution is enabled in production mode and disabled in + // development mode + // * The time required to reach the first byte in development mode is faster + // than in production mode. // // Specific attributes of development mode include: // * The client will receive all of the executed module's logs. - // * It's possible to request specific store snapshots in the execution tree (via `debug_initial_store_snapshot_for_modules`). + // * It's possible to request specific store snapshots in the execution tree + // (via `debug_initial_store_snapshot_for_modules`). // * Multiple module's output is possible. // - // With production mode`, however, you trade off functionality for high speed enabling forward - // parallel execution of module ahead of time. + // With production mode`, however, you trade off functionality for high speed + // enabling forward parallel execution of module ahead of time. bool production_mode = 5; string output_module = 6; @@ -47,23 +54,24 @@ message Request { repeated string debug_initial_store_snapshot_for_modules = 10; } - message Response { oneof message { - SessionInit session = 1; // Always sent first - ModulesProgress progress = 2; // Progress of data preparation, before sending in the stream of `data` events. + SessionInit session = 1; // Always sent first + ModulesProgress progress = 2; // Progress of data preparation, before + // sending in the stream of `data` events. BlockScopedData block_scoped_data = 3; BlockUndoSignal block_undo_signal = 4; Error fatal_error = 5; - // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + // Available only in developer mode, and only if + // `debug_initial_store_snapshot_for_modules` is set. InitialSnapshotData debug_snapshot_data = 10; - // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + // Available only in developer mode, and only if + // `debug_initial_store_snapshot_for_modules` is set. InitialSnapshotComplete debug_snapshot_complete = 11; } } - // BlockUndoSignal informs you that every bit of data // with a block number above 'last_valid_block' has been reverted // on-chain. Delete that data and restart from 'last_valid_cursor' @@ -84,16 +92,14 @@ message BlockScopedData { repeated StoreModuleOutput debug_store_outputs = 11; } -message SessionInit { +message SessionInit { string trace_id = 1; uint64 resolved_start_block = 2; uint64 linear_handoff_block = 3; uint64 max_parallel_workers = 4; } -message InitialSnapshotComplete { - string cursor = 1; -} +message InitialSnapshotComplete { string cursor = 1; } message InitialSnapshotData { string module_name = 1; @@ -110,9 +116,9 @@ message MapModuleOutput { } // StoreModuleOutput are produced for store modules in development mode. -// It is not possible to retrieve store models in production, with parallelization -// enabled. If you need the deltas directly, write a pass through mapper module -// that will get them down to you. +// It is not possible to retrieve store models in production, with +// parallelization enabled. If you need the deltas directly, write a pass +// through mapper module that will get them down to you. message StoreModuleOutput { string name = 1; repeated StoreDelta debug_store_deltas = 2; @@ -121,8 +127,9 @@ message StoreModuleOutput { message OutputDebugInfo { repeated string logs = 1; - // LogsTruncated is a flag that tells you if you received all the logs or if they - // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + // LogsTruncated is a flag that tells you if you received all the logs or if + // they were truncated because you logged too much (fixed limit currently is + // set to 128 KiB). bool logs_truncated = 2; bool cached = 3; } @@ -130,7 +137,8 @@ message OutputDebugInfo { // ModulesProgress is a message that is sent every 500ms message ModulesProgress { // previously: repeated ModuleProgress modules = 1; - // these previous `modules` messages were sent in bursts and are not sent anymore. + // these previous `modules` messages were sent in bursts and are not sent + // anymore. reserved 1; // List of jobs running on tier2 servers repeated Job running_jobs = 2; @@ -147,73 +155,82 @@ message ProcessedBytes { uint64 total_bytes_written = 2; } - message Error { string module = 1; string reason = 2; repeated string logs = 3; - // FailureLogsTruncated is a flag that tells you if you received all the logs or if they - // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + // FailureLogsTruncated is a flag that tells you if you received all the logs + // or if they were truncated because you logged too much (fixed limit + // currently is set to 128 KiB). bool logs_truncated = 4; } - message Job { - uint32 stage = 1; - uint64 start_block = 2; - uint64 stop_block = 3; - uint64 processed_blocks = 4; - uint64 duration_ms = 5; + uint32 stage = 1; + uint64 start_block = 2; + uint64 stop_block = 3; + uint64 processed_blocks = 4; + uint64 duration_ms = 5; } message Stage { - repeated string modules = 1; - repeated BlockRange completed_ranges = 2; + repeated string modules = 1; + repeated BlockRange completed_ranges = 2; } -// ModuleStats gathers metrics and statistics from each module, running on tier1 or tier2 -// All the 'count' and 'time_ms' values may include duplicate for each stage going over that module +// ModuleStats gathers metrics and statistics from each module, running on tier1 +// or tier2 All the 'count' and 'time_ms' values may include duplicate for each +// stage going over that module message ModuleStats { - // name of the module - string name = 1; + // name of the module + string name = 1; - // total_processed_blocks is the sum of blocks sent to that module code - uint64 total_processed_block_count = 2; - // total_processing_time_ms is the sum of all time spent running that module code - uint64 total_processing_time_ms = 3; + // total_processed_blocks is the sum of blocks sent to that module code + uint64 total_processed_block_count = 2; + // total_processing_time_ms is the sum of all time spent running that module + // code + uint64 total_processing_time_ms = 3; - //// external_calls are chain-specific intrinsics, like "Ethereum RPC calls". - repeated ExternalCallMetric external_call_metrics = 4; + //// external_calls are chain-specific intrinsics, like "Ethereum RPC calls". + repeated ExternalCallMetric external_call_metrics = 4; - // total_store_operation_time_ms is the sum of all time spent running that module code waiting for a store operation (ex: read, write, delete...) - uint64 total_store_operation_time_ms = 5; - // total_store_read_count is the sum of all the store Read operations called from that module code - uint64 total_store_read_count = 6; + // total_store_operation_time_ms is the sum of all time spent running that + // module code waiting for a store operation (ex: read, write, delete...) + uint64 total_store_operation_time_ms = 5; + // total_store_read_count is the sum of all the store Read operations called + // from that module code + uint64 total_store_read_count = 6; - // total_store_write_count is the sum of all store Write operations called from that module code (store-only) - uint64 total_store_write_count = 10; + // total_store_write_count is the sum of all store Write operations called + // from that module code (store-only) + uint64 total_store_write_count = 10; - // total_store_deleteprefix_count is the sum of all store DeletePrefix operations called from that module code (store-only) - // note that DeletePrefix can be a costly operation on large stores - uint64 total_store_deleteprefix_count = 11; + // total_store_deleteprefix_count is the sum of all store DeletePrefix + // operations called from that module code (store-only) note that DeletePrefix + // can be a costly operation on large stores + uint64 total_store_deleteprefix_count = 11; - // store_size_bytes is the uncompressed size of the full KV store for that module, from the last 'merge' operation (store-only) - uint64 store_size_bytes = 12; + // store_size_bytes is the uncompressed size of the full KV store for that + // module, from the last 'merge' operation (store-only) + uint64 store_size_bytes = 12; - // total_store_merging_time_ms is the time spent merging partial stores into a full KV store for that module (store-only) - uint64 total_store_merging_time_ms = 13; + // total_store_merging_time_ms is the time spent merging partial stores into a + // full KV store for that module (store-only) + uint64 total_store_merging_time_ms = 13; - // store_currently_merging is true if there is a merging operation (partial store to full KV store) on the way. - bool store_currently_merging = 14; + // store_currently_merging is true if there is a merging operation (partial + // store to full KV store) on the way. + bool store_currently_merging = 14; - // highest_contiguous_block is the highest block in the highest merged full KV store of that module (store-only) - uint64 highest_contiguous_block = 15; + // highest_contiguous_block is the highest block in the highest merged full KV + // store of that module (store-only) + uint64 highest_contiguous_block = 15; } message ExternalCallMetric { - string name = 1; - uint64 count = 2; - uint64 time_ms = 3; + string name = 1; + uint64 count = 2; + uint64 time_ms = 3; } message StoreDelta { diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 2e3c60e88b0..25a7d147540 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -565,9 +565,6 @@ pub enum BlockchainKind { /// NEAR chains (Mainnet, Testnet) or chains that are compatible Near, - /// Cosmos chains - Cosmos, - Substreams, } @@ -577,7 +574,6 @@ impl fmt::Display for BlockchainKind { BlockchainKind::Arweave => "arweave", BlockchainKind::Ethereum => "ethereum", BlockchainKind::Near => "near", - BlockchainKind::Cosmos => "cosmos", BlockchainKind::Substreams => "substreams", }; write!(f, "{}", value) @@ -592,7 +588,6 @@ impl FromStr for BlockchainKind { "arweave" => Ok(BlockchainKind::Arweave), "ethereum" => Ok(BlockchainKind::Ethereum), "near" => Ok(BlockchainKind::Near), - "cosmos" => Ok(BlockchainKind::Cosmos), "substreams" => Ok(BlockchainKind::Substreams), "subgraph" => Ok(BlockchainKind::Ethereum), // TODO(krishna): We should detect the blockchain kind from the source subgraph _ => Err(anyhow!("unknown blockchain kind {}", s)), diff --git a/graph/src/components/graphql.rs b/graph/src/components/graphql.rs index c5abf39b275..b5fc4273860 100644 --- a/graph/src/components/graphql.rs +++ b/graph/src/components/graphql.rs @@ -1,17 +1,11 @@ use crate::data::query::QueryResults; use crate::data::query::{Query, QueryTarget}; -use crate::data::subscription::{Subscription, SubscriptionError, SubscriptionResult}; use crate::prelude::DeploymentHash; use async_trait::async_trait; -use futures01::Future; use std::sync::Arc; use std::time::Duration; -/// Future for subscription results. -pub type SubscriptionResultFuture = - Box + Send>; - pub enum GraphQlTarget { SubgraphName(String), Deployment(DeploymentHash), @@ -33,13 +27,6 @@ pub trait GraphQlRunner: Send + Sync + 'static { max_skip: Option, ) -> QueryResults; - /// Runs a GraphQL subscription and returns a stream of results. - async fn run_subscription( - self: Arc, - subscription: Subscription, - target: QueryTarget, - ) -> Result; - fn metrics(&self) -> Arc; } diff --git a/graph/src/components/metrics/registry.rs b/graph/src/components/metrics/registry.rs index e010d3a89fa..93cf51b3bd1 100644 --- a/graph/src/components/metrics/registry.rs +++ b/graph/src/components/metrics/registry.rs @@ -3,7 +3,7 @@ use std::sync::{Arc, RwLock}; use prometheus::IntGauge; use prometheus::{labels, Histogram, IntCounterVec}; -use slog::info; +use slog::debug; use crate::components::metrics::{counter_with_labels, gauge_with_labels}; use crate::prelude::Collector; @@ -133,7 +133,7 @@ impl MetricsRegistry { let mut result = self.registry.register(collector.clone()); if matches!(result, Err(PrometheusError::AlreadyReg)) { - info!(logger, "Resolving duplicate metric registration"); + debug!(logger, "Resolving duplicate metric registration"); // Since the current metric is a duplicate, // we can use it to unregister the previous registration. @@ -144,7 +144,6 @@ impl MetricsRegistry { match result { Ok(()) => { - info!(logger, "Successfully registered a new metric"); self.registered_metrics.inc(); } Err(err) => { diff --git a/graph/src/components/server/mod.rs b/graph/src/components/server/mod.rs index 404a9bfcf4f..89323b9c8b1 100644 --- a/graph/src/components/server/mod.rs +++ b/graph/src/components/server/mod.rs @@ -1,9 +1,6 @@ /// Component for running GraphQL queries over HTTP. pub mod query; -/// Component for running GraphQL subscriptions over WebSockets. -pub mod subscription; - /// Component for the index node server. pub mod index_node; diff --git a/graph/src/components/server/subscription.rs b/graph/src/components/server/subscription.rs deleted file mode 100644 index dae619356b6..00000000000 --- a/graph/src/components/server/subscription.rs +++ /dev/null @@ -1,8 +0,0 @@ -use async_trait::async_trait; - -/// Common trait for GraphQL subscription servers. -#[async_trait] -pub trait SubscriptionServer { - /// Returns a Future that, when spawned, brings up the GraphQL subscription server. - async fn serve(self, port: u16); -} diff --git a/graph/src/components/store/err.rs b/graph/src/components/store/err.rs index 3aa65c3ecb2..76be7c311ce 100644 --- a/graph/src/components/store/err.rs +++ b/graph/src/components/store/err.rs @@ -74,6 +74,8 @@ pub enum StoreError { UnsupportedFilter(String, String), #[error("writing {0} entities at block {1} failed: {2} Query: {3}")] WriteFailure(String, BlockNumber, String, String), + #[error("database query timed out")] + StatementTimeout, } // Convenience to report a constraint violation @@ -133,25 +135,29 @@ impl Clone for StoreError { Self::WriteFailure(arg0, arg1, arg2, arg3) => { Self::WriteFailure(arg0.clone(), arg1.clone(), arg2.clone(), arg3.clone()) } + Self::StatementTimeout => Self::StatementTimeout, } } } impl StoreError { - fn database_unavailable(e: &DieselError) -> Option { - // When the error is caused by a closed connection, treat the error - // as 'database unavailable'. When this happens during indexing, the - // indexing machinery will retry in that case rather than fail the - // subgraph - if let DieselError::DatabaseError(_, info) = e { - if info - .message() - .contains("server closed the connection unexpectedly") - { - return Some(Self::DatabaseUnavailable); - } + pub fn from_diesel_error(e: &DieselError) -> Option { + const CONN_CLOSE: &str = "server closed the connection unexpectedly"; + const STMT_TIMEOUT: &str = "canceling statement due to statement timeout"; + let DieselError::DatabaseError(_, info) = e else { + return None; + }; + if info.message().contains(CONN_CLOSE) { + // When the error is caused by a closed connection, treat the error + // as 'database unavailable'. When this happens during indexing, the + // indexing machinery will retry in that case rather than fail the + // subgraph + Some(StoreError::DatabaseUnavailable) + } else if info.message().contains(STMT_TIMEOUT) { + Some(StoreError::StatementTimeout) + } else { + None } - None } pub fn write_failure( @@ -160,19 +166,15 @@ impl StoreError { block: BlockNumber, query: String, ) -> Self { - match Self::database_unavailable(&error) { - Some(e) => return e, - None => StoreError::WriteFailure(entity.to_string(), block, error.to_string(), query), - } + Self::from_diesel_error(&error).unwrap_or_else(|| { + StoreError::WriteFailure(entity.to_string(), block, error.to_string(), query) + }) } } impl From for StoreError { fn from(e: DieselError) -> Self { - match Self::database_unavailable(&e) { - Some(e) => return e, - None => StoreError::Unknown(e.into()), - } + Self::from_diesel_error(&e).unwrap_or_else(|| StoreError::Unknown(e.into())) } } diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index 31b0e62cfae..b64f8b35964 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -4,8 +4,7 @@ mod traits; pub mod write; pub use entity_cache::{EntityCache, EntityLfuCache, GetScope, ModificationsAndCache}; -use futures03::future::{FutureExt, TryFutureExt}; -use slog::{trace, Logger}; +use slog::Logger; pub use super::subgraph::Entity; pub use err::StoreError; @@ -14,8 +13,7 @@ use strum_macros::Display; pub use traits::*; pub use write::Batch; -use futures01::stream::poll_fn; -use futures01::{Async, Poll, Stream}; +use futures01::{Async, Stream}; use serde::{Deserialize, Serialize}; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashSet}; @@ -35,7 +33,7 @@ use crate::data::value::Word; use crate::data_source::CausalityRegion; use crate::derive::CheapClone; use crate::env::ENV_VARS; -use crate::prelude::{s, Attribute, DeploymentHash, SubscriptionFilter, ValueType}; +use crate::prelude::{s, Attribute, DeploymentHash, ValueType}; use crate::schema::{ast as sast, EntityKey, EntityType, InputSchema}; use crate::util::stats::MovingStats; @@ -539,58 +537,41 @@ impl EntityQuery { } } -/// Operation types that lead to entity changes. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +/// Operation types that lead to changes in assignments +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "lowercase")] -pub enum EntityChangeOperation { - /// An entity was added or updated +pub enum AssignmentOperation { + /// An assignment was added or updated Set, - /// An existing entity was removed. + /// An assignment was removed. Removed, } -/// Entity change events emitted by [Store](trait.Store.html) implementations. +/// Assignment change events emitted by [Store](trait.Store.html) implementations. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub enum EntityChange { - Data { - subgraph_id: DeploymentHash, - /// Entity type name of the changed entity. - entity_type: String, - }, - Assignment { - deployment: DeploymentLocator, - operation: EntityChangeOperation, - }, +pub struct AssignmentChange { + deployment: DeploymentLocator, + operation: AssignmentOperation, } -impl EntityChange { - pub fn for_data(subgraph_id: DeploymentHash, key: EntityKey) -> Self { - Self::Data { - subgraph_id, - entity_type: key.entity_type.to_string(), - } - } - - pub fn for_assignment(deployment: DeploymentLocator, operation: EntityChangeOperation) -> Self { - Self::Assignment { +impl AssignmentChange { + fn new(deployment: DeploymentLocator, operation: AssignmentOperation) -> Self { + Self { deployment, operation, } } - pub fn as_filter(&self, schema: &InputSchema) -> SubscriptionFilter { - use EntityChange::*; - match self { - Data { - subgraph_id, - entity_type, - .. - } => SubscriptionFilter::Entities( - subgraph_id.clone(), - schema.entity_type(entity_type).unwrap(), - ), - Assignment { .. } => SubscriptionFilter::Assignment, - } + pub fn set(deployment: DeploymentLocator) -> Self { + Self::new(deployment, AssignmentOperation::Set) + } + + pub fn removed(deployment: DeploymentLocator) -> Self { + Self::new(deployment, AssignmentOperation::Removed) + } + + pub fn into_parts(self) -> (DeploymentLocator, AssignmentOperation) { + (self.deployment, self.operation) } } @@ -607,74 +588,26 @@ pub struct StoreEvent { // The tag is only there to make it easier to track StoreEvents in the // logs as they flow through the system pub tag: usize, - pub changes: HashSet, + pub changes: HashSet, } impl StoreEvent { - pub fn new(changes: Vec) -> StoreEvent { + pub fn new(changes: Vec) -> StoreEvent { let changes = changes.into_iter().collect(); StoreEvent::from_set(changes) } - fn from_set(changes: HashSet) -> StoreEvent { + fn from_set(changes: HashSet) -> StoreEvent { static NEXT_TAG: AtomicUsize = AtomicUsize::new(0); let tag = NEXT_TAG.fetch_add(1, Ordering::Relaxed); StoreEvent { tag, changes } } - pub fn from_mods<'a, I: IntoIterator>( - subgraph_id: &DeploymentHash, - mods: I, - ) -> Self { - let changes: Vec<_> = mods - .into_iter() - .map(|op| { - use EntityModification::*; - match op { - Insert { key, .. } | Overwrite { key, .. } | Remove { key, .. } => { - EntityChange::for_data(subgraph_id.clone(), key.clone()) - } - } - }) - .collect(); - StoreEvent::new(changes) - } - - pub fn from_types(deployment: &DeploymentHash, entity_types: HashSet) -> Self { - let changes = - HashSet::from_iter( - entity_types - .into_iter() - .map(|entity_type| EntityChange::Data { - subgraph_id: deployment.clone(), - entity_type: entity_type.to_string(), - }), - ); - Self::from_set(changes) - } - - /// Extend `ev1` with `ev2`. If `ev1` is `None`, just set it to `ev2` - fn accumulate(logger: &Logger, ev1: &mut Option, ev2: StoreEvent) { - if let Some(e) = ev1 { - trace!(logger, "Adding changes to event"; - "from" => ev2.tag, "to" => e.tag); - e.changes.extend(ev2.changes); - } else { - *ev1 = Some(ev2); - } - } - pub fn extend(mut self, other: StoreEvent) -> Self { self.changes.extend(other.changes); self } - - pub fn matches(&self, filters: &BTreeSet) -> bool { - self.changes - .iter() - .any(|change| filters.iter().any(|filter| filter.matches(change))) - } } impl fmt::Display for StoreEvent { @@ -705,8 +638,6 @@ pub struct StoreEventStream { pub type StoreEventStreamBox = StoreEventStream, Error = ()> + Send>>; -pub type UnitStream = Box + Unpin + Send + Sync>; - impl Stream for StoreEventStream where S: Stream, Error = ()> + Send, @@ -727,101 +658,6 @@ where pub fn new(source: S) -> Self { StoreEventStream { source } } - - /// Filter a `StoreEventStream` by subgraph and entity. Only events that have - /// at least one change to one of the given (subgraph, entity) combinations - /// will be delivered by the filtered stream. - pub fn filter_by_entities(self, filters: BTreeSet) -> StoreEventStreamBox { - let source = self.source.filter(move |event| event.matches(&filters)); - - StoreEventStream::new(Box::new(source)) - } - - /// Reduce the frequency with which events are generated while a - /// subgraph deployment is syncing. While the given `deployment` is not - /// synced yet, events from `source` are reported at most every - /// `interval`. At the same time, no event is held for longer than - /// `interval`. The `StoreEvents` that arrive during an interval appear - /// on the returned stream as a single `StoreEvent`; the events are - /// combined by using the maximum of all sources and the concatenation - /// of the changes of the `StoreEvents` received during the interval. - // - // Currently unused, needs to be made compatible with `subscribe_no_payload`. - pub async fn throttle_while_syncing( - self, - logger: &Logger, - store: Arc, - interval: Duration, - ) -> StoreEventStreamBox { - // Check whether a deployment is marked as synced in the store. Note that in the moment a - // subgraph becomes synced any existing subscriptions will continue to be throttled since - // this is not re-checked. - let synced = store.is_deployment_synced().await.unwrap_or(false); - - let mut pending_event: Option = None; - let mut source = self.source.fuse(); - let mut had_err = false; - let mut delay = tokio::time::sleep(interval).unit_error().boxed().compat(); - let logger = logger.clone(); - - let source = Box::new(poll_fn(move || -> Poll>, ()> { - if had_err { - // We had an error the last time through, but returned the pending - // event first. Indicate the error now - had_err = false; - return Err(()); - } - - if synced { - return source.poll(); - } - - // Check if interval has passed since the last time we sent something. - // If it has, start a new delay timer - let should_send = match futures01::future::Future::poll(&mut delay) { - Ok(Async::NotReady) => false, - // Timer errors are harmless. Treat them as if the timer had - // become ready. - Ok(Async::Ready(())) | Err(_) => { - delay = tokio::time::sleep(interval).unit_error().boxed().compat(); - true - } - }; - - // Get as many events as we can off of the source stream - loop { - match source.poll() { - Ok(Async::NotReady) => { - if should_send && pending_event.is_some() { - let event = pending_event.take().map(Arc::new); - return Ok(Async::Ready(event)); - } else { - return Ok(Async::NotReady); - } - } - Ok(Async::Ready(None)) => { - let event = pending_event.take().map(Arc::new); - return Ok(Async::Ready(event)); - } - Ok(Async::Ready(Some(event))) => { - StoreEvent::accumulate(&logger, &mut pending_event, (*event).clone()); - } - Err(()) => { - // Before we report the error, deliver what we have accumulated so far. - // We will report the error the next time poll() is called - if pending_event.is_some() { - had_err = true; - let event = pending_event.take().map(Arc::new); - return Ok(Async::Ready(event)); - } else { - return Err(()); - } - } - }; - } - })); - StoreEventStream::new(source) - } } /// An entity operation that can be transacted into the store. diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 2292a1f61f5..73cb22269fe 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -25,10 +25,7 @@ pub trait SubscriptionManager: Send + Sync + 'static { /// Subscribe to changes for specific subgraphs and entities. /// /// Returns a stream of store events that match the input arguments. - fn subscribe(&self, entities: BTreeSet) -> StoreEventStreamBox; - - /// If the payload is not required, use for a more efficient subscription mechanism backed by a watcher. - fn subscribe_no_payload(&self, entities: BTreeSet) -> UnitStream; + fn subscribe(&self) -> StoreEventStreamBox; } /// Subgraph forking is the process of lazily fetching entities @@ -368,8 +365,6 @@ pub trait WritableStore: ReadStore + DeploymentCursorTracker { /// Set subgraph status to failed with the given error as the cause. async fn fail_subgraph(&self, error: SubgraphError) -> Result<(), StoreError>; - async fn supports_proof_of_indexing(&self) -> Result; - /// Transact the entity changes from a single block atomically into the store, and update the /// subgraph block pointer to `block_ptr_to`, and update the firehose cursor to `firehose_cursor` /// @@ -442,12 +437,9 @@ pub trait QueryStoreManager: Send + Sync + 'static { /// which deployment will be queried. It is not possible to use the id of the /// metadata subgraph, though the resulting store can be used to query /// metadata about the deployment `id` (but not metadata about other deployments). - /// - /// If `for_subscription` is true, the main replica will always be used. async fn query_store( &self, target: QueryTarget, - for_subscription: bool, ) -> Result, QueryExecutionError>; } @@ -663,7 +655,7 @@ pub trait QueryStore: Send + Sync { block_hash: &BlockHash, ) -> Result, Option)>, StoreError>; - fn wait_stats(&self) -> Result; + fn wait_stats(&self) -> PoolWaitStats; /// Find the current state for the subgraph deployment `id` and /// return details about it needed for executing queries @@ -676,7 +668,7 @@ pub trait QueryStore: Send + Sync { fn network_name(&self) -> &str; /// A permit should be acquired before starting query execution. - async fn query_permit(&self) -> Result; + async fn query_permit(&self) -> QueryPermit; /// Report the name of the shard in which the subgraph is stored. This /// should only be used for reporting and monitoring @@ -691,7 +683,7 @@ pub trait QueryStore: Send + Sync { #[async_trait] pub trait StatusStore: Send + Sync + 'static { /// A permit should be acquired before starting query execution. - async fn query_permit(&self) -> Result; + async fn query_permit(&self) -> QueryPermit; fn status(&self, filter: status::Filter) -> Result, StoreError>; diff --git a/graph/src/components/store/write.rs b/graph/src/components/store/write.rs index 721e3d80bc1..6f899633bd8 100644 --- a/graph/src/components/store/write.rs +++ b/graph/src/components/store/write.rs @@ -9,11 +9,10 @@ use crate::{ data::{store::Id, subgraph::schema::SubgraphError}, data_source::CausalityRegion, derive::CacheWeight, - prelude::DeploymentHash, util::cache_weight::CacheWeight, }; -use super::{BlockNumber, EntityKey, EntityType, StoreError, StoreEvent, StoredDynamicDataSource}; +use super::{BlockNumber, EntityKey, EntityType, StoreError, StoredDynamicDataSource}; /// A data structure similar to `EntityModification`, but tagged with a /// block. We might eventually replace `EntityModification` with this, but @@ -440,7 +439,7 @@ impl RowGroup { // clamping an old version match (&*prev_row, &row) { (Insert { end: None, .. } | Overwrite { end: None, .. }, Insert { .. }) - | (Remove { .. }, Overwrite { .. } | Remove { .. }) + | (Remove { .. }, Overwrite { .. }) | ( Insert { end: Some(_), .. } | Overwrite { end: Some(_), .. }, Overwrite { .. } | Remove { .. }, @@ -451,6 +450,11 @@ impl RowGroup { row )) } + (Remove { .. }, Remove { .. }) => { + // Ignore the new row, since prev_row is already a + // delete. This can happen when subgraphs delete + // entities without checking if they even exist + } ( Insert { end: Some(_), .. } | Overwrite { end: Some(_), .. } | Remove { .. }, Insert { .. }, @@ -779,17 +783,6 @@ impl Batch { }) } - /// Generate a store event for all the changes that this batch makes - pub fn store_event(&self, deployment: &DeploymentHash) -> StoreEvent { - let entity_types = HashSet::from_iter( - self.mods - .groups - .iter() - .map(|group| group.entity_type.clone()), - ); - StoreEvent::from_types(deployment, entity_types) - } - pub fn groups<'a>(&'a self) -> impl Iterator { self.mods.groups.iter() } diff --git a/graph/src/components/subgraph/proof_of_indexing/mod.rs b/graph/src/components/subgraph/proof_of_indexing/mod.rs index c8dd8c13314..36eeabc22cd 100644 --- a/graph/src/components/subgraph/proof_of_indexing/mod.rs +++ b/graph/src/components/subgraph/proof_of_indexing/mod.rs @@ -3,11 +3,15 @@ mod online; mod reference; pub use event::ProofOfIndexingEvent; +use graph_derive::CheapClone; pub use online::{ProofOfIndexing, ProofOfIndexingFinisher}; pub use reference::PoICausalityRegion; use atomic_refcell::AtomicRefCell; -use std::sync::Arc; +use slog::Logger; +use std::{ops::Deref, sync::Arc}; + +use crate::prelude::BlockNumber; #[derive(Copy, Clone, Debug)] pub enum ProofOfIndexingVersion { @@ -22,15 +26,57 @@ pub enum ProofOfIndexingVersion { /// intentionally disallowed - PoI requires sequential access to the hash /// function within a given causality region even if ownership is shared across /// multiple mapping contexts. -/// -/// The Option<_> is because not all subgraphs support PoI until re-deployed. -/// Eventually this can be removed. -/// -/// This is not a great place to define this type, since the ProofOfIndexing -/// shouldn't "know" these details about wasmtime and subgraph re-deployments, -/// but the APIs that would make use of this are in graph/components so this -/// lives here for lack of a better choice. -pub type SharedProofOfIndexing = Option>>; +#[derive(Clone, CheapClone)] +pub struct SharedProofOfIndexing { + poi: Option>>, +} + +impl SharedProofOfIndexing { + pub fn new(block: BlockNumber, version: ProofOfIndexingVersion) -> Self { + SharedProofOfIndexing { + poi: Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( + block, version, + )))), + } + } + + pub fn ignored() -> Self { + SharedProofOfIndexing { poi: None } + } + + pub fn write_event( + &self, + poi_event: &ProofOfIndexingEvent, + causality_region: &str, + logger: &Logger, + ) { + if let Some(poi) = &self.poi { + let mut poi = poi.deref().borrow_mut(); + poi.write(logger, causality_region, poi_event); + } + } + + pub fn start_handler(&self, causality_region: &str) { + if let Some(poi) = &self.poi { + let mut poi = poi.deref().borrow_mut(); + poi.start_handler(causality_region); + } + } + + pub fn write_deterministic_error(&self, logger: &Logger, causality_region: &str) { + if let Some(proof_of_indexing) = &self.poi { + proof_of_indexing + .deref() + .borrow_mut() + .write_deterministic_error(logger, causality_region); + } + } + + pub fn into_inner(self) -> Option { + self.poi + .map(|poi| Arc::try_unwrap(poi).unwrap().into_inner()) + } +} #[cfg(test)] mod tests { diff --git a/graph/src/components/subgraph/proof_of_indexing/online.rs b/graph/src/components/subgraph/proof_of_indexing/online.rs index d47f08b0a8f..ebf7a65e2f9 100644 --- a/graph/src/components/subgraph/proof_of_indexing/online.rs +++ b/graph/src/components/subgraph/proof_of_indexing/online.rs @@ -9,6 +9,7 @@ use crate::{ prelude::{debug, BlockNumber, DeploymentHash, Logger, ENV_VARS}, util::stable_hash_glue::AsBytes, }; +use sha2::{Digest, Sha256}; use stable_hash::{fast::FastStableHasher, FieldAddress, StableHash, StableHasher}; use stable_hash_legacy::crypto::{Blake3SeqNo, SetHasher}; use stable_hash_legacy::prelude::{ @@ -31,6 +32,8 @@ enum Hashers { Legacy(SetHasher), } +const STABLE_HASH_LEN: usize = 32; + impl Hashers { fn new(version: ProofOfIndexingVersion) -> Self { match version { @@ -132,9 +135,14 @@ impl BlockEventStream { } Hashers::Fast(mut digest) => { if let Some(prev) = prev { - let prev = prev - .try_into() - .expect("Expected valid fast stable hash representation"); + let prev = if prev.len() == STABLE_HASH_LEN { + prev.try_into() + .expect("Expected valid fast stable hash representation") + } else { + let mut hasher = Sha256::new(); + hasher.update(prev); + hasher.finalize().into() + }; let prev = FastStableHasher::from_bytes(prev); digest.mixin(&prev); } diff --git a/graph/src/data/graphql/ext.rs b/graph/src/data/graphql/ext.rs index 7e873353984..271ace79237 100644 --- a/graph/src/data/graphql/ext.rs +++ b/graph/src/data/graphql/ext.rs @@ -52,8 +52,6 @@ pub trait DocumentExt { fn get_root_query_type(&self) -> Option<&ObjectType>; - fn get_root_subscription_type(&self) -> Option<&ObjectType>; - fn object_or_interface(&self, name: &str) -> Option>; fn get_named_type(&self, name: &str) -> Option<&TypeDefinition>; @@ -159,21 +157,6 @@ impl DocumentExt for Document { .next() } - fn get_root_subscription_type(&self) -> Option<&ObjectType> { - self.definitions - .iter() - .filter_map(|d| match d { - Definition::TypeDefinition(TypeDefinition::Object(t)) - if t.name == "Subscription" => - { - Some(t) - } - _ => None, - }) - .peekable() - .next() - } - fn object_or_interface(&self, name: &str) -> Option> { match self.get_named_type(name) { Some(TypeDefinition::Object(t)) => Some(t.into()), diff --git a/graph/src/data/graphql/visitor.rs b/graph/src/data/graphql/visitor.rs deleted file mode 100644 index 94d26c08644..00000000000 --- a/graph/src/data/graphql/visitor.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::prelude::q; - -pub trait Visitor { - fn enter_field(&mut self, _: &q::Field) -> Result<(), E> { - Ok(()) - } - fn leave_field(&mut self, _: &mut q::Field) -> Result<(), E> { - Ok(()) - } - - fn enter_query(&mut self, _: &q::Query) -> Result<(), E> { - Ok(()) - } - fn leave_query(&mut self, _: &mut q::Query) -> Result<(), E> { - Ok(()) - } - - fn visit_fragment_spread(&mut self, _: &q::FragmentSpread) -> Result<(), E> { - Ok(()) - } -} - -pub fn visit(visitor: &mut dyn Visitor, doc: &mut q::Document) -> Result<(), E> { - for def in &mut doc.definitions { - match def { - q::Definition::Operation(op) => match op { - q::OperationDefinition::SelectionSet(set) => { - visit_selection_set(visitor, set)?; - } - q::OperationDefinition::Query(query) => { - visitor.enter_query(query)?; - visit_selection_set(visitor, &mut query.selection_set)?; - visitor.leave_query(query)?; - } - q::OperationDefinition::Mutation(_) => todo!(), - q::OperationDefinition::Subscription(_) => todo!(), - }, - q::Definition::Fragment(frag) => {} - } - } - Ok(()) -} - -fn visit_selection_set( - visitor: &mut dyn Visitor, - set: &mut q::SelectionSet, -) -> Result<(), E> { - for sel in &mut set.items { - match sel { - q::Selection::Field(field) => { - visitor.enter_field(field)?; - visit_selection_set(visitor, &mut field.selection_set)?; - visitor.leave_field(field)?; - } - q::Selection::FragmentSpread(frag) => { - visitor.visit_fragment_spread(frag)?; - } - q::Selection::InlineFragment(frag) => {} - } - } - Ok(()) -} diff --git a/graph/src/data/mod.rs b/graph/src/data/mod.rs index 45f085c96fa..246d4cdba12 100644 --- a/graph/src/data/mod.rs +++ b/graph/src/data/mod.rs @@ -7,9 +7,6 @@ pub mod query; /// Data types for dealing with storing entities. pub mod store; -/// Data types for dealing with GraphQL subscriptions. -pub mod subscription; - /// Data types for dealing with GraphQL values. pub mod graphql; diff --git a/graph/src/data/query/error.rs b/graph/src/data/query/error.rs index e8b63422cdf..65fc1bcd259 100644 --- a/graph/src/data/query/error.rs +++ b/graph/src/data/query/error.rs @@ -26,7 +26,6 @@ pub enum QueryExecutionError { OperationNameRequired, OperationNotFound(String), NotSupported(String), - NoRootSubscriptionObjectType, NonNullError(Pos, String), ListValueError(Pos, String), NamedTypeError(String), @@ -42,7 +41,6 @@ pub enum QueryExecutionError { FilterNotSupportedError(String, String), UnknownField(Pos, String, String), EmptyQuery, - MultipleSubscriptionFields, SubgraphDeploymentIdError(String), RangeArgumentsError(&'static str, u32, i64), InvalidFilterError, @@ -67,7 +65,6 @@ pub enum QueryExecutionError { Throttled, UndefinedFragment(String), Panic(String), - EventStreamError, FulltextQueryRequiresFilter, FulltextQueryInvalidSyntax(String), DeploymentReverted, @@ -87,7 +84,6 @@ impl QueryExecutionError { OperationNameRequired | OperationNotFound(_) | NotSupported(_) - | NoRootSubscriptionObjectType | NonNullError(_, _) | NamedTypeError(_) | AbstractTypeError(_) @@ -101,7 +97,6 @@ impl QueryExecutionError { | ChildFilterNestingNotSupportedError(_, _) | UnknownField(_, _, _) | EmptyQuery - | MultipleSubscriptionFields | SubgraphDeploymentIdError(_) | InvalidFilterError | EntityFieldError(_, _) @@ -127,7 +122,6 @@ impl QueryExecutionError { | TooComplex(_, _) | TooDeep(_) | Panic(_) - | EventStreamError | TooExpensive | Throttled | DeploymentReverted @@ -166,9 +160,6 @@ impl fmt::Display for QueryExecutionError { write!(f, "{}", message) } NotSupported(s) => write!(f, "Not supported: {}", s), - NoRootSubscriptionObjectType => { - write!(f, "No root Subscription type defined in the schema") - } NonNullError(_, s) => { write!(f, "Null value resolved for non-null field `{}`", s) } @@ -212,10 +203,6 @@ impl fmt::Display for QueryExecutionError { write!(f, "Type `{}` has no field `{}`", t, s) } EmptyQuery => write!(f, "The query is empty"), - MultipleSubscriptionFields => write!( - f, - "Only a single top-level field is allowed in subscriptions" - ), SubgraphDeploymentIdError(s) => { write!(f, "Failed to get subgraph ID from type: `{}`", s) } @@ -276,7 +263,6 @@ impl fmt::Display for QueryExecutionError { CyclicalFragment(name) =>write!(f, "query has fragment cycle including `{}`", name), UndefinedFragment(frag_name) => write!(f, "fragment `{}` is not defined", frag_name), Panic(msg) => write!(f, "panic processing query: {}", msg), - EventStreamError => write!(f, "error in the subscription event stream"), FulltextQueryRequiresFilter => write!(f, "fulltext search queries can only use EntityFilter::Equal"), FulltextQueryInvalidSyntax(msg) => write!(f, "Invalid fulltext search query syntax. Error: {}. Hint: Search terms with spaces need to be enclosed in single quotes", msg), TooExpensive => write!(f, "query is too expensive"), diff --git a/graph/src/data/query/trace.rs b/graph/src/data/query/trace.rs index cf2d153dca4..256c9cdeaf6 100644 --- a/graph/src/data/query/trace.rs +++ b/graph/src/data/query/trace.rs @@ -118,11 +118,8 @@ impl Trace { } } - pub fn query_done(&mut self, dur: Duration, permit: &Result) { - let permit_dur = match permit { - Ok(permit) => permit.wait, - Err(_) => Duration::from_millis(0), - }; + pub fn query_done(&mut self, dur: Duration, permit: &QueryPermit) { + let permit_dur = permit.wait; match self { Trace::None => { /* nothing to do */ } Trace::Root { .. } => { diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index 25c0d3e0813..c8786e9b473 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -3,11 +3,10 @@ use crate::{ derive::CacheWeight, prelude::{lazy_static, q, r, s, CacheWeight, QueryExecutionError}, runtime::gas::{Gas, GasSizeOf}, - schema::{input::VID_FIELD, EntityKey, EntityType}, + schema::{input::VID_FIELD, EntityKey}, util::intern::{self, AtomPool}, util::intern::{Error as InternError, NullValue, Object}, }; -use crate::{data::subgraph::DeploymentHash, prelude::EntityChange}; use anyhow::{anyhow, Error}; use itertools::Itertools; use serde::de; @@ -36,33 +35,6 @@ pub mod ethereum; /// Conversion of values to/from SQL pub mod sql; -/// Filter subscriptions -#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum SubscriptionFilter { - /// Receive updates about all entities from the given deployment of the - /// given type - Entities(DeploymentHash, EntityType), - /// Subscripe to changes in deployment assignments - Assignment, -} - -impl SubscriptionFilter { - pub fn matches(&self, change: &EntityChange) -> bool { - match (self, change) { - ( - Self::Entities(eid, etype), - EntityChange::Data { - subgraph_id, - entity_type, - .. - }, - ) => subgraph_id == eid && entity_type == etype.typename(), - (Self::Assignment, EntityChange::Assignment { .. }) => true, - _ => false, - } - } -} - #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct NodeId(String); @@ -1165,6 +1137,8 @@ fn value_bigint() { #[test] fn entity_validation() { + use crate::data::subgraph::DeploymentHash; + use crate::schema::EntityType; use crate::schema::InputSchema; const DOCUMENT: &str = " diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index d14b2c89b29..77c8ba67d36 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -348,7 +348,7 @@ pub enum SubgraphManifestValidationError { MultipleEthereumNetworks, #[error("subgraph must have at least one Ethereum network data source")] EthereumNetworkRequired, - #[error("the specified block must exist on the Ethereum network")] + #[error("the specified block {0} must exist on the Ethereum network")] BlockNotFound(String), #[error("schema validation failed: {0:?}")] SchemaValidationError(Vec), @@ -370,7 +370,7 @@ pub enum SubgraphManifestResolveError { NonUtf8, #[error("subgraph is not valid YAML")] InvalidFormat, - #[error("resolve error: {0}")] + #[error("resolve error: {0:#}")] ResolveError(#[from] anyhow::Error), } @@ -504,9 +504,9 @@ impl Graft { // The graft point must be at least `reorg_threshold` blocks // behind the subgraph head so that a reorg can not affect the // data that we copy for grafting - (Some(ptr), true) if self.block + ENV_VARS.reorg_threshold > ptr.number => Err(GraftBaseInvalid(format!( + (Some(ptr), true) if self.block + ENV_VARS.reorg_threshold() > ptr.number => Err(GraftBaseInvalid(format!( "failed to graft onto `{}` at block {} since it's only at block {} which is within the reorg threshold of {} blocks", - self.base, self.block, ptr.number, ENV_VARS.reorg_threshold + self.base, self.block, ptr.number, ENV_VARS.reorg_threshold() ))), // If the base deployment is failed *and* the `graft.block` is not // less than the `base.block`, the graft shouldn't be permitted. diff --git a/graph/src/data/subscription/error.rs b/graph/src/data/subscription/error.rs deleted file mode 100644 index 20cf3f3af73..00000000000 --- a/graph/src/data/subscription/error.rs +++ /dev/null @@ -1,34 +0,0 @@ -use serde::ser::*; - -use crate::prelude::QueryExecutionError; -use thiserror::Error; - -/// Error caused while processing a [Subscription](struct.Subscription.html) request. -#[derive(Debug, Error)] -pub enum SubscriptionError { - #[error("GraphQL error: {0:?}")] - GraphQLError(Vec), -} - -impl From for SubscriptionError { - fn from(e: QueryExecutionError) -> Self { - SubscriptionError::GraphQLError(vec![e]) - } -} - -impl From> for SubscriptionError { - fn from(e: Vec) -> Self { - SubscriptionError::GraphQLError(e) - } -} -impl Serialize for SubscriptionError { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map = serializer.serialize_map(Some(1))?; - let msg = format!("{}", self); - map.serialize_entry("message", msg.as_str())?; - map.end() - } -} diff --git a/graph/src/data/subscription/mod.rs b/graph/src/data/subscription/mod.rs deleted file mode 100644 index 093c0008728..00000000000 --- a/graph/src/data/subscription/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod error; -mod result; -mod subscription; - -pub use self::error::SubscriptionError; -pub use self::result::{QueryResultStream, SubscriptionResult}; -pub use self::subscription::Subscription; diff --git a/graph/src/data/subscription/result.rs b/graph/src/data/subscription/result.rs deleted file mode 100644 index 648ce79ac52..00000000000 --- a/graph/src/data/subscription/result.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::prelude::QueryResult; -use std::pin::Pin; -use std::sync::Arc; - -/// A stream of query results for a subscription. -pub type QueryResultStream = - Pin> + Send>>; - -/// The result of running a subscription, if successful. -pub type SubscriptionResult = QueryResultStream; diff --git a/graph/src/data/subscription/subscription.rs b/graph/src/data/subscription/subscription.rs deleted file mode 100644 index 8ae6b872fba..00000000000 --- a/graph/src/data/subscription/subscription.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::prelude::Query; - -/// A GraphQL subscription made by a client. -/// -/// At the moment, this only contains the GraphQL query submitted as the -/// subscription payload. -#[derive(Clone, Debug)] -pub struct Subscription { - /// The GraphQL subscription query. - pub query: Query, -} diff --git a/graph/src/data_source/common.rs b/graph/src/data_source/common.rs index a70f0ab8e17..57781815f5f 100644 --- a/graph/src/data_source/common.rs +++ b/graph/src/data_source/common.rs @@ -81,7 +81,8 @@ impl UnresolvedMappingABI { self.name, self.file.link ) })?; - let contract = Contract::load(&*contract_bytes)?; + let contract = Contract::load(&*contract_bytes) + .with_context(|| format!("failed to load ABI {}", self.name))?; Ok(MappingABI { name: self.name, contract, diff --git a/graph/src/data_source/subgraph.rs b/graph/src/data_source/subgraph.rs index 9e120a4c82c..d8ef847aee4 100644 --- a/graph/src/data_source/subgraph.rs +++ b/graph/src/data_source/subgraph.rs @@ -239,7 +239,16 @@ impl UnresolvedDataSource { None => { return Err(anyhow!("Entity {} not found in source manifest", entity)); } - Some(TypeKind::Object) => {} + Some(TypeKind::Object) => { + // Check if the entity is immutable + let entity_type = source_manifest.schema.entity_type(entity)?; + if !entity_type.is_immutable() { + return Err(anyhow!( + "Entity {} is not immutable and cannot be used as a mapping entity", + entity + )); + } + } } } Ok(()) @@ -270,6 +279,55 @@ impl UnresolvedDataSource { .map(Arc::new) } + /// Recursively verifies that all grafts in the chain meet the minimum spec version requirement for a subgraph source + async fn verify_graft_chain_sourcable( + manifest: Arc>, + resolver: &Arc, + logger: &Logger, + graft_chain: &mut Vec, + ) -> Result<(), Error> { + // Add current manifest to graft chain + graft_chain.push(manifest.id.to_string()); + + // Check if current manifest meets spec version requirement + if manifest.spec_version < SPEC_VERSION_1_3_0 { + return Err(anyhow!( + "Subgraph with a spec version {} is not supported for a subgraph source, minimum supported version is {}. Graft chain: {}", + manifest.spec_version, + SPEC_VERSION_1_3_0, + graft_chain.join(" -> ") + )); + } + + // If there's a graft, recursively verify it + if let Some(graft) = &manifest.graft { + let graft_raw = resolver + .cat(logger, &graft.base.to_ipfs_link()) + .await + .context("Failed to resolve graft base manifest")?; + + let graft_raw: serde_yaml::Mapping = serde_yaml::from_slice(&graft_raw) + .context("Failed to parse graft base manifest as YAML")?; + + let graft_manifest = + UnresolvedSubgraphManifest::::parse(graft.base.clone(), graft_raw) + .context("Failed to parse graft base manifest")? + .resolve(resolver, logger, LATEST_VERSION.clone()) + .await + .context("Failed to resolve graft base manifest")?; + + Box::pin(Self::verify_graft_chain_sourcable( + Arc::new(graft_manifest), + resolver, + logger, + graft_chain, + )) + .await?; + } + + Ok(()) + } + #[allow(dead_code)] pub(super) async fn resolve( self, @@ -286,15 +344,6 @@ impl UnresolvedDataSource { let kind = self.kind.clone(); let source_manifest = self.resolve_source_manifest::(resolver, logger).await?; let source_spec_version = &source_manifest.spec_version; - - if source_manifest - .data_sources - .iter() - .any(|ds| matches!(ds, crate::data_source::DataSource::Subgraph(_))) - { - return Err(anyhow!("Nested subgraph data sources are not supported.")); - } - if source_spec_version < &SPEC_VERSION_1_3_0 { return Err(anyhow!( "Source subgraph manifest spec version {} is not supported, minimum supported version is {}", @@ -303,15 +352,22 @@ impl UnresolvedDataSource { )); } - let pruning_enabled = match source_manifest.indexer_hints.as_ref() { - None => false, - Some(hints) => hints.prune.is_some(), - }; + // Verify the entire graft chain meets spec version requirements + let mut graft_chain = Vec::new(); + Self::verify_graft_chain_sourcable( + source_manifest.clone(), + resolver, + logger, + &mut graft_chain, + ) + .await?; - if pruning_enabled { - return Err(anyhow!( - "Pruning is enabled for source subgraph, which is not supported" - )); + if source_manifest + .data_sources + .iter() + .any(|ds| matches!(ds, crate::data_source::DataSource::Subgraph(_))) + { + return Err(anyhow!("Nested subgraph data sources are not supported.")); } let mapping_entities: Vec = self diff --git a/graph/src/env/graphql.rs b/graph/src/env/graphql.rs index 23fab23cd49..4f1f9896488 100644 --- a/graph/src/env/graphql.rs +++ b/graph/src/env/graphql.rs @@ -86,9 +86,6 @@ pub struct EnvVarsGraphQl { /// Set by the environment variable `GRAPH_GRAPHQL_ERROR_RESULT_SIZE`. The /// default value is [`usize::MAX`]. pub error_result_size: usize, - /// Set by the flag `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`. - /// Defaults to 1000. - pub max_operations_per_connection: usize, /// Set by the flag `GRAPH_GRAPHQL_DISABLE_BOOL_FILTERS`. Off by default. /// Disables AND/OR filters pub disable_bool_filters: bool, @@ -144,7 +141,6 @@ impl From for EnvVarsGraphQl { allow_deployment_change: x.allow_deployment_change.0, warn_result_size: x.warn_result_size.0 .0, error_result_size: x.error_result_size.0 .0, - max_operations_per_connection: x.max_operations_per_connection, disable_bool_filters: x.disable_bool_filters.0, disable_child_sorting: x.disable_child_sorting.0, query_trace_token: x.query_trace_token, @@ -192,8 +188,6 @@ pub struct InnerGraphQl { warn_result_size: WithDefaultUsize, { usize::MAX }>, #[envconfig(from = "GRAPH_GRAPHQL_ERROR_RESULT_SIZE", default = "")] error_result_size: WithDefaultUsize, { usize::MAX }>, - #[envconfig(from = "GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION", default = "1000")] - max_operations_per_connection: usize, #[envconfig(from = "GRAPH_GRAPHQL_DISABLE_BOOL_FILTERS", default = "false")] pub disable_bool_filters: EnvVarBoolean, #[envconfig(from = "GRAPH_GRAPHQL_DISABLE_CHILD_SORTING", default = "false")] diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs index 4383ce17b5c..eff0ebea16e 100644 --- a/graph/src/env/mod.rs +++ b/graph/src/env/mod.rs @@ -15,9 +15,16 @@ use crate::{ runtime::gas::CONST_MAX_GAS_PER_HANDLER, }; +#[cfg(debug_assertions)] +use std::sync::Mutex; + lazy_static! { pub static ref ENV_VARS: EnvVars = EnvVars::from_env().unwrap(); } +#[cfg(debug_assertions)] +lazy_static! { + pub static ref TEST_WITH_NO_REORG: Mutex = Mutex::new(false); +} /// Panics if: /// - The value is not UTF8. @@ -181,7 +188,7 @@ pub struct EnvVars { pub static_filters_threshold: usize, /// Set by the environment variable `ETHEREUM_REORG_THRESHOLD`. The default /// value is 250 blocks. - pub reorg_threshold: BlockNumber, + reorg_threshold: BlockNumber, /// The time to wait between polls when using polling block ingestor. /// The value is set by `ETHERUM_POLLING_INTERVAL` in millis and the /// default is 1000. @@ -247,24 +254,17 @@ pub struct EnvVars { /// Set by the environment variable `GRAPH_FIREHOSE_FETCH_BLOCK_TIMEOUT_SECS`. /// The default value is 60 seconds. pub firehose_block_fetch_timeout: u64, + /// Set by the environment variable `GRAPH_FIREHOSE_BLOCK_BATCH_SIZE`. + /// The default value is 10. + pub firehose_block_batch_size: usize, } impl EnvVars { - pub fn from_env() -> Result { + pub fn from_env() -> Result { let inner = Inner::init_from_env()?; let graphql = InnerGraphQl::init_from_env()?.into(); let mapping_handlers = InnerMappingHandlers::init_from_env()?.into(); - let store = InnerStore::init_from_env()?.into(); - - // The default reorganization (reorg) threshold is set to 250. - // For testing purposes, we need to set this threshold to 0 because: - // 1. Many tests involve reverting blocks. - // 2. Blocks cannot be reverted below the reorg threshold. - // Therefore, during tests, we want to set the reorg threshold to 0. - let reorg_threshold = - inner - .reorg_threshold - .unwrap_or_else(|| if cfg!(debug_assertions) { 0 } else { 250 }); + let store = InnerStore::init_from_env()?.try_into()?; Ok(Self { graphql, @@ -319,13 +319,15 @@ impl EnvVars { external_http_base_url: inner.external_http_base_url, external_ws_base_url: inner.external_ws_base_url, static_filters_threshold: inner.static_filters_threshold, - reorg_threshold, + reorg_threshold: inner.reorg_threshold, ingestor_polling_interval: Duration::from_millis(inner.ingestor_polling_interval), subgraph_settings: inner.subgraph_settings, prefer_substreams_block_streams: inner.prefer_substreams_block_streams, enable_dips_metrics: inner.enable_dips_metrics.0, history_blocks_override: inner.history_blocks_override, - min_history_blocks: inner.min_history_blocks.unwrap_or(2 * reorg_threshold), + min_history_blocks: inner + .min_history_blocks + .unwrap_or(2 * inner.reorg_threshold), dips_metrics_object_store_url: inner.dips_metrics_object_store_url, section_map: inner.section_map, firehose_grpc_max_decode_size_mb: inner.firehose_grpc_max_decode_size_mb, @@ -339,6 +341,7 @@ impl EnvVars { block_write_capacity: inner.block_write_capacity.0, firehose_block_fetch_retry_limit: inner.firehose_block_fetch_retry_limit, firehose_block_fetch_timeout: inner.firehose_block_fetch_timeout, + firehose_block_batch_size: inner.firehose_block_fetch_batch_size, }) } @@ -371,6 +374,23 @@ impl EnvVars { .filter(|x| !x.is_empty()) .collect() } + #[cfg(debug_assertions)] + pub fn reorg_threshold(&self) -> i32 { + // The default reorganization (reorg) threshold is set to 250. + // For testing purposes, we need to set this threshold to 0 because: + // 1. Many tests involve reverting blocks. + // 2. Blocks cannot be reverted below the reorg threshold. + // Therefore, during tests, we want to set the reorg threshold to 0. + if *TEST_WITH_NO_REORG.lock().unwrap() { + 0 + } else { + self.reorg_threshold + } + } + #[cfg(not(debug_assertions))] + pub fn reorg_threshold(&self) -> i32 { + self.reorg_threshold + } } impl Default for EnvVars { @@ -469,8 +489,8 @@ struct Inner { #[envconfig(from = "GRAPH_STATIC_FILTERS_THRESHOLD", default = "10000")] static_filters_threshold: usize, // JSON-RPC specific. - #[envconfig(from = "ETHEREUM_REORG_THRESHOLD")] - reorg_threshold: Option, + #[envconfig(from = "ETHEREUM_REORG_THRESHOLD", default = "250")] + reorg_threshold: BlockNumber, #[envconfig(from = "ETHEREUM_POLLING_INTERVAL", default = "1000")] ingestor_polling_interval: u64, #[envconfig(from = "GRAPH_EXPERIMENTAL_SUBGRAPH_SETTINGS")] @@ -506,6 +526,8 @@ struct Inner { firehose_block_fetch_retry_limit: usize, #[envconfig(from = "GRAPH_FIREHOSE_FETCH_BLOCK_TIMEOUT_SECS", default = "60")] firehose_block_fetch_timeout: u64, + #[envconfig(from = "GRAPH_FIREHOSE_FETCH_BLOCK_BATCH_SIZE", default = "10")] + firehose_block_fetch_batch_size: usize, } #[derive(Clone, Debug)] diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index af0f076978f..8197d07b6bc 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -49,13 +49,6 @@ pub struct EnvVarsStore { /// only as an emergency setting for the hosted service. Remove after /// 2022-07-01 if hosted service had no issues with it being `true` pub order_by_block_range: bool, - /// Whether to disable the notifications that feed GraphQL - /// subscriptions. When the flag is set, no updates - /// about entity changes will be sent to query nodes. - /// - /// Set by the flag `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`. Not set - /// by default. - pub disable_subscription_notifications: bool, /// Set by the environment variable `GRAPH_REMOVE_UNUSED_INTERVAL` /// (expressed in minutes). The default value is 360 minutes. pub remove_unused_interval: chrono::Duration, @@ -88,6 +81,22 @@ pub struct EnvVarsStore { /// The default is 180s. pub batch_target_duration: Duration, + /// Cancel and reset a batch copy operation if it takes longer than + /// this. Set by `GRAPH_STORE_BATCH_TIMEOUT`. Unlimited by default + pub batch_timeout: Option, + + /// The number of workers to use for batch operations. If there are idle + /// connections, each subgraph copy operation will use up to this many + /// workers to copy tables in parallel. Defaults to 1 and must be at + /// least 1 + pub batch_workers: usize, + + /// How long to wait to get an additional connection for a batch worker. + /// This should just be big enough to allow the connection pool to + /// establish a connection. Set by `GRAPH_STORE_BATCH_WORKER_WAIT`. + /// Value is in ms and defaults to 2000ms + pub batch_worker_wait: Duration, + /// Prune tables where we will remove at least this fraction of entity /// versions by rebuilding the table. Set by /// `GRAPH_STORE_HISTORY_REBUILD_THRESHOLD`. The default is 0.5 @@ -120,19 +129,14 @@ pub struct EnvVarsStore { pub use_brin_for_all_query_types: bool, /// Temporary env var to disable certain lookups in the chain store pub disable_block_cache_for_lookup: bool, - /// Temporary env var to fall back to the old broken way of determining - /// the time of the last rollup from the POI table instead of the new - /// way that fixes - /// https://github.com/graphprotocol/graph-node/issues/5530 Remove this - /// and all code that is dead as a consequence once this has been vetted - /// sufficiently, probably after 2024-12-01 - /// Defaults to `false`, i.e. using the new fixed behavior - pub last_rollup_from_poi: bool, /// Safety switch to increase the number of columns used when /// calculating the chunk size in `InsertQuery::chunk_size`. This can be /// used to work around Postgres errors complaining 'number of /// parameters must be between 0 and 65535' when inserting entities pub insert_extra_cols: usize, + /// The number of rows to fetch from the foreign data wrapper in one go, + /// this will be set as the option 'fetch_size' on all foreign servers + pub fdw_fetch_size: usize, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -142,9 +146,11 @@ impl fmt::Debug for EnvVarsStore { } } -impl From for EnvVarsStore { - fn from(x: InnerStore) -> Self { - Self { +impl TryFrom for EnvVarsStore { + type Error = anyhow::Error; + + fn try_from(x: InnerStore) -> Result { + let vars = Self { chain_head_watcher_timeout: Duration::from_secs(x.chain_head_watcher_timeout_in_secs), query_stats_refresh_interval: Duration::from_secs( x.query_stats_refresh_interval_in_secs, @@ -163,7 +169,6 @@ impl From for EnvVarsStore { typea_batch_size: x.typea_batch_size, typed_children_set_size: x.typed_children_set_size, order_by_block_range: x.order_by_block_range.0, - disable_subscription_notifications: x.disable_subscription_notifications.0, remove_unused_interval: chrono::Duration::minutes( x.remove_unused_interval_in_minutes as i64, ), @@ -173,6 +178,9 @@ impl From for EnvVarsStore { connection_idle_timeout: Duration::from_secs(x.connection_idle_timeout_in_secs), write_queue_size: x.write_queue_size, batch_target_duration: Duration::from_secs(x.batch_target_duration_in_secs), + batch_timeout: x.batch_timeout_in_secs.map(Duration::from_secs), + batch_workers: x.batch_workers, + batch_worker_wait: Duration::from_millis(x.batch_worker_wait), rebuild_threshold: x.rebuild_threshold.0, delete_threshold: x.delete_threshold.0, history_slack_factor: x.history_slack_factor.0, @@ -181,9 +189,20 @@ impl From for EnvVarsStore { create_gin_indexes: x.create_gin_indexes, use_brin_for_all_query_types: x.use_brin_for_all_query_types, disable_block_cache_for_lookup: x.disable_block_cache_for_lookup, - last_rollup_from_poi: x.last_rollup_from_poi, insert_extra_cols: x.insert_extra_cols, + fdw_fetch_size: x.fdw_fetch_size, + }; + if let Some(timeout) = vars.batch_timeout { + if timeout < 2 * vars.batch_target_duration { + bail!( + "GRAPH_STORE_BATCH_TIMEOUT must be greater than 2*GRAPH_STORE_BATCH_TARGET_DURATION" + ); + } + } + if vars.batch_workers < 1 { + bail!("GRAPH_STORE_BATCH_WORKERS must be at least 1"); } + Ok(vars) } } @@ -207,8 +226,6 @@ pub struct InnerStore { typed_children_set_size: usize, #[envconfig(from = "ORDER_BY_BLOCK_RANGE", default = "true")] order_by_block_range: EnvVarBoolean, - #[envconfig(from = "GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS", default = "false")] - disable_subscription_notifications: EnvVarBoolean, #[envconfig(from = "GRAPH_REMOVE_UNUSED_INTERVAL", default = "360")] remove_unused_interval_in_minutes: u64, #[envconfig(from = "GRAPH_STORE_RECENT_BLOCKS_CACHE_CAPACITY", default = "10")] @@ -228,6 +245,12 @@ pub struct InnerStore { write_queue_size: usize, #[envconfig(from = "GRAPH_STORE_BATCH_TARGET_DURATION", default = "180")] batch_target_duration_in_secs: u64, + #[envconfig(from = "GRAPH_STORE_BATCH_TIMEOUT")] + batch_timeout_in_secs: Option, + #[envconfig(from = "GRAPH_STORE_BATCH_WORKERS", default = "1")] + batch_workers: usize, + #[envconfig(from = "GRAPH_STORE_BATCH_WORKER_WAIT", default = "2000")] + batch_worker_wait: u64, #[envconfig(from = "GRAPH_STORE_HISTORY_REBUILD_THRESHOLD", default = "0.5")] rebuild_threshold: ZeroToOneF64, #[envconfig(from = "GRAPH_STORE_HISTORY_DELETE_THRESHOLD", default = "0.05")] @@ -244,10 +267,10 @@ pub struct InnerStore { use_brin_for_all_query_types: bool, #[envconfig(from = "GRAPH_STORE_DISABLE_BLOCK_CACHE_FOR_LOOKUP", default = "false")] disable_block_cache_for_lookup: bool, - #[envconfig(from = "GRAPH_STORE_LAST_ROLLUP_FROM_POI", default = "false")] - last_rollup_from_poi: bool, #[envconfig(from = "GRAPH_STORE_INSERT_EXTRA_COLS", default = "0")] insert_extra_cols: usize, + #[envconfig(from = "GRAPH_STORE_FDW_FETCH_SIZE", default = "1000")] + fdw_fetch_size: usize, } #[derive(Clone, Copy, Debug)] diff --git a/graph/src/firehose/codec.rs b/graph/src/firehose/codec.rs index 5537dba153b..3768f3acf45 100644 --- a/graph/src/firehose/codec.rs +++ b/graph/src/firehose/codec.rs @@ -10,11 +10,6 @@ mod pbethereum; #[path = "sf.near.transform.v1.rs"] mod pbnear; -#[rustfmt::skip] -#[path = "sf.cosmos.transform.v1.rs"] -mod pbcosmos; - -pub use pbcosmos::*; pub use pbethereum::*; pub use pbfirehose::*; pub use pbnear::*; diff --git a/graph/src/firehose/endpoints.rs b/graph/src/firehose/endpoints.rs index ebf27faa5a1..448eb845496 100644 --- a/graph/src/firehose/endpoints.rs +++ b/graph/src/firehose/endpoints.rs @@ -1,3 +1,4 @@ +use crate::firehose::codec::InfoRequest; use crate::firehose::fetch_client::FetchClient; use crate::firehose::interceptors::AuthInterceptor; use crate::{ @@ -12,9 +13,10 @@ use crate::{ prelude::{anyhow, debug, DeploymentHash}, substreams_rpc, }; +use anyhow::Context; use async_trait::async_trait; -use futures03::StreamExt; -use http0::uri::{Scheme, Uri}; +use futures03::{StreamExt, TryStreamExt}; +use http::uri::{Scheme, Uri}; use itertools::Itertools; use slog::{error, info, trace, Logger}; use std::{collections::HashMap, fmt::Display, ops::ControlFlow, sync::Arc, time::Duration}; @@ -51,6 +53,7 @@ pub struct FirehoseEndpoint { pub filters_enabled: bool, pub compression_enabled: bool, pub subgraph_limit: SubgraphLimit, + is_substreams: bool, endpoint_metrics: Arc, channel: Channel, @@ -79,8 +82,15 @@ impl NetworkDetails for Arc { async fn provides_extended_blocks(&self) -> anyhow::Result { let info = self.clone().info().await?; + let pred = if info.chain_name.contains("arbitrum-one") + || info.chain_name.contains("optimism-mainnet") + { + |x: &String| x.starts_with("extended") || x == "hybrid" + } else { + |x: &String| x == "extended" + }; - Ok(info.block_features.iter().all(|x| x == "extended")) + Ok(info.block_features.iter().any(pred)) } } @@ -175,6 +185,7 @@ impl FirehoseEndpoint { compression_enabled: bool, subgraph_limit: SubgraphLimit, endpoint_metrics: Arc, + is_substreams_endpoint: bool, ) -> Self { let uri = url .as_ref() @@ -183,9 +194,14 @@ impl FirehoseEndpoint { let endpoint_builder = match uri.scheme().unwrap_or(&Scheme::HTTP).as_str() { "http" => Channel::builder(uri), - "https" => Channel::builder(uri) - .tls_config(ClientTlsConfig::new()) - .expect("TLS config on this host is invalid"), + "https" => { + let mut tls = ClientTlsConfig::new(); + tls = tls.with_native_roots(); + + Channel::builder(uri) + .tls_config(tls) + .expect("TLS config on this host is invalid") + } _ => panic!("invalid uri scheme for firehose endpoint"), }; @@ -238,6 +254,7 @@ impl FirehoseEndpoint { subgraph_limit, endpoint_metrics, info_response: OnceCell::new(), + is_substreams: is_substreams_endpoint, } } @@ -306,7 +323,44 @@ impl FirehoseEndpoint { client } - fn new_substreams_client( + fn new_firehose_info_client(&self) -> crate::firehose::endpoint_info::Client { + let metrics = self.metrics_interceptor(); + let auth = self.auth.clone(); + + let mut client = crate::firehose::endpoint_info::Client::new(metrics, auth); + + if self.compression_enabled { + client = client.with_compression(); + } + + client = client.with_max_message_size(self.max_message_size()); + client + } + + fn new_substreams_info_client( + &self, + ) -> crate::substreams_rpc::endpoint_info_client::EndpointInfoClient< + InterceptedService, impl tonic::service::Interceptor>, + > { + let metrics = self.metrics_interceptor(); + + let mut client = + crate::substreams_rpc::endpoint_info_client::EndpointInfoClient::with_interceptor( + metrics, + self.auth.clone(), + ) + .accept_compressed(CompressionEncoding::Gzip); + + if self.compression_enabled { + client = client.send_compressed(CompressionEncoding::Gzip); + } + + client = client.max_decoding_message_size(self.max_message_size()); + + client + } + + fn new_substreams_streaming_client( &self, ) -> substreams_rpc::stream_client::StreamClient< InterceptedService, impl tonic::service::Interceptor>, @@ -395,15 +449,47 @@ impl FirehoseEndpoint { } } - pub async fn get_block_by_number( - &self, - number: u64, + pub async fn get_block_by_ptr_with_retry( + self: Arc, + ptr: &BlockPtr, logger: &Logger, ) -> Result where M: prost::Message + BlockchainBlock + Default + 'static, { - debug!( + let retry_log_message = format!("get_block_by_ptr for block {}", ptr); + let endpoint = self.cheap_clone(); + let logger = logger.cheap_clone(); + let ptr_for_retry = ptr.clone(); + + retry(retry_log_message, &logger) + .limit(ENV_VARS.firehose_block_fetch_retry_limit) + .timeout_secs(ENV_VARS.firehose_block_fetch_timeout) + .run(move || { + let endpoint = endpoint.cheap_clone(); + let logger = logger.cheap_clone(); + let ptr = ptr_for_retry.clone(); + async move { + endpoint + .get_block_by_ptr::(&ptr, &logger) + .await + .context(format!( + "Failed to fetch block by ptr {} from firehose", + ptr + )) + } + }) + .await + .map_err(move |e| { + anyhow::anyhow!("Failed to fetch block by ptr {} from firehose: {}", ptr, e) + }) + } + + async fn get_block_by_number(&self, number: u64, logger: &Logger) -> Result + where + M: prost::Message + BlockchainBlock + Default + 'static, + { + trace!( logger, "Connecting to firehose to retrieve block for number {}", number; "provider" => self.provider.as_str(), @@ -425,6 +511,44 @@ impl FirehoseEndpoint { } } + pub async fn get_block_by_number_with_retry( + self: Arc, + number: u64, + logger: &Logger, + ) -> Result + where + M: prost::Message + BlockchainBlock + Default + 'static, + { + let retry_log_message = format!("get_block_by_number for block {}", number); + let endpoint = self.cheap_clone(); + let logger = logger.cheap_clone(); + + retry(retry_log_message, &logger) + .limit(ENV_VARS.firehose_block_fetch_retry_limit) + .timeout_secs(ENV_VARS.firehose_block_fetch_timeout) + .run(move || { + let endpoint = endpoint.cheap_clone(); + let logger = logger.cheap_clone(); + async move { + endpoint + .get_block_by_number::(number, &logger) + .await + .context(format!( + "Failed to fetch block by number {} from firehose", + number + )) + } + }) + .await + .map_err(|e| { + anyhow::anyhow!( + "Failed to fetch block by number {} from firehose: {}", + number, + e + ) + }) + } + pub async fn load_blocks_by_numbers( self: Arc, numbers: Vec, @@ -433,51 +557,24 @@ impl FirehoseEndpoint { where M: prost::Message + BlockchainBlock + Default + 'static, { - let mut blocks = Vec::with_capacity(numbers.len()); - - for number in numbers { - let provider_name = self.provider.as_str(); + let logger = logger.clone(); + let logger_for_error = logger.clone(); + + let blocks_stream = futures03::stream::iter(numbers) + .map(move |number| { + let e = self.cheap_clone(); + let l = logger.clone(); + async move { e.get_block_by_number_with_retry::(number, &l).await } + }) + .buffered(ENV_VARS.firehose_block_batch_size); - trace!( - logger, - "Loading block for block number {}", number; - "provider" => provider_name, + let blocks = blocks_stream.try_collect::>().await.map_err(|e| { + error!( + logger_for_error, + "Failed to load blocks from firehose: {}", e; ); - - let retry_log_message = format!("get_block_by_number for block {}", number); - let endpoint_for_retry = self.cheap_clone(); - - let logger_for_retry = logger.clone(); - let logger_for_error = logger.clone(); - - let block = retry(retry_log_message, &logger_for_retry) - .limit(ENV_VARS.firehose_block_fetch_retry_limit) - .timeout_secs(ENV_VARS.firehose_block_fetch_timeout) - .run(move || { - let e = endpoint_for_retry.cheap_clone(); - let l = logger_for_retry.clone(); - async move { e.get_block_by_number::(number, &l).await } - }) - .await; - - match block { - Ok(block) => { - blocks.push(block); - } - Err(e) => { - error!( - logger_for_error, - "Failed to load block number {}: {}", number, e; - "provider" => provider_name, - ); - return Err(anyhow::format_err!( - "failed to load block number {}: {}", - number, - e - )); - } - } - } + anyhow::format_err!("failed to load blocks from firehose: {}", e) + })?; Ok(blocks) } @@ -595,7 +692,7 @@ impl FirehoseEndpoint { request: substreams_rpc::Request, headers: &ConnectionHeaders, ) -> Result, anyhow::Error> { - let mut client = self.new_substreams_client(); + let mut client = self.new_substreams_streaming_client(); let request = headers.add_to_request(request); let response_stream = client.blocks(request).await?; let block_stream = response_stream.into_inner(); @@ -610,18 +707,20 @@ impl FirehoseEndpoint { self.info_response .get_or_try_init(move || async move { - let metrics = endpoint.metrics_interceptor(); - let auth = endpoint.auth.clone(); - - let mut client = crate::firehose::endpoint_info::Client::new(metrics, auth); + if endpoint.is_substreams { + let mut client = endpoint.new_substreams_info_client(); + + client + .info(InfoRequest {}) + .await + .map(|r| r.into_inner()) + .map_err(anyhow::Error::from) + .and_then(|e| e.try_into()) + } else { + let mut client = endpoint.new_firehose_info_client(); - if endpoint.compression_enabled { - client = client.with_compression(); + client.info().await } - - client = client.with_max_message_size(endpoint.max_message_size()); - - client.info().await }) .await .map(ToOwned::to_owned) @@ -709,6 +808,7 @@ mod test { false, SubgraphLimit::Unlimited, Arc::new(EndpointMetrics::mock()), + false, ))]; let endpoints = FirehoseEndpoints::for_testing(endpoint); @@ -741,6 +841,7 @@ mod test { false, SubgraphLimit::Limit(2), Arc::new(EndpointMetrics::mock()), + false, ))]; let endpoints = FirehoseEndpoints::for_testing(endpoint); @@ -768,6 +869,7 @@ mod test { false, SubgraphLimit::Disabled, Arc::new(EndpointMetrics::mock()), + false, ))]; let endpoints = FirehoseEndpoints::for_testing(endpoint); @@ -794,6 +896,7 @@ mod test { false, SubgraphLimit::Unlimited, endpoint_metrics.clone(), + false, )); let high_error_adapter2 = Arc::new(FirehoseEndpoint::new( "high_error".to_string(), @@ -804,6 +907,7 @@ mod test { false, SubgraphLimit::Unlimited, endpoint_metrics.clone(), + false, )); let low_availability = Arc::new(FirehoseEndpoint::new( "low availability".to_string(), @@ -814,6 +918,7 @@ mod test { false, SubgraphLimit::Limit(2), endpoint_metrics.clone(), + false, )); let high_availability = Arc::new(FirehoseEndpoint::new( "high availability".to_string(), @@ -824,6 +929,7 @@ mod test { false, SubgraphLimit::Unlimited, endpoint_metrics.clone(), + false, )); endpoint_metrics.report_for_test(&high_error_adapter1.provider, false); diff --git a/graph/src/firehose/sf.cosmos.transform.v1.rs b/graph/src/firehose/sf.cosmos.transform.v1.rs deleted file mode 100644 index 5bde2c0e996..00000000000 --- a/graph/src/firehose/sf.cosmos.transform.v1.rs +++ /dev/null @@ -1,7 +0,0 @@ -// This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventTypeFilter { - #[prost(string, repeated, tag = "1")] - pub event_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} diff --git a/graph/src/firehose/sf.ethereum.transform.v1.rs b/graph/src/firehose/sf.ethereum.transform.v1.rs index 1f313e956e0..8f80ce08ea3 100644 --- a/graph/src/firehose/sf.ethereum.transform.v1.rs +++ b/graph/src/firehose/sf.ethereum.transform.v1.rs @@ -17,7 +17,6 @@ /// the "block index" is always produced after the merged-blocks files /// are produced. Therefore, the "live" blocks are never filtered out. /// -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CombinedFilter { #[prost(message, repeated, tag = "1")] @@ -30,7 +29,6 @@ pub struct CombinedFilter { pub send_all_block_headers: bool, } /// MultiLogFilter concatenates the results of each LogFilter (inclusive OR) -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MultiLogFilter { #[prost(message, repeated, tag = "1")] @@ -41,7 +39,6 @@ pub struct MultiLogFilter { /// * the event signature (topic.0) is one of the provided event_signatures -- OR event_signatures is empty -- /// /// a LogFilter with both empty addresses and event_signatures lists is invalid and will fail. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct LogFilter { #[prost(bytes = "vec", repeated, tag = "1")] @@ -51,7 +48,6 @@ pub struct LogFilter { pub event_signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } /// MultiCallToFilter concatenates the results of each CallToFilter (inclusive OR) -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MultiCallToFilter { #[prost(message, repeated, tag = "1")] @@ -62,7 +58,6 @@ pub struct MultiCallToFilter { /// * the method signature (in 4-bytes format) is one of the provided signatures -- OR signatures is empty -- /// /// a CallToFilter with both empty addresses and signatures lists is invalid and will fail. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CallToFilter { #[prost(bytes = "vec", repeated, tag = "1")] @@ -72,8 +67,7 @@ pub struct CallToFilter { } /// Deprecated: LightBlock is deprecated, replaced by HeaderOnly, note however that the new transform /// does not have any transactions traces returned, so it's not a direct replacement. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct LightBlock {} /// HeaderOnly returns only the block's header and few top-level core information for the block. Useful /// for cases where no transactions information is required at all. @@ -91,6 +85,5 @@ pub struct LightBlock {} /// ``` /// /// Everything else will be empty. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct HeaderOnly {} diff --git a/graph/src/firehose/sf.firehose.v2.rs b/graph/src/firehose/sf.firehose.v2.rs index b0980b35531..bca61385c71 100644 --- a/graph/src/firehose/sf.firehose.v2.rs +++ b/graph/src/firehose/sf.firehose.v2.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SingleBlockRequest { #[prost(message, repeated, tag = "6")] @@ -10,14 +9,12 @@ pub struct SingleBlockRequest { /// Nested message and enum types in `SingleBlockRequest`. pub mod single_block_request { /// Get the current known canonical version of a block at with this number - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct BlockNumber { #[prost(uint64, tag = "1")] pub num: u64, } /// Get the current block with specific hash and number - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockHashAndNumber { #[prost(uint64, tag = "1")] @@ -26,13 +23,11 @@ pub mod single_block_request { pub hash: ::prost::alloc::string::String, } /// Get the block that generated a specific cursor - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Cursor { #[prost(string, tag = "1")] pub cursor: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Reference { #[prost(message, tag = "3")] @@ -43,13 +38,11 @@ pub mod single_block_request { Cursor(Cursor), } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SingleBlockResponse { #[prost(message, optional, tag = "1")] pub block: ::core::option::Option<::prost_types::Any>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { /// Controls where the stream of blocks will start. @@ -90,7 +83,6 @@ pub struct Request { #[prost(message, repeated, tag = "10")] pub transforms: ::prost::alloc::vec::Vec<::prost_types::Any>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Response { /// Chain specific block payload, ex: @@ -104,10 +96,8 @@ pub struct Response { #[prost(string, tag = "10")] pub cursor: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct InfoRequest {} -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InfoResponse { /// Canonical chain name from (ex: matic, mainnet ...). @@ -160,12 +150,12 @@ pub mod info_response { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - BlockIdEncoding::Unset => "BLOCK_ID_ENCODING_UNSET", - BlockIdEncoding::Hex => "BLOCK_ID_ENCODING_HEX", - BlockIdEncoding::BlockIdEncoding0xHex => "BLOCK_ID_ENCODING_0X_HEX", - BlockIdEncoding::Base58 => "BLOCK_ID_ENCODING_BASE58", - BlockIdEncoding::Base64 => "BLOCK_ID_ENCODING_BASE64", - BlockIdEncoding::Base64url => "BLOCK_ID_ENCODING_BASE64URL", + Self::Unset => "BLOCK_ID_ENCODING_UNSET", + Self::Hex => "BLOCK_ID_ENCODING_HEX", + Self::BlockIdEncoding0xHex => "BLOCK_ID_ENCODING_0X_HEX", + Self::Base58 => "BLOCK_ID_ENCODING_BASE58", + Self::Base64 => "BLOCK_ID_ENCODING_BASE64", + Self::Base64url => "BLOCK_ID_ENCODING_BASE64URL", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -201,10 +191,10 @@ impl ForkStep { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - ForkStep::StepUnset => "STEP_UNSET", - ForkStep::StepNew => "STEP_NEW", - ForkStep::StepUndo => "STEP_UNDO", - ForkStep::StepFinal => "STEP_FINAL", + Self::StepUnset => "STEP_UNSET", + Self::StepNew => "STEP_NEW", + Self::StepUndo => "STEP_UNDO", + Self::StepFinal => "STEP_FINAL", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -220,7 +210,13 @@ impl ForkStep { } /// Generated client implementations. pub mod stream_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -242,8 +238,8 @@ pub mod stream_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -268,7 +264,7 @@ pub mod stream_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { StreamClient::new(InterceptedService::new(inner, interceptor)) } @@ -314,8 +310,7 @@ pub mod stream_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -332,7 +327,13 @@ pub mod stream_client { } /// Generated client implementations. pub mod fetch_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -354,8 +355,8 @@ pub mod fetch_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -380,7 +381,7 @@ pub mod fetch_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { FetchClient::new(InterceptedService::new(inner, interceptor)) } @@ -426,8 +427,7 @@ pub mod fetch_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -444,7 +444,13 @@ pub mod fetch_client { } /// Generated client implementations. pub mod endpoint_info_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -466,8 +472,8 @@ pub mod endpoint_info_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -492,7 +498,7 @@ pub mod endpoint_info_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) } @@ -535,8 +541,7 @@ pub mod endpoint_info_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -553,16 +558,22 @@ pub mod endpoint_info_client { } /// Generated server implementations. pub mod stream_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. #[async_trait] - pub trait Stream: Send + Sync + 'static { + pub trait Stream: std::marker::Send + std::marker::Sync + 'static { /// Server streaming response type for the Blocks method. type BlocksStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, > - + Send + + std::marker::Send + 'static; async fn blocks( &self, @@ -570,20 +581,18 @@ pub mod stream_server { ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct StreamServer { - inner: _Inner, + pub struct StreamServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl StreamServer { + impl StreamServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -633,8 +642,8 @@ pub mod stream_server { impl tonic::codegen::Service> for StreamServer where T: Stream, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -646,7 +655,6 @@ pub mod stream_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.Stream/Blocks" => { #[allow(non_camel_case_types)] @@ -676,7 +684,6 @@ pub mod stream_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlocksSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -695,20 +702,25 @@ pub mod stream_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for StreamServer { + impl Clone for StreamServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -720,27 +732,25 @@ pub mod stream_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for StreamServer { - const NAME: &'static str = "sf.firehose.v2.Stream"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Stream"; + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod fetch_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with FetchServer. #[async_trait] - pub trait Fetch: Send + Sync + 'static { + pub trait Fetch: std::marker::Send + std::marker::Sync + 'static { async fn block( &self, request: tonic::Request, @@ -750,20 +760,18 @@ pub mod fetch_server { >; } #[derive(Debug)] - pub struct FetchServer { - inner: _Inner, + pub struct FetchServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl FetchServer { + impl FetchServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -813,8 +821,8 @@ pub mod fetch_server { impl tonic::codegen::Service> for FetchServer where T: Fetch, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -826,7 +834,6 @@ pub mod fetch_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.Fetch/Block" => { #[allow(non_camel_case_types)] @@ -855,7 +862,6 @@ pub mod fetch_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -874,20 +880,25 @@ pub mod fetch_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for FetchServer { + impl Clone for FetchServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -899,47 +910,43 @@ pub mod fetch_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for FetchServer { - const NAME: &'static str = "sf.firehose.v2.Fetch"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Fetch"; + impl tonic::server::NamedService for FetchServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod endpoint_info_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. #[async_trait] - pub trait EndpointInfo: Send + Sync + 'static { + pub trait EndpointInfo: std::marker::Send + std::marker::Sync + 'static { async fn info( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct EndpointInfoServer { - inner: _Inner, + pub struct EndpointInfoServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl EndpointInfoServer { + impl EndpointInfoServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -989,8 +996,8 @@ pub mod endpoint_info_server { impl tonic::codegen::Service> for EndpointInfoServer where T: EndpointInfo, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -1002,7 +1009,6 @@ pub mod endpoint_info_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.EndpointInfo/Info" => { #[allow(non_camel_case_types)] @@ -1031,7 +1037,6 @@ pub mod endpoint_info_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = InfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -1050,20 +1055,25 @@ pub mod endpoint_info_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for EndpointInfoServer { + impl Clone for EndpointInfoServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -1075,17 +1085,9 @@ pub mod endpoint_info_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for EndpointInfoServer { - const NAME: &'static str = "sf.firehose.v2.EndpointInfo"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.EndpointInfo"; + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = SERVICE_NAME; } } diff --git a/graph/src/firehose/sf.near.transform.v1.rs b/graph/src/firehose/sf.near.transform.v1.rs index f76839cbd4c..2ec950da40b 100644 --- a/graph/src/firehose/sf.near.transform.v1.rs +++ b/graph/src/firehose/sf.near.transform.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BasicReceiptFilter { #[prost(string, repeated, tag = "1")] @@ -14,7 +13,6 @@ pub struct BasicReceiptFilter { /// * {prefix="",suffix=""} is invalid /// /// Note that the suffix will usually have a TLD, ex: "mydomain.near" or "mydomain.testnet" -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PrefixSuffixPair { #[prost(string, tag = "1")] diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 04872aab196..7d1536929bd 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -117,7 +117,7 @@ pub mod prelude { EthereumBlock, EthereumBlockWithCalls, EthereumCall, LightEthereumBlock, LightEthereumBlockExt, }; - pub use crate::components::graphql::{GraphQLMetrics, GraphQlRunner, SubscriptionResultFuture}; + pub use crate::components::graphql::{GraphQLMetrics, GraphQlRunner}; pub use crate::components::link_resolver::{ IpfsResolver, JsonStreamValue, JsonValueStream, LinkResolver, }; @@ -125,10 +125,9 @@ pub mod prelude { stopwatch::StopwatchMetrics, subgraph::*, Collector, Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, MetricsRegistry, Opts, PrometheusError, Registry, }; - pub use crate::components::server::subscription::SubscriptionServer; pub use crate::components::store::{ - write::EntityModification, AttributeNames, BlockNumber, CachedEthereumCall, ChainStore, - Child, ChildMultiplicity, EntityCache, EntityChange, EntityChangeOperation, + write::EntityModification, AssignmentChange, AssignmentOperation, AttributeNames, + BlockNumber, CachedEthereumCall, ChainStore, Child, ChildMultiplicity, EntityCache, EntityCollection, EntityFilter, EntityLink, EntityOperation, EntityOrder, EntityOrderByChild, EntityOrderByChildInfo, EntityQuery, EntityRange, EntityWindow, EthereumCallCache, ParentLink, PartialBlockPtr, PoolWaitStats, QueryStore, @@ -153,9 +152,7 @@ pub mod prelude { Query, QueryError, QueryExecutionError, QueryResult, QueryTarget, QueryVariables, }; pub use crate::data::store::scalar::{BigDecimal, BigInt, BigIntSign}; - pub use crate::data::store::{ - AssignmentEvent, Attribute, Entity, NodeId, SubscriptionFilter, Value, ValueType, - }; + pub use crate::data::store::{AssignmentEvent, Attribute, Entity, NodeId, Value, ValueType}; pub use crate::data::subgraph::schema::SubgraphDeploymentEntity; pub use crate::data::subgraph::{ CreateSubgraphResult, DataSourceContext, DeploymentHash, DeploymentState, Link, @@ -163,9 +160,6 @@ pub mod prelude { SubgraphManifestValidationError, SubgraphName, SubgraphRegistrarError, UnvalidatedSubgraphManifest, }; - pub use crate::data::subscription::{ - QueryResultStream, Subscription, SubscriptionError, SubscriptionResult, - }; pub use crate::data_source::DataSourceTemplateInfo; pub use crate::ext::futures::{ CancelGuard, CancelHandle, CancelToken, CancelableError, FutureExtension, diff --git a/graph/src/runtime/mod.rs b/graph/src/runtime/mod.rs index 5622d37c100..6732a57429e 100644 --- a/graph/src/runtime/mod.rs +++ b/graph/src/runtime/mod.rs @@ -268,77 +268,7 @@ pub enum IndexForAscTypeId { // ... // LastEthereumType = 1499, - // Reserved discriminant space for Cosmos type IDs: [1,500, 2,499] - CosmosAny = 1500, - CosmosAnyArray = 1501, - CosmosBytesArray = 1502, - CosmosCoinArray = 1503, - CosmosCommitSigArray = 1504, - CosmosEventArray = 1505, - CosmosEventAttributeArray = 1506, - CosmosEvidenceArray = 1507, - CosmosModeInfoArray = 1508, - CosmosSignerInfoArray = 1509, - CosmosTxResultArray = 1510, - CosmosValidatorArray = 1511, - CosmosValidatorUpdateArray = 1512, - CosmosAuthInfo = 1513, - CosmosBlock = 1514, - CosmosBlockId = 1515, - CosmosBlockIdFlagEnum = 1516, - CosmosBlockParams = 1517, - CosmosCoin = 1518, - CosmosCommit = 1519, - CosmosCommitSig = 1520, - CosmosCompactBitArray = 1521, - CosmosConsensus = 1522, - CosmosConsensusParams = 1523, - CosmosDuplicateVoteEvidence = 1524, - CosmosDuration = 1525, - CosmosEvent = 1526, - CosmosEventAttribute = 1527, - CosmosEventData = 1528, - CosmosEventVote = 1529, - CosmosEvidence = 1530, - CosmosEvidenceList = 1531, - CosmosEvidenceParams = 1532, - CosmosFee = 1533, - CosmosHeader = 1534, - CosmosHeaderOnlyBlock = 1535, - CosmosLightBlock = 1536, - CosmosLightClientAttackEvidence = 1537, - CosmosModeInfo = 1538, - CosmosModeInfoMulti = 1539, - CosmosModeInfoSingle = 1540, - CosmosPartSetHeader = 1541, - CosmosPublicKey = 1542, - CosmosResponseBeginBlock = 1543, - CosmosResponseDeliverTx = 1544, - CosmosResponseEndBlock = 1545, - CosmosSignModeEnum = 1546, - CosmosSignedHeader = 1547, - CosmosSignedMsgTypeEnum = 1548, - CosmosSignerInfo = 1549, - CosmosTimestamp = 1550, - CosmosTip = 1551, - CosmosTransactionData = 1552, - CosmosTx = 1553, - CosmosTxBody = 1554, - CosmosTxResult = 1555, - CosmosValidator = 1556, - CosmosValidatorParams = 1557, - CosmosValidatorSet = 1558, - CosmosValidatorSetUpdates = 1559, - CosmosValidatorUpdate = 1560, - CosmosVersionParams = 1561, - CosmosMessageData = 1562, - CosmosTransactionContext = 1563, - // Continue to add more Cosmos type IDs here. - // e.g.: - // NextCosmosType = 1564, - // AnotherCosmosType = 1565, - // ... - // LastCosmosType = 2499, + // Discriminant space [1,500, 2,499] was reserved for Cosmos, which has been removed // Arweave types ArweaveBlock = 2500, diff --git a/graph/src/schema/api.rs b/graph/src/schema/api.rs index 6d936177b67..7fe29806a3f 100644 --- a/graph/src/schema/api.rs +++ b/graph/src/schema/api.rs @@ -90,8 +90,8 @@ impl TryFrom<&r::Value> for ErrorPolicy { /// /// (2) By parsing an appropriate GraphQL schema from text and calling /// `from_graphql_schema`. In that case, it's the caller's responsibility to -/// make sure that the schema has all the types needed for querying, like -/// `Query` and `Subscription` +/// make sure that the schema has all the types needed for querying, in +/// particular `Query` /// /// Because of the second point, once constructed, it can not be assumed /// that an `ApiSchema` is based on an `InputSchema` and it can only be used @@ -102,7 +102,6 @@ pub struct ApiSchema { // Root types for the api schema. pub query_type: Arc, - pub subscription_type: Option>, object_types: HashMap>, } @@ -121,11 +120,6 @@ impl ApiSchema { .get_root_query_type() .context("no root `Query` in the schema")? .clone(); - let subscription_type = schema - .document - .get_root_subscription_type() - .cloned() - .map(Arc::new); let object_types = HashMap::from_iter( schema @@ -138,7 +132,6 @@ impl ApiSchema { Ok(Self { schema, query_type: Arc::new(query_type), - subscription_type, object_types, }) } @@ -360,7 +353,6 @@ pub(in crate::schema) fn api_schema( add_types_for_interface_types(&mut api, input_schema)?; add_types_for_aggregation_types(&mut api, input_schema)?; add_query_type(&mut api.document, input_schema)?; - add_subscription_type(&mut api.document, input_schema)?; Ok(api.document) } @@ -493,6 +485,12 @@ fn add_types_for_aggregation_types( input_schema: &InputSchema, ) -> Result<(), APISchemaError> { for (name, agg_type) in input_schema.aggregation_types() { + // Combine regular fields and aggregate fields for ordering + let mut all_fields = agg_type.fields.to_vec(); + for agg in agg_type.aggregates.iter() { + all_fields.push(agg.as_agg_field()); + } + add_order_by_type(&mut api.document, name, &all_fields)?; add_aggregation_filter_type(api, name, agg_type)?; } Ok(()) @@ -686,13 +684,25 @@ impl FilterOps { s::Type::NamedType("OrderDirection".to_string()), ), ], - FilterOps::Aggregation => vec![input_value( - "interval", - "", - s::Type::NonNullType(Box::new(s::Type::NamedType( - "Aggregation_interval".to_string(), - ))), - )], + FilterOps::Aggregation => vec![ + input_value( + "interval", + "", + s::Type::NonNullType(Box::new(s::Type::NamedType( + "Aggregation_interval".to_string(), + ))), + ), + input_value( + "orderBy", + "", + s::Type::NamedType(format!("{}_orderBy", type_name)), + ), + input_value( + "orderDirection", + "", + s::Type::NamedType("OrderDirection".to_string()), + ), + ], }; let mut args = vec![skip, first]; @@ -1135,44 +1145,6 @@ fn query_field_for_fulltext(fulltext: &s::Directive) -> Option { }) } -/// Adds a root `Subscription` object type to the schema. -fn add_subscription_type( - api: &mut s::Document, - input_schema: &InputSchema, -) -> Result<(), APISchemaError> { - let type_name = String::from("Subscription"); - - if api.get_named_type(&type_name).is_some() { - return Err(APISchemaError::TypeExists(type_name)); - } - - let mut fields: Vec = input_schema - .object_types() - .map(|(name, _)| name) - .chain(input_schema.interface_types().map(|(name, _)| name)) - .flat_map(|name| query_fields_for_type(name, FilterOps::Object)) - .collect(); - let mut agg_fields = input_schema - .aggregation_types() - .map(|(name, _)| name) - .flat_map(query_fields_for_agg_type) - .collect::>(); - fields.append(&mut agg_fields); - fields.push(meta_field()); - - let typedef = s::TypeDefinition::Object(s::ObjectType { - position: Pos::default(), - description: None, - name: type_name, - implements_interfaces: vec![], - directives: vec![], - fields, - }); - let def = s::Definition::TypeDefinition(typedef); - api.definitions.push(def); - Ok(()) -} - fn block_argument() -> s::InputValue { s::InputValue { position: Pos::default(), diff --git a/graph/src/schema/input/mod.rs b/graph/src/schema/input/mod.rs index 5ff8ddcda2f..634930c5731 100644 --- a/graph/src/schema/input/mod.rs +++ b/graph/src/schema/input/mod.rs @@ -824,7 +824,7 @@ impl Aggregate { /// The field needed for the finalised aggregation for hourly/daily /// values - fn as_agg_field(&self) -> Field { + pub fn as_agg_field(&self) -> Field { Field { name: self.name.clone(), field_type: self.field_type.clone(), diff --git a/graph/src/substreams/sf.substreams.v1.rs b/graph/src/substreams/sf.substreams.v1.rs index e27ed7b346d..dd6b8930293 100644 --- a/graph/src/substreams/sf.substreams.v1.rs +++ b/graph/src/substreams/sf.substreams.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Package { /// Needs to be one so this file can be used _directly_ as a @@ -22,7 +21,6 @@ pub struct Package { #[prost(string, tag = "11")] pub sink_module: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PackageMetadata { #[prost(string, tag = "1")] @@ -34,7 +32,6 @@ pub struct PackageMetadata { #[prost(string, tag = "4")] pub doc: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModuleMetadata { /// Corresponds to the index in `Package.metadata.package_meta` @@ -43,7 +40,6 @@ pub struct ModuleMetadata { #[prost(string, tag = "2")] pub doc: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Modules { #[prost(message, repeated, tag = "1")] @@ -52,7 +48,6 @@ pub struct Modules { pub binaries: ::prost::alloc::vec::Vec, } /// Binary represents some code compiled to its binary form. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Binary { #[prost(string, tag = "1")] @@ -60,7 +55,6 @@ pub struct Binary { #[prost(bytes = "vec", tag = "2")] pub content: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Module { #[prost(string, tag = "1")] @@ -82,7 +76,6 @@ pub struct Module { } /// Nested message and enum types in `Module`. pub mod module { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockFilter { #[prost(string, tag = "1")] @@ -92,7 +85,6 @@ pub mod module { } /// Nested message and enum types in `BlockFilter`. pub mod block_filter { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Query { #[prost(string, tag = "2")] @@ -101,16 +93,13 @@ pub mod module { QueryFromParams(super::QueryFromParams), } } - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct QueryFromParams {} - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct KindMap { #[prost(string, tag = "1")] pub output_type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct KindStore { /// The `update_policy` determines the functions available to mutate the store @@ -164,14 +153,14 @@ pub mod module { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - UpdatePolicy::Unset => "UPDATE_POLICY_UNSET", - UpdatePolicy::Set => "UPDATE_POLICY_SET", - UpdatePolicy::SetIfNotExists => "UPDATE_POLICY_SET_IF_NOT_EXISTS", - UpdatePolicy::Add => "UPDATE_POLICY_ADD", - UpdatePolicy::Min => "UPDATE_POLICY_MIN", - UpdatePolicy::Max => "UPDATE_POLICY_MAX", - UpdatePolicy::Append => "UPDATE_POLICY_APPEND", - UpdatePolicy::SetSum => "UPDATE_POLICY_SET_SUM", + Self::Unset => "UPDATE_POLICY_UNSET", + Self::Set => "UPDATE_POLICY_SET", + Self::SetIfNotExists => "UPDATE_POLICY_SET_IF_NOT_EXISTS", + Self::Add => "UPDATE_POLICY_ADD", + Self::Min => "UPDATE_POLICY_MIN", + Self::Max => "UPDATE_POLICY_MAX", + Self::Append => "UPDATE_POLICY_APPEND", + Self::SetSum => "UPDATE_POLICY_SET_SUM", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -190,13 +179,11 @@ pub mod module { } } } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct KindBlockIndex { #[prost(string, tag = "1")] pub output_type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Input { #[prost(oneof = "input::Input", tags = "1, 2, 3, 4")] @@ -204,21 +191,18 @@ pub mod module { } /// Nested message and enum types in `Input`. pub mod input { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Source { /// ex: "sf.ethereum.type.v1.Block" #[prost(string, tag = "1")] pub r#type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Map { /// ex: "block_to_pairs" #[prost(string, tag = "1")] pub module_name: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Store { #[prost(string, tag = "1")] @@ -252,9 +236,9 @@ pub mod module { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Mode::Unset => "UNSET", - Mode::Get => "GET", - Mode::Deltas => "DELTAS", + Self::Unset => "UNSET", + Self::Get => "GET", + Self::Deltas => "DELTAS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -268,13 +252,11 @@ pub mod module { } } } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Params { #[prost(string, tag = "1")] pub value: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Input { #[prost(message, tag = "1")] @@ -287,13 +269,11 @@ pub mod module { Params(Params), } } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Output { #[prost(string, tag = "1")] pub r#type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Kind { #[prost(message, tag = "2")] @@ -305,7 +285,6 @@ pub mod module { } } /// Clock is a pointer to a block with added timestamp -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Clock { #[prost(string, tag = "1")] @@ -316,7 +295,6 @@ pub struct Clock { pub timestamp: ::core::option::Option<::prost_types::Timestamp>, } /// BlockRef is a pointer to a block to which we don't know the timestamp -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockRef { #[prost(string, tag = "1")] diff --git a/graph/src/substreams_rpc/sf.firehose.v2.rs b/graph/src/substreams_rpc/sf.firehose.v2.rs new file mode 100644 index 00000000000..905a7038bf5 --- /dev/null +++ b/graph/src/substreams_rpc/sf.firehose.v2.rs @@ -0,0 +1,896 @@ +// This file is @generated by prost-build. +/// Generated client implementations. +pub mod stream_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct StreamClient { + inner: tonic::client::Grpc, + } + impl StreamClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl StreamClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> StreamClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + StreamClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn blocks( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v2.Stream/Blocks", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.firehose.v2.Stream", "Blocks")); + self.inner.server_streaming(req, path, codec).await + } + } +} +/// Generated client implementations. +pub mod fetch_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct FetchClient { + inner: tonic::client::Grpc, + } + impl FetchClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl FetchClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> FetchClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + FetchClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn block( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v2.Fetch/Block", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.firehose.v2.Fetch", "Block")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated client implementations. +pub mod endpoint_info_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct EndpointInfoClient { + inner: tonic::client::Grpc, + } + impl EndpointInfoClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl EndpointInfoClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> EndpointInfoClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v2.EndpointInfo/Info", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.firehose.v2.EndpointInfo", "Info")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod stream_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. + #[async_trait] + pub trait Stream: std::marker::Send + std::marker::Sync + 'static { + /// Server streaming response type for the Blocks method. + type BlocksStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result, + > + + std::marker::Send + + 'static; + async fn blocks( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct StreamServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl StreamServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for StreamServer + where + T: Stream, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sf.firehose.v2.Stream/Blocks" => { + #[allow(non_camel_case_types)] + struct BlocksSvc(pub Arc); + impl< + T: Stream, + > tonic::server::ServerStreamingService + for BlocksSvc { + type Response = crate::firehose::Response; + type ResponseStream = T::BlocksStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::blocks(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = BlocksSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for StreamServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Stream"; + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated server implementations. +pub mod fetch_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with FetchServer. + #[async_trait] + pub trait Fetch: std::marker::Send + std::marker::Sync + 'static { + async fn block( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct FetchServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl FetchServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for FetchServer + where + T: Fetch, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sf.firehose.v2.Fetch/Block" => { + #[allow(non_camel_case_types)] + struct BlockSvc(pub Arc); + impl< + T: Fetch, + > tonic::server::UnaryService + for BlockSvc { + type Response = crate::firehose::SingleBlockResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::block(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = BlockSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for FetchServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Fetch"; + impl tonic::server::NamedService for FetchServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated server implementations. +pub mod endpoint_info_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. + #[async_trait] + pub trait EndpointInfo: std::marker::Send + std::marker::Sync + 'static { + async fn info( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct EndpointInfoServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl EndpointInfoServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for EndpointInfoServer + where + T: EndpointInfo, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sf.firehose.v2.EndpointInfo/Info" => { + #[allow(non_camel_case_types)] + struct InfoSvc(pub Arc); + impl< + T: EndpointInfo, + > tonic::server::UnaryService + for InfoSvc { + type Response = crate::firehose::InfoResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = InfoSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for EndpointInfoServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.EndpointInfo"; + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs b/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs index 19b8d0493f0..ff69b343d29 100644 --- a/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs +++ b/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { #[prost(int64, tag = "1")] @@ -9,28 +8,32 @@ pub struct Request { #[prost(uint64, tag = "3")] pub stop_block_num: u64, /// With final_block_only, you only receive blocks that are irreversible: - /// 'final_block_height' will be equal to current block and no 'undo_signal' will ever be sent + /// 'final_block_height' will be equal to current block and no 'undo_signal' + /// will ever be sent #[prost(bool, tag = "4")] pub final_blocks_only: bool, - /// Substreams has two mode when executing your module(s) either development mode or production - /// mode. Development and production modes impact the execution of Substreams, important aspects - /// of execution include: + /// Substreams has two mode when executing your module(s) either development + /// mode or production mode. Development and production modes impact the + /// execution of Substreams, important aspects of execution include: /// * The time required to reach the first byte. /// * The speed that large ranges get executed. /// * The module logs and outputs sent back to the client. /// - /// By default, the engine runs in developer mode, with richer and deeper output. Differences - /// between production and development modes include: - /// * Forward parallel execution is enabled in production mode and disabled in development mode - /// * The time required to reach the first byte in development mode is faster than in production mode. + /// By default, the engine runs in developer mode, with richer and deeper + /// output. Differences between production and development modes include: + /// * Forward parallel execution is enabled in production mode and disabled in + /// development mode + /// * The time required to reach the first byte in development mode is faster + /// than in production mode. /// /// Specific attributes of development mode include: /// * The client will receive all of the executed module's logs. - /// * It's possible to request specific store snapshots in the execution tree (via `debug_initial_store_snapshot_for_modules`). + /// * It's possible to request specific store snapshots in the execution tree + /// (via `debug_initial_store_snapshot_for_modules`). /// * Multiple module's output is possible. /// - /// With production mode`, however, you trade off functionality for high speed enabling forward - /// parallel execution of module ahead of time. + /// With production mode`, however, you trade off functionality for high speed + /// enabling forward parallel execution of module ahead of time. #[prost(bool, tag = "5")] pub production_mode: bool, #[prost(string, tag = "6")] @@ -43,7 +46,6 @@ pub struct Request { ::prost::alloc::string::String, >, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Response { #[prost(oneof = "response::Message", tags = "1, 2, 3, 4, 5, 10, 11")] @@ -51,25 +53,27 @@ pub struct Response { } /// Nested message and enum types in `Response`. pub mod response { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Message { /// Always sent first #[prost(message, tag = "1")] Session(super::SessionInit), - /// Progress of data preparation, before sending in the stream of `data` events. + /// Progress of data preparation, before #[prost(message, tag = "2")] Progress(super::ModulesProgress), + /// sending in the stream of `data` events. #[prost(message, tag = "3")] BlockScopedData(super::BlockScopedData), #[prost(message, tag = "4")] BlockUndoSignal(super::BlockUndoSignal), #[prost(message, tag = "5")] FatalError(super::Error), - /// Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + /// Available only in developer mode, and only if + /// `debug_initial_store_snapshot_for_modules` is set. #[prost(message, tag = "10")] DebugSnapshotData(super::InitialSnapshotData), - /// Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + /// Available only in developer mode, and only if + /// `debug_initial_store_snapshot_for_modules` is set. #[prost(message, tag = "11")] DebugSnapshotComplete(super::InitialSnapshotComplete), } @@ -77,7 +81,6 @@ pub mod response { /// BlockUndoSignal informs you that every bit of data /// with a block number above 'last_valid_block' has been reverted /// on-chain. Delete that data and restart from 'last_valid_cursor' -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockUndoSignal { #[prost(message, optional, tag = "1")] @@ -85,7 +88,6 @@ pub struct BlockUndoSignal { #[prost(string, tag = "2")] pub last_valid_cursor: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockScopedData { #[prost(message, optional, tag = "1")] @@ -102,7 +104,6 @@ pub struct BlockScopedData { #[prost(message, repeated, tag = "11")] pub debug_store_outputs: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SessionInit { #[prost(string, tag = "1")] @@ -114,13 +115,11 @@ pub struct SessionInit { #[prost(uint64, tag = "4")] pub max_parallel_workers: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InitialSnapshotComplete { #[prost(string, tag = "1")] pub cursor: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InitialSnapshotData { #[prost(string, tag = "1")] @@ -132,7 +131,6 @@ pub struct InitialSnapshotData { #[prost(uint64, tag = "3")] pub total_keys: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MapModuleOutput { #[prost(string, tag = "1")] @@ -144,10 +142,9 @@ pub struct MapModuleOutput { pub debug_info: ::core::option::Option, } /// StoreModuleOutput are produced for store modules in development mode. -/// It is not possible to retrieve store models in production, with parallelization -/// enabled. If you need the deltas directly, write a pass through mapper module -/// that will get them down to you. -#[allow(clippy::derive_partial_eq_without_eq)] +/// It is not possible to retrieve store models in production, with +/// parallelization enabled. If you need the deltas directly, write a pass +/// through mapper module that will get them down to you. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreModuleOutput { #[prost(string, tag = "1")] @@ -157,20 +154,19 @@ pub struct StoreModuleOutput { #[prost(message, optional, tag = "10")] pub debug_info: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct OutputDebugInfo { #[prost(string, repeated, tag = "1")] pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// LogsTruncated is a flag that tells you if you received all the logs or if they - /// were truncated because you logged too much (fixed limit currently is set to 128 KiB). + /// LogsTruncated is a flag that tells you if you received all the logs or if + /// they were truncated because you logged too much (fixed limit currently is + /// set to 128 KiB). #[prost(bool, tag = "2")] pub logs_truncated: bool, #[prost(bool, tag = "3")] pub cached: bool, } /// ModulesProgress is a message that is sent every 500ms -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModulesProgress { /// List of jobs running on tier2 servers @@ -185,15 +181,13 @@ pub struct ModulesProgress { #[prost(message, optional, tag = "5")] pub processed_bytes: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProcessedBytes { #[prost(uint64, tag = "1")] pub total_bytes_read: u64, #[prost(uint64, tag = "2")] pub total_bytes_written: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Error { #[prost(string, tag = "1")] @@ -202,13 +196,13 @@ pub struct Error { pub reason: ::prost::alloc::string::String, #[prost(string, repeated, tag = "3")] pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// FailureLogsTruncated is a flag that tells you if you received all the logs or if they - /// were truncated because you logged too much (fixed limit currently is set to 128 KiB). + /// FailureLogsTruncated is a flag that tells you if you received all the logs + /// or if they were truncated because you logged too much (fixed limit + /// currently is set to 128 KiB). #[prost(bool, tag = "4")] pub logs_truncated: bool, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Job { #[prost(uint32, tag = "1")] pub stage: u32, @@ -221,7 +215,6 @@ pub struct Job { #[prost(uint64, tag = "5")] pub duration_ms: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Stage { #[prost(string, repeated, tag = "1")] @@ -229,9 +222,9 @@ pub struct Stage { #[prost(message, repeated, tag = "2")] pub completed_ranges: ::prost::alloc::vec::Vec, } -/// ModuleStats gathers metrics and statistics from each module, running on tier1 or tier2 -/// All the 'count' and 'time_ms' values may include duplicate for each stage going over that module -#[allow(clippy::derive_partial_eq_without_eq)] +/// ModuleStats gathers metrics and statistics from each module, running on tier1 +/// or tier2 All the 'count' and 'time_ms' values may include duplicate for each +/// stage going over that module #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModuleStats { /// name of the module @@ -240,39 +233,47 @@ pub struct ModuleStats { /// total_processed_blocks is the sum of blocks sent to that module code #[prost(uint64, tag = "2")] pub total_processed_block_count: u64, - /// total_processing_time_ms is the sum of all time spent running that module code + /// total_processing_time_ms is the sum of all time spent running that module + /// code #[prost(uint64, tag = "3")] pub total_processing_time_ms: u64, /// // external_calls are chain-specific intrinsics, like "Ethereum RPC calls". #[prost(message, repeated, tag = "4")] pub external_call_metrics: ::prost::alloc::vec::Vec, - /// total_store_operation_time_ms is the sum of all time spent running that module code waiting for a store operation (ex: read, write, delete...) + /// total_store_operation_time_ms is the sum of all time spent running that + /// module code waiting for a store operation (ex: read, write, delete...) #[prost(uint64, tag = "5")] pub total_store_operation_time_ms: u64, - /// total_store_read_count is the sum of all the store Read operations called from that module code + /// total_store_read_count is the sum of all the store Read operations called + /// from that module code #[prost(uint64, tag = "6")] pub total_store_read_count: u64, - /// total_store_write_count is the sum of all store Write operations called from that module code (store-only) + /// total_store_write_count is the sum of all store Write operations called + /// from that module code (store-only) #[prost(uint64, tag = "10")] pub total_store_write_count: u64, - /// total_store_deleteprefix_count is the sum of all store DeletePrefix operations called from that module code (store-only) - /// note that DeletePrefix can be a costly operation on large stores + /// total_store_deleteprefix_count is the sum of all store DeletePrefix + /// operations called from that module code (store-only) note that DeletePrefix + /// can be a costly operation on large stores #[prost(uint64, tag = "11")] pub total_store_deleteprefix_count: u64, - /// store_size_bytes is the uncompressed size of the full KV store for that module, from the last 'merge' operation (store-only) + /// store_size_bytes is the uncompressed size of the full KV store for that + /// module, from the last 'merge' operation (store-only) #[prost(uint64, tag = "12")] pub store_size_bytes: u64, - /// total_store_merging_time_ms is the time spent merging partial stores into a full KV store for that module (store-only) + /// total_store_merging_time_ms is the time spent merging partial stores into a + /// full KV store for that module (store-only) #[prost(uint64, tag = "13")] pub total_store_merging_time_ms: u64, - /// store_currently_merging is true if there is a merging operation (partial store to full KV store) on the way. + /// store_currently_merging is true if there is a merging operation (partial + /// store to full KV store) on the way. #[prost(bool, tag = "14")] pub store_currently_merging: bool, - /// highest_contiguous_block is the highest block in the highest merged full KV store of that module (store-only) + /// highest_contiguous_block is the highest block in the highest merged full KV + /// store of that module (store-only) #[prost(uint64, tag = "15")] pub highest_contiguous_block: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ExternalCallMetric { #[prost(string, tag = "1")] @@ -282,7 +283,6 @@ pub struct ExternalCallMetric { #[prost(uint64, tag = "3")] pub time_ms: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreDelta { #[prost(enumeration = "store_delta::Operation", tag = "1")] @@ -323,10 +323,10 @@ pub mod store_delta { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Operation::Unset => "UNSET", - Operation::Create => "CREATE", - Operation::Update => "UPDATE", - Operation::Delete => "DELETE", + Self::Unset => "UNSET", + Self::Create => "CREATE", + Self::Update => "UPDATE", + Self::Delete => "DELETE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -341,8 +341,7 @@ pub mod store_delta { } } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct BlockRange { #[prost(uint64, tag = "2")] pub start_block: u64, @@ -350,8 +349,131 @@ pub struct BlockRange { pub end_block: u64, } /// Generated client implementations. +pub mod endpoint_info_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct EndpointInfoClient { + inner: tonic::client::Grpc, + } + impl EndpointInfoClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl EndpointInfoClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> EndpointInfoClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.substreams.rpc.v2.EndpointInfo/Info", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.substreams.rpc.v2.EndpointInfo", "Info")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated client implementations. pub mod stream_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -373,8 +495,8 @@ pub mod stream_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -399,7 +521,7 @@ pub mod stream_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { StreamClient::new(InterceptedService::new(inner, interceptor)) } @@ -445,8 +567,7 @@ pub mod stream_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -462,17 +583,203 @@ pub mod stream_client { } } /// Generated server implementations. +pub mod endpoint_info_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. + #[async_trait] + pub trait EndpointInfo: std::marker::Send + std::marker::Sync + 'static { + async fn info( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct EndpointInfoServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl EndpointInfoServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for EndpointInfoServer + where + T: EndpointInfo, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sf.substreams.rpc.v2.EndpointInfo/Info" => { + #[allow(non_camel_case_types)] + struct InfoSvc(pub Arc); + impl< + T: EndpointInfo, + > tonic::server::UnaryService + for InfoSvc { + type Response = crate::firehose::InfoResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = InfoSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for EndpointInfoServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.substreams.rpc.v2.EndpointInfo"; + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = SERVICE_NAME; + } +} +/// Generated server implementations. pub mod stream_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. #[async_trait] - pub trait Stream: Send + Sync + 'static { + pub trait Stream: std::marker::Send + std::marker::Sync + 'static { /// Server streaming response type for the Blocks method. type BlocksStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, > - + Send + + std::marker::Send + 'static; async fn blocks( &self, @@ -480,20 +787,18 @@ pub mod stream_server { ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct StreamServer { - inner: _Inner, + pub struct StreamServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl StreamServer { + impl StreamServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -543,8 +848,8 @@ pub mod stream_server { impl tonic::codegen::Service> for StreamServer where T: Stream, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -556,7 +861,6 @@ pub mod stream_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.substreams.rpc.v2.Stream/Blocks" => { #[allow(non_camel_case_types)] @@ -586,7 +890,6 @@ pub mod stream_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlocksSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -605,20 +908,25 @@ pub mod stream_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for StreamServer { + impl Clone for StreamServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -630,17 +938,9 @@ pub mod stream_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for StreamServer { - const NAME: &'static str = "sf.substreams.rpc.v2.Stream"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.substreams.rpc.v2.Stream"; + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = SERVICE_NAME; } } diff --git a/graph/src/task_spawn.rs b/graph/src/task_spawn.rs index 09055ad5381..dd1477bb1c8 100644 --- a/graph/src/task_spawn.rs +++ b/graph/src/task_spawn.rs @@ -57,10 +57,11 @@ pub fn block_on(f: impl Future03) -> T { } /// Spawns a thread with access to the tokio runtime. Panics if the thread cannot be spawned. -pub fn spawn_thread( - name: impl Into, - f: impl 'static + FnOnce() + Send, -) -> std::thread::JoinHandle<()> { +pub fn spawn_thread(name: impl Into, f: F) -> std::thread::JoinHandle +where + F: 'static + FnOnce() -> R + Send, + R: 'static + Send, +{ let conf = std::thread::Builder::new().name(name.into()); let runtime = tokio::runtime::Handle::current(); conf.spawn(move || { diff --git a/graph/src/util/futures.rs b/graph/src/util/futures.rs index d742457dcd1..7c49806c53a 100644 --- a/graph/src/util/futures.rs +++ b/graph/src/util/futures.rs @@ -1,5 +1,7 @@ use crate::ext::futures::FutureExtension; use futures03::{Future, FutureExt, TryFutureExt}; +use lazy_static::lazy_static; +use regex::Regex; use slog::{debug, trace, warn, Logger}; use std::fmt::Debug; use std::marker::PhantomData; @@ -61,6 +63,7 @@ pub fn retry(operation_name: impl ToString, logger: &Logger) -> RetryConfi log_after: 1, warn_after: 10, limit: RetryConfigProperty::Unknown, + redact_log_urls: false, phantom_item: PhantomData, phantom_error: PhantomData, } @@ -75,6 +78,7 @@ pub struct RetryConfig { limit: RetryConfigProperty, phantom_item: PhantomData, phantom_error: PhantomData, + redact_log_urls: bool, } impl RetryConfig @@ -125,6 +129,12 @@ where self } + /// Redact alphanumeric URLs from log messages. + pub fn redact_log_urls(mut self, redact_log_urls: bool) -> Self { + self.redact_log_urls = redact_log_urls; + self + } + /// Set how long (in seconds) to wait for an attempt to complete before giving up on that /// attempt. pub fn timeout_secs(self, timeout_secs: u64) -> RetryConfigWithTimeout { @@ -173,6 +183,7 @@ where let log_after = self.inner.log_after; let warn_after = self.inner.warn_after; let limit_opt = self.inner.limit.unwrap(&operation_name, "limit"); + let redact_log_urls = self.inner.redact_log_urls; let timeout = self.timeout; trace!(logger, "Run with retry: {}", operation_name); @@ -184,6 +195,7 @@ where log_after, warn_after, limit_opt, + redact_log_urls, move || { try_it() .timeout(timeout) @@ -214,6 +226,7 @@ impl RetryConfigNoTimeout { let log_after = self.inner.log_after; let warn_after = self.inner.warn_after; let limit_opt = self.inner.limit.unwrap(&operation_name, "limit"); + let redact_log_urls = self.inner.redact_log_urls; trace!(logger, "Run with retry: {}", operation_name); @@ -224,6 +237,7 @@ impl RetryConfigNoTimeout { log_after, warn_after, limit_opt, + redact_log_urls, // No timeout, so all errors are inner errors move || try_it().map_err(TimeoutError::Inner), ) @@ -265,6 +279,7 @@ fn run_retry( log_after: u64, warn_after: u64, limit_opt: Option, + redact_log_urls: bool, mut try_it_with_timeout: F, ) -> impl Future>> + Send where @@ -311,25 +326,38 @@ where // If needs retry if condition.check(&result) { + let result_str = || { + if redact_log_urls { + lazy_static! { + static ref RE: Regex = + Regex::new(r#"https?://[a-zA-Z0-9\-\._:/\?#&=]+"#).unwrap(); + } + let e = format!("{result:?}"); + RE.replace_all(&e, "[REDACTED]").into_owned() + } else { + format!("{result:?}") + } + }; + if attempt_count >= warn_after { // This looks like it would be nice to de-duplicate, but if we try // to use log! slog complains about requiring a const for the log level // See also b05e1594-e408-4047-aefb-71fc60d70e8f warn!( logger, - "Trying again after {} failed (attempt #{}) with result {:?}", + "Trying again after {} failed (attempt #{}) with result {}", &operation_name, attempt_count, - result + result_str(), ); } else if attempt_count >= log_after { // See also b05e1594-e408-4047-aefb-71fc60d70e8f debug!( logger, - "Trying again after {} failed (attempt #{}) with result {:?}", + "Trying again after {} failed (attempt #{}) with result {}", &operation_name, attempt_count, - result + result_str(), ); } diff --git a/graphql/src/execution/query.rs b/graphql/src/execution/query.rs index 4dfdd6c25b0..e8593f27fba 100644 --- a/graphql/src/execution/query.rs +++ b/graphql/src/execution/query.rs @@ -35,7 +35,6 @@ lazy_static! { vec![ Box::new(UniqueOperationNames::new()), Box::new(LoneAnonymousOperation::new()), - Box::new(SingleFieldSubscriptions::new()), Box::new(KnownTypeNames::new()), Box::new(FragmentsOnCompositeTypes::new()), Box::new(VariablesAreInputTypes::new()), @@ -69,12 +68,6 @@ pub enum ComplexityError { CyclicalFragment(String), } -#[derive(Copy, Clone)] -enum Kind { - Query, - Subscription, -} - /// Helper to log the fields in a `SelectionSet` without cloning. Writes /// a list of field names from the selection set separated by ';'. Using /// ';' as a separator makes parsing the log a little easier since slog @@ -130,8 +123,6 @@ pub struct Query { start: Instant, - kind: Kind, - /// Used only for logging; if logging is configured off, these will /// have dummy values pub query_text: Arc, @@ -226,14 +217,14 @@ impl Query { let operation = operation.ok_or(QueryExecutionError::OperationNameRequired)?; let variables = coerce_variables(schema.as_ref(), &operation, query.variables)?; - let (kind, selection_set) = match operation { - q::OperationDefinition::Query(q::Query { selection_set, .. }) => { - (Kind::Query, selection_set) - } + let selection_set = match operation { + q::OperationDefinition::Query(q::Query { selection_set, .. }) => selection_set, // Queries can be run by just sending a selection set - q::OperationDefinition::SelectionSet(selection_set) => (Kind::Query, selection_set), - q::OperationDefinition::Subscription(q::Subscription { selection_set, .. }) => { - (Kind::Subscription, selection_set) + q::OperationDefinition::SelectionSet(selection_set) => selection_set, + q::OperationDefinition::Subscription(_) => { + return Err(vec![QueryExecutionError::NotSupported( + "Subscriptions are not supported".to_owned(), + )]) } q::OperationDefinition::Mutation(_) => { return Err(vec![QueryExecutionError::NotSupported( @@ -243,10 +234,8 @@ impl Query { }; let start = Instant::now(); - let root_type = match kind { - Kind::Query => schema.query_type.as_ref(), - Kind::Subscription => schema.subscription_type.as_ref().unwrap(), - }; + let root_type = schema.query_type.as_ref(); + // Use an intermediate struct so we can modify the query before // enclosing it in an Arc let raw_query = RawQuery { @@ -269,7 +258,6 @@ impl Query { schema, selection_set: Arc::new(selection_set), shape_hash: query.shape_hash, - kind, network, logger, start, @@ -345,23 +333,6 @@ impl Query { Ok(bcs) } - /// Return `true` if this is a query, and not a subscription or - /// mutation - pub fn is_query(&self) -> bool { - match self.kind { - Kind::Query => true, - Kind::Subscription => false, - } - } - - /// Return `true` if this is a subscription, not a query or a mutation - pub fn is_subscription(&self) -> bool { - match self.kind { - Kind::Subscription => true, - Kind::Query => false, - } - } - /// Log details about the overall execution of the query pub fn log_execution(&self, block: BlockNumber) { if ENV_VARS.log_gql_timing() { diff --git a/graphql/src/execution/resolver.rs b/graphql/src/execution/resolver.rs index 1b139c65828..0074eb124d8 100644 --- a/graphql/src/execution/resolver.rs +++ b/graphql/src/execution/resolver.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use graph::components::store::{QueryPermit, UnitStream}; +use graph::components::store::QueryPermit; use graph::data::query::{CacheStatus, Trace}; use graph::prelude::{async_trait, s, Error, QueryExecutionError}; use graph::schema::ApiSchema; @@ -18,7 +18,7 @@ use super::Query; pub trait Resolver: Sized + Send + Sync + 'static { const CACHEABLE: bool; - async fn query_permit(&self) -> Result; + async fn query_permit(&self) -> QueryPermit; /// Prepare for executing a query by prefetching as much data as possible fn prefetch( @@ -111,18 +111,6 @@ pub trait Resolver: Sized + Send + Sync + 'static { } } - // Resolves a change stream for a given field. - fn resolve_field_stream( - &self, - _schema: &ApiSchema, - _object_type: &s::ObjectType, - _field: &a::Field, - ) -> Result { - Err(QueryExecutionError::NotSupported(String::from( - "Resolving field streams is not supported by this resolver", - ))) - } - fn post_process(&self, _result: &mut QueryResult) -> Result<(), Error> { Ok(()) } diff --git a/graphql/src/introspection/resolver.rs b/graphql/src/introspection/resolver.rs index 534cb6aa729..765b0399695 100644 --- a/graphql/src/introspection/resolver.rs +++ b/graphql/src/introspection/resolver.rs @@ -332,10 +332,7 @@ impl IntrospectionResolver { self.type_objects .get(&String::from("Query")) .cloned(), - subscriptionType: - self.type_objects - .get(&String::from("Subscription")) - .cloned(), + subscriptionType: r::Value::Null, mutationType: r::Value::Null, types: self.type_objects.values().cloned().collect::>(), directives: self.directives.clone(), @@ -359,7 +356,7 @@ impl Resolver for IntrospectionResolver { // see `fn as_introspection_context`, so this value is irrelevant. const CACHEABLE: bool = false; - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { unreachable!() } diff --git a/graphql/src/lib.rs b/graphql/src/lib.rs index 7a3070b3844..03626eb907e 100644 --- a/graphql/src/lib.rs +++ b/graphql/src/lib.rs @@ -7,9 +7,6 @@ mod execution; /// Utilities for executing GraphQL queries and working with query ASTs. pub mod query; -/// Utilities for executing GraphQL subscriptions. -pub mod subscription; - /// Utilities for working with GraphQL values. mod values; @@ -28,7 +25,6 @@ pub mod prelude { pub use super::introspection::IntrospectionResolver; pub use super::query::{execute_query, ext::BlockConstraint, QueryExecutionOptions}; pub use super::store::StoreResolver; - pub use super::subscription::SubscriptionExecutionOptions; pub use super::values::MaybeCoercible; pub use super::metrics::GraphQLMetrics; diff --git a/graphql/src/query/mod.rs b/graphql/src/query/mod.rs index a5a01f39ca3..641eb4581bb 100644 --- a/graphql/src/query/mod.rs +++ b/graphql/src/query/mod.rs @@ -1,6 +1,6 @@ use graph::{ data::query::CacheStatus, - prelude::{BlockPtr, CheapClone, QueryExecutionError, QueryResult}, + prelude::{BlockPtr, CheapClone, QueryResult}, }; use std::sync::Arc; use std::time::Instant; @@ -54,14 +54,6 @@ where trace: options.trace, }); - if !query.is_query() { - return ( - Arc::new( - QueryExecutionError::NotSupported("Only queries are supported".to_string()).into(), - ), - CacheStatus::default(), - ); - } let selection_set = selection_set .map(Arc::new) .unwrap_or_else(|| query.selection_set.cheap_clone()); diff --git a/graphql/src/runner.rs b/graphql/src/runner.rs index 79a13b0e04e..d2f0bc9c96c 100644 --- a/graphql/src/runner.rs +++ b/graphql/src/runner.rs @@ -2,18 +2,13 @@ use std::sync::Arc; use std::time::Instant; use crate::metrics::GraphQLMetrics; -use crate::prelude::{QueryExecutionOptions, StoreResolver, SubscriptionExecutionOptions}; +use crate::prelude::{QueryExecutionOptions, StoreResolver}; use crate::query::execute_query; -use crate::subscription::execute_prepared_subscription; use graph::futures03::future; use graph::prelude::MetricsRegistry; -use graph::{ - components::store::SubscriptionManager, - prelude::{ - async_trait, o, CheapClone, DeploymentState, GraphQLMetrics as GraphQLMetricsTrait, - GraphQlRunner as GraphQlRunnerTrait, Logger, Query, QueryExecutionError, Subscription, - SubscriptionError, SubscriptionResult, ENV_VARS, - }, +use graph::prelude::{ + async_trait, o, CheapClone, DeploymentState, GraphQLMetrics as GraphQLMetricsTrait, + GraphQlRunner as GraphQlRunnerTrait, Logger, Query, QueryExecutionError, ENV_VARS, }; use graph::{data::graphql::load_manager::LoadManager, prelude::QueryStoreManager}; use graph::{ @@ -22,10 +17,9 @@ use graph::{ }; /// GraphQL runner implementation for The Graph. -pub struct GraphQlRunner { +pub struct GraphQlRunner { logger: Logger, store: Arc, - subscription_manager: Arc, load_manager: Arc, graphql_metrics: Arc, } @@ -36,16 +30,14 @@ lazy_static::lazy_static! { pub static ref INITIAL_DEPLOYMENT_STATE_FOR_TESTS: std::sync::Mutex> = std::sync::Mutex::new(None); } -impl GraphQlRunner +impl GraphQlRunner where S: QueryStoreManager, - SM: SubscriptionManager, { /// Creates a new query runner. pub fn new( logger: &Logger, store: Arc, - subscription_manager: Arc, load_manager: Arc, registry: Arc, ) -> Self { @@ -54,7 +46,6 @@ where GraphQlRunner { logger, store, - subscription_manager, load_manager, graphql_metrics, } @@ -112,7 +103,7 @@ where // point, and everything needs to go through the `store` we are // setting up here - let store = self.store.query_store(target.clone(), false).await?; + let store = self.store.query_store(target.clone()).await?; let state = store.deployment_state().await?; let network = Some(store.network_name().to_string()); let schema = store.api_schema()?; @@ -152,7 +143,7 @@ where )?; self.load_manager .decide( - &store.wait_stats().map_err(QueryExecutionError::from)?, + &store.wait_stats(), store.shard(), store.deployment_id(), query.shape_hash, @@ -173,7 +164,6 @@ where &self.logger, store.cheap_clone(), &state, - self.subscription_manager.cheap_clone(), ptr, error_policy, query.schema.id().clone(), @@ -220,10 +210,9 @@ where } #[async_trait] -impl GraphQlRunnerTrait for GraphQlRunner +impl GraphQlRunnerTrait for GraphQlRunner where S: QueryStoreManager, - SM: SubscriptionManager, { async fn run_query(self: Arc, query: Query, target: QueryTarget) -> QueryResults { self.run_query_with_complexity( @@ -259,56 +248,6 @@ where .unwrap_or_else(|e| e) } - async fn run_subscription( - self: Arc, - subscription: Subscription, - target: QueryTarget, - ) -> Result { - let store = self.store.query_store(target.clone(), true).await?; - let schema = store.api_schema()?; - let network = store.network_name().to_string(); - - let query = crate::execution::Query::new( - &self.logger, - schema, - Some(network), - subscription.query, - ENV_VARS.graphql.max_complexity, - ENV_VARS.graphql.max_depth, - self.graphql_metrics.cheap_clone(), - )?; - - if let Err(err) = self - .load_manager - .decide( - &store.wait_stats().map_err(QueryExecutionError::from)?, - store.shard(), - store.deployment_id(), - query.shape_hash, - query.query_text.as_ref(), - ) - .to_result() - { - return Err(SubscriptionError::GraphQLError(vec![err])); - } - - execute_prepared_subscription( - query, - SubscriptionExecutionOptions { - logger: self.logger.clone(), - store, - subscription_manager: self.subscription_manager.cheap_clone(), - timeout: ENV_VARS.graphql.query_timeout, - max_complexity: ENV_VARS.graphql.max_complexity, - max_depth: ENV_VARS.graphql.max_depth, - max_first: ENV_VARS.graphql.max_first, - max_skip: ENV_VARS.graphql.max_skip, - graphql_metrics: self.graphql_metrics.clone(), - load_manager: self.load_manager.cheap_clone(), - }, - ) - } - fn metrics(&self) -> Arc { self.graphql_metrics.clone() } diff --git a/graphql/src/store/prefetch.rs b/graphql/src/store/prefetch.rs index de97b243227..33f0b67452b 100644 --- a/graphql/src/store/prefetch.rs +++ b/graphql/src/store/prefetch.rs @@ -713,8 +713,8 @@ impl<'a> Loader<'a> { // that causes unnecessary work in the database query.order = EntityOrder::Unordered; } - // Aggregations are always ordered by (timestamp, id) - if child_type.is_aggregation() { + // Apply default timestamp ordering for aggregations if no custom order is specified + if child_type.is_aggregation() && matches!(query.order, EntityOrder::Default) { let ts = child_type.field(kw::TIMESTAMP).unwrap(); query.order = EntityOrder::Descending(ts.name.to_string(), ts.value_type); } diff --git a/graphql/src/store/query.rs b/graphql/src/store/query.rs index 2c139152f86..bd74141c421 100644 --- a/graphql/src/store/query.rs +++ b/graphql/src/store/query.rs @@ -1,4 +1,3 @@ -use std::collections::{BTreeSet, HashSet, VecDeque}; use std::mem::discriminant; use graph::cheap_clone::CheapClone; @@ -8,12 +7,12 @@ use graph::components::store::{ }; use graph::data::graphql::TypeExt as _; use graph::data::query::QueryExecutionError; -use graph::data::store::{Attribute, SubscriptionFilter, Value, ValueType}; +use graph::data::store::{Attribute, Value, ValueType}; use graph::data::value::Object; use graph::data::value::Value as DataValue; -use graph::prelude::{r, s, TryFromValue, ENV_VARS}; +use graph::prelude::{r, TryFromValue, ENV_VARS}; use graph::schema::ast::{self as sast, FilterOp}; -use graph::schema::{ApiSchema, EntityType, InputSchema, ObjectOrInterface}; +use graph::schema::{EntityType, InputSchema, ObjectOrInterface}; use crate::execution::ast as a; @@ -652,58 +651,6 @@ fn build_order_direction(field: &a::Field) -> Result Result, QueryExecutionError> { - // Output entities - let mut entities = HashSet::new(); - - // List of objects/fields to visit next - let mut queue = VecDeque::new(); - queue.push_back((object_type, field)); - - while let Some((object_type, field)) = queue.pop_front() { - // Check if the field exists on the object type - if let Some(field_type) = sast::get_field(&object_type, &field.name) { - // Check if the field type corresponds to a type definition (in a valid schema, - // this should always be the case) - if let Some(type_definition) = schema.get_type_definition_from_field(field_type) { - // If the field's type definition is an object type, extract that type - if let s::TypeDefinition::Object(object_type) = type_definition { - // Only collect whether the field's type has an @entity directive - if sast::get_object_type_directive(object_type, String::from("entity")) - .is_some() - { - entities - .insert((input_schema.id().cheap_clone(), object_type.name.clone())); - } - - // If the query field has a non-empty selection set, this means we - // need to recursively process it - let object_type = schema.object_type(object_type).into(); - for sub_field in field.selection_set.fields_for(&object_type)? { - queue.push_back((object_type.cheap_clone(), sub_field)) - } - } - } - } - } - - entities - .into_iter() - .map(|(id, entity_type)| { - input_schema - .entity_type(&entity_type) - .map(|entity_type| SubscriptionFilter::Entities(id, entity_type)) - }) - .collect::>() - .map_err(Into::into) -} - #[cfg(test)] mod tests { use graph::components::store::EntityQuery; diff --git a/graphql/src/store/resolver.rs b/graphql/src/store/resolver.rs index a112fc97ae3..d7032740768 100644 --- a/graphql/src/store/resolver.rs +++ b/graphql/src/store/resolver.rs @@ -1,9 +1,8 @@ use std::collections::BTreeMap; -use std::result; use std::sync::Arc; use graph::components::graphql::GraphQLMetrics as _; -use graph::components::store::{QueryPermit, SubscriptionManager, UnitStream}; +use graph::components::store::QueryPermit; use graph::data::graphql::load_manager::LoadManager; use graph::data::graphql::{object, ObjectOrInterface}; use graph::data::query::{CacheStatus, QueryResults, Trace}; @@ -12,8 +11,8 @@ use graph::data::value::{Object, Word}; use graph::derive::CheapClone; use graph::prelude::*; use graph::schema::{ - ast as sast, ApiSchema, INTROSPECTION_SCHEMA_FIELD_NAME, INTROSPECTION_TYPE_FIELD_NAME, - META_FIELD_NAME, META_FIELD_TYPE, + ast as sast, INTROSPECTION_SCHEMA_FIELD_NAME, INTROSPECTION_TYPE_FIELD_NAME, META_FIELD_NAME, + META_FIELD_TYPE, }; use graph::schema::{ErrorPolicy, BLOCK_FIELD_TYPE}; @@ -21,7 +20,6 @@ use crate::execution::{ast as a, Query}; use crate::metrics::GraphQLMetrics; use crate::prelude::{ExecutionContext, Resolver}; use crate::query::ext::BlockConstraint; -use crate::store::query::collect_entities_from_query_field; /// A resolver that fetches entities from a `Store`. #[derive(Clone, CheapClone)] @@ -29,7 +27,6 @@ pub struct StoreResolver { #[allow(dead_code)] logger: Logger, pub(crate) store: Arc, - subscription_manager: Arc, pub(crate) block_ptr: Option, deployment: DeploymentHash, has_non_fatal_errors: bool, @@ -39,33 +36,6 @@ pub struct StoreResolver { } impl StoreResolver { - /// Create a resolver that looks up entities at whatever block is the - /// latest when the query is run. That means that multiple calls to find - /// entities into this resolver might return entities from different - /// blocks - pub fn for_subscription( - logger: &Logger, - deployment: DeploymentHash, - store: Arc, - subscription_manager: Arc, - graphql_metrics: Arc, - load_manager: Arc, - ) -> Self { - StoreResolver { - logger: logger.new(o!("component" => "StoreResolver")), - store, - subscription_manager, - block_ptr: None, - deployment, - - // Checking for non-fatal errors does not work with subscriptions. - has_non_fatal_errors: false, - error_policy: ErrorPolicy::Deny, - graphql_metrics, - load_manager, - } - } - /// Create a resolver that looks up entities at the block specified /// by `bc`. Any calls to find objects will always return entities as /// of that block. Note that if `bc` is `BlockConstraint::Latest` we use @@ -75,7 +45,6 @@ impl StoreResolver { logger: &Logger, store: Arc, state: &DeploymentState, - subscription_manager: Arc, block_ptr: BlockPtr, error_policy: ErrorPolicy, deployment: DeploymentHash, @@ -90,7 +59,6 @@ impl StoreResolver { let resolver = StoreResolver { logger: logger.new(o!("component" => "StoreResolver")), store, - subscription_manager, block_ptr: Some(block_ptr), deployment, has_non_fatal_errors, @@ -288,8 +256,8 @@ impl StoreResolver { impl Resolver for StoreResolver { const CACHEABLE: bool = true; - async fn query_permit(&self) -> Result { - self.store.query_permit().await.map_err(Into::into) + async fn query_permit(&self) -> QueryPermit { + self.store.query_permit().await } fn prefetch( @@ -380,22 +348,6 @@ impl Resolver for StoreResolver { } } - fn resolve_field_stream( - &self, - schema: &ApiSchema, - object_type: &s::ObjectType, - field: &a::Field, - ) -> result::Result { - // Collect all entities involved in the query field - let object_type = schema.object_type(object_type).into(); - let input_schema = self.store.input_schema()?; - let entities = - collect_entities_from_query_field(&input_schema, schema, object_type, field)?; - - // Subscribe to the store and return the entity change stream - Ok(self.subscription_manager.subscribe_no_payload(entities)) - } - fn post_process(&self, result: &mut QueryResult) -> Result<(), anyhow::Error> { // Post-processing is only necessary for queries with indexing errors, and no query errors. if !self.has_non_fatal_errors || result.has_errors() { diff --git a/graphql/src/subscription/mod.rs b/graphql/src/subscription/mod.rs deleted file mode 100644 index ef0fc7b53ce..00000000000 --- a/graphql/src/subscription/mod.rs +++ /dev/null @@ -1,256 +0,0 @@ -use std::result::Result; -use std::time::{Duration, Instant}; - -use graph::components::store::UnitStream; -use graph::data::graphql::load_manager::LoadManager; -use graph::futures03::future::FutureExt; -use graph::futures03::stream::StreamExt; -use graph::schema::ApiSchema; -use graph::{components::store::SubscriptionManager, prelude::*, schema::ErrorPolicy}; - -use crate::metrics::GraphQLMetrics; -use crate::{execution::ast as a, execution::*, prelude::StoreResolver}; - -/// Options available for subscription execution. -pub struct SubscriptionExecutionOptions { - /// The logger to use during subscription execution. - pub logger: Logger, - - /// The store to use. - pub store: Arc, - - pub subscription_manager: Arc, - - /// Individual timeout for each subscription query. - pub timeout: Option, - - /// Maximum complexity for a subscription query. - pub max_complexity: Option, - - /// Maximum depth for a subscription query. - pub max_depth: u8, - - /// Maximum value for the `first` argument. - pub max_first: u32, - - /// Maximum value for the `skip` argument. - pub max_skip: u32, - - pub graphql_metrics: Arc, - - pub load_manager: Arc, -} - -pub fn execute_subscription( - subscription: Subscription, - schema: Arc, - options: SubscriptionExecutionOptions, -) -> Result { - let query = crate::execution::Query::new( - &options.logger, - schema, - None, - subscription.query, - options.max_complexity, - options.max_depth, - options.graphql_metrics.cheap_clone(), - )?; - execute_prepared_subscription(query, options) -} - -pub(crate) fn execute_prepared_subscription( - query: Arc, - options: SubscriptionExecutionOptions, -) -> Result { - if !query.is_subscription() { - return Err(SubscriptionError::from(QueryExecutionError::NotSupported( - "Only subscriptions are supported".to_string(), - ))); - } - - info!( - options.logger, - "Execute subscription"; - "query" => &query.query_text, - ); - - let source_stream = create_source_event_stream(query.clone(), &options)?; - let response_stream = map_source_to_response_stream(query, options, source_stream); - Ok(response_stream) -} - -fn create_source_event_stream( - query: Arc, - options: &SubscriptionExecutionOptions, -) -> Result { - let resolver = StoreResolver::for_subscription( - &options.logger, - query.schema.id().clone(), - options.store.clone(), - options.subscription_manager.cheap_clone(), - options.graphql_metrics.cheap_clone(), - options.load_manager.cheap_clone(), - ); - let ctx = ExecutionContext { - logger: options.logger.cheap_clone(), - resolver, - query, - deadline: None, - max_first: options.max_first, - max_skip: options.max_skip, - cache_status: Default::default(), - trace: ENV_VARS.log_sql_timing(), - }; - - let subscription_type = ctx - .query - .schema - .subscription_type - .as_ref() - .ok_or(QueryExecutionError::NoRootSubscriptionObjectType)?; - - let field = if ctx.query.selection_set.is_empty() { - return Err(SubscriptionError::from(QueryExecutionError::EmptyQuery)); - } else { - match ctx.query.selection_set.single_field() { - Some(field) => field, - None => { - return Err(SubscriptionError::from( - QueryExecutionError::MultipleSubscriptionFields, - )); - } - } - }; - - resolve_field_stream(&ctx, subscription_type, field) -} - -fn resolve_field_stream( - ctx: &ExecutionContext, - object_type: &s::ObjectType, - field: &a::Field, -) -> Result { - ctx.resolver - .resolve_field_stream(&ctx.query.schema, object_type, field) - .map_err(SubscriptionError::from) -} - -fn map_source_to_response_stream( - query: Arc, - options: SubscriptionExecutionOptions, - source_stream: UnitStream, -) -> QueryResultStream { - // Create a stream with a single empty event. By chaining this in front - // of the real events, we trick the subscription into executing its query - // at least once. This satisfies the GraphQL over Websocket protocol - // requirement of "respond[ing] with at least one GQL_DATA message", see - // https://github.com/apollographql/subscriptions-transport-ws/blob/master/PROTOCOL.md#gql_data - let trigger_stream = graph::futures03::stream::once(async {}); - - let SubscriptionExecutionOptions { - logger, - store, - subscription_manager, - timeout, - max_complexity: _, - max_depth: _, - max_first, - max_skip, - graphql_metrics, - load_manager, - } = options; - - trigger_stream - .chain(source_stream) - .then(move |()| { - execute_subscription_event( - logger.clone(), - store.clone(), - subscription_manager.cheap_clone(), - query.clone(), - timeout, - max_first, - max_skip, - graphql_metrics.cheap_clone(), - load_manager.cheap_clone(), - ) - .boxed() - }) - .boxed() -} - -async fn execute_subscription_event( - logger: Logger, - store: Arc, - subscription_manager: Arc, - query: Arc, - timeout: Option, - max_first: u32, - max_skip: u32, - metrics: Arc, - load_manager: Arc, -) -> Arc { - async fn make_resolver( - store: Arc, - logger: &Logger, - subscription_manager: Arc, - query: &Arc, - metrics: Arc, - load_manager: Arc, - ) -> Result { - let state = store.deployment_state().await?; - StoreResolver::at_block( - logger, - store, - &state, - subscription_manager, - state.latest_block.clone(), - ErrorPolicy::Deny, - query.schema.id().clone(), - metrics, - load_manager, - ) - .await - } - - let resolver = match make_resolver( - store, - &logger, - subscription_manager, - &query, - metrics, - load_manager, - ) - .await - { - Ok(resolver) => resolver, - Err(e) => return Arc::new(e.into()), - }; - - let block_ptr = resolver.block_ptr.clone(); - - // Create a fresh execution context with deadline. - let ctx = Arc::new(ExecutionContext { - logger, - resolver, - query, - deadline: timeout.map(|t| Instant::now() + t), - max_first, - max_skip, - cache_status: Default::default(), - trace: ENV_VARS.log_sql_timing(), - }); - - let subscription_type = match ctx.query.schema.subscription_type.as_ref() { - Some(t) => t.cheap_clone(), - None => return Arc::new(QueryExecutionError::NoRootSubscriptionObjectType.into()), - }; - - execute_root_selection_set( - ctx.cheap_clone(), - ctx.query.selection_set.cheap_clone(), - subscription_type.into(), - block_ptr, - ) - .await -} diff --git a/node/Cargo.toml b/node/Cargo.toml index 820ed8405a8..444b18784fc 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -19,19 +19,17 @@ clap.workspace = true git-testament = "0.2" itertools = { workspace = true } lazy_static = "1.5.0" -url = "2.5.2" +url = "2.5.4" graph = { path = "../graph" } graph-core = { path = "../core" } graph-chain-arweave = { path = "../chain/arweave" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-near = { path = "../chain/near" } -graph-chain-cosmos = { path = "../chain/cosmos" } graph-chain-substreams = { path = "../chain/substreams" } graph-graphql = { path = "../graphql" } graph-server-http = { path = "../server/http" } graph-server-index-node = { path = "../server/index-node" } graph-server-json-rpc = { path = "../server/json-rpc" } -graph-server-websocket = { path = "../server/websocket" } graph-server-metrics = { path = "../server/metrics" } graph-store-postgres = { path = "../store/postgres" } graphman-server = { workspace = true } diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index 902241d3d54..50ee9b61958 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -24,9 +24,7 @@ use graph_node::manager::color::Terminal; use graph_node::manager::commands; use graph_node::network_setup::Networks; use graph_node::{ - manager::{deployment::DeploymentSearch, PanicSubscriptionManager}, - store_builder::StoreBuilder, - MetricsContext, + manager::deployment::DeploymentSearch, store_builder::StoreBuilder, MetricsContext, }; use graph_store_postgres::connection_pool::PoolCoordinator; use graph_store_postgres::ChainStore; @@ -36,6 +34,7 @@ use graph_store_postgres::{ }; use itertools::Itertools; use lazy_static::lazy_static; +use std::env; use std::str::FromStr; use std::{collections::HashMap, num::ParseIntError, sync::Arc, time::Duration}; const VERSION_LABEL_KEY: &str = "version"; @@ -469,14 +468,8 @@ pub enum ConfigCommand { pub enum ListenCommand { /// Listen only to assignment events Assignments, - /// Listen to events for entities in a specific deployment - Entities { - /// The deployment (see `help info`). - deployment: DeploymentSearch, - /// The entity types for which to print change notifications - entity_types: Vec, - }, } + #[derive(Clone, Debug, Subcommand)] pub enum CopyCommand { /// Create a copy of an existing subgraph @@ -905,7 +898,7 @@ impl Context { fn primary_pool(self) -> ConnectionPool { let primary = self.config.primary_store(); - let coord = Arc::new(PoolCoordinator::new(Arc::new(vec![]))); + let coord = Arc::new(PoolCoordinator::new(&self.logger, Arc::new(vec![]))); let pool = StoreBuilder::main_pool( &self.logger, &self.node_id, @@ -932,13 +925,6 @@ impl Context { )) } - fn primary_and_subscription_manager(self) -> (ConnectionPool, Arc) { - let mgr = self.subscription_manager(); - let primary_pool = self.primary_pool(); - - (primary_pool, mgr) - } - fn store(&self) -> Arc { let (store, _) = self.store_and_pools(); store @@ -998,22 +984,15 @@ impl Context { (store.block_store(), primary.clone()) } - fn graphql_runner(self) -> Arc> { + fn graphql_runner(self) -> Arc> { let logger = self.logger.clone(); let registry = self.registry.clone(); let store = self.store(); - let subscription_manager = Arc::new(PanicSubscriptionManager); let load_manager = Arc::new(LoadManager::new(&logger, vec![], vec![], registry.clone())); - Arc::new(GraphQlRunner::new( - &logger, - store, - subscription_manager, - load_manager, - registry, - )) + Arc::new(GraphQlRunner::new(&logger, store, load_manager, registry)) } async fn networks(&self) -> anyhow::Result { @@ -1052,6 +1031,9 @@ impl Context { #[tokio::main] async fn main() -> anyhow::Result<()> { + // Disable load management for graphman commands + env::set_var("GRAPH_LOAD_THRESHOLD", "0"); + let opt = Opt::parse(); Terminal::set_color_preference(&opt.color); @@ -1319,13 +1301,6 @@ async fn main() -> anyhow::Result<()> { use ListenCommand::*; match cmd { Assignments => commands::listen::assignments(ctx.subscription_manager()).await, - Entities { - deployment, - entity_types, - } => { - let (primary, mgr) = ctx.primary_and_subscription_manager(); - commands::listen::entities(primary, mgr, &deployment, entity_types).await - } } } Copy(cmd) => { diff --git a/node/src/chain.rs b/node/src/chain.rs index 289c0580c2e..4ff45b8211a 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -90,6 +90,7 @@ pub fn create_substreams_networks( firehose.compression_enabled(), SubgraphLimit::Unlimited, endpoint_metrics.clone(), + true, ))); } } @@ -157,6 +158,7 @@ pub fn create_firehose_networks( firehose.compression_enabled(), firehose.limit_for(&config.node), endpoint_metrics.cheap_clone(), + false, ))); } } @@ -439,6 +441,7 @@ pub async fn networks_as_chains( client.clone(), metrics_registry.clone(), chain_store.clone(), + eth_adapters.clone(), ); let call_cache = chain_store.cheap_clone(); @@ -457,7 +460,7 @@ pub async fn networks_as_chains( Arc::new(adapter_selector), Arc::new(EthereumRuntimeAdapterBuilder {}), eth_adapters, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), polling_interval, true, ); @@ -504,33 +507,6 @@ pub async fn networks_as_chains( ) .await; } - BlockchainKind::Cosmos => { - let firehose_endpoints = networks.firehose_endpoints(chain_id.clone()); - blockchain_map.insert::( - chain_id.clone(), - Arc::new( - BasicBlockchainBuilder { - logger_factory: logger_factory.clone(), - name: chain_id.clone(), - chain_store: chain_store.cheap_clone(), - firehose_endpoints, - metrics_registry: metrics_registry.clone(), - } - .build(config) - .await, - ), - ); - add_substreams::( - networks, - config, - chain_id.clone(), - blockchain_map, - logger_factory.clone(), - chain_store, - metrics_registry.clone(), - ) - .await; - } BlockchainKind::Substreams => { let substreams_endpoints = networks.substreams_endpoints(chain_id.clone()); blockchain_map.insert::( diff --git a/node/src/main.rs b/node/src/main.rs index 870cce97318..b2003dff28f 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -28,7 +28,6 @@ use graph_server_http::GraphQLServer as GraphQLQueryServer; use graph_server_index_node::IndexNodeServer; use graph_server_json_rpc::JsonRpcServer; use graph_server_metrics::PrometheusMetricsServer; -use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer; use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::Store; use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender}; @@ -79,8 +78,21 @@ fn read_expensive_queries( Ok(queries) } -#[tokio::main] -async fn main() { +fn main() { + let max_blocking: usize = std::env::var("GRAPH_MAX_BLOCKING_THREADS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(512); + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .max_blocking_threads(max_blocking) + .build() + .unwrap() + .block_on(async { main_inner().await }) +} + +async fn main_inner() { env_logger::init(); let env_vars = Arc::new(EnvVars::from_env().unwrap()); @@ -142,7 +154,6 @@ async fn main() { // Obtain ports to use for the GraphQL server(s) let http_port = opt.http_port; - let ws_port = opt.ws_port; // Obtain JSON-RPC server port let json_rpc_port = opt.admin_port; @@ -357,13 +368,10 @@ async fn main() { let graphql_runner = Arc::new(GraphQlRunner::new( &logger, network_store.clone(), - subscription_manager.clone(), load_manager, graphql_metrics_registry, )); let graphql_server = GraphQLQueryServer::new(&logger_factory, graphql_runner.clone()); - let subscription_server = - GraphQLSubscriptionServer::new(&logger, graphql_runner.clone(), network_store.clone()); let index_node_server = IndexNodeServer::new( &logger_factory, @@ -446,7 +454,6 @@ async fn main() { let json_rpc_server = JsonRpcServer::serve( json_rpc_port, http_port, - ws_port, subgraph_registrar.clone(), node_id.clone(), logger.clone(), @@ -508,10 +515,7 @@ async fn main() { } // Serve GraphQL queries over HTTP - graph::spawn(async move { graphql_server.start(http_port, ws_port).await }); - - // Serve GraphQL subscriptions over WebSockets - graph::spawn(subscription_server.serve(ws_port)); + graph::spawn(async move { graphql_server.start(http_port).await }); // Run the index node server graph::spawn(async move { index_node_server.start(index_node_port).await }); diff --git a/node/src/manager/commands/copy.rs b/node/src/manager/commands/copy.rs index 620e27eef6c..9ca80bc9b20 100644 --- a/node/src/manager/commands/copy.rs +++ b/node/src/manager/commands/copy.rs @@ -2,7 +2,7 @@ use diesel::{ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, RunQuery use std::{collections::HashMap, sync::Arc, time::SystemTime}; use graph::{ - components::store::{BlockStore as _, DeploymentId}, + components::store::{BlockStore as _, DeploymentId, DeploymentLocator}, data::query::QueryTarget, prelude::{ anyhow::{anyhow, bail, Error}, @@ -84,10 +84,9 @@ impl CopyState { } } -pub async fn create( +async fn create_inner( store: Arc, - primary: ConnectionPool, - src: DeploymentSearch, + src: &DeploymentLocator, shard: String, shards: Vec, node: String, @@ -104,12 +103,11 @@ pub async fn create( }; let subgraph_store = store.subgraph_store(); - let src = src.locate_unique(&primary)?; let query_store = store - .query_store( - QueryTarget::Deployment(src.hash.clone(), Default::default()), - true, - ) + .query_store(QueryTarget::Deployment( + src.hash.clone(), + Default::default(), + )) .await?; let network = query_store.network_name(); @@ -154,6 +152,32 @@ pub async fn create( Ok(()) } +pub async fn create( + store: Arc, + primary: ConnectionPool, + src: DeploymentSearch, + shard: String, + shards: Vec, + node: String, + block_offset: u32, + activate: bool, + replace: bool, +) -> Result<(), Error> { + let src = src.locate_unique(&primary)?; + create_inner( + store, + &src, + shard, + shards, + node, + block_offset, + activate, + replace, + ) + .await + .map_err(|e| anyhow!("cannot copy {src}: {e}")) +} + pub fn activate(store: Arc, deployment: String, shard: String) -> Result<(), Error> { let shard = Shard::new(shard)?; let deployment = @@ -231,13 +255,11 @@ pub fn list(pools: HashMap) -> Result<(), Error> { } pub fn status(pools: HashMap, dst: &DeploymentSearch) -> Result<(), Error> { + const CHECK: &str = "✓"; + use catalog::active_copies as ac; use catalog::deployment_schemas as ds; - fn done(ts: &Option) -> String { - ts.map(|_| "✓").unwrap_or(".").to_string() - } - fn duration(start: &UtcDateTime, end: &Option) -> String { let start = *start; let end = *end; @@ -290,7 +312,7 @@ pub fn status(pools: HashMap, dst: &DeploymentSearch) -> }; let progress = match &state.finished_at { - Some(_) => done(&state.finished_at), + Some(_) => CHECK.to_string(), None => { let target: i64 = tables.iter().map(|table| table.target_vid).sum(); let next: i64 = tables.iter().map(|table| table.next_vid).sum(); @@ -339,13 +361,15 @@ pub fn status(pools: HashMap, dst: &DeploymentSearch) -> ); println!("{:-<74}", "-"); for table in tables { - let status = if table.next_vid > 0 && table.next_vid < table.target_vid { - ">".to_string() - } else if table.target_vid < 0 { + let status = match &table.finished_at { + // table finished + Some(_) => CHECK, // empty source table - "✓".to_string() - } else { - done(&table.finished_at) + None if table.target_vid < 0 => CHECK, + // copying in progress + None if table.duration_ms > 0 => ">", + // not started + None => ".", }; println!( "{} {:<28} | {:>8} | {:>8} | {:>8} | {:>8}", diff --git a/node/src/manager/commands/drop.rs b/node/src/manager/commands/drop.rs index 30d724575c5..2c86e88e23a 100644 --- a/node/src/manager/commands/drop.rs +++ b/node/src/manager/commands/drop.rs @@ -9,8 +9,8 @@ use std::sync::Arc; /// Finds, unassigns, record and remove matching deployments. /// -/// Asks for confirmation before removing any data. -/// This is a convenience fuction that to call a series of other graphman commands. +/// Asks for confirmation before removing any data. This is a convenience +/// function that to call a series of other graphman commands. pub async fn run( primary_pool: ConnectionPool, subgraph_store: Arc, diff --git a/node/src/manager/commands/listen.rs b/node/src/manager/commands/listen.rs index feee8350797..69c3ff93cbf 100644 --- a/node/src/manager/commands/listen.rs +++ b/node/src/manager/commands/listen.rs @@ -1,24 +1,16 @@ -use std::iter::FromIterator; +use std::io::Write; use std::sync::Arc; -use std::{collections::BTreeSet, io::Write}; -use crate::manager::deployment::DeploymentSearch; use graph::futures01::Stream as _; use graph::futures03::compat::Future01CompatExt; -use graph::prelude::DeploymentHash; -use graph::schema::{EntityType, InputSchema}; use graph::{ components::store::SubscriptionManager as _, - prelude::{serde_json, Error, SubscriptionFilter}, + prelude::{serde_json, Error}, }; -use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::SubscriptionManager; -async fn listen( - mgr: Arc, - filter: BTreeSet, -) -> Result<(), Error> { - let events = mgr.subscribe(filter); +async fn listen(mgr: Arc) -> Result<(), Error> { + let events = mgr.subscribe(); println!("press ctrl-c to stop"); let res = events .inspect(move |event| { @@ -44,51 +36,7 @@ async fn listen( pub async fn assignments(mgr: Arc) -> Result<(), Error> { println!("waiting for assignment events"); - listen( - mgr, - FromIterator::from_iter([SubscriptionFilter::Assignment]), - ) - .await?; - - Ok(()) -} - -pub async fn entities( - primary_pool: ConnectionPool, - mgr: Arc, - search: &DeploymentSearch, - entity_types: Vec, -) -> Result<(), Error> { - // We convert the entity type names into entity types in this very - // awkward way to avoid needing to have a SubgraphStore from which we - // load the input schema - fn as_entity_types( - entity_types: Vec, - id: &DeploymentHash, - ) -> Result, Error> { - use std::fmt::Write; - - let schema = entity_types - .iter() - .fold(String::new(), |mut buf, entity_type| { - writeln!(buf, "type {entity_type} @entity {{ id: ID! }}").unwrap(); - buf - }); - let schema = InputSchema::parse_latest(&schema, id.clone()).unwrap(); - entity_types - .iter() - .map(|et| schema.entity_type(et)) - .collect::>() - } - - let locator = search.locate_unique(&primary_pool)?; - let filter = as_entity_types(entity_types, &locator.hash)? - .into_iter() - .map(|et| SubscriptionFilter::Entities(locator.hash.clone(), et)) - .collect(); - - println!("waiting for store events from {}", locator); - listen(mgr, filter).await?; + listen(mgr).await?; Ok(()) } diff --git a/node/src/manager/commands/prune.rs b/node/src/manager/commands/prune.rs index c169577ee65..2c3c2ae2386 100644 --- a/node/src/manager/commands/prune.rs +++ b/node/src/manager/commands/prune.rs @@ -188,13 +188,13 @@ pub async fn run( println!("prune {deployment}"); println!(" latest: {latest}"); - println!(" final: {}", latest - ENV_VARS.reorg_threshold); + println!(" final: {}", latest - ENV_VARS.reorg_threshold()); println!(" earliest: {}\n", latest - history); let mut req = PruneRequest::new( &deployment, history, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), status.earliest_block_number, latest, )?; @@ -217,7 +217,7 @@ pub async fn run( store.subgraph_store().set_history_blocks( &deployment, history, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), )?; } diff --git a/node/src/manager/commands/query.rs b/node/src/manager/commands/query.rs index 879e0eaf4a4..6339b7bf9cc 100644 --- a/node/src/manager/commands/query.rs +++ b/node/src/manager/commands/query.rs @@ -16,10 +16,8 @@ use graph::{ use graph_graphql::prelude::GraphQlRunner; use graph_store_postgres::Store; -use crate::manager::PanicSubscriptionManager; - pub async fn run( - runner: Arc>, + runner: Arc>, target: String, query: String, vars: Vec, diff --git a/node/src/manager/commands/rewind.rs b/node/src/manager/commands/rewind.rs index 339f2ec979a..629c4b6e70f 100644 --- a/node/src/manager/commands/rewind.rs +++ b/node/src/manager/commands/rewind.rs @@ -133,13 +133,13 @@ pub async fn run( let deployment_details = deployment_store.deployment_details_for_id(locator)?; let block_number_to = block_ptr_to.as_ref().map(|b| b.number).unwrap_or(0); - if block_number_to < deployment_details.earliest_block_number + ENV_VARS.reorg_threshold { + if block_number_to < deployment_details.earliest_block_number + ENV_VARS.reorg_threshold() { bail!( "The block number {} is not safe to rewind to for deployment {}. The earliest block number of this deployment is {}. You can only safely rewind to block number {}", block_ptr_to.as_ref().map(|b| b.number).unwrap_or(0), locator, deployment_details.earliest_block_number, - deployment_details.earliest_block_number + ENV_VARS.reorg_threshold + deployment_details.earliest_block_number + ENV_VARS.reorg_threshold() ); } } diff --git a/node/src/manager/mod.rs b/node/src/manager/mod.rs index b2eccaf6e9a..6a332653ca8 100644 --- a/node/src/manager/mod.rs +++ b/node/src/manager/mod.rs @@ -1,8 +1,6 @@ -use std::collections::BTreeSet; - use graph::{ - components::store::{SubscriptionManager, UnitStream}, - prelude::{anyhow, StoreEventStreamBox, SubscriptionFilter}, + components::store::SubscriptionManager, + prelude::{anyhow, StoreEventStreamBox}, }; pub mod catalog; @@ -16,13 +14,9 @@ pub mod prompt; pub struct PanicSubscriptionManager; impl SubscriptionManager for PanicSubscriptionManager { - fn subscribe(&self, _: BTreeSet) -> StoreEventStreamBox { + fn subscribe(&self) -> StoreEventStreamBox { panic!("we were never meant to call `subscribe`"); } - - fn subscribe_no_payload(&self, _: BTreeSet) -> UnitStream { - panic!("we were never meant to call `subscribe_no_payload`"); - } } pub type CmdResult = Result<(), anyhow::Error>; diff --git a/node/src/network_setup.rs b/node/src/network_setup.rs index 4a8b4cedca1..1ebe2b5109c 100644 --- a/node/src/network_setup.rs +++ b/node/src/network_setup.rs @@ -338,9 +338,6 @@ impl Networks { BlockchainKind::Near => { block_ingestor::(logger, id, chain, &mut res).await? } - BlockchainKind::Cosmos => { - block_ingestor::(logger, id, chain, &mut res).await? - } BlockchainKind::Substreams => {} } } diff --git a/node/src/opt.rs b/node/src/opt.rs index e4dc44ba92a..9928144396a 100644 --- a/node/src/opt.rs +++ b/node/src/opt.rs @@ -132,14 +132,6 @@ pub struct Opt { help = "Port for the index node server" )] pub index_node_port: u16, - #[clap( - long, - default_value = "8001", - value_name = "PORT", - help = "Port for the GraphQL WebSocket server", - env = "GRAPH_GRAPHQL_WS_PORT" - )] - pub ws_port: u16, #[clap( long, default_value = "8020", diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index 7fadf6b92c2..5294179f8eb 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -1,15 +1,15 @@ use std::iter::FromIterator; use std::{collections::HashMap, sync::Arc}; -use graph::futures03::future::join_all; use graph::prelude::{o, MetricsRegistry, NodeId}; +use graph::slog::warn; use graph::url::Url; use graph::{ prelude::{info, CheapClone, Logger}, util::security::SafeDisplay, }; use graph_store_postgres::connection_pool::{ - ConnectionPool, ForeignServer, PoolCoordinator, PoolName, + ConnectionPool, ForeignServer, PoolCoordinator, PoolRole, }; use graph_store_postgres::{ BlockStore as DieselBlockStore, ChainHeadUpdateListener as PostgresChainHeadUpdateListener, @@ -62,7 +62,7 @@ impl StoreBuilder { // attempt doesn't work for all of them because the database is // unavailable, they will try again later in the normal course of // using the pool - join_all(pools.values().map(|pool| pool.setup())).await; + coord.setup_all(logger).await; let chains = HashMap::from_iter(config.chains.chains.iter().map(|(name, chain)| { let shard = ShardName::new(chain.shard.to_string()) @@ -111,13 +111,28 @@ impl StoreBuilder { .collect::, _>>() .expect("connection url's contain enough detail"); let servers = Arc::new(servers); - let coord = Arc::new(PoolCoordinator::new(servers)); + let coord = Arc::new(PoolCoordinator::new(logger, servers)); let shards: Vec<_> = config .stores .iter() - .map(|(name, shard)| { + .filter_map(|(name, shard)| { let logger = logger.new(o!("shard" => name.to_string())); + let pool_size = shard.pool_size.size_for(node, name).unwrap_or_else(|_| { + panic!("cannot determine the pool size for store {}", name) + }); + if pool_size == 0 { + if name == PRIMARY_SHARD.as_str() { + panic!("pool size for primary shard must be greater than 0"); + } else { + warn!( + logger, + "pool size for shard {} is 0, ignoring this shard", name + ); + return None; + } + } + let conn_pool = Self::main_pool( &logger, node, @@ -138,7 +153,7 @@ impl StoreBuilder { let name = ShardName::new(name.to_string()).expect("shard names have been validated"); - (name, conn_pool, read_only_conn_pools, weights) + Some((name, conn_pool, read_only_conn_pools, weights)) }) .collect(); @@ -196,8 +211,8 @@ impl StoreBuilder { Arc::new(DieselStore::new(subgraph_store, block_store)) } - /// Create a connection pool for the main database of the primary shard - /// without connecting to all the other configured databases + /// Create a connection pool for the main (non-replica) database of a + /// shard pub fn main_pool( logger: &Logger, node: &NodeId, @@ -225,7 +240,7 @@ impl StoreBuilder { coord.create_pool( &logger, name, - PoolName::Main, + PoolRole::Main, shard.connection.clone(), pool_size, Some(fdw_pool_size), @@ -265,7 +280,7 @@ impl StoreBuilder { coord.clone().create_pool( &logger, name, - PoolName::Replica(pool), + PoolRole::Replica(pool), replica.connection.clone(), pool_size, None, diff --git a/runtime/derive/src/generate_array_type.rs b/runtime/derive/src/generate_array_type.rs deleted file mode 100644 index 1e674c182c7..00000000000 --- a/runtime/derive/src/generate_array_type.rs +++ /dev/null @@ -1,78 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, ItemStruct}; - -pub fn generate_array_type(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let name = item_struct.ident.clone(); - - let asc_name = Ident::new(&format!("Asc{}", name), Span::call_site()); - let asc_name_array = Ident::new(&format!("Asc{}Array", name), Span::call_site()); - - let args = { - let mut args = Vec::new(); - let parser = syn::meta::parser(|meta| { - if let Some(ident) = meta.path.get_ident() { - args.push(ident.to_string()); - } - Ok(()) - }); - parse_macro_input!(metadata with parser); - args - }; - - assert!( - !args.is_empty(), - "arguments not found! generate_array_type()" - ); - - let no_asc_name = if name.to_string().to_uppercase().starts_with("ASC") { - name.to_string()[3..].to_owned() - } else { - name.to_string() - }; - - let index_asc_type_id_array = format!("{}{}Array", args[0], no_asc_name) - .parse::() - .unwrap(); - - quote! { - #item_struct - - #[automatically_derived] - pub struct #asc_name_array(pub graph_runtime_wasm::asc_abi::class::Array>); - - impl graph::runtime::ToAscObj<#asc_name_array> for Vec<#name> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &graph::runtime::gas::GasCounter, - ) -> Result<#asc_name_array, graph::runtime::HostExportError> { - let content: Result, _> = self.iter().map(|x| graph::runtime::asc_new(heap, x, gas)).collect(); - - Ok(#asc_name_array(graph_runtime_wasm::asc_abi::class::Array::new(&content?, heap, gas)?)) - } - } - - impl graph::runtime::AscType for #asc_name_array { - fn to_asc_bytes(&self) -> Result, graph::runtime::DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &graph::semver::Version, - ) -> Result { - Ok(Self(graph_runtime_wasm::asc_abi::class::Array::from_asc_bytes(asc_obj, api_version)?)) - } - } - - #[automatically_derived] - impl graph::runtime::AscIndexId for #asc_name_array { - const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::#index_asc_type_id_array ; - } - - } - .into() -} diff --git a/runtime/derive/src/generate_asc_type.rs b/runtime/derive/src/generate_asc_type.rs deleted file mode 100644 index 0d79f08482c..00000000000 --- a/runtime/derive/src/generate_asc_type.rs +++ /dev/null @@ -1,172 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, Field, ItemStruct}; - -pub fn generate_asc_type(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let args = parse_macro_input!(metadata as super::Args); - - let name = item_struct.ident.clone(); - let asc_name = Ident::new(&format!("Asc{}", name), Span::call_site()); - - let enum_names = args - .vars - .iter() - .filter(|f| f.ident != super::REQUIRED_IDENT_NAME) - .map(|f| f.ident.to_string()) - .collect::>(); - - //struct's fields -> need to skip enum fields - let mut fields = item_struct - .fields - .iter() - .filter(|f| !enum_names.contains(&f.ident.as_ref().unwrap().to_string())) - .collect::>(); - - //extend fields list with enum's variants - args.vars - .iter() - .filter(|f| f.ident != super::REQUIRED_IDENT_NAME) - .flat_map(|f| f.fields.named.iter()) - .for_each(|f| fields.push(f)); - - let m_fields: Vec = fields - .iter() - .map(|f| { - let fld_name = f.ident.clone().unwrap(); - let typ = field_type_map(field_type(f)); - let fld_type = typ.parse::().unwrap(); - - quote! { - pub #fld_name : #fld_type , - } - }) - .collect(); - - let expanded = quote! { - - #item_struct - - #[automatically_derived] - - #[repr(C)] - #[derive(graph_runtime_derive::AscType)] - #[derive(Debug, Default)] - pub struct #asc_name { - #(#m_fields)* - } - }; - - expanded.into() -} - -fn is_scalar(nm: &str) -> bool { - match nm { - "i8" | "u8" => true, - "i16" | "u16" => true, - "i32" | "u32" => true, - "i64" | "u64" => true, - "usize" | "isize" => true, - "bool" => true, - _ => false, - } -} - -fn field_type_map(tp: String) -> String { - if is_scalar(&tp) { - tp - } else { - match tp.as_ref() { - "String" => "graph_runtime_wasm::asc_abi::class::AscString".into(), - _ => tp.clone(), - } - } -} - -fn field_type(fld: &syn::Field) -> String { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(ps) = tp.path.segments.last() { - let name = ps.ident.to_string(); - //TODO - this must be optimized - match name.as_ref() { - "Vec" => match &ps.arguments { - syn::PathArguments::AngleBracketed(v) => { - if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { - let nm = path_to_string(&p.path); - - match nm.as_ref(){ - "u8" => "graph::runtime::AscPtr".to_owned(), - "Vec" => "graph::runtime::AscPtr".to_owned(), - "String" => "graph::runtime::AscPtr>>".to_owned(), - _ => format!("graph::runtime::AscPtr", path_to_string(&p.path)) - } - } else { - name - } - } - - syn::PathArguments::None => name, - syn::PathArguments::Parenthesized(_v) => { - panic!("syn::PathArguments::Parenthesized is not implemented") - } - }, - "Option" => match &ps.arguments { - syn::PathArguments::AngleBracketed(v) => { - if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { - let tp_nm = path_to_string(&p.path); - if is_scalar(&tp_nm) { - format!("Option<{}>", tp_nm) - } else { - format!("graph::runtime::AscPtr", tp_nm) - } - } else { - name - } - } - - syn::PathArguments::None => name, - syn::PathArguments::Parenthesized(_v) => { - panic!("syn::PathArguments::Parenthesized is not implemented") - } - }, - "String" => { - //format!("graph::runtime::AscPtr", name) - "graph::runtime::AscPtr" - .to_owned() - } - - _ => { - if is_scalar(&name) { - name - } else { - format!("graph::runtime::AscPtr", name) - } - } - } - } else { - "N/A".into() - } - } else { - "N/A".into() - } -} - -//recursive -fn path_to_string(path: &syn::Path) -> String { - if let Some(ps) = path.segments.last() { - let nm = ps.ident.to_string(); - - if let syn::PathArguments::AngleBracketed(v) = &ps.arguments { - if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { - format!("{}<{}>", nm, path_to_string(&p.path)) - } else { - nm - } - } else { - nm - } - } else { - panic!("path_to_string - can't get last segment!") - } -} diff --git a/runtime/derive/src/generate_from_rust_type.rs b/runtime/derive/src/generate_from_rust_type.rs deleted file mode 100644 index 6e24ad78c8c..00000000000 --- a/runtime/derive/src/generate_from_rust_type.rs +++ /dev/null @@ -1,228 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, Field, ItemStruct}; - -pub fn generate_from_rust_type(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let args = parse_macro_input!(metadata as super::Args); - - let enum_names = args - .vars - .iter() - .filter(|f| f.ident != super::REQUIRED_IDENT_NAME) - .map(|f| f.ident.to_string()) - .collect::>(); - - let required_flds = args - .vars - .iter() - .filter(|f| f.ident == super::REQUIRED_IDENT_NAME) - .flat_map(|f| f.fields.named.iter()) - .map(|f| f.ident.as_ref().unwrap().to_string()) - .collect::>(); - - //struct's standard fields - let fields = item_struct - .fields - .iter() - .filter(|f| { - let nm = f.ident.as_ref().unwrap().to_string(); - !enum_names.contains(&nm) && !nm.starts_with('_') - }) - .collect::>(); - - //struct's enum fields - let enum_fields = item_struct - .fields - .iter() - .filter(|f| enum_names.contains(&f.ident.as_ref().unwrap().to_string())) - .collect::>(); - - //module name - let mod_name = Ident::new( - &format!("__{}__", item_struct.ident.to_string().to_lowercase()), - item_struct.ident.span(), - ); - - let name = item_struct.ident.clone(); - let asc_name = Ident::new(&format!("Asc{}", name), Span::call_site()); - - //generate enum fields validator - let enum_validation = enum_fields.iter().map(|f|{ - let fld_name = f.ident.as_ref().unwrap(); //empty, maybe call it "sum"? - let type_nm = format!("\"{}\"", name).parse::().unwrap(); - let fld_nm = format!("\"{}\"", fld_name).parse::().unwrap(); - - quote! { - let #fld_name = self.#fld_name.as_ref() - .ok_or_else(|| graph::runtime::HostExportError::from(graph::runtime::DeterministicHostError::from(anyhow::anyhow!("{} missing {}", #type_nm, #fld_nm))))?; - } - }); - - let mut methods:Vec = - fields.iter().map(|f| { - let fld_name = f.ident.as_ref().unwrap(); - let self_ref = - if is_byte_array(f){ - quote! { graph_runtime_wasm::asc_abi::class::Bytes(&self.#fld_name) } - }else{ - quote!{ self.#fld_name } - }; - - let is_required = is_required(f, &required_flds); - - let setter = - if is_nullable(f) { - if is_required{ - let type_nm = format!("\"{}\"", name).parse::().unwrap(); - let fld_nm = format!("\"{}\"", fld_name).parse::().unwrap(); - - quote! { - #fld_name: graph::runtime::asc_new_or_missing(heap, &#self_ref, gas, #type_nm, #fld_nm)?, - } - }else{ - quote! { - #fld_name: graph::runtime::asc_new_or_null(heap, &#self_ref, gas)?, - } - } - } else if is_scalar(&field_type(f)){ - quote!{ - #fld_name: #self_ref, - } - }else{ - quote! { - #fld_name: graph::runtime::asc_new(heap, &#self_ref, gas)?, - } - }; - setter - }) - .collect(); - - for var in args.vars { - let var_nm = var.ident.to_string(); - if var_nm == super::REQUIRED_IDENT_NAME { - continue; - } - - let mut c = var_nm.chars(); - let var_type_name = c.next().unwrap().to_uppercase().collect::() + c.as_str(); - - var.fields.named.iter().map(|f|{ - - let fld_nm = f.ident.as_ref().unwrap(); - let var_nm = var.ident.clone(); - - use heck::{ToUpperCamelCase, ToSnakeCase}; - - let varian_type_name = fld_nm.to_string().to_upper_camel_case(); - let mod_name = item_struct.ident.to_string().to_snake_case(); - let varian_type_name = format!("{}::{}::{}",mod_name, var_type_name, varian_type_name).parse::().unwrap(); - - if is_byte_array(f){ - quote! { - #fld_nm: if let #varian_type_name(v) = #var_nm {graph::runtime::asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(v), gas)? } else {graph::runtime::AscPtr::null()}, - } - }else{ - quote! { - #fld_nm: if let #varian_type_name(v) = #var_nm {graph::runtime::asc_new(heap, v, gas)? } else {graph::runtime::AscPtr::null()}, - } - } - }) - .for_each(|ts| methods.push(ts)); - } - - let expanded = quote! { - #item_struct - - #[automatically_derived] - mod #mod_name{ - use super::*; - - use crate::protobuf::*; - - impl graph::runtime::ToAscObj<#asc_name> for #name { - - #[allow(unused_variables)] - fn to_asc_obj( - &self, - heap: &mut H, - gas: &graph::runtime::gas::GasCounter, - ) -> Result<#asc_name, graph::runtime::HostExportError> { - - #(#enum_validation)* - - Ok( - #asc_name { - #(#methods)* - ..Default::default() - } - ) - } - } - } // -------- end of mod - - - }; - - expanded.into() -} - -fn is_scalar(fld: &str) -> bool { - match fld { - "i8" | "u8" => true, - "i16" | "u16" => true, - "i32" | "u32" => true, - "i64" | "u64" => true, - "usize" | "isize" => true, - "bool" => true, - _ => false, - } -} - -fn field_type(fld: &syn::Field) -> String { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(ps) = tp.path.segments.last() { - ps.ident.to_string() - } else { - "N/A".into() - } - } else { - "N/A".into() - } -} - -fn is_required(fld: &syn::Field, req_list: &[String]) -> bool { - let fld_name = fld.ident.as_ref().unwrap().to_string(); - req_list.iter().any(|r| r == &fld_name) -} - -fn is_nullable(fld: &syn::Field) -> bool { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(last) = tp.path.segments.last() { - return last.ident == "Option"; - } - } - false -} - -fn is_byte_array(fld: &syn::Field) -> bool { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(last) = tp.path.segments.last() { - if last.ident == "Vec" { - if let syn::PathArguments::AngleBracketed(ref v) = last.arguments { - if let Some(last) = v.args.last() { - if let syn::GenericArgument::Type(t) = last { - if let syn::Type::Path(p) = t { - if let Some(a) = p.path.segments.last() { - return a.ident == "u8"; - } - } - } - } - } - } - } - } - false -} diff --git a/runtime/derive/src/generate_network_type_id.rs b/runtime/derive/src/generate_network_type_id.rs deleted file mode 100644 index 15a586fa6f1..00000000000 --- a/runtime/derive/src/generate_network_type_id.rs +++ /dev/null @@ -1,54 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, ItemStruct}; - -pub fn generate_network_type_id(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let name = item_struct.ident.clone(); - - let asc_name = if name.to_string().to_uppercase().starts_with("ASC") { - name.clone() - } else { - Ident::new(&format!("Asc{}", name), Span::call_site()) - }; - - let no_asc_name = if name.to_string().to_uppercase().starts_with("ASC") { - name.to_string()[3..].to_owned() - } else { - name.to_string() - }; - - let args = { - let mut args = Vec::new(); - let parser = syn::meta::parser(|meta| { - if let Some(ident) = meta.path.get_ident() { - args.push(ident.to_string()); - } - Ok(()) - }); - parse_macro_input!(metadata with parser); - args - }; - - assert!( - !args.is_empty(), - "arguments not found! generate_network_type_id()" - ); - - //type_id variant name - let index_asc_type_id = format!("{}{}", args[0], no_asc_name) - .parse::() - .unwrap(); - - let expanded = quote! { - #item_struct - - #[automatically_derived] - impl graph::runtime::AscIndexId for #asc_name { - const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::#index_asc_type_id ; - } - }; - - expanded.into() -} diff --git a/runtime/derive/src/lib.rs b/runtime/derive/src/lib.rs index 3974ea343b5..6238797ce50 100644 --- a/runtime/derive/src/lib.rs +++ b/runtime/derive/src/lib.rs @@ -4,127 +4,7 @@ extern crate proc_macro; use proc_macro::TokenStream; use quote::quote; -use syn::{ - parse::{Parse, ParseStream}, - Fields, FieldsNamed, Ident, Item, ItemEnum, ItemStruct, Token, -}; - -const REQUIRED_IDENT_NAME: &str = "__required__"; - -struct Args { - vars: Vec, -} - -struct ArgsField { - ident: Ident, - fields: FieldsNamed, -} - -impl Parse for Args { - fn parse(input: ParseStream) -> syn::Result { - let mut idents = Vec::::new(); - - while input.peek(syn::Ident) { - let ident = input.call(Ident::parse)?; - idents.push(ArgsField { - ident, - fields: input.call(FieldsNamed::parse)?, - }); - let _: Option = input.parse()?; - } - - Ok(Args { vars: idents }) - } -} - -//generates graph::runtime::ToAscObj implementation for the type -//takes optional optional list of required fields '__required__{name:TypeName}' and enumerations field decraration with types, i.e. sum{single: ModeInfoSingle,multi: ModeInfoMulti} -//intended use is in build.rs with tonic_build's type_attribute(<...>, <...>) to generate type implementation of graph::runtime::ToAscObj -//Annotation example: -//#[graph_runtime_derive::generate_from_rust_type(...)] -// pub struct MyMessageType { -// .. -// } -//the above annotation will produce following implementation -// impl graph::runtime::ToAscObj for MyMessageType { -// ... -// } -mod generate_from_rust_type; -#[proc_macro_attribute] -pub fn generate_from_rust_type(args: TokenStream, input: TokenStream) -> TokenStream { - generate_from_rust_type::generate_from_rust_type(args, input) -} - -//generates graph::runtime::AscIndexId implementation for the type -//takes required network name attribute to form variant name graph::runtime::IndexForAscTypeId::+ -//Annotation example: -//#[graph_runtime_derive::generate_network_type_id(Cosmos)] -// pub struct MyMessageType { -// .. -// } -//the above annotation will produce following implementation -// impl graph::runtime::AscIndexId for AscMyMessageType { -// const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::CosmosMyMessageType ; -// } - -mod generate_network_type_id; -#[proc_macro_attribute] -pub fn generate_network_type_id(args: TokenStream, input: TokenStream) -> TokenStream { - generate_network_type_id::generate_network_type_id(args, input) -} - -//generates AscType for Type. Takes optional list of non-optional field+type -//Annotation example: -//#[graph_runtime_derive::generate_asc_type(non-optional-field-name: non-optional-field-type,...)] -// pub struct MyMessageType { -// .. -// } -//the above annotation will produce following implementation -// #[repr(C)] -// #[derive(graph_runtime_derive::AscType)] -// #[derive(Debug, Default)] -// pub struct AscMyMessageType { -// ... -// } -// -//Note: this macro makes heavy reliance on types to be available via crate::protobuf (network chain crate root/src/protobuf/lib.rs) -//please see usage exmple in chain::cosmos crate... lib.rs imports generates protobuf bindings, as well as any other needed types -mod generate_asc_type; -#[proc_macro_attribute] -pub fn generate_asc_type(args: TokenStream, input: TokenStream) -> TokenStream { - generate_asc_type::generate_asc_type(args, input) -} - -//generates array type for a type. -//Annotation example: -// #[graph_runtime_derive::generate_array_type(>)] -// pub struct MyMessageType { -// .. -// } -//the above annoation will generate code for MyMessageType type -//Example: -// pub struct AscMyMessageTypeArray(pub graph_runtime_wasm::asc_abi::class::Array>) -//where "AscMyMessageTypeArray" is an array type for "AscMyMessageType" (AscMyMessageType is generated by asc_type derive macro above) -//Macro, also, will generate code for the following 3 trait implementations -//1. graph::runtime::ToAscObj trait -//Example: -// impl graph::runtime::ToAscObj for Vec { -// ... -// } -//2. graph::runtime::AscType -//Example: -// impl graph::runtime::AscType for AscMyMessageTypeArray { -// ... -// } -//3. graph::runtime::AscIndexId (adding expected >Array (CosmosMyMessageTypeArray) variant to graph::runtime::IndexForAscTypeId is manual step) -//impl graph::runtime::AscIndexId for MyMessageTypeArray { -// const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::CosmosMyMessageTypeArray ; -//} -mod generate_array_type; -#[proc_macro_attribute] -pub fn generate_array_type(args: TokenStream, input: TokenStream) -> TokenStream { - generate_array_type::generate_array_type(args, input) -} +use syn::{Fields, Item, ItemEnum, ItemStruct}; #[proc_macro_derive(AscType)] pub fn asc_type_derive(input: TokenStream) -> TokenStream { diff --git a/runtime/test/src/common.rs b/runtime/test/src/common.rs index 7641dd06d8b..461d4a08256 100644 --- a/runtime/test/src/common.rs +++ b/runtime/test/src/common.rs @@ -1,6 +1,7 @@ use ethabi::Contract; use graph::blockchain::BlockTime; use graph::components::store::DeploymentLocator; +use graph::components::subgraph::SharedProofOfIndexing; use graph::data::subgraph::*; use graph::data_source; use graph::data_source::common::MappingABI; @@ -127,7 +128,7 @@ pub fn mock_context( .unwrap(), Default::default(), ), - proof_of_indexing: None, + proof_of_indexing: SharedProofOfIndexing::ignored(), host_fns: Arc::new(Vec::new()), debug_fork: None, mapping_logger: Logger::root(slog::Discard, o!()), diff --git a/runtime/test/src/test_padding.rs b/runtime/test/src/test_padding.rs index ae244be67b3..a68f27f8c61 100644 --- a/runtime/test/src/test_padding.rs +++ b/runtime/test/src/test_padding.rs @@ -17,56 +17,6 @@ fn rnd_sub_graph_name(size: usize) -> String { } pub mod data { - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeBool { - pub str_pref: String, - pub under_test: bool, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeI8 { - pub str_pref: String, - pub under_test: i8, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeU16 { - pub str_pref: String, - pub under_test: u16, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeU32 { - pub str_pref: String, - pub under_test: u32, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - pub struct Bad { pub nonce: u64, pub str_suff: String, @@ -212,46 +162,6 @@ async fn test_v4_manual_padding_should_fail() { manual_padding_should_fail(super::test::API_VERSION_0_0_4).await } -#[tokio::test] -async fn test_v5_bool_padding_ok() { - bool_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_bool_padding_ok() { - bool_padding_ok(super::test::API_VERSION_0_0_4).await -} - -#[tokio::test] -async fn test_v5_i8_padding_ok() { - i8_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_i8_padding_ok() { - i8_padding_ok(super::test::API_VERSION_0_0_4).await -} - -#[tokio::test] -async fn test_v5_u16_padding_ok() { - u16_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_u16_padding_ok() { - u16_padding_ok(super::test::API_VERSION_0_0_4).await -} - -#[tokio::test] -async fn test_v5_u32_padding_ok() { - u32_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_u32_padding_ok() { - u32_padding_ok(super::test::API_VERSION_0_0_4).await -} - async fn manual_padding_should_fail(api_version: semver::Version) { let mut instance = super::test::test_module( &rnd_sub_graph_name(12), @@ -314,131 +224,3 @@ async fn manual_padding_manualy_fixed_ok(api_version: semver::Version) { assert!(res.is_ok(), "{:?}", res.err()); } - -async fn bool_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeBool { - str_pref: "pref".into(), - under_test: true, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_bool") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} - -async fn i8_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeI8 { - str_pref: "pref".into(), - under_test: i8::MAX, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_i8") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} - -async fn u16_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeU16 { - str_pref: "pref".into(), - under_test: i16::MAX as u16, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_i16") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} - -async fn u32_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeU32 { - str_pref: "pref".into(), - under_test: i32::MAX as u32, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_i32") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} diff --git a/runtime/wasm/Cargo.toml b/runtime/wasm/Cargo.toml index 3e74e9f985e..dfbd7983b1c 100644 --- a/runtime/wasm/Cargo.toml +++ b/runtime/wasm/Cargo.toml @@ -11,7 +11,6 @@ graph = { path = "../../graph" } bs58 = "0.4.0" graph-runtime-derive = { path = "../derive" } semver = "1.0.23" -uuid = { version = "1.9.1", features = ["v4"] } anyhow = "1.0" never = "0.1" diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index bd1c8706c4a..12099c55b7e 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::ops::Deref; use std::str::FromStr; use std::time::{Duration, Instant}; @@ -33,18 +32,6 @@ use crate::{error::DeterminismLevel, module::IntoTrap}; use super::module::WasmInstanceData; -fn write_poi_event( - proof_of_indexing: &SharedProofOfIndexing, - poi_event: &ProofOfIndexingEvent, - causality_region: &str, - logger: &Logger, -) { - if let Some(proof_of_indexing) = proof_of_indexing { - let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); - proof_of_indexing.write(logger, causality_region, poi_event); - } -} - impl IntoTrap for HostExportError { fn determinism_level(&self) -> DeterminismLevel { match self { @@ -336,8 +323,7 @@ impl HostExports { .map_err(|e| HostExportError::Deterministic(anyhow!(e)))?; let poi_section = stopwatch.start_section("host_export_store_set__proof_of_indexing"); - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::SetEntity { entity_type: &key.entity_type.typename(), id: &key.entity_id.to_string(), @@ -369,8 +355,7 @@ impl HostExports { entity_id: String, gas: &GasCounter, ) -> Result<(), HostExportError> { - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::RemoveEntity { entity_type: &entity_type, id: &entity_id, diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index 8086051961a..c5678bf2aa2 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -12,6 +12,7 @@ use graph::runtime::gas::Gas; use parity_wasm::elements::ExportEntry; use std::collections::BTreeMap; use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::{panic, thread}; @@ -28,6 +29,8 @@ pub fn spawn_module( where ::MappingTrigger: ToAscPtr, { + static THREAD_COUNT: AtomicUsize = AtomicUsize::new(0); + let valid_module = Arc::new(ValidModule::new(&logger, raw_module, timeout)?); // Create channel for event handling requests @@ -39,8 +42,8 @@ where // In case of failure, this thread may panic or simply terminate, // dropping the `mapping_request_receiver` which ultimately causes the // subgraph to fail the next time it tries to handle an event. - let conf = - thread::Builder::new().name(format!("mapping-{}-{}", &subgraph_id, uuid::Uuid::new_v4())); + let next_id = THREAD_COUNT.fetch_add(1, Ordering::SeqCst); + let conf = thread::Builder::new().name(format!("mapping-{}-{:0>4}", &subgraph_id, next_id)); conf.spawn(move || { let _runtime_guard = runtime.enter(); diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index 4b01b3a5fd8..b911542ffe5 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -70,23 +70,13 @@ impl ToAscPtr for offchain::TriggerData { } } -impl ToAscPtr for subgraph::TriggerData { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - asc_new(heap, &self.entity, gas).map(|ptr| ptr.erase()) - } -} - impl ToAscPtr for subgraph::MappingEntityTrigger { fn to_asc_ptr( self, heap: &mut H, gas: &GasCounter, ) -> Result, HostExportError> { - asc_new(heap, &self.data.entity, gas).map(|ptr| ptr.erase()) + asc_new(heap, &self.data.entity.entity.sorted_ref(), gas).map(|ptr| ptr.erase()) } } diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 9bbe0298abc..6bb7122613f 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -1,13 +1,11 @@ use ethabi; -use graph::blockchain::block_stream::{EntityOperationKind, EntitySourceOperation}; use graph::data::store::scalar::Timestamp; use graph::data::value::Word; use graph::prelude::{BigDecimal, BigInt}; use graph::runtime::gas::GasCounter; use graph::runtime::{ - asc_get, asc_new, AscIndexId, AscPtr, AscType, AscValue, HostExportError, IndexForAscTypeId, - ToAscObj, + asc_get, asc_new, AscIndexId, AscPtr, AscType, AscValue, HostExportError, ToAscObj, }; use graph::{data::store, runtime::DeterministicHostError}; use graph::{prelude::serde_json, runtime::FromAscObj}; @@ -474,39 +472,6 @@ pub enum AscSubgraphEntityOp { Delete, } -#[derive(AscType)] -pub struct AscEntityTrigger { - pub entity_op: AscSubgraphEntityOp, - pub entity_type: AscPtr, - pub entity: AscPtr, - pub vid: i64, -} - -impl ToAscObj for EntitySourceOperation { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let entity_op = match self.entity_op { - EntityOperationKind::Create => AscSubgraphEntityOp::Create, - EntityOperationKind::Modify => AscSubgraphEntityOp::Modify, - EntityOperationKind::Delete => AscSubgraphEntityOp::Delete, - }; - - Ok(AscEntityTrigger { - entity_op, - entity_type: asc_new(heap, &self.entity_type.as_str(), gas)?, - entity: asc_new(heap, &self.entity.sorted_ref(), gas)?, - vid: self.vid, - }) - } -} - -impl AscIndexId for AscEntityTrigger { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::AscEntityTrigger; -} - impl ToAscObj> for serde_yaml::Value { fn to_asc_obj( &self, diff --git a/server/graphman/tests/deployment_query.rs b/server/graphman/tests/deployment_query.rs index 87ac04c3ca3..ee66323716c 100644 --- a/server/graphman/tests/deployment_query.rs +++ b/server/graphman/tests/deployment_query.rs @@ -60,10 +60,10 @@ fn graphql_returns_deployment_info() { let namespace = format!("sgd{}", locator.id); let node = SUBGRAPH_STORE.assigned_node(&locator).unwrap().unwrap(); let qs = STORE - .query_store( - QueryTarget::Deployment(locator.hash.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + locator.hash.clone(), + Default::default(), + )) .await .expect("could get a query store"); let shard = qs.shard(); diff --git a/server/http/src/server.rs b/server/http/src/server.rs index e02fb54fade..f5868cff5b8 100644 --- a/server/http/src/server.rs +++ b/server/http/src/server.rs @@ -32,7 +32,7 @@ impl GraphQLServer { } } - pub async fn start(&self, port: u16, ws_port: u16) -> Result { + pub async fn start(&self, port: u16) -> Result { let logger = self.logger.clone(); info!( @@ -42,7 +42,7 @@ impl GraphQLServer { let graphql_runner = self.graphql_runner.clone(); - let service = Arc::new(GraphQLService::new(logger.clone(), graphql_runner, ws_port)); + let service = Arc::new(GraphQLService::new(logger.clone(), graphql_runner)); start(logger, port, move |req| { let service = service.cheap_clone(); diff --git a/server/http/src/service.rs b/server/http/src/service.rs index c806b9f2b65..8e2237b86ff 100644 --- a/server/http/src/service.rs +++ b/server/http/src/service.rs @@ -48,7 +48,6 @@ fn client_error(msg: impl Into) -> ServerResponse { pub struct GraphQLService { logger: Logger, graphql_runner: Arc, - ws_port: u16, } impl GraphQLService @@ -56,17 +55,15 @@ where Q: GraphQlRunner, { /// Creates a new GraphQL service. - pub fn new(logger: Logger, graphql_runner: Arc, ws_port: u16) -> Self { + pub fn new(logger: Logger, graphql_runner: Arc) -> Self { GraphQLService { logger, graphql_runner, - ws_port, } } fn graphiql_html(&self) -> String { - include_str!("../assets/index.html") - .replace("__WS_PORT__", format!("{}", self.ws_port).as_str()) + include_str!("../assets/index.html").to_string() } async fn index(&self) -> ServerResult { @@ -449,14 +446,6 @@ mod tests { )) } - async fn run_subscription( - self: Arc, - _subscription: Subscription, - _target: QueryTarget, - ) -> Result { - unreachable!(); - } - fn metrics(&self) -> Arc { Arc::new(TestGraphQLMetrics) } @@ -467,7 +456,7 @@ mod tests { let logger = Logger::root(slog::Discard, o!()); let graphql_runner = Arc::new(TestGraphQlRunner); - let service = GraphQLService::new(logger, graphql_runner, 8001); + let service = GraphQLService::new(logger, graphql_runner); let request: Request> = Request::builder() .method(Method::GET) @@ -499,7 +488,7 @@ mod tests { let subgraph_id = USERS.clone(); let graphql_runner = Arc::new(TestGraphQlRunner); - let service = GraphQLService::new(logger, graphql_runner, 8001); + let service = GraphQLService::new(logger, graphql_runner); let request: Request> = Request::builder() .method(Method::POST) @@ -531,7 +520,7 @@ mod tests { let subgraph_id = USERS.clone(); let graphql_runner = Arc::new(TestGraphQlRunner); - let service = GraphQLService::new(logger, graphql_runner, 8001); + let service = GraphQLService::new(logger, graphql_runner); let request: Request> = Request::builder() .method(Method::POST) diff --git a/server/http/tests/server.rs b/server/http/tests/server.rs index a62a27a6c59..3ad78138437 100644 --- a/server/http/tests/server.rs +++ b/server/http/tests/server.rs @@ -63,14 +63,6 @@ impl GraphQlRunner for TestGraphQlRunner { .into() } - async fn run_subscription( - self: Arc, - _subscription: Subscription, - _target: QueryTarget, - ) -> Result { - unreachable!(); - } - fn metrics(&self) -> Arc { Arc::new(TestGraphQLMetrics) } @@ -173,7 +165,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8007, 8008) + .start(8007) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { @@ -205,7 +197,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8002, 8003) + .start(8002) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { @@ -277,7 +269,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8003, 8004) + .start(8003) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { @@ -314,7 +306,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8005, 8006) + .start(8005) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { diff --git a/server/index-node/Cargo.toml b/server/index-node/Cargo.toml index edc438d1279..72b7ff869f7 100644 --- a/server/index-node/Cargo.toml +++ b/server/index-node/Cargo.toml @@ -4,12 +4,11 @@ version.workspace = true edition.workspace = true [dependencies] -blake3 = "1.5" +blake3 = "1.6" graph = { path = "../../graph" } graph-graphql = { path = "../../graphql" } graph-chain-arweave = { path = "../../chain/arweave" } graph-chain-ethereum = { path = "../../chain/ethereum" } graph-chain-near = { path = "../../chain/near" } -graph-chain-cosmos = { path = "../../chain/cosmos" } graph-chain-substreams = { path = "../../chain/substreams" } git-testament = "0.2.5" diff --git a/server/index-node/src/resolver.rs b/server/index-node/src/resolver.rs index 6603d296509..7974afe41db 100644 --- a/server/index-node/src/resolver.rs +++ b/server/index-node/src/resolver.rs @@ -523,23 +523,6 @@ impl IndexNodeResolver { ) .await? } - BlockchainKind::Cosmos => { - let unvalidated_subgraph_manifest = - UnvalidatedSubgraphManifest::::resolve( - deployment_hash.clone(), - raw_yaml, - &self.link_resolver, - &self.logger, - max_spec_version, - ) - .await?; - - Self::validate_and_extract_features( - &self.store.subgraph_store(), - unvalidated_subgraph_manifest, - ) - .await? - } BlockchainKind::Near => { let unvalidated_subgraph_manifest = UnvalidatedSubgraphManifest::::resolve( @@ -698,7 +681,6 @@ impl IndexNodeResolver { // so this seems like the next best thing. try_resolve_for_chain!(graph_chain_ethereum::Chain); try_resolve_for_chain!(graph_chain_arweave::Chain); - try_resolve_for_chain!(graph_chain_cosmos::Chain); try_resolve_for_chain!(graph_chain_near::Chain); // If you're adding support for a new chain and this `match` clause just @@ -710,7 +692,6 @@ impl IndexNodeResolver { BlockchainKind::Substreams | BlockchainKind::Arweave | BlockchainKind::Ethereum - | BlockchainKind::Cosmos | BlockchainKind::Near => (), } @@ -796,8 +777,8 @@ fn entity_changes_to_graphql(entity_changes: Vec) -> r::Value { impl Resolver for IndexNodeResolver { const CACHEABLE: bool = false; - async fn query_permit(&self) -> Result { - self.store.query_permit().await.map_err(Into::into) + async fn query_permit(&self) -> QueryPermit { + self.store.query_permit().await } fn prefetch( diff --git a/server/json-rpc/src/lib.rs b/server/json-rpc/src/lib.rs index b8e5b0330b7..103d36f806c 100644 --- a/server/json-rpc/src/lib.rs +++ b/server/json-rpc/src/lib.rs @@ -21,7 +21,6 @@ impl JsonRpcServer { pub async fn serve( port: u16, http_port: u16, - ws_port: u16, registrar: Arc, node_id: NodeId, logger: Logger, @@ -39,7 +38,6 @@ impl JsonRpcServer { let state = ServerState { registrar, http_port, - ws_port, node_id, logger, }; @@ -87,7 +85,6 @@ impl JsonRpcServer { struct ServerState { registrar: Arc, http_port: u16, - ws_port: u16, node_id: NodeId, logger: Logger, } @@ -123,7 +120,7 @@ impl ServerState { info!(&self.logger, "Received subgraph_deploy request"; "params" => format!("{:?}", params)); let node_id = params.node_id.clone().unwrap_or(self.node_id.clone()); - let routes = subgraph_routes(¶ms.name, self.http_port, self.ws_port); + let routes = subgraph_routes(¶ms.name, self.http_port); match self .registrar .create_subgraph_version( @@ -243,15 +240,11 @@ fn json_rpc_error( ))) } -fn subgraph_routes(name: &SubgraphName, http_port: u16, ws_port: u16) -> JsonValue { +fn subgraph_routes(name: &SubgraphName, http_port: u16) -> JsonValue { let http_base_url = ENV_VARS .external_http_base_url .clone() .unwrap_or_else(|| format!(":{}", http_port)); - let ws_base_url = ENV_VARS - .external_ws_base_url - .clone() - .unwrap_or_else(|| format!(":{}", ws_port)); let mut map = BTreeMap::new(); map.insert( @@ -262,10 +255,6 @@ fn subgraph_routes(name: &SubgraphName, http_port: u16, ws_port: u16) -> JsonVal "queries", format!("{}/subgraphs/name/{}", http_base_url, name), ); - map.insert( - "subscriptions", - format!("{}/subgraphs/name/{}", ws_base_url, name), - ); serde_json::to_value(map).expect("invalid subgraph routes") } diff --git a/server/websocket/Cargo.toml b/server/websocket/Cargo.toml deleted file mode 100644 index 622682732c1..00000000000 --- a/server/websocket/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "graph-server-websocket" -version.workspace = true -edition.workspace = true - -[dependencies] -graph = { path = "../../graph" } -serde = { workspace = true } -serde_derive = { workspace = true } -tokio-tungstenite = "0.23" -uuid = { version = "1.9.1", features = ["v4"] } diff --git a/server/websocket/src/connection.rs b/server/websocket/src/connection.rs deleted file mode 100644 index 571817703f9..00000000000 --- a/server/websocket/src/connection.rs +++ /dev/null @@ -1,436 +0,0 @@ -use graph::futures01::sync::mpsc; -use graph::futures01::{Future, IntoFuture, Sink as _, Stream as _}; -use graph::futures03::future::TryFutureExt; -use graph::futures03::sink::SinkExt; -use graph::futures03::stream::{SplitStream, StreamExt, TryStreamExt}; -use std::collections::HashMap; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio_tungstenite::tungstenite::{ - http::Response as WsResponse, http::StatusCode, Error as WsError, Message as WsMessage, -}; -use tokio_tungstenite::WebSocketStream; -use uuid::Uuid; - -use graph::futures03::compat::Future01CompatExt; -use graph::{data::query::QueryTarget, prelude::*}; - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -struct StartPayload { - query: String, - variables: Option, - operation_name: Option, -} - -/// GraphQL/WebSocket message received from a client. -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum IncomingMessage { - ConnectionInit { - #[allow(dead_code)] - payload: Option, - }, - ConnectionTerminate, - Start { - id: String, - payload: StartPayload, - }, - Stop { - id: String, - }, -} - -impl IncomingMessage { - pub fn from_ws_message(msg: WsMessage) -> Result { - let text = msg.into_text()?; - serde_json::from_str(text.as_str()).map_err(|e| { - let msg = - format!("Invalid GraphQL over WebSocket message: {}: {}", text, e).into_bytes(); - WsError::Http(WsResponse::new(Some(msg))) - }) - } -} - -/// GraphQL/WebSocket message to be sent to the client. -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum OutgoingMessage { - ConnectionAck, - Error { - id: String, - payload: String, - }, - Data { - id: String, - payload: Arc, - }, - Complete { - id: String, - }, -} - -impl OutgoingMessage { - pub fn from_query_result(id: String, result: Arc) -> Self { - OutgoingMessage::Data { - id, - payload: result, - } - } - - pub fn from_error_string(id: String, s: String) -> Self { - OutgoingMessage::Error { id, payload: s } - } -} - -impl From for WsMessage { - fn from(msg: OutgoingMessage) -> Self { - WsMessage::text(serde_json::to_string(&msg).expect("invalid GraphQL/WebSocket message")) - } -} - -/// Helper function to send outgoing messages. -fn send_message( - sink: &mpsc::UnboundedSender, - msg: OutgoingMessage, -) -> Result<(), WsError> { - sink.unbounded_send(msg.into()).map_err(|_| { - let mut response = WsResponse::new(None); - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - WsError::Http(response) - }) -} - -/// Helper function to send error messages. -fn send_error_string( - sink: &mpsc::UnboundedSender, - operation_id: String, - error: String, -) -> Result<(), WsError> { - sink.unbounded_send(OutgoingMessage::from_error_string(operation_id, error).into()) - .map_err(|_| { - let mut response = WsResponse::new(None); - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - WsError::Http(response) - }) -} - -/// Responsible for recording operation ids and stopping them. -/// On drop, cancels all operations. -struct Operations { - operations: HashMap, - msg_sink: mpsc::UnboundedSender, -} - -impl Operations { - fn new(msg_sink: mpsc::UnboundedSender) -> Self { - Self { - operations: HashMap::new(), - msg_sink, - } - } - - fn contains(&self, id: &str) -> bool { - self.operations.contains_key(id) - } - - fn insert(&mut self, id: String, guard: CancelGuard) { - self.operations.insert(id, guard); - } - - fn stop(&mut self, operation_id: String) -> Result<(), WsError> { - // Remove the operation with this ID from the known operations. - match self.operations.remove(&operation_id) { - Some(stopper) => { - // Cancel the subscription result stream. - stopper.cancel(); - - // Send a GQL_COMPLETE to indicate the operation is been completed. - send_message( - &self.msg_sink, - OutgoingMessage::Complete { - id: operation_id.clone(), - }, - ) - } - None => send_error_string( - &self.msg_sink, - operation_id.clone(), - format!("Unknown operation ID: {}", operation_id), - ), - } - } -} - -impl Drop for Operations { - fn drop(&mut self) { - let ids = Vec::from_iter(self.operations.keys().cloned()); - for id in ids { - // Discard errors, the connection is being shutdown anyways. - let _ = self.stop(id); - } - } -} - -/// A WebSocket connection implementing the GraphQL over WebSocket protocol. -pub struct GraphQlConnection { - id: String, - logger: Logger, - graphql_runner: Arc, - stream: WebSocketStream, - deployment: DeploymentHash, -} - -impl GraphQlConnection -where - Q: GraphQlRunner, - S: AsyncRead + AsyncWrite + Send + 'static + Unpin, -{ - /// Creates a new GraphQL subscription service. - pub(crate) fn new( - logger: &Logger, - deployment: DeploymentHash, - stream: WebSocketStream, - graphql_runner: Arc, - ) -> Self { - GraphQlConnection { - id: Uuid::new_v4().to_string(), - logger: logger.new(o!("component" => "GraphQlConnection")), - graphql_runner, - stream, - deployment, - } - } - - async fn handle_incoming_messages( - mut ws_stream: SplitStream>, - mut msg_sink: mpsc::UnboundedSender, - logger: Logger, - connection_id: String, - deployment: DeploymentHash, - graphql_runner: Arc, - ) -> Result<(), WsError> { - let mut operations = Operations::new(msg_sink.clone()); - - // Process incoming messages as long as the WebSocket is open - while let Some(ws_msg) = ws_stream.try_next().await? { - use self::IncomingMessage::*; - use self::OutgoingMessage::*; - - debug!(logger, "Received message"; - "connection" => &connection_id, - "msg" => format!("{}", ws_msg).as_str()); - - let msg = IncomingMessage::from_ws_message(ws_msg.clone())?; - - debug!(logger, "GraphQL/WebSocket message"; - "connection" => &connection_id, - "msg" => format!("{:?}", msg).as_str()); - - match msg { - // Always accept connection init requests - ConnectionInit { payload: _ } => send_message(&msg_sink, ConnectionAck), - - // When receiving a connection termination request - ConnectionTerminate => { - // Close the message sink - msg_sink.close().unwrap(); - - // Return an error here to terminate the connection - Err(WsError::ConnectionClosed) - } - - // When receiving a stop request - Stop { id } => operations.stop(id), - - // When receiving a start request - Start { id, payload } => { - // Respond with a GQL_ERROR if we already have an operation with this ID - if operations.contains(&id) { - return send_error_string( - &msg_sink, - id.clone(), - format!("Operation with ID already started: {}", id), - ); - } - - let max_ops = ENV_VARS.graphql.max_operations_per_connection; - if operations.operations.len() >= max_ops { - return send_error_string( - &msg_sink, - id, - format!("Reached the limit of {} operations per connection", max_ops), - ); - } - - // Parse the GraphQL query document; respond with a GQL_ERROR if - // the query is invalid - let query = match q::parse_query(&payload.query) { - Ok(query) => query.into_static(), - Err(e) => { - return send_error_string( - &msg_sink, - id, - format!("Invalid query: {}: {}", payload.query, e), - ); - } - }; - - // Parse the query variables, if present - let variables = match payload.variables { - None | Some(serde_json::Value::Null) => None, - Some(variables @ serde_json::Value::Object(_)) => { - match serde_json::from_value(variables.clone()) { - Ok(variables) => Some(variables), - Err(e) => { - return send_error_string( - &msg_sink, - id, - format!("Invalid variables provided: {}", e), - ); - } - } - } - _ => { - return send_error_string( - &msg_sink, - id, - "Invalid variables provided (must be an object)".to_string(), - ); - } - }; - - // Construct a subscription - let target = QueryTarget::Deployment(deployment.clone(), Default::default()); - let subscription = Subscription { - // Subscriptions currently do not benefit from the generational cache - // anyways, so don't bother passing a network. - query: Query::new(query, variables, false), - }; - - debug!(logger, "Start operation"; - "connection" => &connection_id, - "id" => &id); - - // Execute the GraphQL subscription - let error_sink = msg_sink.clone(); - let result_sink = msg_sink.clone(); - let result_id = id.clone(); - let err_id = id.clone(); - let err_connection_id = connection_id.clone(); - let err_logger = logger.clone(); - let run_subscription = graphql_runner - .cheap_clone() - .run_subscription(subscription, target) - .compat() - .map_err(move |e| { - debug!(err_logger, "Subscription error"; - "connection" => &err_connection_id, - "id" => &err_id, - "error" => format!("{:?}", e)); - - // Send errors back to the client as GQL_DATA - match e { - SubscriptionError::GraphQLError(e) => { - // Don't bug clients with transient `TooExpensive` errors, - // simply skip updating them - if !e - .iter() - .any(|err| matches!(err, QueryExecutionError::TooExpensive)) - { - let result = Arc::new(QueryResult::from(e)); - let msg = OutgoingMessage::from_query_result( - err_id.clone(), - result, - ); - - // An error means the client closed the websocket, ignore - // and let it be handled in the websocket loop above. - let _ = error_sink.unbounded_send(msg.into()); - } - } - }; - }) - .and_then(move |result_stream| { - // Send results back to the client as GQL_DATA - result_stream - .map(move |result| { - OutgoingMessage::from_query_result(result_id.clone(), result) - }) - .map(WsMessage::from) - .map(Ok) - .compat() - .forward(result_sink.sink_map_err(|_| ())) - .map(|_| ()) - }); - - // Setup cancelation. - let guard = CancelGuard::new(); - let logger = logger.clone(); - let cancel_id = id.clone(); - let connection_id = connection_id.clone(); - let run_subscription = - run_subscription.compat().cancelable(&guard, move || { - debug!(logger, "Stopped operation"; - "connection" => &connection_id, - "id" => &cancel_id); - Ok(()) - }); - operations.insert(id, guard); - - graph::spawn_allow_panic(run_subscription); - Ok(()) - } - }? - } - Ok(()) - } -} - -impl IntoFuture for GraphQlConnection -where - Q: GraphQlRunner, - S: AsyncRead + AsyncWrite + Send + 'static + Unpin, -{ - type Future = Box + Send>; - type Item = (); - type Error = (); - - fn into_future(self) -> Self::Future { - debug!(self.logger, "GraphQL over WebSocket connection opened"; "id" => &self.id); - - // Obtain sink/stream pair to send and receive WebSocket messages - let (ws_sink, ws_stream) = self.stream.split(); - - // Allocate a channel for writing - let (msg_sink, msg_stream) = mpsc::unbounded(); - - // Handle incoming messages asynchronously - let ws_reader = Self::handle_incoming_messages( - ws_stream, - msg_sink, - self.logger.clone(), - self.id.clone(), - self.deployment.clone(), - self.graphql_runner.clone(), - ); - - // Send outgoing messages asynchronously - let ws_writer = msg_stream.forward(ws_sink.compat().sink_map_err(|_| ())); - - // Silently swallow internal send results and errors. There is nothing - // we can do about these errors ourselves. Clients will be disconnected - // as a result of this but most will try to reconnect (GraphiQL for sure, - // Apollo maybe). - let ws_writer = ws_writer.map(|_| ()); - let ws_reader = Box::pin(ws_reader.map_err(|_| ())); - - // Return a future that is fulfilled when either we or the client close - // our/their end of the WebSocket stream - let logger = self.logger.clone(); - let id = self.id.clone(); - Box::new(ws_reader.compat().select(ws_writer).then(move |_| { - debug!(logger, "GraphQL over WebSocket connection closed"; "connection" => id); - Ok(()) - })) - } -} diff --git a/server/websocket/src/lib.rs b/server/websocket/src/lib.rs deleted file mode 100644 index 887fed506fe..00000000000 --- a/server/websocket/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod connection; -mod server; - -pub use self::server::SubscriptionServer; diff --git a/server/websocket/src/server.rs b/server/websocket/src/server.rs deleted file mode 100644 index 9e1178cf0d0..00000000000 --- a/server/websocket/src/server.rs +++ /dev/null @@ -1,217 +0,0 @@ -use crate::connection::GraphQlConnection; -use graph::futures01::IntoFuture as _; -use graph::futures03::compat::Future01CompatExt; -use graph::futures03::future::FutureExt; -use graph::{ - data::query::QueryTarget, - prelude::{SubscriptionServer as SubscriptionServerTrait, *}, -}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::Mutex; -use tokio::net::TcpListener; -use tokio_tungstenite::accept_hdr_async; -use tokio_tungstenite::tungstenite::handshake::server::Request; -use tokio_tungstenite::tungstenite::http::{ - header::ACCESS_CONTROL_ALLOW_ORIGIN, header::CONTENT_TYPE, HeaderValue, Response, StatusCode, -}; - -/// A GraphQL subscription server based on Hyper / Websockets. -pub struct SubscriptionServer { - logger: Logger, - graphql_runner: Arc, - store: Arc, -} - -impl SubscriptionServer -where - Q: GraphQlRunner, - S: QueryStoreManager, -{ - pub fn new(logger: &Logger, graphql_runner: Arc, store: Arc) -> Self { - SubscriptionServer { - logger: logger.new(o!("component" => "SubscriptionServer")), - graphql_runner, - store, - } - } - - async fn subgraph_id_from_url_path( - store: Arc, - path: &str, - ) -> Result, Error> { - fn target_from_name(name: String, api_version: ApiVersion) -> Option { - SubgraphName::new(name) - .ok() - .map(|sub_name| QueryTarget::Name(sub_name, api_version)) - } - - fn target_from_id(id: &str, api_version: ApiVersion) -> Option { - DeploymentHash::new(id) - .ok() - .map(|hash| QueryTarget::Deployment(hash, api_version)) - } - - async fn state( - store: Arc, - target: Option, - ) -> Option { - let target = match target { - Some(target) => target, - None => return None, - }; - match store.query_store(target, false).await.ok() { - Some(query_store) => query_store.deployment_state().await.ok(), - None => None, - } - } - - let path_segments = { - let mut segments = path.split('/'); - - // Remove leading '/' - let first_segment = segments.next(); - if first_segment != Some("") { - return Ok(None); - } - - segments.collect::>() - }; - - match path_segments.as_slice() { - &["subgraphs", "id", subgraph_id] => { - Ok(state(store, target_from_id(subgraph_id, ApiVersion::default())).await) - } - &["subgraphs", "name", _] | &["subgraphs", "name", _, _] => Ok(state( - store, - target_from_name(path_segments[2..].join("/"), ApiVersion::default()), // TODO: version - ) - .await), - &["subgraphs", "network", _, _] => Ok(state( - store, - target_from_name(path_segments[1..].join("/"), ApiVersion::default()), // TODO: version - ) - .await), - _ => Ok(None), - } - } -} - -#[async_trait] -impl SubscriptionServerTrait for SubscriptionServer -where - Q: GraphQlRunner, - S: QueryStoreManager, -{ - async fn serve(self, port: u16) { - info!( - self.logger, - "Starting GraphQL WebSocket server at: ws://localhost:{}", port - ); - - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port); - let socket = TcpListener::bind(&addr) - .await - .expect("Failed to bind WebSocket port"); - - loop { - let stream = match socket.accept().await { - Ok((stream, _)) => stream, - Err(e) => { - trace!(self.logger, "Connection error: {}", e); - continue; - } - }; - let logger = self.logger.clone(); - let logger2 = self.logger.clone(); - let graphql_runner = self.graphql_runner.clone(); - let store = self.store.clone(); - - // Subgraph that the request is resolved to (if any) - let subgraph_id = Arc::new(Mutex::new(None)); - let accept_subgraph_id = subgraph_id.clone(); - - accept_hdr_async(stream, move |request: &Request, mut response: Response<()>| { - // Try to obtain the subgraph ID or name from the URL path. - // Return a 404 if the URL path contains no name/ID segment. - let path = request.uri().path(); - - // `block_in_place` is not recommended but in this case we have no alternative since - // we're in an async context but `tokio_tungstenite` doesn't allow this callback - // to be a future. - let state = tokio::task::block_in_place(|| { - graph::block_on(Self::subgraph_id_from_url_path( - store.clone(), - path, - )) - }) - .map_err(|e| { - error!( - logger, - "Error resolving subgraph ID from URL path"; - "error" => e.to_string() - ); - - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(CONTENT_TYPE, "text/plain") - .body(None) - .unwrap() - }) - .and_then(|state| { - state.ok_or_else(|| { - Response::builder() - .status(StatusCode::NOT_FOUND) - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(CONTENT_TYPE, "text/plain") - .body(None) - .unwrap() - }) - })?; - - // Check if the subgraph is deployed - if !state.is_deployed() { - error!(logger, "Failed to establish WS connection, no data found for subgraph"; - "subgraph_id" => state.id.to_string(), - ); - return Err(Response::builder() - .status(StatusCode::NOT_FOUND) - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(CONTENT_TYPE, "text/plain") - .body(None) - .unwrap()); - } - - *accept_subgraph_id.lock().unwrap() = Some(state.id); - response.headers_mut().insert( - "Sec-WebSocket-Protocol", - HeaderValue::from_static("graphql-ws"), - ); - Ok(response) - }) - .then(move |result| async move { - match result { - Ok(ws_stream) => { - // Obtain the subgraph ID or name that we resolved the request to - let subgraph_id = subgraph_id.lock().unwrap().clone().unwrap(); - - // Spawn a GraphQL over WebSocket connection - let service = GraphQlConnection::new( - &logger2, - subgraph_id, - ws_stream, - graphql_runner.clone(), - ); - - graph::spawn_allow_panic(service.into_future().compat()); - } - Err(e) => { - // We gracefully skip over failed connection attempts rather - // than tearing down the entire stream - trace!(logger2, "Failed to establish WebSocket connection: {}", e); - } - } - }).await - } - } -} diff --git a/store/postgres/Cargo.toml b/store/postgres/Cargo.toml index fa9ea5a20c5..9a746646807 100644 --- a/store/postgres/Cargo.toml +++ b/store/postgres/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [dependencies] async-trait = "0.1.50" -blake3 = "1.5" +blake3 = "1.6" chrono = { workspace = true } derive_more = { version = "0.99.18" } diesel = { workspace = true } @@ -21,12 +21,11 @@ lazy_static = "1.5" lru_time_cache = "0.11" maybe-owned = "0.3.4" postgres = "0.19.1" -openssl = "0.10.64" +openssl = "0.10.71" postgres-openssl = "0.5.0" rand = "0.8.4" serde = { workspace = true } serde_json = { workspace = true } -uuid = { version = "1.9.1", features = ["v4"] } stable-hash_legacy = { git = "https://github.com/graphprotocol/stable-hash", branch = "old", package = "stable-hash" } anyhow = "1.0.86" git-testament = "0.2.5" diff --git a/store/postgres/src/advisory_lock.rs b/store/postgres/src/advisory_lock.rs index bd60d34c634..85e2cf5a4ae 100644 --- a/store/postgres/src/advisory_lock.rs +++ b/store/postgres/src/advisory_lock.rs @@ -6,7 +6,7 @@ //! has more details on advisory locks. //! //! We use the following 64 bit locks: -//! * 1,2: to synchronize on migratons +//! * 1: to synchronize on migratons //! //! We use the following 2x 32-bit locks //! * 1, n: to lock copying of the deployment with id n in the destination @@ -69,17 +69,31 @@ const COPY: Scope = Scope { id: 1 }; const WRITE: Scope = Scope { id: 2 }; const PRUNE: Scope = Scope { id: 3 }; -/// Get a lock for running migrations. Blocks until we get the lock. -pub(crate) fn lock_migration(conn: &mut PgConnection) -> Result<(), StoreError> { - sql_query("select pg_advisory_lock(1)").execute(conn)?; +/// Block until we can get the migration lock, then run `f` and unlock when +/// it is done. This is used to make sure that only one node runs setup at a +/// time. +pub(crate) async fn with_migration_lock( + conn: &mut PgConnection, + f: F, +) -> Result +where + F: FnOnce(&mut PgConnection) -> Fut, + Fut: std::future::Future>, +{ + fn execute(conn: &mut PgConnection, query: &str, msg: &str) -> Result<(), StoreError> { + sql_query(query).execute(conn).map(|_| ()).map_err(|e| { + StoreError::from_diesel_error(&e) + .unwrap_or_else(|| StoreError::Unknown(anyhow::anyhow!("{}: {}", msg, e))) + }) + } - Ok(()) -} + const LOCK: &str = "select pg_advisory_lock(1)"; + const UNLOCK: &str = "select pg_advisory_unlock(1)"; -/// Release the migration lock. -pub(crate) fn unlock_migration(conn: &mut PgConnection) -> Result<(), StoreError> { - sql_query("select pg_advisory_unlock(1)").execute(conn)?; - Ok(()) + execute(conn, LOCK, "failed to acquire migration lock")?; + let res = f(conn).await; + execute(conn, UNLOCK, "failed to release migration lock")?; + res } /// Take the lock used to keep two copy operations to run simultaneously on diff --git a/store/postgres/src/block_store.rs b/store/postgres/src/block_store.rs index 9af40b8d2a0..762a2642524 100644 --- a/store/postgres/src/block_store.rs +++ b/store/postgres/src/block_store.rs @@ -319,11 +319,7 @@ impl BlockStore { } pub(crate) async fn query_permit_primary(&self) -> QueryPermit { - self.mirror - .primary() - .query_permit() - .await - .expect("the primary is never disabled") + self.mirror.primary().query_permit().await } pub fn allocate_chain( @@ -507,7 +503,7 @@ impl BlockStore { }; if let Some(head_block) = store.remove_cursor(&&store.chain)? { - let lower_bound = head_block.saturating_sub(ENV_VARS.reorg_threshold * 2); + let lower_bound = head_block.saturating_sub(ENV_VARS.reorg_threshold() * 2); info!(&self.logger, "Removed cursor for non-firehose chain, now cleaning shallow blocks"; "network" => &store.chain, "lower_bound" => lower_bound); store.cleanup_shallow_blocks(lower_bound)?; } diff --git a/store/postgres/src/catalog.rs b/store/postgres/src/catalog.rs index 8e988e31522..ba532dd53ff 100644 --- a/store/postgres/src/catalog.rs +++ b/store/postgres/src/catalog.rs @@ -398,6 +398,16 @@ pub fn drop_foreign_schema(conn: &mut PgConnection, src: &Site) -> Result<(), St Ok(()) } +pub fn foreign_tables(conn: &mut PgConnection, nsp: &str) -> Result, StoreError> { + use foreign_tables as ft; + + ft::table + .filter(ft::foreign_table_schema.eq(nsp)) + .select(ft::foreign_table_name) + .get_results::(conn) + .map_err(StoreError::from) +} + /// Drop the schema `nsp` and all its contents if it exists, and create it /// again so that `nsp` is an empty schema pub fn recreate_schema(conn: &mut PgConnection, nsp: &str) -> Result<(), StoreError> { @@ -626,6 +636,58 @@ pub fn create_foreign_table( Ok(query) } +/// Create a SQL statement unioning imported tables from all shards, +/// something like +/// +/// ```sql +/// create view "dst_nsp"."src_table" as +/// select 'shard1' as shard, "col1", "col2" from "shard_shard1_subgraphs"."table_name" +/// union all +/// ... +/// ```` +/// +/// The list `shard_nsps` consists of pairs `(name, namespace)` where `name` +/// is the name of the shard and `namespace` is the namespace where the +/// `src_table` is mapped +pub fn create_cross_shard_view( + conn: &mut PgConnection, + src_nsp: &str, + src_table: &str, + dst_nsp: &str, + shard_nsps: &[(&str, String)], +) -> Result { + fn build_query( + columns: &[table_schema::Column], + table_name: &str, + dst_nsp: &str, + shard_nsps: &[(&str, String)], + ) -> Result { + let mut query = String::new(); + write!(query, "create view \"{}\".\"{}\" as ", dst_nsp, table_name)?; + for (idx, (name, nsp)) in shard_nsps.into_iter().enumerate() { + if idx > 0 { + write!(query, " union all ")?; + } + write!(query, "select '{name}' as shard")?; + for column in columns { + write!(query, ", \"{}\"", column.column_name)?; + } + writeln!(query, " from \"{}\".\"{}\"", nsp, table_name)?; + } + Ok(query) + } + + let columns = table_schema::columns(conn, src_nsp, src_table)?; + let query = build_query(&columns, src_table, dst_nsp, shard_nsps).map_err(|_| { + anyhow!( + "failed to generate 'create foreign table' query for {}.{}", + dst_nsp, + src_table + ) + })?; + Ok(query) +} + /// Checks in the database if a given index is valid. pub(crate) fn check_index_is_valid( conn: &mut PgConnection, diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index c2795fca5db..6ff46649494 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -10,6 +10,9 @@ use diesel_migrations::{EmbeddedMigrations, HarnessWithOutput}; use graph::cheap_clone::CheapClone; use graph::components::store::QueryPermit; use graph::constraint_violation; +use graph::derive::CheapClone; +use graph::futures03::future::join_all; +use graph::futures03::FutureExt as _; use graph::prelude::tokio::time::Instant; use graph::prelude::{tokio, MetricsRegistry}; use graph::slog::warn; @@ -33,10 +36,58 @@ use std::{collections::HashMap, sync::RwLock}; use postgres::config::{Config, Host}; -use crate::primary::{self, NAMESPACE_PUBLIC}; -use crate::{advisory_lock, catalog}; +use crate::advisory_lock::with_migration_lock; +use crate::catalog; +use crate::primary::{self, Mirror, Namespace, NAMESPACE_PUBLIC}; use crate::{Shard, PRIMARY_SHARD}; +/// Tables that we map from the primary into `primary_public` in each shard +const PRIMARY_TABLES: [&str; 3] = ["deployment_schemas", "chains", "active_copies"]; + +/// Tables that we map from each shard into each other shard into the +/// `shard__subgraphs` namespace +const SHARDED_TABLES: [(&str, &[&str]); 2] = [ + ("public", &["ethereum_networks"]), + ( + "subgraphs", + &[ + "copy_state", + "copy_table_state", + "dynamic_ethereum_contract_data_source", + "subgraph_deployment", + "subgraph_error", + "subgraph_manifest", + "table_stats", + "subgraph", + "subgraph_version", + "subgraph_deployment_assignment", + ], + ), +]; + +/// Make sure that the tables that `jobs::MirrorJob` wants to mirror are +/// actually mapped into the various shards. A failure here is simply a +/// coding mistake +fn check_mirrored_tables() { + for table in Mirror::PUBLIC_TABLES { + if !PRIMARY_TABLES.contains(&table) { + panic!("table {} is not in PRIMARY_TABLES", table); + } + } + + let subgraphs_tables = *SHARDED_TABLES + .iter() + .find(|(nsp, _)| *nsp == "subgraphs") + .map(|(_, tables)| tables) + .unwrap(); + + for table in Mirror::SUBGRAPHS_TABLES { + if !subgraphs_tables.contains(&table) { + panic!("table {} is not in SHARDED_TABLES[subgraphs]", table); + } + } +} + pub struct ForeignServer { pub name: String, pub shard: Shard, @@ -49,6 +100,7 @@ pub struct ForeignServer { impl ForeignServer { pub(crate) const PRIMARY_PUBLIC: &'static str = "primary_public"; + pub(crate) const CROSS_SHARD_NSP: &'static str = "sharded"; /// The name of the foreign server under which data for `shard` is /// accessible @@ -132,7 +184,11 @@ impl ForeignServer { "\ create server \"{name}\" foreign data wrapper postgres_fdw - options (host '{remote_host}', port '{remote_port}', dbname '{remote_db}', updatable 'false'); + options (host '{remote_host}', \ + port '{remote_port}', \ + dbname '{remote_db}', \ + fetch_size '{fetch_size}', \ + updatable 'false'); create user mapping for current_user server \"{name}\" options (user '{remote_user}', password '{remote_password}');", @@ -142,6 +198,7 @@ impl ForeignServer { remote_db = self.dbname, remote_user = self.user, remote_password = self.password, + fetch_size = ENV_VARS.store.fdw_fetch_size, ); Ok(conn.batch_execute(&query)?) } @@ -160,17 +217,22 @@ impl ForeignServer { let query = format!( "\ alter server \"{name}\" - options (set host '{remote_host}', {set_port} port '{remote_port}', set dbname '{remote_db}'); + options (set host '{remote_host}', \ + {set_port} port '{remote_port}', \ + set dbname '{remote_db}', \ + {set_fetch_size} fetch_size '{fetch_size}'); alter user mapping for current_user server \"{name}\" options (set user '{remote_user}', set password '{remote_password}');", name = self.name, remote_host = self.host, set_port = set_or_add("port"), + set_fetch_size = set_or_add("fetch_size"), remote_port = self.port, remote_db = self.dbname, remote_user = self.user, remote_password = self.password, + fetch_size = ENV_VARS.store.fdw_fetch_size, ); Ok(conn.batch_execute(&query)?) } @@ -181,7 +243,7 @@ impl ForeignServer { catalog::recreate_schema(conn, Self::PRIMARY_PUBLIC)?; let mut query = String::new(); - for table_name in ["deployment_schemas", "chains", "active_copies"] { + for table_name in PRIMARY_TABLES { let create_stmt = if shard == &*PRIMARY_SHARD { format!( "create view {nsp}.{table_name} as select * from public.{table_name};", @@ -209,29 +271,57 @@ impl ForeignServer { let nsp = Self::metadata_schema(&self.shard); catalog::recreate_schema(conn, &nsp)?; let mut query = String::new(); - for table_name in [ - "subgraph_error", - "dynamic_ethereum_contract_data_source", - "table_stats", - "subgraph_deployment_assignment", - "subgraph", - "subgraph_version", - "subgraph_deployment", - "subgraph_manifest", - ] { - let create_stmt = - catalog::create_foreign_table(conn, "subgraphs", table_name, &nsp, &self.name)?; - write!(query, "{}", create_stmt)?; + for (src_nsp, src_tables) in SHARDED_TABLES { + for src_table in src_tables { + let create_stmt = + catalog::create_foreign_table(conn, src_nsp, src_table, &nsp, &self.name)?; + write!(query, "{}", create_stmt)?; + } } Ok(conn.batch_execute(&query)?) } + + fn needs_remap(&self, conn: &mut PgConnection) -> Result { + fn different(mut existing: Vec, mut needed: Vec) -> bool { + existing.sort(); + needed.sort(); + existing != needed + } + + if &self.shard == &*PRIMARY_SHARD { + let existing = catalog::foreign_tables(conn, Self::PRIMARY_PUBLIC)?; + let needed = PRIMARY_TABLES + .into_iter() + .map(String::from) + .collect::>(); + if different(existing, needed) { + return Ok(true); + } + } + + let existing = catalog::foreign_tables(conn, &Self::metadata_schema(&self.shard))?; + let needed = SHARDED_TABLES + .iter() + .flat_map(|(_, tables)| *tables) + .map(|table| table.to_string()) + .collect::>(); + Ok(different(existing, needed)) + } } /// How long to keep connections in the `fdw_pool` around before closing /// them on idle. This is much shorter than the default of 10 minutes. const FDW_IDLE_TIMEOUT: Duration = Duration::from_secs(60); -/// A pool goes through several states, and this enum tracks what state we +enum PoolStateInner { + /// A connection pool, and all the servers for which we need to + /// establish fdw mappings when we call `setup` on the pool + Created(Arc, Arc), + /// The pool has been successfully set up + Ready(Arc), +} + +/// A pool goes through several states, and this struct tracks what state we /// are in, together with the `state_tracker` field on `ConnectionPool`. /// When first created, the pool is in state `Created`; once we successfully /// called `setup` on it, it moves to state `Ready`. During use, we use the @@ -241,20 +331,95 @@ const FDW_IDLE_TIMEOUT: Duration = Duration::from_secs(60); /// database connection. That avoids overall undesirable states like buildup /// of queries; instead of queueing them until the database is available, /// they return almost immediately with an error -enum PoolState { - /// A connection pool, and all the servers for which we need to - /// establish fdw mappings when we call `setup` on the pool - Created(Arc, Arc), - /// The pool has been successfully set up - Ready(Arc), - /// The pool has been disabled by setting its size to 0 - Disabled, +#[derive(Clone, CheapClone)] +struct PoolState { + logger: Logger, + inner: Arc>, } +impl PoolState { + fn new(logger: Logger, inner: PoolStateInner, name: String) -> Self { + let pool_name = format!("pool-{}", name); + Self { + logger, + inner: Arc::new(TimedMutex::new(inner, pool_name)), + } + } + + fn created(pool: Arc, coord: Arc) -> Self { + let logger = pool.logger.clone(); + let name = pool.shard.to_string(); + let inner = PoolStateInner::Created(pool, coord); + Self::new(logger, inner, name) + } + + fn ready(pool: Arc) -> Self { + let logger = pool.logger.clone(); + let name = pool.shard.to_string(); + let inner = PoolStateInner::Ready(pool); + Self::new(logger, inner, name) + } + + fn set_ready(&self) { + use PoolStateInner::*; + + let mut guard = self.inner.lock(&self.logger); + match &*guard { + Created(pool, _) => *guard = Ready(pool.clone()), + Ready(_) => { /* nothing to do */ } + } + } + + /// Get a connection pool that is ready, i.e., has been through setup + /// and running migrations + fn get_ready(&self) -> Result, StoreError> { + // We have to be careful here that we do not hold a lock when we + // call `setup_bg`, otherwise we will deadlock + let (pool, coord) = { + let guard = self.inner.lock(&self.logger); + + use PoolStateInner::*; + match &*guard { + Created(pool, coord) => (pool.cheap_clone(), coord.cheap_clone()), + Ready(pool) => return Ok(pool.clone()), + } + }; + + // self is `Created` and needs to have setup run + coord.setup_bg(self.cheap_clone())?; + + // We just tried to set up the pool; if it is still not set up and + // we didn't have an error, it means the database is not available + if self.needs_setup() { + return Err(StoreError::DatabaseUnavailable); + } else { + Ok(pool) + } + } + + /// Get the inner pool, regardless of whether it has been set up or not. + /// Most uses should use `get_ready` instead + fn get_unready(&self) -> Arc { + use PoolStateInner::*; + + match &*self.inner.lock(&self.logger) { + Created(pool, _) | Ready(pool) => pool.cheap_clone(), + } + } + + fn needs_setup(&self) -> bool { + let guard = self.inner.lock(&self.logger); + + use PoolStateInner::*; + match &*guard { + Created(_, _) => true, + Ready(_) => false, + } + } +} #[derive(Clone)] pub struct ConnectionPool { - inner: Arc>, - logger: Logger, + inner: PoolState, pub shard: Shard, state_tracker: PoolStateTracker, } @@ -267,27 +432,27 @@ impl fmt::Debug for ConnectionPool { } } -/// The name of the pool, mostly for logging, and what purpose it serves. +/// The role of the pool, mostly for logging, and what purpose it serves. /// The main pool will always be called `main`, and can be used for reading /// and writing. Replica pools can only be used for reading, and don't /// require any setup (migrations etc.) -pub enum PoolName { +pub enum PoolRole { Main, Replica(String), } -impl PoolName { +impl PoolRole { fn as_str(&self) -> &str { match self { - PoolName::Main => "main", - PoolName::Replica(name) => name, + PoolRole::Main => "main", + PoolRole::Replica(name) => name, } } fn is_replica(&self) -> bool { match self { - PoolName::Main => false, - PoolName::Replica(_) => true, + PoolRole::Main => false, + PoolRole::Replica(_) => true, } } } @@ -295,12 +460,14 @@ impl PoolName { #[derive(Clone)] struct PoolStateTracker { available: Arc, + ignore_timeout: Arc, } impl PoolStateTracker { fn new() -> Self { Self { available: Arc::new(AtomicBool::new(true)), + ignore_timeout: Arc::new(AtomicBool::new(false)), } } @@ -315,12 +482,26 @@ impl PoolStateTracker { fn is_available(&self) -> bool { self.available.load(Ordering::Relaxed) } + + fn timeout_is_ignored(&self) -> bool { + self.ignore_timeout.load(Ordering::Relaxed) + } + + fn ignore_timeout(&self, f: F) -> R + where + F: FnOnce() -> R, + { + self.ignore_timeout.store(true, Ordering::Relaxed); + let res = f(); + self.ignore_timeout.store(false, Ordering::Relaxed); + res + } } impl ConnectionPool { fn create( shard_name: &str, - pool_name: PoolName, + pool_name: PoolRole, postgres_url: String, pool_size: u32, fdw_pool_size: Option, @@ -331,30 +512,25 @@ impl ConnectionPool { let state_tracker = PoolStateTracker::new(); let shard = Shard::new(shard_name.to_string()).expect("shard_name is a valid name for a shard"); - let pool_state = { - if pool_size == 0 { - PoolState::Disabled + let inner = { + let pool = PoolInner::create( + shard.clone(), + pool_name.as_str(), + postgres_url, + pool_size, + fdw_pool_size, + logger, + registry, + state_tracker.clone(), + ); + if pool_name.is_replica() { + PoolState::ready(Arc::new(pool)) } else { - let pool = PoolInner::create( - shard.clone(), - pool_name.as_str(), - postgres_url, - pool_size, - fdw_pool_size, - logger, - registry, - state_tracker.clone(), - ); - if pool_name.is_replica() { - PoolState::Ready(Arc::new(pool)) - } else { - PoolState::Created(Arc::new(pool), coord) - } + PoolState::created(Arc::new(pool), coord) } }; ConnectionPool { - inner: Arc::new(TimedMutex::new(pool_state, format!("pool-{}", shard_name))), - logger: logger.clone(), + inner, shard, state_tracker, } @@ -363,11 +539,7 @@ impl ConnectionPool { /// This is only used for `graphman` to ensure it doesn't run migrations /// or other setup steps pub fn skip_setup(&self) { - let mut guard = self.inner.lock(&self.logger); - match &*guard { - PoolState::Created(pool, _) => *guard = PoolState::Ready(pool.clone()), - PoolState::Ready(_) | PoolState::Disabled => { /* nothing to do */ } - } + self.inner.set_ready(); } /// Return a pool that is ready, i.e., connected to the database. If the @@ -375,7 +547,6 @@ impl ConnectionPool { /// or the pool is marked as unavailable, return /// `StoreError::DatabaseUnavailable` fn get_ready(&self) -> Result, StoreError> { - let mut guard = self.inner.lock(&self.logger); if !self.state_tracker.is_available() { // We know that trying to use this pool is pointless since the // database is not available, and will only lead to other @@ -384,16 +555,12 @@ impl ConnectionPool { return Err(StoreError::DatabaseUnavailable); } - match &*guard { - PoolState::Created(pool, servers) => { - pool.setup(servers.clone())?; - let pool2 = pool.clone(); - *guard = PoolState::Ready(pool.clone()); + match self.inner.get_ready() { + Ok(pool) => { self.state_tracker.mark_available(); - Ok(pool2) + Ok(pool) } - PoolState::Ready(pool) => Ok(pool.clone()), - PoolState::Disabled => Err(StoreError::DatabaseDisabled), + Err(e) => Err(e), } } @@ -472,53 +639,32 @@ impl ConnectionPool { self.get_ready()?.get_fdw(logger, timeout) } - pub fn connection_detail(&self) -> Result { - let pool = self.get_ready()?; - ForeignServer::new(pool.shard.clone(), &pool.postgres_url).map_err(|e| e.into()) - } - - /// Check that we can connect to the database - pub fn check(&self) -> bool { - true - } - - /// Setup the database for this pool. This includes configuring foreign - /// data wrappers for cross-shard communication, and running any pending - /// schema migrations for this database. - /// - /// # Panics - /// - /// If any errors happen during the migration, the process panics - pub async fn setup(&self) { - let pool = self.clone(); - graph::spawn_blocking_allow_panic(move || { - pool.get_ready().ok(); - }) - .await - // propagate panics - .unwrap(); + /// Get a connection from the pool for foreign data wrapper access if + /// one is available + pub fn try_get_fdw( + &self, + logger: &Logger, + timeout: Duration, + ) -> Option>> { + let Ok(inner) = self.get_ready() else { + return None; + }; + self.state_tracker + .ignore_timeout(|| inner.try_get_fdw(logger, timeout)) } - pub(crate) async fn query_permit(&self) -> Result { - let pool = match &*self.inner.lock(&self.logger) { - PoolState::Created(pool, _) | PoolState::Ready(pool) => pool.clone(), - PoolState::Disabled => { - return Err(StoreError::DatabaseDisabled); - } - }; + pub(crate) async fn query_permit(&self) -> QueryPermit { + let pool = self.inner.get_unready(); let start = Instant::now(); let permit = pool.query_permit().await; - Ok(QueryPermit { + QueryPermit { permit, wait: start.elapsed(), - }) + } } - pub(crate) fn wait_stats(&self) -> Result { - match &*self.inner.lock(&self.logger) { - PoolState::Created(pool, _) | PoolState::Ready(pool) => Ok(pool.wait_stats.clone()), - PoolState::Disabled => Err(StoreError::DatabaseDisabled), - } + pub(crate) fn wait_stats(&self) -> PoolWaitStats { + self.inner.get_unready().wait_stats.cheap_clone() } /// Mirror key tables from the primary into our own schema. We do this @@ -678,6 +824,9 @@ impl HandleEvent for EventHandler { } fn handle_timeout(&self, event: e::TimeoutEvent) { + if self.state_tracker.timeout_is_ignored() { + return; + } self.add_conn_wait_time(event.timeout()); if self.state_tracker.is_available() { error!(self.logger, "Connection checkout timed out"; @@ -732,6 +881,8 @@ impl PoolInner { registry: Arc, state_tracker: PoolStateTracker, ) -> PoolInner { + check_mirrored_tables(); + let logger_store = logger.new(o!("component" => "Store")); let logger_pool = logger.new(o!("component" => "ConnectionPool")); let const_labels = { @@ -914,18 +1065,21 @@ impl PoolInner { self.pool.get().map_err(|_| StoreError::DatabaseUnavailable) } - pub fn get_with_timeout_warning( + /// Get the pool for fdw connections. It is an error if none is configured + fn fdw_pool( &self, logger: &Logger, - ) -> Result>, StoreError> { - loop { - match self.pool.get_timeout(ENV_VARS.store.connection_timeout) { - Ok(conn) => return Ok(conn), - Err(e) => error!(logger, "Error checking out connection, retrying"; - "error" => brief_error_msg(&e), - ), + ) -> Result<&Pool>, StoreError> { + let pool = match &self.fdw_pool { + Some(pool) => pool, + None => { + const MSG: &str = + "internal error: trying to get fdw connection on a pool that doesn't have any"; + error!(logger, "{}", MSG); + return Err(constraint_violation!(MSG)); } - } + }; + Ok(pool) } /// Get a connection from the pool for foreign data wrapper access; @@ -943,15 +1097,7 @@ impl PoolInner { where F: FnMut() -> bool, { - let pool = match &self.fdw_pool { - Some(pool) => pool, - None => { - const MSG: &str = - "internal error: trying to get fdw connection on a pool that doesn't have any"; - error!(logger, "{}", MSG); - return Err(constraint_violation!(MSG)); - } - }; + let pool = self.fdw_pool(logger)?; loop { match pool.get() { Ok(conn) => return Ok(conn), @@ -964,6 +1110,27 @@ impl PoolInner { } } + /// Get a connection from the fdw pool if one is available. We wait for + /// `timeout` for a connection which should be set just big enough to + /// allow establishing a connection + pub fn try_get_fdw( + &self, + logger: &Logger, + timeout: Duration, + ) -> Option>> { + // Any error trying to get a connection is treated as "couldn't get + // a connection in time". If there is a serious error with the + // database, e.g., because it's not available, the next database + // operation will run into it and report it. + let Ok(fdw_pool) = self.fdw_pool(logger) else { + return None; + }; + let Ok(conn) = fdw_pool.get_timeout(timeout) else { + return None; + }; + Some(conn) + } + pub fn connection_detail(&self) -> Result { ForeignServer::new(self.shard.clone(), &self.postgres_url).map_err(|e| e.into()) } @@ -977,68 +1144,25 @@ impl PoolInner { .unwrap_or(false) } - /// Setup the database for this pool. This includes configuring foreign - /// data wrappers for cross-shard communication, and running any pending - /// schema migrations for this database. - /// - /// Returns `StoreError::DatabaseUnavailable` if we can't connect to the - /// database. Any other error causes a panic. - /// - /// # Panics - /// - /// If any errors happen during the migration, the process panics - fn setup(&self, coord: Arc) -> Result<(), StoreError> { - fn die(logger: &Logger, msg: &'static str, err: &dyn std::fmt::Display) -> ! { - crit!(logger, "{}", msg; "error" => format!("{:#}", err)); - panic!("{}: {}", msg, err); - } - - let pool = self.clone(); - let mut conn = self.get().map_err(|_| StoreError::DatabaseUnavailable)?; - - let start = Instant::now(); - - advisory_lock::lock_migration(&mut conn) - .unwrap_or_else(|err| die(&pool.logger, "failed to get migration lock", &err)); - // This code can cause a race in database setup: if pool A has had - // schema changes and pool B then tries to map tables from pool A, - // but does so before the concurrent thread running this code for - // pool B has at least finished `configure_fdw`, mapping tables will - // fail. In that case, the node must be restarted. The restart is - // guaranteed because this failure will lead to a panic in the setup - // for pool A - // - // This code can also leave the table mappings in a state where they - // have not been updated if the process is killed after migrating - // the schema but before finishing remapping in all shards. - // Addressing that would require keeping track of the need to remap - // in the database instead of just in memory - let result = pool - .configure_fdw(coord.servers.as_ref()) - .and_then(|()| migrate_schema(&pool.logger, &mut conn)) - .and_then(|count| coord.propagate(&pool, count)); - debug!(&pool.logger, "Release migration lock"); - advisory_lock::unlock_migration(&mut conn).unwrap_or_else(|err| { - die(&pool.logger, "failed to release migration lock", &err); - }); - result.unwrap_or_else(|err| die(&pool.logger, "migrations failed", &err)); - - // Locale check - if let Err(msg) = catalog::Locale::load(&mut conn)?.suitable() { - if &self.shard == &*PRIMARY_SHARD && primary::is_empty(&mut conn)? { - die( - &pool.logger, + fn locale_check( + &self, + logger: &Logger, + mut conn: PooledConnection>, + ) -> Result<(), StoreError> { + Ok( + if let Err(msg) = catalog::Locale::load(&mut conn)?.suitable() { + if &self.shard == &*PRIMARY_SHARD && primary::is_empty(&mut conn)? { + const MSG: &str = "Database does not use C locale. \ - Please check the graph-node documentation for how to set up the database locale", - &msg, - ); - } else { - warn!(pool.logger, "{}.\nPlease check the graph-node documentation for how to set up the database locale", msg); - } - } + Please check the graph-node documentation for how to set up the database locale"; - debug!(&pool.logger, "Setup finished"; "setup_time_s" => start.elapsed().as_secs()); - Ok(()) + crit!(logger, "{}: {}", MSG, msg); + panic!("{}: {}", MSG, msg); + } else { + warn!(logger, "{}.\nPlease check the graph-node documentation for how to set up the database locale", msg); + } + }, + ) } pub(crate) async fn query_permit(&self) -> tokio::sync::OwnedSemaphorePermit { @@ -1068,6 +1192,101 @@ impl PoolInner { }) } + /// Do the part of database setup that only affects this pool. Those + /// steps are + /// 1. Configuring foreign servers and user mappings for talking to the + /// other shards + /// 2. Migrating the schema to the latest version + /// 3. Checking that the locale is set to C + async fn migrate( + self: Arc, + servers: &[ForeignServer], + ) -> Result { + self.configure_fdw(servers)?; + let mut conn = self.get()?; + let (this, count) = conn.transaction(|conn| -> Result<_, StoreError> { + let count = migrate_schema(&self.logger, conn)?; + Ok((self, count)) + })?; + + this.locale_check(&this.logger, conn)?; + + Ok(count) + } + + /// If this is the primary shard, drop the namespace `CROSS_SHARD_NSP` + fn drop_cross_shard_views(&self) -> Result<(), StoreError> { + if self.shard != *PRIMARY_SHARD { + return Ok(()); + } + + info!(&self.logger, "Dropping cross-shard views"); + let mut conn = self.get()?; + conn.transaction(|conn| { + let query = format!( + "drop schema if exists {} cascade", + ForeignServer::CROSS_SHARD_NSP + ); + conn.batch_execute(&query)?; + Ok(()) + }) + } + + /// If this is the primary shard, create the namespace `CROSS_SHARD_NSP` + /// and populate it with tables that union various imported tables + fn create_cross_shard_views(&self, servers: &[ForeignServer]) -> Result<(), StoreError> { + fn shard_nsp_pairs<'a>( + current: &Shard, + local_nsp: &str, + servers: &'a [ForeignServer], + ) -> Vec<(&'a str, String)> { + servers + .into_iter() + .map(|server| { + let nsp = if &server.shard == current { + local_nsp.to_string() + } else { + ForeignServer::metadata_schema(&server.shard) + }; + (server.shard.as_str(), nsp) + }) + .collect::>() + } + + if self.shard != *PRIMARY_SHARD { + return Ok(()); + } + + let mut conn = self.get()?; + let sharded = Namespace::special(ForeignServer::CROSS_SHARD_NSP); + if catalog::has_namespace(&mut conn, &sharded)? { + // We dropped the namespace before, but another node must have + // recreated it in the meantime so we don't need to do anything + return Ok(()); + } + + info!(&self.logger, "Creating cross-shard views"); + conn.transaction(|conn| { + let query = format!("create schema {}", ForeignServer::CROSS_SHARD_NSP); + conn.batch_execute(&query)?; + for (src_nsp, src_tables) in SHARDED_TABLES { + // Pairs of (shard, nsp) for all servers + let nsps = shard_nsp_pairs(&self.shard, src_nsp, servers); + for src_table in src_tables { + let create_view = catalog::create_cross_shard_view( + conn, + src_nsp, + src_table, + ForeignServer::CROSS_SHARD_NSP, + &nsps, + )?; + conn.batch_execute(&create_view)?; + } + } + Ok(()) + }) + } + /// Copy the data from key tables in the primary into our local schema /// so it can be used as a fallback when the primary goes down pub async fn mirror_primary_tables(&self) -> Result<(), StoreError> { @@ -1082,9 +1301,9 @@ impl PoolInner { .await } - // The foreign server `server` had schema changes, and we therefore need - // to remap anything that we are importing via fdw to make sure we are - // using this updated schema + /// The foreign server `server` had schema changes, and we therefore + /// need to remap anything that we are importing via fdw to make sure we + /// are using this updated schema pub fn remap(&self, server: &ForeignServer) -> Result<(), StoreError> { if &server.shard == &*PRIMARY_SHARD { info!(&self.logger, "Mapping primary"); @@ -1102,6 +1321,15 @@ impl PoolInner { } Ok(()) } + + pub fn needs_remap(&self, server: &ForeignServer) -> Result { + if &server.shard == &self.shard { + return Ok(false); + } + + let mut conn = self.get()?; + server.needs_remap(&mut conn) + } } pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); @@ -1115,10 +1343,6 @@ impl MigrationCount { fn had_migrations(&self) -> bool { self.old != self.new } - - fn is_new(&self) -> bool { - self.old == 0 - } } /// Run all schema migrations. @@ -1158,11 +1382,6 @@ fn migrate_schema(logger: &Logger, conn: &mut PgConnection) -> Result Result>>, + logger: Logger, + pools: Mutex>, servers: Arc>, } impl PoolCoordinator { - pub fn new(servers: Arc>) -> Self { + pub fn new(logger: &Logger, servers: Arc>) -> Self { + let logger = logger.new(o!("component" => "ConnectionPool", "component" => "Coordinator")); Self { + logger, pools: Mutex::new(HashMap::new()), servers, } @@ -1190,7 +1412,7 @@ impl PoolCoordinator { self: Arc, logger: &Logger, name: &str, - pool_name: PoolName, + pool_name: PoolRole, postgres_url: String, pool_size: u32, fdw_pool_size: Option, @@ -1212,20 +1434,12 @@ impl PoolCoordinator { // Ignore non-writable pools (replicas), there is no need (and no // way) to coordinate schema changes with them if is_writable { - // It is safe to take this lock here since nobody has seen the pool - // yet. We remember the `PoolInner` so that later, when we have to - // call `remap()`, we do not have to take this lock as that will be - // already held in `get_ready()` - match &*pool.inner.lock(logger) { - PoolState::Created(inner, _) | PoolState::Ready(inner) => { - self.pools - .lock() - .unwrap() - .insert(pool.shard.clone(), inner.clone()); - } - PoolState::Disabled => { /* nothing to do */ } - } + self.pools + .lock() + .unwrap() + .insert(pool.shard.clone(), pool.inner.cheap_clone()); } + pool } @@ -1233,18 +1447,33 @@ impl PoolCoordinator { /// other pools will then recreate any tables that they imported from /// `shard`. If `pool` is a new shard, we also map all other shards into /// it. + /// + /// This tries to take the migration lock and must therefore be run from + /// code that does _not_ hold the migration lock as it will otherwise + /// deadlock fn propagate(&self, pool: &PoolInner, count: MigrationCount) -> Result<(), StoreError> { - // pool is a new shard, map all other shards into it - if count.is_new() { - for server in self.servers.iter() { + // We need to remap all these servers into `pool` if the list of + // tables that are mapped have changed from the code of the previous + // version. Since dropping and recreating the foreign table + // definitions can slow the startup of other nodes down because of + // locking, we try to only do this when it is actually needed + for server in self.servers.iter() { + if pool.needs_remap(server)? { pool.remap(server)?; } } - // pool had schema changes, refresh the import from pool into all other shards + + // pool had schema changes, refresh the import from pool into all + // other shards. This makes sure that schema changes to + // already-mapped tables are propagated to all other shards. Since + // we run `propagate` after migrations have been applied to `pool`, + // we can be sure that these mappings use the correct schema if count.had_migrations() { let server = self.server(&pool.shard)?; for pool in self.pools.lock().unwrap().values() { - if let Err(e) = pool.remap(server) { + let pool = pool.get_unready(); + let remap_res = pool.remap(server); + if let Err(e) = remap_res { error!(pool.logger, "Failed to map imports from {}", server.shard; "error" => e.to_string()); return Err(e); } @@ -1253,8 +1482,15 @@ impl PoolCoordinator { Ok(()) } + /// Return a list of all pools, regardless of whether they are ready or + /// not. pub fn pools(&self) -> Vec> { - self.pools.lock().unwrap().values().cloned().collect() + self.pools + .lock() + .unwrap() + .values() + .map(|state| state.get_unready()) + .collect::>() } pub fn servers(&self) -> Arc> { @@ -1267,4 +1503,188 @@ impl PoolCoordinator { .find(|server| &server.shard == shard) .ok_or_else(|| constraint_violation!("unknown shard {shard}")) } + + fn primary(&self) -> Result, StoreError> { + let map = self.pools.lock().unwrap(); + let pool_state = map.get(&*&PRIMARY_SHARD).ok_or_else(|| { + constraint_violation!("internal error: primary shard not found in pool coordinator") + })?; + + Ok(pool_state.get_unready()) + } + + /// Setup all pools the coordinator knows about and return the number of + /// pools that were successfully set up. + /// + /// # Panics + /// + /// If any errors besides a database not being available happen during + /// the migration, the process panics + pub async fn setup_all(&self, logger: &Logger) -> usize { + let pools = self + .pools + .lock() + .unwrap() + .values() + .cloned() + .collect::>(); + + let res = self.setup(pools).await; + + match res { + Ok(count) => { + info!(logger, "Setup finished"; "shards" => count); + count + } + Err(e) => { + crit!(logger, "database setup failed"; "error" => format!("{e}")); + panic!("database setup failed: {}", e); + } + } + } + + /// A helper to call `setup` from a non-async context. Returns `true` if + /// the setup was actually run, i.e. if `pool` was available + fn setup_bg(self: Arc, pool: PoolState) -> Result { + let migrated = graph::spawn_thread("database-setup", move || { + graph::block_on(self.setup(vec![pool.clone()])) + }) + .join() + // unwrap: propagate panics + .unwrap()?; + Ok(migrated == 1) + } + + /// Setup all pools by doing the following steps: + /// 1. Get the migration lock in the primary. This makes sure that only + /// one node runs migrations + /// 2. Remove the views in `sharded` as they might interfere with + /// running migrations + /// 3. In parallel, do the following in each pool: + /// 1. Configure fdw servers + /// 2. Run migrations in all pools in parallel + /// 4. In parallel, do the following in each pool: + /// 1. Create/update the mappings in `shard__subgraphs` and in + /// `primary_public` + /// 5. Create the views in `sharded` again + /// 6. Release the migration lock + /// + /// This method tolerates databases that are not available and will + /// simply ignore them. The returned count is the number of pools that + /// were successfully set up. + /// + /// When this method returns, the entries from `states` that were + /// successfully set up will be marked as ready. The method returns the + /// number of pools that were set up + async fn setup(&self, states: Vec) -> Result { + type MigrationCounts = Vec<(PoolState, MigrationCount)>; + + /// Filter out pools that are not available. We don't want to fail + /// because one of the pools is not available. We will just ignore + /// them and continue with the others. + fn filter_unavailable( + (state, res): (PoolState, Result), + ) -> Option> { + if let Err(StoreError::DatabaseUnavailable) = res { + error!( + state.logger, + "migrations failed because database was unavailable" + ); + None + } else { + Some(res.map(|count| (state, count))) + } + } + + /// Migrate all pools in parallel + async fn migrate( + pools: &[PoolState], + servers: &[ForeignServer], + ) -> Result { + let futures = pools + .iter() + .map(|state| { + state + .get_unready() + .cheap_clone() + .migrate(servers) + .map(|res| (state.cheap_clone(), res)) + }) + .collect::>(); + join_all(futures) + .await + .into_iter() + .filter_map(filter_unavailable) + .collect::, _>>() + } + + /// Propagate the schema changes to all other pools in parallel + async fn propagate( + this: &PoolCoordinator, + migrated: MigrationCounts, + ) -> Result, StoreError> { + let futures = migrated + .into_iter() + .map(|(state, count)| async move { + let pool = state.get_unready(); + let res = this.propagate(&pool, count); + (state.cheap_clone(), res) + }) + .collect::>(); + join_all(futures) + .await + .into_iter() + .filter_map(filter_unavailable) + .map(|res| res.map(|(state, ())| state)) + .collect::, _>>() + } + + let primary = self.primary()?; + + let mut pconn = primary.get().map_err(|_| StoreError::DatabaseUnavailable)?; + + let states: Vec<_> = states + .into_iter() + .filter(|pool| pool.needs_setup()) + .collect(); + if states.is_empty() { + return Ok(0); + } + + // Everything here happens under the migration lock. Anything called + // from here should not try to get that lock, otherwise the process + // will deadlock + debug!(self.logger, "Waiting for migration lock"); + let res = with_migration_lock(&mut pconn, |_| async { + debug!(self.logger, "Migration lock acquired"); + + // While we were waiting for the migration lock, another thread + // might have already run this + let states: Vec<_> = states + .into_iter() + .filter(|pool| pool.needs_setup()) + .collect(); + if states.is_empty() { + debug!(self.logger, "No pools to set up"); + return Ok(0); + } + + primary.drop_cross_shard_views()?; + + let migrated = migrate(&states, self.servers.as_ref()).await?; + + let propagated = propagate(&self, migrated).await?; + + primary.create_cross_shard_views(&self.servers)?; + + for state in &propagated { + state.set_ready(); + } + Ok(propagated.len()) + }) + .await; + debug!(self.logger, "Database setup finished"); + + res + } } diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 9b64390581b..2e8807b2aa8 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -13,12 +13,17 @@ //! `graph-node` was restarted while the copy was running. use std::{ convert::TryFrom, - ops::DerefMut, - sync::Arc, + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicI64, Ordering}, + Arc, Mutex, + }, time::{Duration, Instant}, }; use diesel::{ + connection::SimpleConnection as _, dsl::sql, insert_into, r2d2::{ConnectionManager, PooledConnection}, @@ -27,15 +32,20 @@ use diesel::{ }; use graph::{ constraint_violation, - prelude::{info, o, warn, BlockNumber, BlockPtr, Logger, StoreError}, + futures03::{future::select_all, FutureExt as _}, + prelude::{ + info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, + }, schema::EntityType, + slog::error, + tokio, }; use itertools::Itertools; use crate::{ - advisory_lock, catalog, + advisory_lock, catalog, deployment, dynds::DataSourcesTable, - primary::{DeploymentId, Site}, + primary::{DeploymentId, Primary, Site}, relational::index::IndexList, vid_batcher::{VidBatcher, VidRange}, }; @@ -54,6 +64,13 @@ const ACCEPTABLE_REPLICATION_LAG: Duration = Duration::from_secs(30); /// the lag again const REPLICATION_SLEEP: Duration = Duration::from_secs(10); +lazy_static! { + static ref STATEMENT_TIMEOUT: Option = ENV_VARS + .store + .batch_timeout + .map(|duration| format!("set local statement_timeout={}", duration.as_millis())); +} + table! { subgraphs.copy_state(dst) { // deployment_schemas.id @@ -85,32 +102,24 @@ table! { } } -// This is the same as primary::active_copies, but mapped into each shard -table! { - primary_public.active_copies(dst) { - src -> Integer, - dst -> Integer, - cancelled_at -> Nullable, - } -} - -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum Status { Finished, Cancelled, } -#[allow(dead_code)] struct CopyState { src: Arc, dst: Arc, target_block: BlockPtr, - tables: Vec, + finished: Vec, + unfinished: Vec, } impl CopyState { fn new( conn: &mut PgConnection, + primary: Primary, src: Arc, dst: Arc, target_block: BlockPtr, @@ -149,9 +158,9 @@ impl CopyState { src.site.id )); } - Self::load(conn, src, dst, target_block) + Self::load(conn, primary, src, dst, target_block) } - None => Self::create(conn, src, dst, target_block), + None => Self::create(conn, primary.cheap_clone(), src, dst, target_block), }?; Ok(state) @@ -159,21 +168,27 @@ impl CopyState { fn load( conn: &mut PgConnection, + primary: Primary, src: Arc, dst: Arc, target_block: BlockPtr, ) -> Result { - let tables = TableState::load(conn, src.as_ref(), dst.as_ref())?; + let tables = TableState::load(conn, primary, src.as_ref(), dst.as_ref())?; + let (finished, mut unfinished): (Vec<_>, Vec<_>) = + tables.into_iter().partition(|table| table.finished()); + unfinished.sort_by_key(|table| table.dst.object.to_string()); Ok(CopyState { src, dst, target_block, - tables, + finished, + unfinished, }) } fn create( conn: &mut PgConnection, + primary: Primary, src: Arc, dst: Arc, target_block: BlockPtr, @@ -190,7 +205,7 @@ impl CopyState { )) .execute(conn)?; - let mut tables: Vec<_> = dst + let mut unfinished: Vec<_> = dst .tables .values() .filter_map(|dst_table| { @@ -199,6 +214,7 @@ impl CopyState { .map(|src_table| { TableState::init( conn, + primary.cheap_clone(), dst.site.clone(), &src, src_table.clone(), @@ -208,17 +224,17 @@ impl CopyState { }) }) .collect::>()?; - tables.sort_by_key(|table| table.batch.dst.object.to_string()); + unfinished.sort_by_key(|table| table.dst.object.to_string()); - let values = tables + let values = unfinished .iter() .map(|table| { ( - cts::entity_type.eq(table.batch.dst.object.as_str()), + cts::entity_type.eq(table.dst.object.as_str()), cts::dst.eq(dst.site.id), - cts::next_vid.eq(table.batch.next_vid()), - cts::target_vid.eq(table.batch.target_vid()), - cts::batch_size.eq(table.batch.batch_size()), + cts::next_vid.eq(table.batcher.next_vid()), + cts::target_vid.eq(table.batcher.target_vid()), + cts::batch_size.eq(table.batcher.batch_size() as i64), ) }) .collect::>(); @@ -228,7 +244,8 @@ impl CopyState { src, dst, target_block, - tables, + finished: Vec::new(), + unfinished, }) } @@ -272,6 +289,10 @@ impl CopyState { } Ok(()) } + + fn all_tables(&self) -> impl Iterator { + self.finished.iter().chain(self.unfinished.iter()) + } } pub(crate) fn source( @@ -294,57 +315,19 @@ pub(crate) fn source( /// so that we can copy rows from one to the other with very little /// transformation. See `CopyEntityBatchQuery` for the details of what /// exactly that means -pub(crate) struct BatchCopy { +struct TableState { + primary: Primary, src: Arc, dst: Arc
, - batcher: VidBatcher, -} - -impl BatchCopy { - pub fn new(batcher: VidBatcher, src: Arc
, dst: Arc
) -> Self { - Self { src, dst, batcher } - } - - /// Copy one batch of entities and update internal state so that the - /// next call to `run` will copy the next batch - pub fn run(&mut self, conn: &mut PgConnection) -> Result { - let (duration, _) = self.batcher.step(|start, end| { - rq::CopyEntityBatchQuery::new(self.dst.as_ref(), &self.src, start, end)? - .execute(conn)?; - Ok(()) - })?; - - Ok(duration) - } - - pub fn finished(&self) -> bool { - self.batcher.finished() - } - - /// The first `vid` that has not been copied yet - pub fn next_vid(&self) -> i64 { - self.batcher.next_vid() - } - - /// The last `vid` that should be copied - pub fn target_vid(&self) -> i64 { - self.batcher.target_vid() - } - - pub fn batch_size(&self) -> i64 { - self.batcher.batch_size() as i64 - } -} - -struct TableState { - batch: BatchCopy, dst_site: Arc, + batcher: VidBatcher, duration_ms: i64, } impl TableState { fn init( conn: &mut PgConnection, + primary: Primary, dst_site: Arc, src_layout: &Layout, src: Arc
, @@ -354,18 +337,22 @@ impl TableState { let vid_range = VidRange::for_copy(conn, &src, target_block)?; let batcher = VidBatcher::load(conn, &src_layout.site.namespace, src.as_ref(), vid_range)?; Ok(Self { - batch: BatchCopy::new(batcher, src, dst), + primary, + src, + dst, dst_site, + batcher, duration_ms: 0, }) } fn finished(&self) -> bool { - self.batch.finished() + self.batcher.finished() } fn load( conn: &mut PgConnection, + primary: Primary, src_layout: &Layout, dst_layout: &Layout, ) -> Result, StoreError> { @@ -427,11 +414,13 @@ impl TableState { VidRange::new(current_vid, target_vid), )? .with_batch_size(size as usize); - let batch = BatchCopy::new(batcher, src, dst); Ok(TableState { - batch, + primary: primary.cheap_clone(), + src, + dst, dst_site: dst_layout.site.clone(), + batcher, duration_ms, }) } @@ -460,20 +449,20 @@ impl TableState { update( cts::table .filter(cts::dst.eq(self.dst_site.id)) - .filter(cts::entity_type.eq(self.batch.dst.object.as_str())) + .filter(cts::entity_type.eq(self.dst.object.as_str())) .filter(cts::duration_ms.eq(0)), ) .set(cts::started_at.eq(sql("now()"))) .execute(conn)?; let values = ( - cts::next_vid.eq(self.batch.next_vid()), - cts::batch_size.eq(self.batch.batch_size()), + cts::next_vid.eq(self.batcher.next_vid()), + cts::batch_size.eq(self.batcher.batch_size() as i64), cts::duration_ms.eq(self.duration_ms), ); update( cts::table .filter(cts::dst.eq(self.dst_site.id)) - .filter(cts::entity_type.eq(self.batch.dst.object.as_str())), + .filter(cts::entity_type.eq(self.dst.object.as_str())), ) .set(values) .execute(conn)?; @@ -486,7 +475,7 @@ impl TableState { update( cts::table .filter(cts::dst.eq(self.dst_site.id)) - .filter(cts::entity_type.eq(self.batch.dst.object.as_str())), + .filter(cts::entity_type.eq(self.dst.object.as_str())), ) .set(cts::finished_at.eq(sql("now()"))) .execute(conn)?; @@ -494,13 +483,8 @@ impl TableState { } fn is_cancelled(&self, conn: &mut PgConnection) -> Result { - use active_copies as ac; - let dst = self.dst_site.as_ref(); - let canceled = ac::table - .filter(ac::dst.eq(dst.id)) - .select(ac::cancelled_at.is_not_null()) - .get_result::(conn)?; + let canceled = self.primary.is_copy_cancelled(dst)?; if canceled { use copy_state as cs; @@ -512,7 +496,17 @@ impl TableState { } fn copy_batch(&mut self, conn: &mut PgConnection) -> Result { - let duration = self.batch.run(conn)?; + let (duration, count) = self.batcher.step(|start, end| { + let count = rq::CopyEntityBatchQuery::new(self.dst.as_ref(), &self.src, start, end)? + .count_current() + .get_result::(conn) + .optional()?; + Ok(count.unwrap_or(0) as i32) + })?; + + let count = count.unwrap_or(0); + + deployment::update_entity_count(conn, &self.dst_site, count)?; self.record_progress(conn, duration)?; @@ -522,37 +516,57 @@ impl TableState { Ok(Status::Finished) } + + fn set_batch_size(&mut self, conn: &mut PgConnection, size: usize) -> Result<(), StoreError> { + use copy_table_state as cts; + + self.batcher.set_batch_size(size); + + update( + cts::table + .filter(cts::dst.eq(self.dst_site.id)) + .filter(cts::entity_type.eq(self.dst.object.as_str())), + ) + .set(cts::batch_size.eq(self.batcher.batch_size() as i64)) + .execute(conn)?; + + Ok(()) + } } -// A helper for logging progress while data is being copied -struct CopyProgress<'a> { - logger: &'a Logger, - last_log: Instant, +// A helper for logging progress while data is being copied and +// communicating across all copy workers +struct CopyProgress { + logger: Logger, + last_log: Arc>, src: Arc, dst: Arc, - current_vid: i64, + /// The sum of all `target_vid` of tables that have finished + current_vid: AtomicI64, target_vid: i64, + cancelled: AtomicBool, } -impl<'a> CopyProgress<'a> { - fn new(logger: &'a Logger, state: &CopyState) -> Self { +impl CopyProgress { + fn new(logger: Logger, state: &CopyState) -> Self { let target_vid: i64 = state - .tables - .iter() - .map(|table| table.batch.target_vid()) + .all_tables() + .map(|table| table.batcher.target_vid()) .sum(); let current_vid = state - .tables - .iter() - .map(|table| table.batch.next_vid()) + .all_tables() + .filter(|table| table.finished()) + .map(|table| table.batcher.next_vid()) .sum(); + let current_vid = AtomicI64::new(current_vid); Self { logger, - last_log: Instant::now(), + last_log: Arc::new(Mutex::new(Instant::now())), src: state.src.site.clone(), dst: state.dst.site.clone(), current_vid, target_vid, + cancelled: AtomicBool::new(false), } } @@ -567,6 +581,16 @@ impl<'a> CopyProgress<'a> { ); } + fn start_table(&self, table: &TableState) { + info!( + self.logger, + "Starting to copy `{}` entities from {} to {}", + table.dst.object, + table.src.qualified_name, + table.dst.qualified_name + ); + } + fn progress_pct(current_vid: i64, target_vid: i64) -> f64 { // When a step is done, current_vid == target_vid + 1; don't report // more than 100% completion @@ -577,23 +601,37 @@ impl<'a> CopyProgress<'a> { } } - fn update(&mut self, batch: &BatchCopy) { - if self.last_log.elapsed() > LOG_INTERVAL { + fn update(&self, entity_type: &EntityType, batcher: &VidBatcher) { + let mut last_log = self.last_log.lock().unwrap_or_else(|err| { + // Better to clear the poison error and skip a log message than + // crash for no important reason + warn!( + self.logger, + "Lock for progress locking was poisoned, skipping a log message" + ); + let mut last_log = err.into_inner(); + *last_log = Instant::now(); + self.last_log.clear_poison(); + last_log + }); + if last_log.elapsed() > LOG_INTERVAL { + let total_current_vid = self.current_vid.load(Ordering::SeqCst) + batcher.next_vid(); info!( self.logger, "Copied {:.2}% of `{}` entities ({}/{} entity versions), {:.2}% of overall data", - Self::progress_pct(batch.next_vid(), batch.target_vid()), - batch.dst.object, - batch.next_vid(), - batch.target_vid(), - Self::progress_pct(self.current_vid + batch.next_vid(), self.target_vid) + Self::progress_pct(batcher.next_vid(), batcher.target_vid()), + entity_type, + batcher.next_vid(), + batcher.target_vid(), + Self::progress_pct(total_current_vid, self.target_vid) ); - self.last_log = Instant::now(); + *last_log = Instant::now(); } } - fn table_finished(&mut self, batch: &BatchCopy) { - self.current_vid += batch.next_vid(); + fn table_finished(&self, batcher: &VidBatcher) { + self.current_vid + .fetch_add(batcher.next_vid(), Ordering::SeqCst); } fn finished(&self) { @@ -602,6 +640,262 @@ impl<'a> CopyProgress<'a> { "Finished copying data into {}[{}]", self.dst.deployment, self.dst.namespace ); } + + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + } + + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } +} + +enum WorkerResult { + Ok(CopyTableWorker), + Err(StoreError), + Wake, +} + +impl From> for WorkerResult { + fn from(result: Result) -> Self { + match result { + Ok(worker) => WorkerResult::Ok(worker), + Err(e) => WorkerResult::Err(e), + } + } +} + +/// We pass connections back and forth between the control loop and various +/// workers. We need to make sure that we end up with the connection that +/// was used to acquire the copy lock in the right place so we can release +/// the copy lock which is only possible with the connection that acquired +/// it. +/// +/// This struct helps us with that. It wraps a connection and tracks whether +/// the connection was used to acquire the copy lock +struct LockTrackingConnection { + inner: PooledConnection>, + has_lock: bool, +} + +impl LockTrackingConnection { + fn new(inner: PooledConnection>) -> Self { + Self { + inner, + has_lock: false, + } + } + + fn transaction(&mut self, f: F) -> Result + where + F: FnOnce(&mut PgConnection) -> Result, + { + let conn = &mut self.inner; + conn.transaction(|conn| f(conn)) + } + + /// Put `self` into `other` if `self` has the lock. + fn extract(self, other: &mut Option) { + if self.has_lock { + *other = Some(self); + } + } + + fn lock(&mut self, logger: &Logger, dst: &Site) -> Result<(), StoreError> { + if self.has_lock { + warn!(logger, "already acquired copy lock for {}", dst); + return Ok(()); + } + advisory_lock::lock_copying(&mut self.inner, dst)?; + self.has_lock = true; + Ok(()) + } + + fn unlock(&mut self, logger: &Logger, dst: &Site) -> Result<(), StoreError> { + if !self.has_lock { + error!( + logger, + "tried to release copy lock for {} even though we are not the owner", dst + ); + return Ok(()); + } + advisory_lock::unlock_copying(&mut self.inner, dst)?; + self.has_lock = false; + Ok(()) + } +} + +/// A helper to run copying of one table. We need to thread `conn` and +/// `table` from the control loop to the background worker and back again to +/// the control loop. This worker facilitates that +struct CopyTableWorker { + conn: LockTrackingConnection, + table: TableState, + result: Result, +} + +impl CopyTableWorker { + fn new(conn: LockTrackingConnection, table: TableState) -> Self { + Self { + conn, + table, + result: Ok(Status::Cancelled), + } + } + + async fn run(mut self, logger: Logger, progress: Arc) -> WorkerResult { + let object = self.table.dst.object.cheap_clone(); + graph::spawn_blocking_allow_panic(move || { + self.result = self.run_inner(logger, &progress); + self + }) + .await + .map_err(|e| constraint_violation!("copy worker for {} panicked: {}", object, e)) + .into() + } + + fn run_inner(&mut self, logger: Logger, progress: &CopyProgress) -> Result { + use Status::*; + + let conn = &mut self.conn.inner; + progress.start_table(&self.table); + while !self.table.finished() { + // It is important that this check happens outside the write + // transaction so that we do not hold on to locks acquired + // by the check + if self.table.is_cancelled(conn)? || progress.is_cancelled() { + progress.cancel(); + return Ok(Cancelled); + } + + // Pause copying if replication is lagging behind to avoid + // overloading replicas + let mut lag = catalog::replication_lag(conn)?; + if lag > MAX_REPLICATION_LAG { + loop { + info!(logger, + "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", + REPLICATION_SLEEP.as_secs(); + "lag_s" => lag.as_secs()); + std::thread::sleep(REPLICATION_SLEEP); + lag = catalog::replication_lag(conn)?; + if lag <= ACCEPTABLE_REPLICATION_LAG { + break; + } + } + } + + let status = { + loop { + if progress.is_cancelled() { + break Cancelled; + } + + match conn.transaction(|conn| { + if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { + conn.batch_execute(timeout)?; + } + self.table.copy_batch(conn) + }) { + Ok(status) => { + break status; + } + Err(StoreError::StatementTimeout) => { + let timeout = ENV_VARS + .store + .batch_timeout + .map(|t| t.as_secs().to_string()) + .unwrap_or_else(|| "unlimted".to_string()); + warn!( + logger, + "Current batch timed out. Retrying with a smaller batch size."; + "timeout_s" => timeout, + "table" => self.table.dst.qualified_name.as_str(), + "current_vid" => self.table.batcher.next_vid(), + "current_batch_size" => self.table.batcher.batch_size(), + ); + } + Err(e) => { + return Err(e); + } + } + // We hit a timeout. Reset the batch size to 1. + // That's small enough that we will make _some_ + // progress, assuming the timeout is set to a + // reasonable value (several minutes) + // + // Our estimation of batch sizes is generally good + // and stays within the prescribed bounds, but there + // are cases where proper estimation of the batch + // size is nearly impossible since the size of the + // rows in the table jumps sharply at some point + // that is hard to predict. This mechanism ensures + // that if our estimation is wrong, the consequences + // aren't too severe. + conn.transaction(|conn| self.table.set_batch_size(conn, 1))?; + } + }; + + if status == Cancelled { + progress.cancel(); + return Ok(Cancelled); + } + progress.update(&self.table.dst.object, &self.table.batcher); + } + progress.table_finished(&self.table.batcher); + Ok(Finished) + } +} + +/// A helper to manage the workers that are copying data. Besides the actual +/// workers it also keeps a worker that wakes us up periodically to give us +/// a chance to create more workers if there are database connections +/// available +struct Workers { + /// The list of workers that are currently running. This will always + /// include a future that wakes us up periodically + futures: Vec>>>, +} + +impl Workers { + fn new() -> Self { + Self { + futures: vec![Self::waker()], + } + } + + fn add(&mut self, worker: Pin>>) { + self.futures.push(worker); + } + + fn has_work(&self) -> bool { + self.futures.len() > 1 + } + + async fn select(&mut self) -> WorkerResult { + use WorkerResult::*; + + let futures = std::mem::take(&mut self.futures); + let (result, _idx, remaining) = select_all(futures).await; + self.futures = remaining; + match result { + Ok(_) | Err(_) => { /* nothing to do */ } + Wake => { + self.futures.push(Self::waker()); + } + } + result + } + + fn waker() -> Pin>> { + let sleep = tokio::time::sleep(ENV_VARS.store.batch_target_duration); + Box::pin(sleep.map(|()| WorkerResult::Wake)) + } + + /// Return the number of workers that are not the waker + fn len(&self) -> usize { + self.futures.len() - 1 + } } /// A helper for copying subgraphs @@ -609,12 +903,25 @@ pub struct Connection { /// The connection pool for the shard that will contain the destination /// of the copy logger: Logger, - conn: PooledConnection>, + /// We always have one database connection to make sure that copy jobs, + /// once started, can eventually finished so that we don't have + /// different copy jobs that are all half done and have to wait for + /// other jobs to finish + /// + /// This is an `Option` because we need to take this connection out of + /// `self` at some point to spawn a background task to copy an + /// individual table. Except for that case, this will always be + /// `Some(..)`. Most code shouldn't access `self.conn` directly, but use + /// `self.transaction` + conn: Option, + pool: ConnectionPool, + primary: Primary, + workers: usize, src: Arc, dst: Arc, target_block: BlockPtr, - src_manifest_idx_and_name: Vec<(i32, String)>, - dst_manifest_idx_and_name: Vec<(i32, String)>, + src_manifest_idx_and_name: Arc>, + dst_manifest_idx_and_name: Arc>, } impl Connection { @@ -626,6 +933,7 @@ impl Connection { /// is available. pub fn new( logger: &Logger, + primary: Primary, pool: ConnectionPool, src: Arc, dst: Arc, @@ -651,9 +959,15 @@ impl Connection { } false })?; + let src_manifest_idx_and_name = Arc::new(src_manifest_idx_and_name); + let dst_manifest_idx_and_name = Arc::new(dst_manifest_idx_and_name); + let conn = Some(LockTrackingConnection::new(conn)); Ok(Self { logger, conn, + pool, + primary, + workers: ENV_VARS.store.batch_workers, src, dst, target_block, @@ -666,110 +980,251 @@ impl Connection { where F: FnOnce(&mut PgConnection) -> Result, { - self.conn.transaction(|conn| f(conn)) + let Some(conn) = self.conn.as_mut() else { + return Err(constraint_violation!( + "copy connection has been handed to background task but not returned yet (transaction)" + )); + }; + conn.transaction(|conn| f(conn)) } /// Copy private data sources if the source uses a schema version that /// has a private data sources table. The copying is done in its own /// transaction. fn copy_private_data_sources(&mut self, state: &CopyState) -> Result<(), StoreError> { + let src_manifest_idx_and_name = self.src_manifest_idx_and_name.cheap_clone(); + let dst_manifest_idx_and_name = self.dst_manifest_idx_and_name.cheap_clone(); if state.src.site.schema_version.private_data_sources() { - let conn = &mut self.conn; - conn.transaction(|conn| { + self.transaction(|conn| { DataSourcesTable::new(state.src.site.namespace.clone()).copy_to( conn, &DataSourcesTable::new(state.dst.site.namespace.clone()), state.target_block.number, - &self.src_manifest_idx_and_name, - &self.dst_manifest_idx_and_name, + &src_manifest_idx_and_name, + &dst_manifest_idx_and_name, ) })?; } Ok(()) } - pub fn copy_data_internal(&mut self, index_list: IndexList) -> Result { + /// Create a worker using the connection in `self.conn`. This may return + /// `None` if there are no more tables that need to be copied. It is an + /// error to call this if `self.conn` is `None` + fn default_worker( + &mut self, + state: &mut CopyState, + progress: &Arc, + ) -> Option>>> { + let Some(conn) = self.conn.take() else { + return None; + }; + let Some(table) = state.unfinished.pop() else { + self.conn = Some(conn); + return None; + }; + + let worker = CopyTableWorker::new(conn, table); + Some(Box::pin( + worker.run(self.logger.cheap_clone(), progress.cheap_clone()), + )) + } + + /// Opportunistically create an extra worker if we have more tables to + /// copy and there are idle fdw connections. If there are no more tables + /// or no idle connections, this will return `None`. + fn extra_worker( + &mut self, + state: &mut CopyState, + progress: &Arc, + ) -> Option>>> { + // It's important that we get the connection before the table since + // we remove the table from the state and could drop it otherwise + let Some(conn) = self + .pool + .try_get_fdw(&self.logger, ENV_VARS.store.batch_worker_wait) + else { + return None; + }; + let Some(table) = state.unfinished.pop() else { + return None; + }; + let conn = LockTrackingConnection::new(conn); + + let worker = CopyTableWorker::new(conn, table); + Some(Box::pin( + worker.run(self.logger.cheap_clone(), progress.cheap_clone()), + )) + } + + /// Check that we can make progress, i.e., that we have at least one + /// worker that copies as long as there are unfinished tables. This is a + /// safety check to guard against `copy_data_internal` looping forever + /// because of some internal inconsistency + fn assert_progress(&self, num_workers: usize, state: &CopyState) -> Result<(), StoreError> { + if num_workers == 0 && !state.unfinished.is_empty() { + // Something bad happened. We should have at least one + // worker if there are still tables to copy + if self.conn.is_none() { + return Err(constraint_violation!( + "copy connection has been handed to background task but not returned yet (copy_data_internal)" + )); + } else { + return Err(constraint_violation!( + "no workers left but still tables to copy" + )); + } + } + Ok(()) + } + + /// Wait for all workers to finish. This is called when we a worker has + /// failed with an error that forces us to abort copying + async fn cancel_workers(&mut self, progress: Arc, mut workers: Workers) { + progress.cancel(); + error!( + self.logger, + "copying encountered an error; waiting for all workers to finish" + ); + while workers.has_work() { + use WorkerResult::*; + let result = workers.select().await; + match result { + Ok(worker) => { + worker.conn.extract(&mut self.conn); + } + Err(e) => { + /* Ignore; we had an error previously */ + error!(self.logger, "copy worker panicked: {}", e); + } + Wake => { /* Ignore; this is just a waker */ } + } + } + } + + async fn copy_data_internal(&mut self, index_list: IndexList) -> Result { let src = self.src.clone(); let dst = self.dst.clone(); let target_block = self.target_block.clone(); - let mut state = self.transaction(|conn| CopyState::new(conn, src, dst, target_block))?; + let primary = self.primary.cheap_clone(); + let mut state = + self.transaction(|conn| CopyState::new(conn, primary, src, dst, target_block))?; - let logger = &self.logger.clone(); - let mut progress = CopyProgress::new(logger, &state); + let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); - for table in state.tables.iter_mut().filter(|table| !table.finished()) { - while !table.finished() { - // It is important that this check happens outside the write - // transaction so that we do not hold on to locks acquired - // by the check - if table.is_cancelled(&mut self.conn)? { - return Ok(Status::Cancelled); + // Run as many copy jobs as we can in parallel, up to `self.workers` + // many. We can always start at least one worker because of the + // connection in `self.conn`. If the fdw pool has idle connections + // and there are more tables to be copied, we can start more + // workers, up to `self.workers` many + // + // The loop has to be very careful about terminating early so that + // we do not ever leave the loop with `self.conn == None` + let mut workers = Workers::new(); + while !state.unfinished.is_empty() || workers.has_work() { + // We usually add at least one job here, except if we are out of + // tables to copy. In that case, we go through the `while` loop + // every time one of the tables we are currently copying + // finishes + if let Some(worker) = self.default_worker(&mut state, &progress) { + workers.add(worker); + } + loop { + if workers.len() >= self.workers { + break; } + let Some(worker) = self.extra_worker(&mut state, &progress) else { + break; + }; + workers.add(worker); + } - // Pause copying if replication is lagging behind to avoid - // overloading replicas - let mut lag = catalog::replication_lag(&mut self.conn)?; - if lag > MAX_REPLICATION_LAG { - loop { - info!(&self.logger, - "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", - REPLICATION_SLEEP.as_secs(); - "lag_s" => lag.as_secs()); - std::thread::sleep(REPLICATION_SLEEP); - lag = catalog::replication_lag(&mut self.conn)?; - if lag <= ACCEPTABLE_REPLICATION_LAG { - break; + self.assert_progress(workers.len(), &state)?; + let result = workers.select().await; + + // Analyze `result` and take another trip through the loop if + // everything is ok; wait for pending workers and return if + // there was an error or if copying was cancelled. + use WorkerResult as W; + match result { + W::Err(e) => { + // This is a panic in the background task. We need to + // cancel all other tasks and return the error + error!(self.logger, "copy worker panicked: {}", e); + self.cancel_workers(progress, workers).await; + return Err(e); + } + W::Ok(worker) => { + // Put the connection back into self.conn so that we can use it + // in the next iteration. + worker.conn.extract(&mut self.conn); + + match (worker.result, progress.is_cancelled()) { + (Ok(Status::Finished), false) => { + // The worker finished successfully, and nothing was + // cancelled; take another trip through the loop + state.finished.push(worker.table); + } + (Ok(Status::Finished), true) => { + state.finished.push(worker.table); + self.cancel_workers(progress, workers).await; + return Ok(Status::Cancelled); + } + (Ok(Status::Cancelled), _) => { + self.cancel_workers(progress, workers).await; + return Ok(Status::Cancelled); + } + (Err(e), _) => { + error!(self.logger, "copy worker had an error: {}", e); + self.cancel_workers(progress, workers).await; + return Err(e); } } } - - let status = self.transaction(|conn| table.copy_batch(conn))?; - if status == Status::Cancelled { - return Ok(status); + W::Wake => { + // nothing to do, just try to create more workers by + // going through the loop again } - progress.update(&table.batch); - } - progress.table_finished(&table.batch); + }; } + debug_assert!(self.conn.is_some()); // Create indexes for all the attributes that were postponed at the start of // the copy/graft operations. // First recreate the indexes that existed in the original subgraph. - let conn = self.conn.deref_mut(); - for table in state.tables.iter() { + for table in state.all_tables() { let arr = index_list.indexes_for_table( &self.dst.site.namespace, - &table.batch.src.name.to_string(), - &table.batch.dst, + &table.src.name.to_string(), + &table.dst, true, + false, true, )?; for (_, sql) in arr { let query = sql_query(format!("{};", sql)); - query.execute(conn)?; + self.transaction(|conn| query.execute(conn).map_err(StoreError::from))?; } } // Second create the indexes for the new fields. // Here we need to skip those created in the first step for the old fields. - for table in state.tables.iter() { + for table in state.all_tables() { let orig_colums = table - .batch .src .columns .iter() .map(|c| c.name.to_string()) .collect_vec(); for sql in table - .batch .dst - .create_postponed_indexes(orig_colums) + .create_postponed_indexes(orig_colums, false) .into_iter() { let query = sql_query(sql); - query.execute(conn)?; + self.transaction(|conn| query.execute(conn).map_err(StoreError::from))?; } } @@ -797,7 +1252,7 @@ impl Connection { /// lower(v1.block_range) => v2.vid > v1.vid` and we can therefore stop /// the copying of each table as soon as we hit `max_vid = max { v.vid | /// lower(v.block_range) <= target_block.number }`. - pub fn copy_data(&mut self, index_list: IndexList) -> Result { + pub async fn copy_data(mut self, index_list: IndexList) -> Result { // We require sole access to the destination site, and that we get a // consistent view of what has been copied so far. In general, that // is always true. It can happen though that this function runs when @@ -810,9 +1265,33 @@ impl Connection { &self.logger, "Obtaining copy lock (this might take a long time if another process is still copying)" ); - advisory_lock::lock_copying(&mut self.conn, self.dst.site.as_ref())?; - let res = self.copy_data_internal(index_list); - advisory_lock::unlock_copying(&mut self.conn, self.dst.site.as_ref())?; + + let dst_site = self.dst.site.cheap_clone(); + let Some(conn) = self.conn.as_mut() else { + return Err(constraint_violation!( + "copy connection went missing (copy_data)" + )); + }; + conn.lock(&self.logger, &dst_site)?; + + let res = self.copy_data_internal(index_list).await; + + match self.conn.as_mut() { + None => { + // A background worker panicked and left us without our + // dedicated connection; we would need to get that + // connection to unlock the advisory lock. We can't do that, + // so we just log an error + warn!( + self.logger, + "can't unlock copy lock since the default worker panicked; lock will linger until session ends" + ); + } + Some(conn) => { + conn.unlock(&self.logger, &dst_site)?; + } + } + if matches!(res, Ok(Status::Cancelled)) { warn!(&self.logger, "Copying was cancelled and is incomplete"); } diff --git a/store/postgres/src/deployment.rs b/store/postgres/src/deployment.rs index 836048912b1..5d83a563181 100644 --- a/store/postgres/src/deployment.rs +++ b/store/postgres/src/deployment.rs @@ -546,10 +546,14 @@ pub fn revert_block_ptr( // Work around a Diesel issue with serializing BigDecimals to numeric let number = format!("{}::numeric", ptr.number); + // Intention is to revert to a block lower than the reorg threshold, on the other + // hand the earliest we can possibly go is genesys block, so go to genesys even + // if it's within the reorg threshold. + let earliest_block = i32::max(ptr.number - ENV_VARS.reorg_threshold(), 0); let affected_rows = update( d::table .filter(d::deployment.eq(id.as_str())) - .filter(d::earliest_block_number.le(ptr.number - ENV_VARS.reorg_threshold)), + .filter(d::earliest_block_number.le(earliest_block)), ) .set(( d::latest_ethereum_block_number.eq(sql(&number)), @@ -1249,17 +1253,12 @@ pub fn update_entity_count( Ok(()) } -/// Set the deployment's entity count to whatever `full_count_query` produces -pub fn set_entity_count( - conn: &mut PgConnection, - site: &Site, - full_count_query: &str, -) -> Result<(), StoreError> { +/// Set the deployment's entity count back to `0` +pub fn clear_entity_count(conn: &mut PgConnection, site: &Site) -> Result<(), StoreError> { use subgraph_deployment as d; - let full_count_query = format!("({})", full_count_query); update(d::table.filter(d::id.eq(site.id))) - .set(d::entity_count.eq(sql(&full_count_query))) + .set(d::entity_count.eq(BigDecimal::from(0))) .execute(conn)?; Ok(()) } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 99e4409d0ab..92de85f316e 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -42,7 +42,7 @@ use graph::data::subgraph::schema::{DeploymentCreate, SubgraphError}; use graph::prelude::{ anyhow, debug, info, o, warn, web3, AttributeNames, BlockNumber, BlockPtr, CheapClone, DeploymentHash, DeploymentState, Entity, EntityQuery, Error, Logger, QueryExecutionError, - StopwatchMetrics, StoreError, StoreEvent, UnfailOutcome, Value, ENV_VARS, + StopwatchMetrics, StoreError, UnfailOutcome, Value, ENV_VARS, }; use graph::schema::{ApiSchema, EntityKey, EntityType, InputSchema}; use web3::types::Address; @@ -51,7 +51,7 @@ use crate::block_range::{BLOCK_COLUMN, BLOCK_RANGE_COLUMN}; use crate::deployment::{self, OnSync}; use crate::detail::ErrorDetail; use crate::dynds::DataSourcesTable; -use crate::primary::DeploymentId; +use crate::primary::{DeploymentId, Primary}; use crate::relational::index::{CreateIndex, IndexList, Method}; use crate::relational::{Layout, LayoutCache, SqlName, Table}; use crate::relational_queries::FromEntityData; @@ -93,6 +93,8 @@ type PruneHandle = JoinHandle>; pub struct StoreInner { logger: Logger, + primary: Primary, + pool: ConnectionPool, read_only_pools: Vec, @@ -130,6 +132,7 @@ impl Deref for DeploymentStore { impl DeploymentStore { pub fn new( logger: &Logger, + primary: Primary, pool: ConnectionPool, read_only_pools: Vec, mut pool_weights: Vec, @@ -160,6 +163,7 @@ impl DeploymentStore { // Create the store let store = StoreInner { logger: logger.clone(), + primary, pool, read_only_pools, replica_order, @@ -415,7 +419,7 @@ impl DeploymentStore { Ok(conn) } - pub(crate) async fn query_permit(&self, replica: ReplicaId) -> Result { + pub(crate) async fn query_permit(&self, replica: ReplicaId) -> QueryPermit { let pool = match replica { ReplicaId::Main => &self.pool, ReplicaId::ReadOnly(idx) => &self.read_only_pools[idx], @@ -423,7 +427,7 @@ impl DeploymentStore { pool.query_permit().await } - pub(crate) fn wait_stats(&self, replica: ReplicaId) -> Result { + pub(crate) fn wait_stats(&self, replica: ReplicaId) -> PoolWaitStats { match replica { ReplicaId::Main => self.pool.wait_stats(), ReplicaId::ReadOnly(idx) => self.read_only_pools[idx].wait_stats(), @@ -900,34 +904,12 @@ impl DeploymentStore { .await } - pub(crate) fn block_time( - &self, - site: Arc, - block: BlockNumber, - ) -> Result, StoreError> { + pub(crate) fn block_time(&self, site: Arc) -> Result, StoreError> { let store = self.cheap_clone(); let mut conn = self.get_conn()?; let layout = store.layout(&mut conn, site.cheap_clone())?; - if ENV_VARS.store.last_rollup_from_poi { - layout.block_time(&mut conn, block) - } else { - layout.last_rollup(&mut conn) - } - } - - pub(crate) async fn supports_proof_of_indexing<'a>( - &self, - site: Arc, - ) -> Result { - let store = self.clone(); - self.with_conn(move |conn, cancel| { - cancel.check_cancel()?; - let layout = store.layout(conn, site)?; - Ok(layout.supports_proof_of_indexing()) - }) - .await - .map_err(Into::into) + layout.last_rollup(&mut conn) } pub(crate) async fn get_proof_of_indexing( @@ -950,10 +932,6 @@ impl DeploymentStore { let layout = store.layout(conn, site.cheap_clone())?; - if !layout.supports_proof_of_indexing() { - return Ok(None); - } - conn.transaction::<_, CancelableError, _>(move |conn| { let mut block_ptr = block.cheap_clone(); let latest_block_ptr = @@ -1119,18 +1097,12 @@ impl DeploymentStore { last_rollup: Option, stopwatch: &StopwatchMetrics, manifest_idx_and_name: &[(u32, String)], - ) -> Result { + ) -> Result<(), StoreError> { let mut conn = { let _section = stopwatch.start_section("transact_blocks_get_conn"); self.get_conn()? }; - // Emit a store event for the changes we are about to make. We - // wait with sending it until we have done all our other work - // so that we do not hold a lock on the notification queue - // for longer than we have to - let event: StoreEvent = batch.store_event(&site.deployment); - let (layout, earliest_block) = deployment::with_lock(&mut conn, &site, |conn| { conn.transaction(|conn| -> Result<_, StoreError> { // Make the changes @@ -1205,7 +1177,7 @@ impl DeploymentStore { )?; } - Ok(event) + Ok(()) } fn spawn_prune( @@ -1258,6 +1230,15 @@ impl DeploymentStore { site: Arc, req: PruneRequest, ) -> Result<(), StoreError> { + { + if store.is_source(&site)? { + debug!( + logger, + "Skipping pruning since this deployment is being copied" + ); + return Ok(()); + } + } let logger2 = logger.cheap_clone(); retry::forever_async(&logger2, "prune", move || { let store = store.cheap_clone(); @@ -1272,7 +1253,7 @@ impl DeploymentStore { let req = PruneRequest::new( &site.as_ref().into(), history_blocks, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), earliest_block, latest_block, )?; @@ -1294,9 +1275,9 @@ impl DeploymentStore { block_ptr_to: BlockPtr, firehose_cursor: &FirehoseCursor, truncate: bool, - ) -> Result { + ) -> Result<(), StoreError> { let logger = self.logger.cheap_clone(); - let event = deployment::with_lock(conn, &site, |conn| { + deployment::with_lock(conn, &site, |conn| { conn.transaction(|conn| -> Result<_, StoreError> { // The revert functions want the number of the first block that we need to get rid of let block = block_ptr_to.number + 1; @@ -1311,15 +1292,13 @@ impl DeploymentStore { // Revert the data let layout = self.layout(conn, site.clone())?; - let event = if truncate { - let event = layout.truncate_tables(conn)?; - deployment::set_entity_count(conn, site.as_ref(), layout.count_query.as_str())?; - event + if truncate { + layout.truncate_tables(conn)?; + deployment::clear_entity_count(conn, site.as_ref())?; } else { - let (event, count) = layout.revert_block(conn, block)?; + let count = layout.revert_block(conn, block)?; deployment::update_entity_count(conn, site.as_ref(), count)?; - event - }; + } // Revert the meta data changes that correspond to this subgraph. // Only certain meta data changes need to be reverted, most @@ -1328,18 +1307,16 @@ impl DeploymentStore { // changes that might need to be reverted Layout::revert_metadata(&logger, conn, &site, block)?; - Ok(event) + Ok(()) }) - })?; - - Ok(event) + }) } pub(crate) fn truncate( &self, site: Arc, block_ptr_to: BlockPtr, - ) -> Result { + ) -> Result<(), StoreError> { let mut conn = self.get_conn()?; let block_ptr_from = Self::block_ptr_with_conn(&mut conn, site.cheap_clone())?; @@ -1365,11 +1342,7 @@ impl DeploymentStore { ) } - pub(crate) fn rewind( - &self, - site: Arc, - block_ptr_to: BlockPtr, - ) -> Result { + pub(crate) fn rewind(&self, site: Arc, block_ptr_to: BlockPtr) -> Result<(), StoreError> { let mut conn = self.get_conn()?; let block_ptr_from = Self::block_ptr_with_conn(&mut conn, site.cheap_clone())?; @@ -1400,7 +1373,7 @@ impl DeploymentStore { site: Arc, block_ptr_to: BlockPtr, firehose_cursor: &FirehoseCursor, - ) -> Result { + ) -> Result<(), StoreError> { let mut conn = self.get_conn()?; // Unwrap: If we are reverting then the block ptr is not `None`. let deployment_head = Self::block_ptr_with_conn(&mut conn, site.cheap_clone())?.unwrap(); @@ -1449,23 +1422,16 @@ impl DeploymentStore { Ok(()) } - pub(crate) fn replica_for_query( - &self, - for_subscription: bool, - ) -> Result { + pub(crate) fn replica_for_query(&self) -> Result { use std::sync::atomic::Ordering; - let replica_id = match for_subscription { - // Pick a weighted ReplicaId. `replica_order` contains a list of - // replicas with repetitions according to their weight - false => { - let weights_count = self.replica_order.len(); - let index = - self.conn_round_robin_counter.fetch_add(1, Ordering::SeqCst) % weights_count; - *self.replica_order.get(index).unwrap() - } - // Subscriptions always go to the main replica. - true => ReplicaId::Main, + // Pick a weighted ReplicaId. `replica_order` contains a list of + // replicas with repetitions according to their weight + let replica_id = { + let weights_count = self.replica_order.len(); + let index = + self.conn_round_robin_counter.fetch_add(1, Ordering::SeqCst) % weights_count; + *self.replica_order.get(index).unwrap() }; Ok(replica_id) @@ -1520,7 +1486,7 @@ impl DeploymentStore { /// to the graph point, so that calling this needlessly with `Some(..)` /// will remove any progress that might have been made since the last /// time the deployment was started. - pub(crate) fn start_subgraph( + pub(crate) async fn start_subgraph( &self, logger: &Logger, site: Arc, @@ -1548,8 +1514,9 @@ impl DeploymentStore { // as adding new tables in `self`; we only need to check that tables // that actually need to be copied from the source are compatible // with the corresponding tables in `self` - let mut copy_conn = crate::copy::Connection::new( + let copy_conn = crate::copy::Connection::new( logger, + self.primary.cheap_clone(), self.pool.clone(), src.clone(), dst.clone(), @@ -1557,7 +1524,7 @@ impl DeploymentStore { src_manifest_idx_and_name, dst_manifest_idx_and_name, )?; - let status = copy_conn.copy_data(index_list)?; + let status = copy_conn.copy_data(index_list).await?; if status == crate::copy::Status::Cancelled { return Err(StoreError::Canceled); } @@ -1582,6 +1549,12 @@ impl DeploymentStore { catalog::copy_account_like(conn, &src.site, &dst.site)?; + // Analyze all tables for this deployment + info!(logger, "Analyzing all {} tables", dst.tables.len()); + for entity_name in dst.tables.keys() { + self.analyze_with_conn(site.cheap_clone(), entity_name.as_str(), conn)?; + } + // Rewind the subgraph so that entity versions that are // clamped in the future (beyond `block`) become valid for // all blocks after `block`. `revert_block` gets rid of @@ -1592,13 +1565,11 @@ impl DeploymentStore { .number .checked_add(1) .expect("block numbers fit into an i32"); - dst.revert_block(conn, block_to_revert)?; - info!(logger, "Rewound subgraph to block {}", block.number; - "time_ms" => start.elapsed().as_millis()); + info!(logger, "Rewinding to block {}", block.number); + let count = dst.revert_block(conn, block_to_revert)?; + deployment::update_entity_count(conn, &dst.site, count)?; - let start = Instant::now(); - deployment::set_entity_count(conn, &dst.site, &dst.count_query)?; - info!(logger, "Counted the entities"; + info!(logger, "Rewound subgraph to block {}", block.number; "time_ms" => start.elapsed().as_millis()); deployment::set_history_blocks( @@ -1607,11 +1578,6 @@ impl DeploymentStore { src_deployment.manifest.history_blocks, )?; - // Analyze all tables for this deployment - for entity_name in dst.tables.keys() { - self.analyze_with_conn(site.cheap_clone(), entity_name.as_str(), conn)?; - } - // The `earliest_block` for `src` might have changed while // we did the copy if `src` was pruned while we copied; // adjusting it very late in the copy process ensures that @@ -1879,6 +1845,10 @@ impl DeploymentStore { }) .await } + + fn is_source(&self, site: &Site) -> Result { + self.primary.is_source(site) + } } /// Tries to fetch a [`Table`] either by its Entity name or its SQL name. diff --git a/store/postgres/src/dynds/private.rs b/store/postgres/src/dynds/private.rs index e8e7f4ce992..243a7dc5a57 100644 --- a/store/postgres/src/dynds/private.rs +++ b/store/postgres/src/dynds/private.rs @@ -1,8 +1,9 @@ -use std::ops::Bound; +use std::{collections::HashMap, i32, ops::Bound}; use diesel::{ - pg::sql_types, + pg::{sql_types, Pg}, prelude::*, + query_builder::{AstPass, QueryFragment, QueryId}, sql_query, sql_types::{Binary, Bool, Integer, Jsonb, Nullable}, PgConnection, QueryDsl, RunQueryDsl, @@ -16,7 +17,7 @@ use graph::{ prelude::{serde_json, BlockNumber, StoreError}, }; -use crate::primary::Namespace; +use crate::{primary::Namespace, relational_queries::POSTGRES_MAX_PARAMETERS}; type DynTable = diesel_dynamic_schema::Table; type DynColumn = diesel_dynamic_schema::Column; @@ -226,16 +227,12 @@ impl DataSourcesTable { return Ok(count as usize); } - type Tuple = ( - (Bound, Bound), - i32, - Option>, - Option, - i32, - Option, - ); + let manifest_map = + ManifestIdxMap::new(src_manifest_idx_and_name, dst_manifest_idx_and_name); - let src_tuples = self + // Load all data sources that were created up to and including + // `target_block` and transform them ready for insertion + let dss: Vec<_> = self .table .clone() .filter( @@ -250,55 +247,18 @@ impl DataSourcesTable { &self.done_at, )) .order_by(&self.vid) - .load::(conn)?; + .load::(conn)? + .into_iter() + .map(|ds| ds.src_to_dst(target_block, &manifest_map, &self.namespace, &dst.namespace)) + .collect::>()?; + // Split all dss into chunks so that we never use more than + // `POSTGRES_MAX_PARAMETERS` bind variables per chunk + let chunk_size = POSTGRES_MAX_PARAMETERS / CopyDsQuery::BIND_PARAMS; let mut count = 0; - for (block_range, src_manifest_idx, param, context, causality_region, done_at) in src_tuples - { - let name = &src_manifest_idx_and_name - .iter() - .find(|(idx, _)| idx == &src_manifest_idx) - .with_context(|| { - anyhow!( - "the source {} does not have a template with index {}", - self.namespace, - src_manifest_idx - ) - })? - .1; - let dst_manifest_idx = dst_manifest_idx_and_name - .iter() - .find(|(_, n)| n == name) - .with_context(|| { - anyhow!( - "the destination {} is missing a template with name {}. The source {} created one at block {:?}", - dst.namespace, - name, self.namespace, block_range.0 - ) - })? - .0; - - let query = format!( - "\ - insert into {dst}(block_range, manifest_idx, param, context, causality_region, done_at) - values(case - when upper($2) <= $1 then $2 - else int4range(lower($2), null) - end, - $3, $4, $5, $6, $7) - ", - dst = dst.qname - ); - - count += sql_query(query) - .bind::(target_block) - .bind::, _>(block_range) - .bind::(dst_manifest_idx) - .bind::, _>(param) - .bind::, _>(context) - .bind::(causality_region) - .bind::, _>(done_at) - .execute(conn)?; + for chunk in dss.chunks(chunk_size) { + let query = CopyDsQuery::new(dst, chunk)?; + count += query.execute(conn)?; } // If the manifest idxes remained constant, we can test that both tables have the same @@ -361,3 +321,141 @@ impl DataSourcesTable { .optional()?) } } + +/// Map src manifest indexes to dst manifest indexes. If the +/// destination is missing an entry, put `None` as the value for the +/// source index +struct ManifestIdxMap { + map: HashMap, String)>, +} + +impl ManifestIdxMap { + fn new(src: &[(i32, String)], dst: &[(i32, String)]) -> Self { + let dst_idx_map: HashMap<&String, i32> = + HashMap::from_iter(dst.iter().map(|(idx, name)| (name, *idx))); + let map = src + .iter() + .map(|(src_idx, src_name)| { + ( + *src_idx, + (dst_idx_map.get(src_name).copied(), src_name.to_string()), + ) + }) + .collect(); + ManifestIdxMap { map } + } + + fn dst_idx( + &self, + src_idx: i32, + src_nsp: &Namespace, + src_created: BlockNumber, + dst_nsp: &Namespace, + ) -> Result { + let (dst_idx, name) = self.map.get(&src_idx).with_context(|| { + anyhow!( + "the source {src_nsp} does not have a template with \ + index {src_idx} but created one at block {src_created}" + ) + })?; + let dst_idx = dst_idx.with_context(|| { + anyhow!( + "the destination {dst_nsp} is missing a template with \ + name {name}. The source {src_nsp} created one at block {src_created}" + ) + })?; + Ok(dst_idx) + } +} + +#[derive(Queryable)] +struct DsForCopy { + block_range: (Bound, Bound), + idx: i32, + param: Option>, + context: Option, + causality_region: i32, + done_at: Option, +} + +impl DsForCopy { + fn src_to_dst( + mut self, + target_block: BlockNumber, + map: &ManifestIdxMap, + src_nsp: &Namespace, + dst_nsp: &Namespace, + ) -> Result { + // unclamp block range if it ends beyond target block + match self.block_range.1 { + Bound::Included(block) if block > target_block => self.block_range.1 = Bound::Unbounded, + Bound::Excluded(block) if block - 1 > target_block => { + self.block_range.1 = Bound::Unbounded + } + _ => { /* use block range as is */ } + } + // Translate manifest index + let src_created = match self.block_range.0 { + Bound::Included(block) => block, + Bound::Excluded(block) => block + 1, + Bound::Unbounded => 0, + }; + self.idx = map.dst_idx(self.idx, src_nsp, src_created, dst_nsp)?; + Ok(self) + } +} + +struct CopyDsQuery<'a> { + dst: &'a DataSourcesTable, + dss: &'a [DsForCopy], +} + +impl<'a> CopyDsQuery<'a> { + const BIND_PARAMS: usize = 6; + + fn new(dst: &'a DataSourcesTable, dss: &'a [DsForCopy]) -> Result { + Ok(CopyDsQuery { dst, dss }) + } +} + +impl<'a> QueryFragment for CopyDsQuery<'a> { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { + out.unsafe_to_cache_prepared(); + out.push_sql("insert into "); + out.push_sql(&self.dst.qname); + out.push_sql( + "(block_range, manifest_idx, param, context, causality_region, done_at) values ", + ); + let mut first = true; + for ds in self.dss.iter() { + if first { + first = false; + } else { + out.push_sql(", "); + } + out.push_sql("("); + out.push_bind_param::, _>(&ds.block_range)?; + out.push_sql(", "); + out.push_bind_param::(&ds.idx)?; + out.push_sql(", "); + out.push_bind_param::, _>(&ds.param)?; + out.push_sql(", "); + out.push_bind_param::, _>(&ds.context)?; + out.push_sql(", "); + out.push_bind_param::(&ds.causality_region)?; + out.push_sql(", "); + out.push_bind_param::, _>(&ds.done_at)?; + out.push_sql(")"); + } + + Ok(()) + } +} + +impl<'a> QueryId for CopyDsQuery<'a> { + type QueryId = (); + + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl<'a, Conn> RunQueryDsl for CopyDsQuery<'a> {} diff --git a/store/postgres/src/primary.rs b/store/postgres/src/primary.rs index d62ab74da06..39df898ba32 100644 --- a/store/postgres/src/primary.rs +++ b/store/postgres/src/primary.rs @@ -36,11 +36,12 @@ use graph::{ store::scalar::ToPrimitive, subgraph::{status, DeploymentFeatures}, }, + derive::CheapClone, prelude::{ anyhow, chrono::{DateTime, Utc}, - serde_json, DeploymentHash, EntityChange, EntityChangeOperation, NodeId, StoreError, - SubgraphName, SubgraphVersionSwitchingMode, + serde_json, AssignmentChange, DeploymentHash, NodeId, StoreError, SubgraphName, + SubgraphVersionSwitchingMode, }, }; use graph::{ @@ -53,9 +54,9 @@ use maybe_owned::MaybeOwnedMut; use std::{ borrow::Borrow, collections::HashMap, - convert::TryFrom, - convert::TryInto, + convert::{TryFrom, TryInto}, fmt, + sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; @@ -266,6 +267,13 @@ impl Namespace { Namespace(format!("prune{id}")) } + /// A namespace that is not a deployment namespace. This is used for + /// special namespaces we use. No checking is done on `s` and the caller + /// must ensure it's a valid namespace name + pub fn special(s: impl Into) -> Self { + Namespace(s.into()) + } + pub fn as_str(&self) -> &str { &self.0 } @@ -790,7 +798,7 @@ impl<'a> Connection<'a> { /// Delete all assignments for deployments that are neither the current nor the /// pending version of a subgraph and return the deployment id's - fn remove_unused_assignments(&mut self) -> Result, StoreError> { + fn remove_unused_assignments(&mut self) -> Result, StoreError> { use deployment_schemas as ds; use subgraph as s; use subgraph_deployment_assignment as a; @@ -827,12 +835,7 @@ impl<'a> Connection<'a> { .into_iter() .map(|(id, hash)| { DeploymentHash::new(hash) - .map(|hash| { - EntityChange::for_assignment( - DeploymentLocator::new(id.into(), hash), - EntityChangeOperation::Removed, - ) - }) + .map(|hash| AssignmentChange::removed(DeploymentLocator::new(id.into(), hash))) .map_err(|id| { StoreError::ConstraintViolation(format!( "invalid id `{}` for deployment assignment", @@ -851,7 +854,7 @@ impl<'a> Connection<'a> { pub fn promote_deployment( &mut self, id: &DeploymentHash, - ) -> Result, StoreError> { + ) -> Result, StoreError> { use subgraph as s; use subgraph_version as v; @@ -926,7 +929,7 @@ impl<'a> Connection<'a> { node_id: NodeId, mode: SubgraphVersionSwitchingMode, exists_and_synced: F, - ) -> Result, StoreError> + ) -> Result, StoreError> where F: Fn(&DeploymentHash) -> Result, { @@ -1034,13 +1037,16 @@ impl<'a> Connection<'a> { // Clean up any assignments we might have displaced let mut changes = self.remove_unused_assignments()?; if new_assignment { - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); changes.push(change); } Ok(changes) } - pub fn remove_subgraph(&mut self, name: SubgraphName) -> Result, StoreError> { + pub fn remove_subgraph( + &mut self, + name: SubgraphName, + ) -> Result, StoreError> { use subgraph as s; use subgraph_version as v; @@ -1062,7 +1068,7 @@ impl<'a> Connection<'a> { } } - pub fn pause_subgraph(&mut self, site: &Site) -> Result, StoreError> { + pub fn pause_subgraph(&mut self, site: &Site) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1073,8 +1079,7 @@ impl<'a> Connection<'a> { match updates { 0 => Err(StoreError::DeploymentNotFound(site.deployment.to_string())), 1 => { - let change = - EntityChange::for_assignment(site.into(), EntityChangeOperation::Removed); + let change = AssignmentChange::removed(site.into()); Ok(vec![change]) } _ => { @@ -1085,7 +1090,7 @@ impl<'a> Connection<'a> { } } - pub fn resume_subgraph(&mut self, site: &Site) -> Result, StoreError> { + pub fn resume_subgraph(&mut self, site: &Site) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1096,7 +1101,7 @@ impl<'a> Connection<'a> { match updates { 0 => Err(StoreError::DeploymentNotFound(site.deployment.to_string())), 1 => { - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); Ok(vec![change]) } _ => { @@ -1111,7 +1116,7 @@ impl<'a> Connection<'a> { &mut self, site: &Site, node: &NodeId, - ) -> Result, StoreError> { + ) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1121,7 +1126,7 @@ impl<'a> Connection<'a> { match updates { 0 => Err(StoreError::DeploymentNotFound(site.deployment.to_string())), 1 => { - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); Ok(vec![change]) } _ => { @@ -1248,7 +1253,7 @@ impl<'a> Connection<'a> { &mut self, site: &Site, node: &NodeId, - ) -> Result, StoreError> { + ) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1256,11 +1261,11 @@ impl<'a> Connection<'a> { .values((a::id.eq(site.id), a::node_id.eq(node.as_str()))) .execute(conn)?; - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); Ok(vec![change]) } - pub fn unassign_subgraph(&mut self, site: &Site) -> Result, StoreError> { + pub fn unassign_subgraph(&mut self, site: &Site) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1271,8 +1276,7 @@ impl<'a> Connection<'a> { match delete_count { 0 => Ok(vec![]), 1 => { - let change = - EntityChange::for_assignment(site.into(), EntityChangeOperation::Removed); + let change = AssignmentChange::removed(site.into()); Ok(vec![change]) } _ => { @@ -1823,6 +1827,52 @@ impl<'a> Connection<'a> { } } +/// A limited interface to query the primary database. +#[derive(Clone, CheapClone)] +pub struct Primary { + pool: Arc, +} + +impl Primary { + pub fn new(pool: Arc) -> Self { + // This really indicates a programming error + if pool.shard != *PRIMARY_SHARD { + panic!("Primary pool must be the primary shard"); + } + + Primary { pool } + } + + /// Return `true` if the site is the source of a copy operation. The copy + /// operation might be just queued or in progress already. This method will + /// block until a fdw connection becomes available. + pub fn is_source(&self, site: &Site) -> Result { + use active_copies as ac; + + let mut conn = self.pool.get()?; + + select(diesel::dsl::exists( + ac::table + .filter(ac::src.eq(site.id)) + .filter(ac::cancelled_at.is_null()), + )) + .get_result::(&mut conn) + .map_err(StoreError::from) + } + + pub fn is_copy_cancelled(&self, dst: &Site) -> Result { + use active_copies as ac; + + let mut conn = self.pool.get()?; + + ac::table + .filter(ac::dst.eq(dst.id)) + .select(ac::cancelled_at.is_not_null()) + .get_result::(&mut conn) + .map_err(StoreError::from) + } +} + /// Return `true` if we deem this installation to be empty, defined as /// having no deployments and no subgraph names in the database pub fn is_empty(conn: &mut PgConnection) -> Result { @@ -1843,6 +1893,20 @@ pub struct Mirror { } impl Mirror { + // The tables that we mirror + // + // `chains` needs to be mirrored before `deployment_schemas` because + // of the fk constraint on `deployment_schemas.network`. We don't + // care much about mirroring `active_copies` but it has a fk + // constraint on `deployment_schemas` and is tiny, therefore it's + // easiest to just mirror it + pub(crate) const PUBLIC_TABLES: [&str; 3] = ["chains", "deployment_schemas", "active_copies"]; + pub(crate) const SUBGRAPHS_TABLES: [&str; 3] = [ + "subgraph_deployment_assignment", + "subgraph", + "subgraph_version", + ]; + pub fn new(pools: &HashMap) -> Mirror { let primary = pools .get(&PRIMARY_SHARD) @@ -1899,18 +1963,6 @@ impl Mirror { conn: &mut PgConnection, handle: &CancelHandle, ) -> Result<(), StoreError> { - // `chains` needs to be mirrored before `deployment_schemas` because - // of the fk constraint on `deployment_schemas.network`. We don't - // care much about mirroring `active_copies` but it has a fk - // constraint on `deployment_schemas` and is tiny, therefore it's - // easiest to just mirror it - const PUBLIC_TABLES: [&str; 3] = ["chains", "deployment_schemas", "active_copies"]; - const SUBGRAPHS_TABLES: [&str; 3] = [ - "subgraph_deployment_assignment", - "subgraph", - "subgraph_version", - ]; - fn run_query(conn: &mut PgConnection, query: String) -> Result<(), StoreError> { conn.batch_execute(&query).map_err(StoreError::from) } @@ -1942,11 +1994,11 @@ impl Mirror { // Truncate all tables at once, otherwise truncation can fail // because of foreign key constraints - let tables = PUBLIC_TABLES + let tables = Self::PUBLIC_TABLES .iter() .map(|name| (NAMESPACE_PUBLIC, name)) .chain( - SUBGRAPHS_TABLES + Self::SUBGRAPHS_TABLES .iter() .map(|name| (NAMESPACE_SUBGRAPHS, name)), ) @@ -1957,7 +2009,7 @@ impl Mirror { check_cancel()?; // Repopulate `PUBLIC_TABLES` by copying their data wholesale - for table_name in PUBLIC_TABLES { + for table_name in Self::PUBLIC_TABLES { copy_table( conn, ForeignServer::PRIMARY_PUBLIC, diff --git a/store/postgres/src/query_store.rs b/store/postgres/src/query_store.rs index 8fc2da822e4..fe7d084030b 100644 --- a/store/postgres/src/query_store.rs +++ b/store/postgres/src/query_store.rs @@ -112,7 +112,7 @@ impl QueryStoreTrait for QueryStore { self.chain_store.block_numbers(block_hashes).await } - fn wait_stats(&self) -> Result { + fn wait_stats(&self) -> PoolWaitStats { self.store.wait_stats(self.replica_id) } @@ -137,7 +137,7 @@ impl QueryStoreTrait for QueryStore { &self.site.network } - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { self.store.query_permit(self.replica_id).await } diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index de7e6895083..fb181b7e74d 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -32,7 +32,6 @@ use graph::blockchain::block_stream::{EntityOperationKind, EntitySourceOperation use graph::blockchain::BlockTime; use graph::cheap_clone::CheapClone; use graph::components::store::write::{RowGroup, WriteChunk}; -use graph::components::subgraph::PoICausalityRegion; use graph::constraint_violation; use graph::data::graphql::TypeExt as _; use graph::data::query::Trace; @@ -69,11 +68,11 @@ use crate::{ }, }; use graph::components::store::{AttributeNames, DerivedEntityQuery}; -use graph::data::store::{Id, IdList, IdType, BYTES_SCALAR}; +use graph::data::store::{IdList, IdType, BYTES_SCALAR}; use graph::data::subgraph::schema::POI_TABLE; use graph::prelude::{ - anyhow, info, BlockNumber, DeploymentHash, Entity, EntityChange, EntityOperation, Logger, - QueryExecutionError, StoreError, StoreEvent, ValueType, BLOCK_NUMBER_MAX, + anyhow, info, BlockNumber, DeploymentHash, Entity, EntityOperation, Logger, + QueryExecutionError, StoreError, ValueType, }; use crate::block_range::{BoundSide, BLOCK_COLUMN, BLOCK_RANGE_COLUMN}; @@ -231,8 +230,6 @@ pub struct Layout { pub tables: HashMap>, /// The database schema for this subgraph pub catalog: Catalog, - /// The query to count all entities - pub count_query: String, /// How many blocks of history the subgraph should keep pub history_blocks: BlockNumber, @@ -290,25 +287,6 @@ impl Layout { )) } - let count_query = tables - .iter() - .map(|table| { - if table.immutable { - format!( - "select count(*) from \"{}\".\"{}\"", - &catalog.site.namespace, table.name - ) - } else { - format!( - "select count(*) from \"{}\".\"{}\" where block_range @> {}", - &catalog.site.namespace, table.name, BLOCK_NUMBER_MAX - ) - } - }) - .collect::>() - .join("\nunion all\n"); - let count_query = format!("select sum(e.count) from ({}) e", count_query); - let tables: HashMap<_, _> = tables .into_iter() .fold(HashMap::new(), |mut tables, table| { @@ -322,7 +300,6 @@ impl Layout { site, catalog, tables, - count_query, history_blocks: i32::MAX, input_schema: schema.cheap_clone(), rollups, @@ -397,10 +374,6 @@ impl Layout { } } - pub fn supports_proof_of_indexing(&self) -> bool { - self.tables.contains_key(&self.input_schema.poi_type()) - } - pub fn create_relational_schema( conn: &mut PgConnection, site: Arc, @@ -1026,23 +999,25 @@ impl Layout { Ok(count) } - pub fn truncate_tables(&self, conn: &mut PgConnection) -> Result { + pub fn truncate_tables(&self, conn: &mut PgConnection) -> Result<(), StoreError> { for table in self.tables.values() { sql_query(&format!("TRUNCATE TABLE {}", table.qualified_name)).execute(conn)?; } - Ok(StoreEvent::new(vec![])) + Ok(()) } /// Revert the block with number `block` and all blocks with higher /// numbers. After this operation, only entity versions inserted or /// updated at blocks with numbers strictly lower than `block` will /// remain + /// + /// The `i32` that is returned is the amount by which the entity count + /// for the subgraph needs to be adjusted pub fn revert_block( &self, conn: &mut PgConnection, block: BlockNumber, - ) -> Result<(StoreEvent, i32), StoreError> { - let mut changes: Vec = Vec::new(); + ) -> Result { let mut count: i32 = 0; for table in self.tables.values() { @@ -1071,23 +1046,8 @@ impl Layout { let deleted = removed.difference(&unclamped).count() as i32; let inserted = unclamped.difference(&removed).count() as i32; count += inserted - deleted; - // EntityChange for versions we just deleted - let deleted = removed - .into_iter() - .filter(|id| !unclamped.contains(id)) - .map(|_| EntityChange::Data { - subgraph_id: self.site.deployment.clone(), - entity_type: table.object.to_string(), - }); - changes.extend(deleted); - // EntityChange for versions that we just updated or inserted - let set = unclamped.into_iter().map(|_| EntityChange::Data { - subgraph_id: self.site.deployment.clone(), - entity_type: table.object.to_string(), - }); - changes.extend(set); } - Ok((StoreEvent::new(changes), count)) + Ok(count) } /// Revert the metadata (dynamic data sources and related entities) for @@ -1152,32 +1112,6 @@ impl Layout { Ok(Arc::new(layout)) } - pub(crate) fn block_time( - &self, - conn: &mut PgConnection, - block: BlockNumber, - ) -> Result, StoreError> { - let block_time_name = self.input_schema.poi_block_time(); - let poi_type = self.input_schema.poi_type(); - let id = Id::String(Word::from(PoICausalityRegion::from_network( - &self.site.network, - ))); - let key = poi_type.key(id); - - let block_time = self - .find(conn, &key, block)? - .and_then(|entity| { - entity.get(&block_time_name).map(|value| { - value - .as_int8() - .ok_or_else(|| constraint_violation!("block_time must have type Int8")) - }) - }) - .transpose()? - .map(|value| BlockTime::since_epoch(value, 0)); - Ok(block_time) - } - /// Find the time of the last rollup for the subgraph. We do this by /// looking for the maximum timestamp in any aggregation table and /// adding a little bit more than the corresponding interval to it. This @@ -1715,7 +1649,7 @@ impl Table { pub fn new_like(&self, namespace: &Namespace, name: &SqlName) -> Arc
{ let other = Table { object: self.object.clone(), - nsp: self.nsp.clone(), + nsp: namespace.clone(), name: name.clone(), qualified_name: SqlName::qualified_name(namespace, name), columns: self.columns.clone(), diff --git a/store/postgres/src/relational/ddl.rs b/store/postgres/src/relational/ddl.rs index 40a02d6051e..a3c4ed6885e 100644 --- a/store/postgres/src/relational/ddl.rs +++ b/store/postgres/src/relational/ddl.rs @@ -269,7 +269,11 @@ impl Table { (method, index_expr) } - pub(crate) fn create_postponed_indexes(&self, skip_colums: Vec) -> Vec { + pub(crate) fn create_postponed_indexes( + &self, + skip_colums: Vec, + concurrently: bool, + ) -> Vec { let mut indexing_queries = vec![]; let columns = self.columns_to_index(); @@ -281,8 +285,9 @@ impl Table { && column.name.as_str() != "id" && !skip_colums.contains(&column.name.to_string()) { + let conc = if concurrently { "concurrently " } else { "" }; let sql = format!( - "create index concurrently if not exists attr_{table_index}_{column_index}_{table_name}_{column_name}\n on {qname} using {method}({index_expr});\n", + "create index {conc}if not exists attr_{table_index}_{column_index}_{table_name}_{column_name}\n on {qname} using {method}({index_expr});\n", table_index = self.position, table_name = self.name, column_name = column.name, @@ -404,11 +409,12 @@ impl Table { let arr = index_def .unwrap() .indexes_for_table( - &catalog.site.namespace, + &self.nsp, &self.name.to_string(), &self, false, false, + false, ) .map_err(|_| fmt::Error)?; for (_, sql) in arr { @@ -416,8 +422,9 @@ impl Table { } } else { self.create_attribute_indexes(out)?; + self.create_aggregate_indexes(schema, out)?; } - self.create_aggregate_indexes(schema, out) + Ok(()) } pub fn exclusion_ddl(&self, out: &mut String) -> fmt::Result { diff --git a/store/postgres/src/relational/ddl_tests.rs b/store/postgres/src/relational/ddl_tests.rs index 86e9f232d49..b15a40cecfb 100644 --- a/store/postgres/src/relational/ddl_tests.rs +++ b/store/postgres/src/relational/ddl_tests.rs @@ -158,7 +158,7 @@ fn generate_postponed_indexes() { let layout = test_layout(THING_GQL); let table = layout.table(&SqlName::from("Scalar")).unwrap(); let skip_colums = vec!["id".to_string()]; - let query_vec = table.create_postponed_indexes(skip_colums); + let query_vec = table.create_postponed_indexes(skip_colums, true); assert!(query_vec.len() == 7); let queries = query_vec.join(" "); check_eqv(THING_POSTPONED_INDEXES, &queries) @@ -352,6 +352,97 @@ fn can_copy_from() { ); } +/// Check that we do not create the index on `block$` twice. There was a bug +/// that if an immutable entity type had a `block` field and index creation +/// was postponed, we would emit the index on `block$` twice, once from +/// `Table.create_time_travel_indexes` and once through +/// `IndexList.indexes_for_table` +#[test] +fn postponed_indexes_with_block_column() { + fn index_list() -> IndexList { + // To generate this list, print the output of `layout.as_ddl(None)`, run + // that in Postgres and do `select indexdef from pg_indexes where + // schemaname = 'sgd0815'` + const INDEX_DEFS: &[&str] = &[ + "CREATE UNIQUE INDEX data_pkey ON sgd0815.data USING btree (vid)", + "CREATE UNIQUE INDEX data_id_key ON sgd0815.data USING btree (id)", + "CREATE INDEX data_block ON sgd0815.data USING btree (block$)", + "CREATE INDEX attr_1_0_data_block ON sgd0815.data USING btree (block, \"block$\")", + ]; + + let mut indexes: HashMap> = HashMap::new(); + indexes.insert( + "data".to_string(), + INDEX_DEFS + .iter() + .map(|def| CreateIndex::parse(def.to_string())) + .collect(), + ); + IndexList { indexes } + } + + fn cr(index: &str) -> String { + format!("create index{}", index) + } + + fn cre(index: &str) -> String { + format!("create index if not exists{}", index) + } + + // Names of the two indexes we are interested in. Not the leading space + // to guard a little against overlapping names + const BLOCK_IDX: &str = " data_block"; + const ATTR_IDX: &str = " attr_1_0_data_block"; + + let layout = test_layout(BLOCK_GQL); + + // Create everything + let sql = layout.as_ddl(None).unwrap(); + assert!(sql.contains(&cr(BLOCK_IDX))); + assert!(sql.contains(&cr(ATTR_IDX))); + + // Defer attribute indexes + let sql = layout.as_ddl(Some(index_list())).unwrap(); + assert!(sql.contains(&cr(BLOCK_IDX))); + assert!(!sql.contains(ATTR_IDX)); + // This used to be duplicated + let count = sql.matches(BLOCK_IDX).count(); + assert_eq!(1, count); + + let table = layout.table(&SqlName::from("Data")).unwrap(); + let sql = table.create_postponed_indexes(vec![], false); + assert_eq!(1, sql.len()); + assert!(!sql[0].contains(BLOCK_IDX)); + assert!(sql[0].contains(&cre(ATTR_IDX))); + + let dst_nsp = Namespace::new("sgd2".to_string()).unwrap(); + let arr = index_list() + .indexes_for_table( + &dst_nsp, + &table.name.to_string(), + &table, + true, + false, + false, + ) + .unwrap(); + assert_eq!(1, arr.len()); + assert!(!arr[0].1.contains(BLOCK_IDX)); + assert!(arr[0].1.contains(&cr(ATTR_IDX))); + + let arr = index_list() + .indexes_for_table( + &dst_nsp, + &table.name.to_string(), + &table, + false, + false, + false, + ) + .unwrap(); + assert_eq!(0, arr.len()); +} + const THING_GQL: &str = r#" type Thing @entity { id: ID! @@ -1109,3 +1200,15 @@ on "sgd0815"."stats_3_day" using btree("volume"); create index stats_3_day_dims on "sgd0815"."stats_3_day"(group_2, group_1, timestamp); "#; + +const BLOCK_GQL: &str = r#" +type Block @entity(immutable: true) { + id: ID! + number: Int! +} + +type Data @entity(immutable: true) { + id: ID! + block: Block! +} +"#; diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 6013a5d9e68..efa82e901f0 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -123,7 +123,7 @@ impl Display for Expr { Expr::Column(s) => write!(f, "{s}")?, Expr::Prefix(s, _) => write!(f, "{s}")?, Expr::Vid => write!(f, "vid")?, - Expr::Block => write!(f, "block")?, + Expr::Block => write!(f, "{BLOCK_COLUMN}")?, Expr::BlockRange => write!(f, "block_range")?, Expr::BlockRangeLower => write!(f, "lower(block_range)")?, Expr::BlockRangeUpper => write!(f, "upper(block_range)")?, @@ -440,7 +440,7 @@ impl CreateIndex { } } - fn with_nsp(&self, nsp2: String) -> Result { + pub fn with_nsp(&self, nsp2: String) -> Result { let s = self.clone(); match s { CreateIndex::Unknown { defn: _ } => Err(anyhow!("Failed to parse the index")), @@ -488,12 +488,29 @@ impl CreateIndex { && columns[1] == Expr::BlockRange } Method::Brin => false, - Method::BTree | Method::Gin => { + Method::Gin => { + // 'using gin()' columns.len() == 1 && columns[0].is_attribute() && cond.is_none() && with.is_none() } + Method::BTree => { + match columns.len() { + 1 => { + // 'using btree()' + columns[0].is_attribute() && cond.is_none() && with.is_none() + } + 2 => { + // 'using btree(, block$)' + columns[0].is_attribute() + && columns[1] == Expr::Block + && cond.is_none() + && with.is_none() + } + _ => false, + } + } Method::Unknown(_) => false, } } @@ -537,6 +554,7 @@ impl CreateIndex { None, ), dummy(false, BTree, &[Expr::BlockRangeUpper], Some(Cond::Closed)), + dummy(false, BTree, &[Expr::Block], None), ] }; } @@ -630,7 +648,7 @@ impl CreateIndex { } pub fn fields_exist_in_dest<'a>(&self, dest_table: &'a Table) -> bool { - fn column_exists<'a>(it: &mut impl Iterator, column_name: &String) -> bool { + fn column_exists<'a>(it: &mut impl Iterator, column_name: &str) -> bool { it.any(|c| *c == *column_name) } @@ -667,7 +685,7 @@ impl CreateIndex { } Expr::Vid => (), Expr::Block => { - if !column_exists(cols, &"block".to_string()) { + if !dest_table.immutable { return false; } } @@ -734,6 +752,16 @@ pub struct IndexList { pub(crate) indexes: HashMap>, } +pub fn load_indexes_from_table( + conn: &mut PgConnection, + table: &Arc
, + schema_name: &str, +) -> Result, StoreError> { + let table_name = table.name.as_str(); + let indexes = catalog::indexes_for_table(conn, schema_name, table_name)?; + Ok(indexes.into_iter().map(CreateIndex::parse).collect()) +} + impl IndexList { pub fn load( conn: &mut PgConnection, @@ -746,10 +774,8 @@ impl IndexList { let schema_name = site.namespace.clone(); let layout = store.layout(conn, site)?; for (_, table) in &layout.tables { - let table_name = table.name.as_str(); - let indexes = catalog::indexes_for_table(conn, schema_name.as_str(), table_name)?; - let collect: Vec = indexes.into_iter().map(CreateIndex::parse).collect(); - list.indexes.insert(table_name.to_string(), collect); + let indexes = load_indexes_from_table(conn, table, schema_name.as_str())?; + list.indexes.insert(table.name.to_string(), indexes); } Ok(list) } @@ -760,7 +786,8 @@ impl IndexList { table_name: &String, dest_table: &Table, postponed: bool, - concurrent_if_not_exist: bool, + concurrent: bool, + if_not_exists: bool, ) -> Result, String)>, Error> { let mut arr = vec![]; if let Some(vec) = self.indexes.get(table_name) { @@ -768,7 +795,7 @@ impl IndexList { // First we check if the fields do exist in the destination subgraph. // In case of grafting that is not given. if ci.fields_exist_in_dest(dest_table) - // Then we check if the index is one of the default indexes not based on + // Then we check if the index is one of the default indexes not based on // the attributes. Those will be created anyway and we should skip them. && !ci.is_default_non_attr_index() // Then ID based indexes in the immutable tables are also created initially @@ -781,7 +808,7 @@ impl IndexList { { if let Ok(sql) = ci .with_nsp(namespace.to_string())? - .to_sql(concurrent_if_not_exist, concurrent_if_not_exist) + .to_sql(concurrent, if_not_exists) { arr.push((ci.name(), sql)) } @@ -805,7 +832,7 @@ impl IndexList { let namespace = &layout.catalog.site.namespace; for table in layout.tables.values() { for (ind_name, create_query) in - self.indexes_for_table(namespace, &table.name.to_string(), table, true, true)? + self.indexes_for_table(namespace, &table.name.to_string(), table, true, true, true)? { if let Some(index_name) = ind_name { let table_name = table.name.clone(); diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index 62632549397..5c3035ce172 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -1,4 +1,4 @@ -use std::{fmt::Write, sync::Arc}; +use std::{collections::HashMap, fmt::Write, sync::Arc}; use diesel::{ connection::SimpleConnection, @@ -23,7 +23,10 @@ use crate::{ vid_batcher::{VidBatcher, VidRange}, }; -use super::{Catalog, Layout, Namespace}; +use super::{ + index::{load_indexes_from_table, CreateIndex, IndexList}, + Catalog, Layout, Namespace, +}; /// Utility to copy relevant data out of a source table and into a new /// destination table and replace the source table with the destination @@ -56,9 +59,18 @@ impl TablePair { if catalog::table_exists(conn, dst_nsp.as_str(), &dst.name)? { writeln!(query, "truncate table {};", dst.qualified_name)?; } else { + let mut list = IndexList { + indexes: HashMap::new(), + }; + let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())? + .into_iter() + .map(|index| index.with_nsp(dst_nsp.to_string())) + .collect::, _>>()?; + list.indexes.insert(src.name.to_string(), indexes); + // In case of pruning we don't do delayed creation of indexes, // as the asumption is that there is not that much data inserted. - dst.as_ddl(schema, catalog, None, &mut query)?; + dst.as_ddl(schema, catalog, Some(&list), &mut query)?; } conn.batch_execute(&query)?; diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index f4b55e89150..028f6044c34 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -53,7 +53,7 @@ use crate::{ const BASE_SQL_COLUMNS: [&str; 2] = ["id", "vid"]; /// The maximum number of bind variables that can be used in a query -const POSTGRES_MAX_PARAMETERS: usize = u16::MAX as usize; // 65535 +pub(crate) const POSTGRES_MAX_PARAMETERS: usize = u16::MAX as usize; // 65535 const SORT_KEY_COLUMN: &str = "sort_key$"; @@ -4799,17 +4799,23 @@ impl<'a> CopyEntityBatchQuery<'a> { last_vid, }) } + + pub fn count_current(self) -> CountCurrentVersionsQuery<'a> { + CountCurrentVersionsQuery::new(self) + } } impl<'a> QueryFragment for CopyEntityBatchQuery<'a> { fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { out.unsafe_to_cache_prepared(); - let has_vid_seq = self.src.object.has_vid_seq(); + let has_vid_seq = self.dst.object.has_vid_seq(); // Construct a query // insert into {dst}({columns}) // select {columns} from {src} + // where vid >= {first_vid} and vid <= {last_vid} + // returning {upper_inf(block_range)|true} out.push_sql("insert into "); out.push_sql(self.dst.qualified_name.as_str()); out.push_sql("("); @@ -4905,6 +4911,12 @@ impl<'a> QueryFragment for CopyEntityBatchQuery<'a> { out.push_bind_param::(&self.first_vid)?; out.push_sql(" and vid <= "); out.push_bind_param::(&self.last_vid)?; + out.push_sql("\n returning "); + if self.dst.immutable { + out.push_sql("true"); + } else { + out.push_sql(BLOCK_RANGE_CURRENT); + } Ok(()) } } @@ -4917,6 +4929,40 @@ impl<'a> QueryId for CopyEntityBatchQuery<'a> { impl<'a, Conn> RunQueryDsl for CopyEntityBatchQuery<'a> {} +#[derive(Debug, Clone)] +pub struct CountCurrentVersionsQuery<'a> { + copy: CopyEntityBatchQuery<'a>, +} + +impl<'a> CountCurrentVersionsQuery<'a> { + pub fn new(copy: CopyEntityBatchQuery<'a>) -> Self { + Self { copy } + } +} +impl<'a> QueryFragment for CountCurrentVersionsQuery<'a> { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { + // Generate a query + // with copy_cte as ( {copy} ) + // select count(*) from copy_cte where {block_range_current} + out.push_sql("with copy_cte(current) as ("); + self.copy.walk_ast(out.reborrow())?; + out.push_sql(")\nselect count(*) from copy_cte where current"); + Ok(()) + } +} + +impl<'a> QueryId for CountCurrentVersionsQuery<'a> { + type QueryId = (); + + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl<'a> Query for CountCurrentVersionsQuery<'a> { + type SqlType = BigInt; +} + +impl<'a, Conn> RunQueryDsl for CountCurrentVersionsQuery<'a> {} + /// Helper struct for returning the id's touched by the RevertRemove and /// RevertExtend queries #[derive(QueryableByName, PartialEq, Eq, Hash)] diff --git a/store/postgres/src/store.rs b/store/postgres/src/store.rs index b663053c3da..7eb428a5058 100644 --- a/store/postgres/src/store.rs +++ b/store/postgres/src/store.rs @@ -70,7 +70,6 @@ impl QueryStoreManager for Store { async fn query_store( &self, target: graph::data::query::QueryTarget, - for_subscription: bool, ) -> Result< Arc, graph::prelude::QueryExecutionError, @@ -80,7 +79,7 @@ impl QueryStoreManager for Store { let target = target.clone(); let (store, site, replica) = graph::spawn_blocking_allow_panic(move || { store - .replica_for_query(target.clone(), for_subscription) + .replica_for_query(target.clone()) .map_err(|e| e.into()) }) .await @@ -168,8 +167,8 @@ impl StatusStore for Store { .await } - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { // Status queries go to the primary shard. - Ok(self.block_store.query_permit_primary().await) + self.block_store.query_permit_primary().await } } diff --git a/store/postgres/src/store_events.rs b/store/postgres/src/store_events.rs index 6370cd3aa92..83b8e3b069b 100644 --- a/store/postgres/src/store_events.rs +++ b/store/postgres/src/store_events.rs @@ -2,19 +2,15 @@ use graph::futures01::Stream; use graph::futures03::compat::Stream01CompatExt; use graph::futures03::stream::StreamExt; use graph::futures03::TryStreamExt; -use graph::parking_lot::Mutex; use graph::tokio_stream::wrappers::ReceiverStream; -use std::collections::BTreeSet; use std::sync::{atomic::Ordering, Arc, RwLock}; use std::{collections::HashMap, sync::atomic::AtomicUsize}; use tokio::sync::mpsc::{channel, Sender}; -use tokio::sync::watch; -use uuid::Uuid; use crate::notification_listener::{NotificationListener, SafeChannelName}; -use graph::components::store::{SubscriptionManager as SubscriptionManagerTrait, UnitStream}; +use graph::components::store::SubscriptionManager as SubscriptionManagerTrait; use graph::prelude::serde_json; -use graph::{prelude::*, tokio_stream}; +use graph::prelude::*; pub struct StoreEventListener { notification_listener: NotificationListener, @@ -89,46 +85,10 @@ impl StoreEventListener { } } -struct Watcher { - sender: Arc>, - receiver: watch::Receiver, -} - -impl Watcher { - fn new(init: T) -> Self { - let (sender, receiver) = watch::channel(init); - Watcher { - sender: Arc::new(sender), - receiver, - } - } - - fn send(&self, v: T) { - // Unwrap: `self` holds a receiver. - self.sender.send(v).unwrap() - } - - fn stream(&self) -> Box + Unpin + Send + Sync> { - Box::new(tokio_stream::wrappers::WatchStream::new( - self.receiver.clone(), - )) - } - - /// Outstanding receivers returned from `Self::stream`. - fn receiver_count(&self) -> usize { - // Do not count the internal receiver. - self.sender.receiver_count() - 1 - } -} - /// Manage subscriptions to the `StoreEvent` stream. Keep a list of /// currently active subscribers and forward new events to each of them pub struct SubscriptionManager { - // These are more efficient since only one entry is stored per filter. - subscriptions_no_payload: Arc, Watcher<()>>>>, - - subscriptions: - Arc>, Sender>)>>>, + subscriptions: Arc>>>>, /// Keep the notification listener alive listener: StoreEventListener, @@ -139,7 +99,6 @@ impl SubscriptionManager { let (listener, store_events) = StoreEventListener::new(logger, postgres_url, registry); let mut manager = SubscriptionManager { - subscriptions_no_payload: Arc::new(Mutex::new(HashMap::new())), subscriptions: Arc::new(RwLock::new(HashMap::new())), listener, }; @@ -161,7 +120,6 @@ impl SubscriptionManager { store_events: Box + Send>, ) { let subscriptions = self.subscriptions.cheap_clone(); - let subscriptions_no_payload = self.subscriptions_no_payload.cheap_clone(); let mut store_events = store_events.compat(); // This channel is constantly receiving things and there are locks involved, @@ -176,34 +134,19 @@ impl SubscriptionManager { // Write change to all matching subscription streams; remove subscriptions // whose receiving end has been dropped - for (id, (_, sender)) in senders - .iter() - .filter(|(_, (filter, _))| event.matches(filter)) - { + for (id, sender) in senders { if sender.send(event.cheap_clone()).await.is_err() { // Receiver was dropped - subscriptions.write().unwrap().remove(id); + subscriptions.write().unwrap().remove(&id); } } } - - // Send to `subscriptions_no_payload`. - { - let watchers = subscriptions_no_payload.lock(); - - // Write change to all matching subscription streams - for (_, watcher) in watchers.iter().filter(|(filter, _)| event.matches(filter)) - { - watcher.send(()); - } - } } }); } fn periodically_clean_up_stale_subscriptions(&self) { let subscriptions = self.subscriptions.cheap_clone(); - let subscriptions_no_payload = self.subscriptions_no_payload.cheap_clone(); // Clean up stale subscriptions every 5s graph::spawn(async move { @@ -218,26 +161,7 @@ impl SubscriptionManager { // Obtain IDs of subscriptions whose receiving end has gone let stale_ids = subscriptions .iter_mut() - .filter_map(|(id, (_, sender))| match sender.is_closed() { - true => Some(id.clone()), - false => None, - }) - .collect::>(); - - // Remove all stale subscriptions - for id in stale_ids { - subscriptions.remove(&id); - } - } - - // Cleanup `subscriptions_no_payload`. - { - let mut subscriptions = subscriptions_no_payload.lock(); - - // Obtain IDs of subscriptions whose receiving end has gone - let stale_ids = subscriptions - .iter_mut() - .filter_map(|(id, watcher)| match watcher.receiver_count() == 0 { + .filter_map(|(id, sender)| match sender.is_closed() { true => Some(id.clone()), false => None, }) @@ -254,28 +178,17 @@ impl SubscriptionManager { } impl SubscriptionManagerTrait for SubscriptionManager { - fn subscribe(&self, entities: BTreeSet) -> StoreEventStreamBox { - let id = Uuid::new_v4().to_string(); + fn subscribe(&self) -> StoreEventStreamBox { + static SUBSCRIPTION_COUNTER: AtomicUsize = AtomicUsize::new(0); + let id = SUBSCRIPTION_COUNTER.fetch_add(1, Ordering::SeqCst); // Prepare the new subscription by creating a channel and a subscription object let (sender, receiver) = channel(100); // Add the new subscription - self.subscriptions - .write() - .unwrap() - .insert(id, (Arc::new(entities.clone()), sender)); + self.subscriptions.write().unwrap().insert(id, sender); // Return the subscription ID and entity change stream StoreEventStream::new(Box::new(ReceiverStream::new(receiver).map(Ok).compat())) - .filter_by_entities(entities) - } - - fn subscribe_no_payload(&self, entities: BTreeSet) -> UnitStream { - self.subscriptions_no_payload - .lock() - .entry(entities) - .or_insert_with(|| Watcher::new(())) - .stream() } } diff --git a/store/postgres/src/subgraph_store.rs b/store/postgres/src/subgraph_store.rs index f6544a79e0d..339c66cee3f 100644 --- a/store/postgres/src/subgraph_store.rs +++ b/store/postgres/src/subgraph_store.rs @@ -39,7 +39,7 @@ use graph::{ use crate::{ connection_pool::ConnectionPool, deployment::{OnSync, SubgraphHealth}, - primary::{self, DeploymentId, Mirror as PrimaryMirror, Site}, + primary::{self, DeploymentId, Mirror as PrimaryMirror, Primary, Site}, relational::{ index::{IndexList, Method}, Layout, @@ -360,6 +360,12 @@ impl SubgraphStoreInner { sender: Arc, registry: Arc, ) -> Self { + let primary = stores + .iter() + .find(|(name, _, _, _)| name == &*PRIMARY_SHARD) + .map(|(_, pool, _, _)| Primary::new(Arc::new(pool.clone()))) + .expect("primary shard must be present"); + let mirror = { let pools = HashMap::from_iter( stores @@ -376,6 +382,7 @@ impl SubgraphStoreInner { name, Arc::new(DeploymentStore::new( &logger, + primary.cheap_clone(), main_pool, read_only_pools, weights, @@ -699,12 +706,6 @@ impl SubgraphStoreInner { ))); } let deployment = src_store.load_deployment(src.clone())?; - if deployment.failed { - return Err(StoreError::Unknown(anyhow!( - "can not copy deployment {} because it has failed", - src_loc - ))); - } let index_def = src_store.load_indexes(src.clone())?; // Transmogrify the deployment into a new one @@ -811,7 +812,6 @@ impl SubgraphStoreInner { pub(crate) fn replica_for_query( &self, target: QueryTarget, - for_subscription: bool, ) -> Result<(Arc, Arc, ReplicaId), StoreError> { let id = match target { QueryTarget::Name(name, _) => self.mirror.current_deployment_for_subgraph(&name)?, @@ -819,7 +819,7 @@ impl SubgraphStoreInner { }; let (store, site) = self.store(&id)?; - let replica = store.replica_for_query(for_subscription)?; + let replica = store.replica_for_query()?; Ok((store.clone(), site, replica)) } @@ -1046,14 +1046,12 @@ impl SubgraphStoreInner { pub fn rewind(&self, id: DeploymentHash, block_ptr_to: BlockPtr) -> Result<(), StoreError> { let (store, site) = self.store(&id)?; - let event = store.rewind(site, block_ptr_to)?; - self.send_store_event(&event) + store.rewind(site, block_ptr_to) } pub fn truncate(&self, id: DeploymentHash, block_ptr_to: BlockPtr) -> Result<(), StoreError> { let (store, site) = self.store(&id)?; - let event = store.truncate(site, block_ptr_to)?; - self.send_store_event(&event) + store.truncate(site, block_ptr_to) } pub(crate) async fn get_proof_of_indexing( diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index 81da5382e3d..93197b5a85d 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -112,20 +112,6 @@ pub(crate) struct VidBatcher { } impl VidBatcher { - fn histogram_bounds( - conn: &mut PgConnection, - nsp: &Namespace, - table: &Table, - range: VidRange, - ) -> Result, StoreError> { - let bounds = catalog::histogram_bounds(conn, nsp, &table.name, VID_COLUMN)? - .into_iter() - .filter(|bound| range.min < *bound && range.max > *bound) - .chain(vec![range.min, range.max].into_iter()) - .collect::>(); - Ok(bounds) - } - /// Initialize a batcher for batching through entries in `table` with /// `vid` in the given `vid_range` /// @@ -138,7 +124,7 @@ impl VidBatcher { table: &Table, vid_range: VidRange, ) -> Result { - let bounds = Self::histogram_bounds(conn, nsp, table, vid_range)?; + let bounds = catalog::histogram_bounds(conn, nsp, &table.name, VID_COLUMN)?; let batch_size = AdaptiveBatchSize::new(table); Self::new(bounds, vid_range, batch_size) } @@ -150,6 +136,26 @@ impl VidBatcher { ) -> Result { let start = range.min; + let bounds = { + // Keep only histogram bounds that are relevent for the range + let mut bounds = bounds + .into_iter() + .filter(|bound| range.min <= *bound && range.max >= *bound) + .collect::>(); + // The first and last entry in `bounds` are Postgres' estimates + // of the min and max `vid` values in the table. We use the + // actual min and max `vid` values from the `vid_range` instead + let len = bounds.len(); + if len > 1 { + bounds[0] = range.min; + bounds[len - 1] = range.max; + } else { + // If Postgres doesn't have a histogram, just use one bucket + // from min to max + bounds = vec![range.min, range.max]; + } + bounds + }; let mut ogive = if range.is_empty() { None } else { @@ -220,13 +226,22 @@ impl VidBatcher { let duration = self.step_timer.elapsed(); let batch_size = self.batch_size.adapt(duration); - self.start = self.end + 1; + // We can't possibly copy farther than `max_vid` + self.start = (self.end + 1).min(self.max_vid + 1); self.end = ogive.next_point(self.start, batch_size as usize)?; Ok((duration, Some(res))) } } } + + pub(crate) fn set_batch_size(&mut self, size: usize) { + self.batch_size.size = size as i64; + self.end = match &self.ogive { + Some(ogive) => ogive.next_point(self.start, size as usize).unwrap(), + None => self.start + size as i64, + }; + } } #[derive(Copy, Clone, QueryableByName)] @@ -248,7 +263,10 @@ impl VidRange { } pub fn is_empty(&self) -> bool { - self.max == -1 + // min > max can happen when we restart a copy job that has finished + // some tables. For those, min (the next_vid) will be larger than + // max (the target_vid) + self.max == -1 || self.min > self.max } pub fn size(&self) -> usize { @@ -371,6 +389,17 @@ mod tests { } } + impl std::fmt::Debug for Batcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Batcher") + .field("start", &self.vid.start) + .field("end", &self.vid.end) + .field("size", &self.vid.batch_size.size) + .field("duration", &self.vid.batch_size.target.as_secs()) + .finish() + } + } + #[test] fn simple() { let bounds = vec![10, 20, 30, 40, 49]; @@ -422,4 +451,23 @@ mod tests { batcher.at(360, 359, 80); batcher.step(360, 359, S010); } + + #[test] + fn vid_batcher_adjusts_bounds() { + // The first and last entry in `bounds` are estimats of the min and + // max that are slightly off compared to the actual min and max we + // put in `vid_range`. Check that `VidBatcher` uses the actual min + // and max from `vid_range`. + let bounds = vec![639, 20_000, 40_000, 60_000, 80_000, 90_000]; + let vid_range = VidRange::new(1, 100_000); + let batch_size = AdaptiveBatchSize { + size: 1000, + target: S100, + }; + + let vid_batcher = VidBatcher::new(bounds, vid_range, batch_size).unwrap(); + let ogive = vid_batcher.ogive.as_ref().unwrap(); + assert_eq!(1, ogive.start()); + assert_eq!(100_000, ogive.end()); + } } diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index a9525ba9eb5..3d85042d07c 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -95,8 +95,8 @@ impl LastRollup { let kind = match (has_aggregations, block) { (false, _) => LastRollup::NotNeeded, (true, None) => LastRollup::Unknown, - (true, Some(block)) => { - let block_time = store.block_time(site, block)?; + (true, Some(_)) => { + let block_time = store.block_time(site)?; block_time .map(|b| LastRollup::Some(b)) .unwrap_or(LastRollup::Unknown) @@ -190,19 +190,6 @@ impl SyncStore { last_rollup, }) } - - /// Try to send a `StoreEvent`; if sending fails, log the error but - /// return `Ok(())` - fn try_send_store_event(&self, event: StoreEvent) -> Result<(), StoreError> { - if !ENV_VARS.store.disable_subscription_notifications { - let _ = self.store.send_store_event(&event).map_err( - |e| error!(self.logger, "Could not send store event"; "error" => e.to_string()), - ); - Ok(()) - } else { - Ok(()) - } - } } // Methods that mirror `WritableStoreTrait` @@ -233,8 +220,10 @@ impl SyncStore { } None => None, }; - self.writable - .start_subgraph(logger, self.site.clone(), graft_base)?; + graph::block_on( + self.writable + .start_subgraph(logger, self.site.clone(), graft_base), + )?; self.store.primary_conn()?.copy_finished(self.site.as_ref()) }) } @@ -245,18 +234,14 @@ impl SyncStore { firehose_cursor: &FirehoseCursor, ) -> Result<(), StoreError> { retry::forever(&self.logger, "revert_block_operations", || { - let event = self.writable.revert_block_operations( + self.writable.revert_block_operations( self.site.clone(), block_ptr_to.clone(), firehose_cursor, )?; - let block_time = self - .writable - .block_time(self.site.cheap_clone(), block_ptr_to.number)?; - self.last_rollup.set(block_time)?; - - self.try_send_store_event(event) + let block_time = self.writable.block_time(self.site.cheap_clone())?; + self.last_rollup.set(block_time) }) } @@ -294,15 +279,6 @@ impl SyncStore { .await } - async fn supports_proof_of_indexing(&self) -> Result { - retry::forever_async(&self.logger, "supports_proof_of_indexing", || async { - self.writable - .supports_proof_of_indexing(self.site.clone()) - .await - }) - .await - } - fn get(&self, key: &EntityKey, block: BlockNumber) -> Result, StoreError> { retry::forever(&self.logger, "get", || { self.writable.get(self.site.cheap_clone(), key, block) @@ -315,7 +291,7 @@ impl SyncStore { stopwatch: &StopwatchMetrics, ) -> Result<(), StoreError> { retry::forever(&self.logger, "transact_block_operations", move || { - let event = self.writable.transact_block_operations( + self.writable.transact_block_operations( &self.logger, self.site.clone(), batch, @@ -326,9 +302,6 @@ impl SyncStore { // unwrap: batch.block_times is never empty let last_block_time = batch.block_times.last().unwrap().1; self.last_rollup.set(Some(last_block_time))?; - - let _section = stopwatch.start_section("send_store_event"); - self.try_send_store_event(event)?; Ok(()) }) } @@ -1685,10 +1658,6 @@ impl WritableStoreTrait for WritableStore { self.store.fail_subgraph(error).await } - async fn supports_proof_of_indexing(&self) -> Result { - self.store.supports_proof_of_indexing().await - } - async fn transact_block_operations( &self, block_ptr_to: BlockPtr, diff --git a/store/test-store/src/store.rs b/store/test-store/src/store.rs index 70fc26a3dde..2fa96148ba9 100644 --- a/store/test-store/src/store.rs +++ b/store/test-store/src/store.rs @@ -68,7 +68,6 @@ lazy_static! { pub static ref PRIMARY_POOL: ConnectionPool = STORE_POOL_CONFIG.1.clone(); pub static ref STORE: Arc = STORE_POOL_CONFIG.0.clone(); static ref CONFIG: Config = STORE_POOL_CONFIG.2.clone(); - pub static ref SUBSCRIPTION_MANAGER: Arc = STORE_POOL_CONFIG.3.clone(); pub static ref NODE_ID: NodeId = NodeId::new("test").unwrap(); pub static ref SUBGRAPH_STORE: Arc = STORE.subgraph_store(); static ref BLOCK_STORE: Arc = STORE.block_store(); @@ -531,7 +530,7 @@ async fn execute_subgraph_query_internal( let deployment = query.schema.id().clone(); let store = STORE .clone() - .query_store(QueryTarget::Deployment(deployment, version.clone()), false) + .query_store(QueryTarget::Deployment(deployment, version.clone())) .await .unwrap(); let state = store.deployment_state().await.unwrap(); @@ -544,7 +543,6 @@ async fn execute_subgraph_query_internal( &logger, store.clone(), &state, - SUBSCRIPTION_MANAGER.clone(), ptr, error_policy, query.schema.id().clone(), @@ -573,10 +571,10 @@ async fn execute_subgraph_query_internal( pub async fn deployment_state(store: &Store, subgraph_id: &DeploymentHash) -> DeploymentState { store - .query_store( - QueryTarget::Deployment(subgraph_id.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + subgraph_id.clone(), + Default::default(), + )) .await .expect("could get a query store") .deployment_state() diff --git a/store/test-store/tests/chain/ethereum/manifest.rs b/store/test-store/tests/chain/ethereum/manifest.rs index 9d094ae5817..02f4e1413f9 100644 --- a/store/test-store/tests/chain/ethereum/manifest.rs +++ b/store/test-store/tests/chain/ethereum/manifest.rs @@ -47,9 +47,10 @@ specVersion: 1.3.0 "; const SOURCE_SUBGRAPH_SCHEMA: &str = " -type TestEntity @entity { id: ID! } -type User @entity { id: ID! } -type Profile @entity { id: ID! } +type TestEntity @entity(immutable: true) { id: ID! } +type MutableEntity @entity { id: ID! } +type User @entity(immutable: true) { id: ID! } +type Profile @entity(immutable: true) { id: ID! } type TokenData @entity(timeseries: true) { id: Int8! @@ -1761,6 +1762,7 @@ specVersion: 1.3.0 let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; assert!(result.is_err()); let err = result.unwrap_err(); + println!("Error: {}", err); assert!(err .to_string() .contains("Subgraph datasources cannot be used alongside onchain datasources")); @@ -1857,3 +1859,72 @@ specVersion: 1.3.0 } }) } + +#[tokio::test] +async fn subgraph_ds_manifest_mutable_entities_should_fail() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: + - name: SubgraphSource + kind: subgraph + entities: + - Gravatar + network: mainnet + source: + address: 'QmSource' + startBlock: 9562480 + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handlers: + - handler: handleEntity + entity: MutableEntity # This is a mutable entity and should fail +specVersion: 1.3.0 +"; + + let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err + .to_string() + .contains("Entity MutableEntity is not immutable and cannot be used as a mapping entity")); +} + +#[tokio::test] +async fn subgraph_ds_manifest_immutable_entities_should_succeed() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: + - name: SubgraphSource + kind: subgraph + entities: + - Gravatar + network: mainnet + source: + address: 'QmSource' + startBlock: 9562480 + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handlers: + - handler: handleEntity + entity: User # This is an immutable entity and should succeed +specVersion: 1.3.0 +"; + + let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; + + assert!(result.is_ok()); +} diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index d54a88751b8..c4353ffb63b 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -121,10 +121,6 @@ impl WritableStore for MockStore { unimplemented!() } - async fn supports_proof_of_indexing(&self) -> Result { - unimplemented!() - } - async fn transact_block_operations( &self, _: BlockPtr, diff --git a/store/test-store/tests/graphql/introspection.rs b/store/test-store/tests/graphql/introspection.rs index 6139e673767..8bc76213e6b 100644 --- a/store/test-store/tests/graphql/introspection.rs +++ b/store/test-store/tests/graphql/introspection.rs @@ -53,15 +53,15 @@ impl Resolver for MockResolver { Ok(r::Value::Null) } - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { let permit = Arc::new(tokio::sync::Semaphore::new(1)) .acquire_owned() .await .unwrap(); - Ok(QueryPermit { + QueryPermit { permit, wait: Duration::from_secs(0), - }) + } } } diff --git a/store/test-store/tests/graphql/mock_introspection.json b/store/test-store/tests/graphql/mock_introspection.json index 11a3ed6fdec..9ebb47acb5d 100644 --- a/store/test-store/tests/graphql/mock_introspection.json +++ b/store/test-store/tests/graphql/mock_introspection.json @@ -4,9 +4,7 @@ "name": "Query" }, "mutationType": null, - "subscriptionType": { - "name": "Subscription" - }, + "subscriptionType": null, "types": [ { "kind": "ENUM", @@ -762,340 +760,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "Subscription", - "description": null, - "fields": [ - { - "name": "user", - "description": null, - "args": [ - { - "name": "id", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "OBJECT", - "name": "User", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "users", - "description": null, - "args": [ - { - "name": "skip", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "0" - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "100" - }, - { - "name": "orderBy", - "description": null, - "type": { - "kind": "ENUM", - "name": "User_orderBy", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "orderDirection", - "description": null, - "type": { - "kind": "ENUM", - "name": "OrderDirection", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "where", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "User_filter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "User", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": null, - "args": [ - { - "name": "id", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "INTERFACE", - "name": "Node", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": null, - "args": [ - { - "name": "skip", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "0" - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "100" - }, - { - "name": "orderBy", - "description": null, - "type": { - "kind": "ENUM", - "name": "Node_orderBy", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "orderDirection", - "description": null, - "type": { - "kind": "ENUM", - "name": "OrderDirection", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "where", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "Node_filter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Node", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "_meta", - "description": "Access to subgraph metadata", - "args": [ - { - "name": "block", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - } - ], - "type": { - "kind": "OBJECT", - "name": "_Meta_", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, { "kind": "SCALAR", "name": "Timestamp", diff --git a/store/test-store/tests/graphql/query.rs b/store/test-store/tests/graphql/query.rs index d7e7dec8f55..d5e82834a1e 100644 --- a/store/test-store/tests/graphql/query.rs +++ b/store/test-store/tests/graphql/query.rs @@ -4,12 +4,12 @@ use graph::data::store::scalar::Timestamp; use graph::data::subgraph::schema::DeploymentCreate; use graph::data::subgraph::LATEST_VERSION; use graph::entity; -use graph::prelude::{SubscriptionResult, Value}; +use graph::prelude::Value; use graph::schema::InputSchema; use std::iter::FromIterator; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use std::{ collections::{BTreeSet, HashMap}, marker::PhantomData, @@ -18,7 +18,6 @@ use test_store::block_store::{ FakeBlock, BLOCK_FOUR, BLOCK_ONE, BLOCK_THREE, BLOCK_TWO, GENESIS_BLOCK, }; -use graph::futures03::stream::StreamExt; use graph::{ components::store::DeploymentLocator, data::graphql::{object, object_value}, @@ -28,17 +27,16 @@ use graph::{ subgraph::SubgraphFeature, }, prelude::{ - lazy_static, o, q, r, serde_json, slog, BlockPtr, DeploymentHash, Entity, EntityOperation, - FutureExtension, GraphQlRunner as _, Logger, NodeId, Query, QueryError, - QueryExecutionError, QueryResult, QueryStoreManager, QueryVariables, SubgraphManifest, - SubgraphName, SubgraphStore, SubgraphVersionSwitchingMode, Subscription, SubscriptionError, + lazy_static, q, r, serde_json, BlockPtr, DeploymentHash, Entity, EntityOperation, + GraphQlRunner as _, NodeId, Query, QueryError, QueryExecutionError, QueryResult, + QueryVariables, SubgraphManifest, SubgraphName, SubgraphStore, + SubgraphVersionSwitchingMode, }, }; -use graph_graphql::{prelude::*, subscription::execute_subscription}; +use graph_graphql::prelude::*; use test_store::{ - deployment_state, execute_subgraph_query, execute_subgraph_query_with_deadline, - graphql_metrics, revert_block, run_test_sequentially, transact_errors, Store, LOAD_MANAGER, - LOGGER, METRICS_REGISTRY, STORE, SUBSCRIPTION_MANAGER, + deployment_state, execute_subgraph_query, execute_subgraph_query_with_deadline, revert_block, + run_test_sequentially, transact_errors, Store, LOAD_MANAGER, LOGGER, METRICS_REGISTRY, STORE, }; /// Ids for the various entities that we create in `insert_entities` and @@ -615,7 +613,6 @@ async fn execute_query_document_with_variables( let runner = Arc::new(GraphQlRunner::new( &LOGGER, STORE.clone(), - SUBSCRIPTION_MANAGER.clone(), LOAD_MANAGER.clone(), METRICS_REGISTRY.clone(), )); @@ -726,7 +723,6 @@ where let runner = Arc::new(GraphQlRunner::new( &LOGGER, STORE.clone(), - SUBSCRIPTION_MANAGER.clone(), LOAD_MANAGER.clone(), METRICS_REGISTRY.clone(), )); @@ -745,43 +741,6 @@ where }) } -/// Helper to run a subscription -async fn run_subscription( - store: &Arc, - query: &str, - max_complexity: Option, -) -> Result { - let deployment = setup_readonly(store.as_ref()).await; - let logger = Logger::root(slog::Discard, o!()); - let query_store = store - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - true, - ) - .await - .unwrap(); - - let query = Query::new(q::parse_query(query).unwrap().into_static(), None, false); - let options = SubscriptionExecutionOptions { - logger: logger.clone(), - store: query_store.clone(), - subscription_manager: SUBSCRIPTION_MANAGER.clone(), - timeout: None, - max_complexity, - max_depth: 100, - max_first: std::u32::MAX, - max_skip: std::u32::MAX, - graphql_metrics: graphql_metrics(), - load_manager: LOAD_MANAGER.clone(), - }; - let schema = STORE - .subgraph_store() - .api_schema(&deployment.hash, &Default::default()) - .unwrap(); - - execute_subscription(Subscription { query }, schema, options) -} - #[test] fn can_query_one_to_one_relationship() { const QUERY: &str = " @@ -1858,58 +1817,6 @@ fn query_complexity() { }) } -#[test] -fn query_complexity_subscriptions() { - run_test_sequentially(|store| async move { - const QUERY1: &str = "subscription { - musicians(orderBy: id) { - name - bands(first: 100, orderBy: id) { - name - members(first: 100, orderBy: id) { - name - } - } - } - }"; - let max_complexity = Some(1_010_100); - - // This query is exactly at the maximum complexity. - // FIXME: Not collecting the stream because that will hang the test. - let _ignore_stream = run_subscription(&store, QUERY1, max_complexity) - .await - .unwrap(); - - const QUERY2: &str = "subscription { - musicians(orderBy: id) { - name - t1: bands(first: 100, orderBy: id) { - name - members(first: 100, orderBy: id) { - name - } - } - t2: bands(first: 200, orderBy: id) { - name - members(first: 100, orderBy: id) { - name - } - } - } - }"; - - let result = run_subscription(&store, QUERY2, max_complexity).await; - - match result { - Err(SubscriptionError::GraphQLError(e)) => match &e[0] { - QueryExecutionError::TooComplex(3_030_100, _) => (), // Expected - e => panic!("did not catch complexity: {:?}", e), - }, - _ => panic!("did not catch complexity"), - } - }) -} - #[test] fn instant_timeout() { run_test_sequentially(|store| async move { @@ -2138,38 +2045,6 @@ fn cannot_filter_by_derved_relationship_fields() { }) } -#[test] -fn subscription_gets_result_even_without_events() { - run_test_sequentially(|store| async move { - const QUERY: &str = "subscription { - musicians(orderBy: id, first: 2) { - name - } - }"; - - // Execute the subscription and expect at least one result to be - // available in the result stream - let stream = run_subscription(&store, QUERY, None).await.unwrap(); - let results: Vec<_> = stream - .take(1) - .collect() - .timeout(Duration::from_secs(3)) - .await - .unwrap(); - - assert_eq!(results.len(), 1); - let result = Arc::try_unwrap(results.into_iter().next().unwrap()).unwrap(); - let data = extract_data!(result).unwrap(); - let exp = object! { - musicians: vec![ - object! { name: "John" }, - object! { name: "Lisa" } - ] - }; - assert_eq!(data, exp); - }) -} - #[test] fn can_use_nested_filter() { const QUERY: &str = " diff --git a/store/test-store/tests/postgres/chain_head.rs b/store/test-store/tests/postgres/chain_head.rs index 3b840ba3566..c93eb624e59 100644 --- a/store/test-store/tests/postgres/chain_head.rs +++ b/store/test-store/tests/postgres/chain_head.rs @@ -219,10 +219,10 @@ fn test_get_block_number() { create_test_subgraph(&subgraph, "type Dummy @entity { id: ID! }").await; let query_store = subgraph_store - .query_store( - QueryTarget::Deployment(subgraph.cheap_clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + subgraph.cheap_clone(), + Default::default(), + )) .await .unwrap(); diff --git a/store/test-store/tests/postgres/store.rs b/store/test-store/tests/postgres/store.rs index 5f2f1e80e6c..28fd05da18f 100644 --- a/store/test-store/tests/postgres/store.rs +++ b/store/test-store/tests/postgres/store.rs @@ -1,17 +1,12 @@ use graph::blockchain::block_stream::FirehoseCursor; use graph::blockchain::BlockTime; use graph::data::graphql::ext::TypeDefinitionExt; -use graph::data::query::QueryTarget; use graph::data::subgraph::schema::DeploymentCreate; use graph::data_source::common::MappingABI; -use graph::futures01::{future, Stream}; -use graph::futures03::compat::Future01CompatExt; use graph::schema::{EntityType, InputSchema}; use graph_chain_ethereum::Mapping; use hex_literal::hex; use lazy_static::lazy_static; -use std::time::Duration; -use std::{collections::HashSet, sync::Mutex}; use std::{marker::PhantomData, str::FromStr}; use test_store::*; @@ -19,10 +14,7 @@ use graph::components::store::{DeploymentLocator, ReadStore, WritableStore}; use graph::data::subgraph::*; use graph::{ blockchain::DataSource, - components::store::{ - BlockStore as _, EntityFilter, EntityOrder, EntityQuery, StatusStore, - SubscriptionManager as _, - }, + components::store::{BlockStore as _, EntityFilter, EntityOrder, EntityQuery, StatusStore}, prelude::ethabi::Contract, }; use graph::{data::store::scalar, semver::Version}; @@ -912,78 +904,7 @@ fn find() { }); } -fn make_entity_change(entity_type: &EntityType) -> EntityChange { - EntityChange::Data { - subgraph_id: TEST_SUBGRAPH_ID.clone(), - entity_type: entity_type.to_string(), - } -} - -// Get as events until we've seen all the expected events or we time out waiting -async fn check_events( - stream: StoreEventStream, Error = ()> + Send>, - expected: Vec, -) { - fn as_set(events: Vec>) -> HashSet { - events.into_iter().fold(HashSet::new(), |mut set, event| { - set.extend(event.changes.iter().cloned()); - set - }) - } - - let expected = Mutex::new(as_set(expected.into_iter().map(Arc::new).collect())); - // Capture extra changes here; this is only needed for debugging, really. - // It's permissible that we get more changes than we expected because of - // how store events group changes together - let extra: Mutex> = Mutex::new(HashSet::new()); - // Get events from the store until we've either seen all the changes we - // expected or we timed out waiting for them - stream - .take_while(|event| { - let mut expected = expected.lock().unwrap(); - for change in &event.changes { - if !expected.remove(change) { - extra.lock().unwrap().insert(change.clone()); - } - } - future::ok(!expected.is_empty()) - }) - .collect() - .compat() - .timeout(Duration::from_secs(3)) - .await - .unwrap_or_else(|_| { - panic!( - "timed out waiting for events\n still waiting for {:?}\n got extra events {:?}", - expected.lock().unwrap().clone(), - extra.lock().unwrap().clone() - ) - }) - .expect("something went wrong getting events"); - // Check again that we really got everything - assert_eq!(HashSet::new(), expected.lock().unwrap().clone()); -} - -// Subscribe to store events -fn subscribe( - subgraph: &DeploymentHash, - entity_type: &EntityType, -) -> StoreEventStream, Error = ()> + Send> { - let subscription = - SUBSCRIPTION_MANAGER.subscribe(FromIterator::from_iter([SubscriptionFilter::Entities( - subgraph.clone(), - entity_type.to_owned(), - )])); - - StoreEventStream::new(subscription) -} - -async fn check_basic_revert( - store: Arc, - expected: StoreEvent, - deployment: &DeploymentLocator, - entity_type: &EntityType, -) { +async fn check_basic_revert(store: Arc, deployment: &DeploymentLocator) { let this_query = user_query() .filter(EntityFilter::Equal( "name".to_owned(), @@ -991,7 +912,6 @@ async fn check_basic_revert( )) .desc("name"); - let subscription = subscribe(&deployment.hash, entity_type); let state = deployment_state(store.as_ref(), &deployment.hash).await; assert_eq!(&deployment.hash, &state.id); @@ -1014,17 +934,13 @@ async fn check_basic_revert( let state = deployment_state(store.as_ref(), &deployment.hash).await; assert_eq!(&deployment.hash, &state.id); - - check_events(subscription, vec![expected]).await } #[test] fn revert_block_basic_user() { run_test(|store, _, deployment| async move { - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - let count = get_entity_count(store.clone(), &deployment.hash); - check_basic_revert(store.clone(), expected, &deployment, &*USER_TYPE).await; + check_basic_revert(store.clone(), &deployment).await; assert_eq!(count, get_entity_count(store.clone(), &deployment.hash)); }) } @@ -1052,8 +968,6 @@ fn revert_block_with_delete() { .await .unwrap(); - let subscription = subscribe(&deployment.hash, &*USER_TYPE); - // Revert deletion let count = get_entity_count(store.clone(), &deployment.hash); revert_block(&store, &deployment, &TEST_BLOCK_2_PTR).await; @@ -1073,12 +987,6 @@ fn revert_block_with_delete() { let test_value = Value::String("dinici@email.com".to_owned()); assert!(returned_name.is_some()); assert_eq!(&test_value, returned_name.unwrap()); - - // Check that the subscription notified us of the changes - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - - // The last event is the one for the reversion - check_events(subscription, vec![expected]).await }) } @@ -1106,8 +1014,6 @@ fn revert_block_with_partial_update() { .await .unwrap(); - let subscription = subscribe(&deployment.hash, &*USER_TYPE); - // Perform revert operation, reversing the partial update let count = get_entity_count(store.clone(), &deployment.hash); revert_block(&store, &deployment, &TEST_BLOCK_2_PTR).await; @@ -1118,11 +1024,6 @@ fn revert_block_with_partial_update() { // Verify that the entity has been returned to its original state assert_eq!(reverted_entity, original_entity); - - // Check that the subscription notified us of the changes - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - - check_events(subscription, vec![expected]).await }) } @@ -1229,8 +1130,6 @@ fn revert_block_with_dynamic_data_source_operations() { **loaded_dds[0].param.as_ref().unwrap() ); - let subscription = subscribe(&deployment.hash, &*USER_TYPE); - // Revert block that added the user and the dynamic data source revert_block(&store, &deployment, &TEST_BLOCK_2_PTR).await; @@ -1246,233 +1145,6 @@ fn revert_block_with_dynamic_data_source_operations() { .await .unwrap(); assert_eq!(0, loaded_dds.len()); - - // Verify that the right change events were emitted for the reversion - let expected_events = vec![StoreEvent { - tag: 3, - changes: HashSet::from_iter( - vec![EntityChange::Data { - subgraph_id: DeploymentHash::new("testsubgraph").unwrap(), - entity_type: USER_TYPE.to_string(), - }] - .into_iter(), - ), - }]; - check_events(subscription, expected_events).await - }) -} - -#[test] -fn entity_changes_are_fired_and_forwarded_to_subscriptions() { - run_test(|store, _, _| async move { - let subgraph_id = DeploymentHash::new("EntityChangeTestSubgraph").unwrap(); - let schema = InputSchema::parse_latest(USER_GQL, subgraph_id.clone()) - .expect("Failed to parse user schema"); - let manifest = SubgraphManifest:: { - id: subgraph_id.clone(), - spec_version: Version::new(1, 3, 0), - features: Default::default(), - description: None, - repository: None, - schema: schema.clone(), - data_sources: vec![], - graft: None, - templates: vec![], - chain: PhantomData, - indexer_hints: None, - }; - - let deployment = - DeploymentCreate::new(String::new(), &manifest, Some(TEST_BLOCK_0_PTR.clone())); - let name = SubgraphName::new("test/entity-changes-are-fired").unwrap(); - let node_id = NodeId::new("test").unwrap(); - let deployment = store - .subgraph_store() - .create_subgraph_deployment( - name, - &schema, - deployment, - node_id, - NETWORK_NAME.to_string(), - SubgraphVersionSwitchingMode::Instant, - ) - .unwrap(); - - let subscription = subscribe(&subgraph_id, &*USER_TYPE); - - // Add two entities to the store - let added_entities = vec![ - ( - "1".to_owned(), - entity! { schema => id: "1", name: "Johnny Boy", vid: 5i64 }, - ), - ( - "2".to_owned(), - entity! { schema => id: "2", name: "Tessa", vid: 6i64 }, - ), - ]; - transact_and_wait( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_1_PTR.clone(), - added_entities - .iter() - .map(|(id, data)| { - let mut data = data.clone(); - data.set_vid_if_empty(); - EntityOperation::Set { - key: USER_TYPE.parse_key(id.as_str()).unwrap(), - data, - } - }) - .collect(), - ) - .await - .unwrap(); - - // Update an entity in the store - let updated_entity = entity! { schema => id: "1", name: "Johnny", vid: 7i64 }; - let update_op = EntityOperation::Set { - key: USER_TYPE.parse_key("1").unwrap(), - data: updated_entity.clone(), - }; - - // Delete an entity in the store - let delete_op = EntityOperation::Remove { - key: USER_TYPE.parse_key("2").unwrap(), - }; - - // Commit update & delete ops - transact_and_wait( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_2_PTR.clone(), - vec![update_op, delete_op], - ) - .await - .unwrap(); - - // We're expecting two events to be written to the subscription stream - let expected = vec![ - StoreEvent::new(vec![ - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - ]), - StoreEvent::new(vec![ - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - ]), - ]; - - check_events(subscription, expected).await - }) -} - -#[test] -fn throttle_subscription_delivers() { - run_test(|store, _, deployment| async move { - let subscription = subscribe(&deployment.hash, &*USER_TYPE) - .throttle_while_syncing( - &LOGGER, - store - .clone() - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - true, - ) - .await - .unwrap(), - Duration::from_millis(500), - ) - .await; - - let user4 = create_test_entity( - "4", - &*USER_TYPE, - "Steve", - "nieve@email.com", - 72_i32, - 120.7, - false, - None, - 7, - ); - - transact_entity_operations( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_3_PTR.clone(), - vec![user4], - ) - .await - .unwrap(); - - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - - check_events(subscription, vec![expected]).await - }) -} - -#[test] -fn throttle_subscription_throttles() { - run_test(|store, _, deployment| async move { - // Throttle for a very long time (30s) - let subscription = subscribe(&deployment.hash, &*USER_TYPE) - .throttle_while_syncing( - &LOGGER, - store - .clone() - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - true, - ) - .await - .unwrap(), - Duration::from_secs(30), - ) - .await; - - let user4 = create_test_entity( - "4", - &*USER_TYPE, - "Steve", - "nieve@email.com", - 72_i32, - 120.7, - false, - None, - 8, - ); - - transact_entity_operations( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_3_PTR.clone(), - vec![user4], - ) - .await - .unwrap(); - - // Make sure we time out waiting for the subscription - let res = subscription - .take(1) - .collect() - .compat() - .timeout(Duration::from_millis(500)) - .await; - assert!(res.is_err()); }) } diff --git a/store/test-store/tests/postgres/subgraph.rs b/store/test-store/tests/postgres/subgraph.rs index 3065c8800ef..c66d34e27c7 100644 --- a/store/test-store/tests/postgres/subgraph.rs +++ b/store/test-store/tests/postgres/subgraph.rs @@ -10,9 +10,8 @@ use graph::{ schema::{DeploymentCreate, SubgraphError}, DeploymentFeatures, }, + prelude::AssignmentChange, prelude::BlockPtr, - prelude::EntityChange, - prelude::EntityChangeOperation, prelude::QueryStoreManager, prelude::StoreEvent, prelude::SubgraphManifest, @@ -59,18 +58,12 @@ const SUBGRAPH_FEATURES_GQL: &str = " } "; -fn assigned(deployment: &DeploymentLocator) -> EntityChange { - EntityChange::Assignment { - deployment: deployment.clone(), - operation: EntityChangeOperation::Set, - } +fn assigned(deployment: &DeploymentLocator) -> AssignmentChange { + AssignmentChange::set(deployment.clone()) } -fn unassigned(deployment: &DeploymentLocator) -> EntityChange { - EntityChange::Assignment { - deployment: deployment.clone(), - operation: EntityChangeOperation::Removed, - } +fn unassigned(deployment: &DeploymentLocator) -> AssignmentChange { + AssignmentChange::removed(deployment.clone()) } fn get_version_info(store: &Store, subgraph_name: &str) -> VersionInfo { @@ -163,7 +156,7 @@ fn create_subgraph() { store: &SubgraphStore, id: &str, mode: SubgraphVersionSwitchingMode, - ) -> (DeploymentLocator, HashSet) { + ) -> (DeploymentLocator, HashSet) { let name = SubgraphName::new(SUBGRAPH_NAME.to_string()).unwrap(); let id = DeploymentHash::new(id.to_string()).unwrap(); let schema = InputSchema::parse_latest(SUBGRAPH_GQL, id.clone()).unwrap(); @@ -203,7 +196,7 @@ fn create_subgraph() { (deployment, events) } - fn deploy_event(deployment: &DeploymentLocator) -> HashSet { + fn deploy_event(deployment: &DeploymentLocator) -> HashSet { let mut changes = HashSet::new(); changes.insert(assigned(deployment)); changes @@ -703,10 +696,10 @@ fn fatal_vs_non_fatal() { run_test_sequentially(|store| async move { let deployment = setup().await; let query_store = store - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + deployment.hash.clone(), + Default::default(), + )) .await .unwrap(); @@ -764,10 +757,10 @@ fn fail_unfail_deterministic_error() { let deployment = setup().await; let query_store = store - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + deployment.hash.clone(), + Default::default(), + )) .await .unwrap(); diff --git a/substreams/substreams-trigger-filter/Cargo.toml b/substreams/substreams-trigger-filter/Cargo.toml index a3c736f2c92..f1880c3412b 100644 --- a/substreams/substreams-trigger-filter/Cargo.toml +++ b/substreams/substreams-trigger-filter/Cargo.toml @@ -10,12 +10,12 @@ crate-type = ["cdylib"] [dependencies] hex = { version = "0.4", default-features = false } -prost = "0.11.9" -substreams = "0.5" -substreams-entity-change = "1.3" -substreams-near-core = "0.10.1" +prost.workspace = true +substreams.workspace = true +substreams-entity-change.workspace = true +substreams-near-core.workspace = true trigger-filters.path = "../trigger-filters" [build-dependencies] -tonic-build = { version = "0.11.0", features = ["prost"] } +tonic-build.workspace = true diff --git a/substreams/substreams-trigger-filter/build.rs b/substreams/substreams-trigger-filter/build.rs index f29e8ecf091..22b972babc5 100644 --- a/substreams/substreams-trigger-filter/build.rs +++ b/substreams/substreams-trigger-filter/build.rs @@ -6,13 +6,7 @@ fn main() { ".sf.near.codec.v1", "::substreams_near_core::pb::sf::near::type::v1", ) - // .extern_path( - // ".sf.ethereum.type.v2", - // "graph_chain_ethereum::codec::pbcodec", - // ) - // .extern_path(".sf.arweave.type.v1", "graph_chain_arweave::codec::pbcodec") - // .extern_path(".sf.cosmos.type.v1", "graph_chain_cosmos::codec") .out_dir("src/pb") - .compile(&["proto/receipts.proto"], &["proto"]) + .compile_protos(&["proto/receipts.proto"], &["proto"]) .expect("Failed to compile Substreams entity proto(s)"); } diff --git a/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs b/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs index dc5b47203ef..76b6d1fe456 100644 --- a/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs +++ b/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockAndReceipts { #[prost(message, optional, tag = "1")] diff --git a/substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs b/substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs deleted file mode 100644 index ed60d39b47e..00000000000 --- a/substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs +++ /dev/null @@ -1,1181 +0,0 @@ -// @generated -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(string, tag="1")] - pub author: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub header: ::core::option::Option, - #[prost(message, repeated, tag="3")] - pub chunk_headers: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub shards: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="5")] - pub state_changes: ::prost::alloc::vec::Vec, -} -/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are -/// removed so that hydrating that object from a \[Block\] bytes payload will -/// drastically reduced allocated memory required to hold the full block. -/// -/// This can be used to unpack a \[Block\] when only the \[BlockHeader\] information -/// is required and greatly reduced required memory. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag="2")] - pub header: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateChangeWithCause { - #[prost(message, optional, tag="1")] - pub value: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub cause: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateChangeCause { - #[prost(oneof="state_change_cause::Cause", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10")] - pub cause: ::core::option::Option, -} -/// Nested message and enum types in `StateChangeCause`. -pub mod state_change_cause { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct NotWritableToDisk { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct InitialState { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct TransactionProcessing { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ActionReceiptProcessingStarted { - #[prost(message, optional, tag="1")] - pub receipt_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ActionReceiptGasReward { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ReceiptProcessing { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct PostponedReceipt { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct UpdatedDelayedReceipts { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ValidatorAccountsUpdate { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct Migration { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Cause { - #[prost(message, tag="1")] - NotWritableToDisk(NotWritableToDisk), - #[prost(message, tag="2")] - InitialState(InitialState), - #[prost(message, tag="3")] - TransactionProcessing(TransactionProcessing), - #[prost(message, tag="4")] - ActionReceiptProcessingStarted(ActionReceiptProcessingStarted), - #[prost(message, tag="5")] - ActionReceiptGasReward(ActionReceiptGasReward), - #[prost(message, tag="6")] - ReceiptProcessing(ReceiptProcessing), - #[prost(message, tag="7")] - PostponedReceipt(PostponedReceipt), - #[prost(message, tag="8")] - UpdatedDelayedReceipts(UpdatedDelayedReceipts), - #[prost(message, tag="9")] - ValidatorAccountsUpdate(ValidatorAccountsUpdate), - #[prost(message, tag="10")] - Migration(Migration), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateChangeValue { - #[prost(oneof="state_change_value::Value", tags="1, 2, 3, 4, 5, 6, 7, 8")] - pub value: ::core::option::Option, -} -/// Nested message and enum types in `StateChangeValue`. -pub mod state_change_value { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccountUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub account: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccountDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccessKeyUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub access_key: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccessKeyDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct DataUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub key: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="3")] - pub value: ::prost::alloc::vec::Vec, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct DataDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub key: ::prost::alloc::vec::Vec, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ContractCodeUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub code: ::prost::alloc::vec::Vec, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ContractCodeDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Value { - #[prost(message, tag="1")] - AccountUpdate(AccountUpdate), - #[prost(message, tag="2")] - AccountDeletion(AccountDeletion), - #[prost(message, tag="3")] - AccessKeyUpdate(AccessKeyUpdate), - #[prost(message, tag="4")] - AccessKeyDeletion(AccessKeyDeletion), - #[prost(message, tag="5")] - DataUpdate(DataUpdate), - #[prost(message, tag="6")] - DataDeletion(DataDeletion), - #[prost(message, tag="7")] - ContractCodeUpdate(ContractCodeUpdate), - #[prost(message, tag="8")] - ContractDeletion(ContractCodeDeletion), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Account { - #[prost(message, optional, tag="1")] - pub amount: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub locked: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub code_hash: ::core::option::Option, - #[prost(uint64, tag="4")] - pub storage_usage: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockHeader { - #[prost(uint64, tag="1")] - pub height: u64, - #[prost(uint64, tag="2")] - pub prev_height: u64, - #[prost(message, optional, tag="3")] - pub epoch_id: ::core::option::Option, - #[prost(message, optional, tag="4")] - pub next_epoch_id: ::core::option::Option, - #[prost(message, optional, tag="5")] - pub hash: ::core::option::Option, - #[prost(message, optional, tag="6")] - pub prev_hash: ::core::option::Option, - #[prost(message, optional, tag="7")] - pub prev_state_root: ::core::option::Option, - #[prost(message, optional, tag="8")] - pub chunk_receipts_root: ::core::option::Option, - #[prost(message, optional, tag="9")] - pub chunk_headers_root: ::core::option::Option, - #[prost(message, optional, tag="10")] - pub chunk_tx_root: ::core::option::Option, - #[prost(message, optional, tag="11")] - pub outcome_root: ::core::option::Option, - #[prost(uint64, tag="12")] - pub chunks_included: u64, - #[prost(message, optional, tag="13")] - pub challenges_root: ::core::option::Option, - #[prost(uint64, tag="14")] - pub timestamp: u64, - #[prost(uint64, tag="15")] - pub timestamp_nanosec: u64, - #[prost(message, optional, tag="16")] - pub random_value: ::core::option::Option, - #[prost(message, repeated, tag="17")] - pub validator_proposals: ::prost::alloc::vec::Vec, - #[prost(bool, repeated, tag="18")] - pub chunk_mask: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="19")] - pub gas_price: ::core::option::Option, - #[prost(uint64, tag="20")] - pub block_ordinal: u64, - #[prost(message, optional, tag="21")] - pub total_supply: ::core::option::Option, - #[prost(message, repeated, tag="22")] - pub challenges_result: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="23")] - pub last_final_block_height: u64, - #[prost(message, optional, tag="24")] - pub last_final_block: ::core::option::Option, - #[prost(uint64, tag="25")] - pub last_ds_final_block_height: u64, - #[prost(message, optional, tag="26")] - pub last_ds_final_block: ::core::option::Option, - #[prost(message, optional, tag="27")] - pub next_bp_hash: ::core::option::Option, - #[prost(message, optional, tag="28")] - pub block_merkle_root: ::core::option::Option, - #[prost(bytes="vec", tag="29")] - pub epoch_sync_data_hash: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="30")] - pub approvals: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="31")] - pub signature: ::core::option::Option, - #[prost(uint32, tag="32")] - pub latest_protocol_version: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BigInt { - #[prost(bytes="vec", tag="1")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CryptoHash { - #[prost(bytes="vec", tag="1")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Signature { - #[prost(enumeration="CurveKind", tag="1")] - pub r#type: i32, - #[prost(bytes="vec", tag="2")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PublicKey { - #[prost(enumeration="CurveKind", tag="1")] - pub r#type: i32, - #[prost(bytes="vec", tag="2")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorStake { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub stake: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SlashedValidator { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bool, tag="2")] - pub is_double_sign: bool, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ChunkHeader { - #[prost(bytes="vec", tag="1")] - pub chunk_hash: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="2")] - pub prev_block_hash: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="3")] - pub outcome_root: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="4")] - pub prev_state_root: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="5")] - pub encoded_merkle_root: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="6")] - pub encoded_length: u64, - #[prost(uint64, tag="7")] - pub height_created: u64, - #[prost(uint64, tag="8")] - pub height_included: u64, - #[prost(uint64, tag="9")] - pub shard_id: u64, - #[prost(uint64, tag="10")] - pub gas_used: u64, - #[prost(uint64, tag="11")] - pub gas_limit: u64, - #[prost(message, optional, tag="12")] - pub validator_reward: ::core::option::Option, - #[prost(message, optional, tag="13")] - pub balance_burnt: ::core::option::Option, - #[prost(bytes="vec", tag="14")] - pub outgoing_receipts_root: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="15")] - pub tx_root: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="16")] - pub validator_proposals: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="17")] - pub signature: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerShard { - #[prost(uint64, tag="1")] - pub shard_id: u64, - #[prost(message, optional, tag="2")] - pub chunk: ::core::option::Option, - #[prost(message, repeated, tag="3")] - pub receipt_execution_outcomes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerExecutionOutcomeWithReceipt { - #[prost(message, optional, tag="1")] - pub execution_outcome: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub receipt: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerChunk { - #[prost(string, tag="1")] - pub author: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub header: ::core::option::Option, - #[prost(message, repeated, tag="3")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub receipts: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerTransactionWithOutcome { - #[prost(message, optional, tag="1")] - pub transaction: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub outcome: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedTransaction { - #[prost(string, tag="1")] - pub signer_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - #[prost(uint64, tag="3")] - pub nonce: u64, - #[prost(string, tag="4")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(message, repeated, tag="5")] - pub actions: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="6")] - pub signature: ::core::option::Option, - #[prost(message, optional, tag="7")] - pub hash: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerExecutionOutcomeWithOptionalReceipt { - #[prost(message, optional, tag="1")] - pub execution_outcome: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub receipt: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Receipt { - #[prost(string, tag="1")] - pub predecessor_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="3")] - pub receipt_id: ::core::option::Option, - #[prost(oneof="receipt::Receipt", tags="10, 11")] - pub receipt: ::core::option::Option, -} -/// Nested message and enum types in `Receipt`. -pub mod receipt { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Receipt { - #[prost(message, tag="10")] - Action(super::ReceiptAction), - #[prost(message, tag="11")] - Data(super::ReceiptData), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ReceiptData { - #[prost(message, optional, tag="1")] - pub data_id: ::core::option::Option, - #[prost(bytes="vec", tag="2")] - pub data: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ReceiptAction { - #[prost(string, tag="1")] - pub signer_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub signer_public_key: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub gas_price: ::core::option::Option, - #[prost(message, repeated, tag="4")] - pub output_data_receivers: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="5")] - pub input_data_ids: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="6")] - pub actions: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DataReceiver { - #[prost(message, optional, tag="1")] - pub data_id: ::core::option::Option, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ExecutionOutcomeWithId { - #[prost(message, optional, tag="1")] - pub proof: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub block_hash: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub id: ::core::option::Option, - #[prost(message, optional, tag="4")] - pub outcome: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ExecutionOutcome { - #[prost(string, repeated, tag="1")] - pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - #[prost(message, repeated, tag="2")] - pub receipt_ids: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="3")] - pub gas_burnt: u64, - #[prost(message, optional, tag="4")] - pub tokens_burnt: ::core::option::Option, - #[prost(string, tag="5")] - pub executor_id: ::prost::alloc::string::String, - #[prost(enumeration="ExecutionMetadata", tag="6")] - pub metadata: i32, - #[prost(oneof="execution_outcome::Status", tags="20, 21, 22, 23")] - pub status: ::core::option::Option, -} -/// Nested message and enum types in `ExecutionOutcome`. -pub mod execution_outcome { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Status { - #[prost(message, tag="20")] - Unknown(super::UnknownExecutionStatus), - #[prost(message, tag="21")] - Failure(super::FailureExecutionStatus), - #[prost(message, tag="22")] - SuccessValue(super::SuccessValueExecutionStatus), - #[prost(message, tag="23")] - SuccessReceiptId(super::SuccessReceiptIdExecutionStatus), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SuccessValueExecutionStatus { - #[prost(bytes="vec", tag="1")] - pub value: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SuccessReceiptIdExecutionStatus { - #[prost(message, optional, tag="1")] - pub id: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct UnknownExecutionStatus { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FailureExecutionStatus { - #[prost(oneof="failure_execution_status::Failure", tags="1, 2")] - pub failure: ::core::option::Option, -} -/// Nested message and enum types in `FailureExecutionStatus`. -pub mod failure_execution_status { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Failure { - #[prost(message, tag="1")] - ActionError(super::ActionError), - #[prost(enumeration="super::InvalidTxError", tag="2")] - InvalidTxError(i32), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ActionError { - #[prost(uint64, tag="1")] - pub index: u64, - #[prost(oneof="action_error::Kind", tags="21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42")] - pub kind: ::core::option::Option, -} -/// Nested message and enum types in `ActionError`. -pub mod action_error { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Kind { - #[prost(message, tag="21")] - AccountAlreadyExist(super::AccountAlreadyExistsErrorKind), - #[prost(message, tag="22")] - AccountDoesNotExist(super::AccountDoesNotExistErrorKind), - #[prost(message, tag="23")] - CreateAccountOnlyByRegistrar(super::CreateAccountOnlyByRegistrarErrorKind), - #[prost(message, tag="24")] - CreateAccountNotAllowed(super::CreateAccountNotAllowedErrorKind), - #[prost(message, tag="25")] - ActorNoPermission(super::ActorNoPermissionErrorKind), - #[prost(message, tag="26")] - DeleteKeyDoesNotExist(super::DeleteKeyDoesNotExistErrorKind), - #[prost(message, tag="27")] - AddKeyAlreadyExists(super::AddKeyAlreadyExistsErrorKind), - #[prost(message, tag="28")] - DeleteAccountStaking(super::DeleteAccountStakingErrorKind), - #[prost(message, tag="29")] - LackBalanceForState(super::LackBalanceForStateErrorKind), - #[prost(message, tag="30")] - TriesToUnstake(super::TriesToUnstakeErrorKind), - #[prost(message, tag="31")] - TriesToStake(super::TriesToStakeErrorKind), - #[prost(message, tag="32")] - InsufficientStake(super::InsufficientStakeErrorKind), - #[prost(message, tag="33")] - FunctionCall(super::FunctionCallErrorKind), - #[prost(message, tag="34")] - NewReceiptValidation(super::NewReceiptValidationErrorKind), - #[prost(message, tag="35")] - OnlyImplicitAccountCreationAllowed(super::OnlyImplicitAccountCreationAllowedErrorKind), - #[prost(message, tag="36")] - DeleteAccountWithLargeState(super::DeleteAccountWithLargeStateErrorKind), - #[prost(message, tag="37")] - DelegateActionInvalidSignature(super::DelegateActionInvalidSignatureKind), - #[prost(message, tag="38")] - DelegateActionSenderDoesNotMatchTxReceiver(super::DelegateActionSenderDoesNotMatchTxReceiverKind), - #[prost(message, tag="39")] - DelegateActionExpired(super::DelegateActionExpiredKind), - #[prost(message, tag="40")] - DelegateActionAccessKeyError(super::DelegateActionAccessKeyErrorKind), - #[prost(message, tag="41")] - DelegateActionInvalidNonce(super::DelegateActionInvalidNonceKind), - #[prost(message, tag="42")] - DelegateActionNonceTooLarge(super::DelegateActionNonceTooLargeKind), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountAlreadyExistsErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountDoesNotExistErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -/// / A top-level account ID can only be created by registrar. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreateAccountOnlyByRegistrarErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub registrar_account_id: ::prost::alloc::string::String, - #[prost(string, tag="3")] - pub predecessor_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreateAccountNotAllowedErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub predecessor_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ActorNoPermissionErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub actor_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteKeyDoesNotExistErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AddKeyAlreadyExistsErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteAccountStakingErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LackBalanceForStateErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub balance: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TriesToUnstakeErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TriesToStakeErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub stake: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub locked: ::core::option::Option, - #[prost(message, optional, tag="4")] - pub balance: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct InsufficientStakeErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub stake: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub minimum_stake: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FunctionCallErrorKind { - #[prost(enumeration="FunctionCallErrorSer", tag="1")] - pub error: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct NewReceiptValidationErrorKind { - #[prost(enumeration="ReceiptValidationError", tag="1")] - pub error: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct OnlyImplicitAccountCreationAllowedErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteAccountWithLargeStateErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionInvalidSignatureKind { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionSenderDoesNotMatchTxReceiverKind { - #[prost(string, tag="1")] - pub sender_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionExpiredKind { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionAccessKeyErrorKind { - /// InvalidAccessKeyError - #[prost(enumeration="InvalidTxError", tag="1")] - pub error: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionInvalidNonceKind { - #[prost(uint64, tag="1")] - pub delegate_nonce: u64, - #[prost(uint64, tag="2")] - pub ak_nonce: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionNonceTooLargeKind { - #[prost(uint64, tag="1")] - pub delegate_nonce: u64, - #[prost(uint64, tag="2")] - pub upper_bound: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MerklePath { - #[prost(message, repeated, tag="1")] - pub path: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MerklePathItem { - #[prost(message, optional, tag="1")] - pub hash: ::core::option::Option, - #[prost(enumeration="Direction", tag="2")] - pub direction: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Action { - #[prost(oneof="action::Action", tags="1, 2, 3, 4, 5, 6, 7, 8, 9")] - pub action: ::core::option::Option, -} -/// Nested message and enum types in `Action`. -pub mod action { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Action { - #[prost(message, tag="1")] - CreateAccount(super::CreateAccountAction), - #[prost(message, tag="2")] - DeployContract(super::DeployContractAction), - #[prost(message, tag="3")] - FunctionCall(super::FunctionCallAction), - #[prost(message, tag="4")] - Transfer(super::TransferAction), - #[prost(message, tag="5")] - Stake(super::StakeAction), - #[prost(message, tag="6")] - AddKey(super::AddKeyAction), - #[prost(message, tag="7")] - DeleteKey(super::DeleteKeyAction), - #[prost(message, tag="8")] - DeleteAccount(super::DeleteAccountAction), - #[prost(message, tag="9")] - Delegate(super::SignedDelegateAction), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreateAccountAction { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeployContractAction { - #[prost(bytes="vec", tag="1")] - pub code: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FunctionCallAction { - #[prost(string, tag="1")] - pub method_name: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub args: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="3")] - pub gas: u64, - #[prost(message, optional, tag="4")] - pub deposit: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransferAction { - #[prost(message, optional, tag="1")] - pub deposit: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StakeAction { - #[prost(message, optional, tag="1")] - pub stake: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AddKeyAction { - #[prost(message, optional, tag="1")] - pub public_key: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub access_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteKeyAction { - #[prost(message, optional, tag="1")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteAccountAction { - #[prost(string, tag="1")] - pub beneficiary_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedDelegateAction { - #[prost(message, optional, tag="1")] - pub signature: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub delegate_action: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateAction { - #[prost(string, tag="1")] - pub sender_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(message, repeated, tag="3")] - pub actions: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="4")] - pub nonce: u64, - #[prost(uint64, tag="5")] - pub max_block_height: u64, - #[prost(message, optional, tag="6")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccessKey { - #[prost(uint64, tag="1")] - pub nonce: u64, - #[prost(message, optional, tag="2")] - pub permission: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccessKeyPermission { - #[prost(oneof="access_key_permission::Permission", tags="1, 2")] - pub permission: ::core::option::Option, -} -/// Nested message and enum types in `AccessKeyPermission`. -pub mod access_key_permission { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Permission { - #[prost(message, tag="1")] - FunctionCall(super::FunctionCallPermission), - #[prost(message, tag="2")] - FullAccess(super::FullAccessPermission), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FunctionCallPermission { - #[prost(message, optional, tag="1")] - pub allowance: ::core::option::Option, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(string, repeated, tag="3")] - pub method_names: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FullAccessPermission { -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum CurveKind { - Ed25519 = 0, - Secp256k1 = 1, -} -impl CurveKind { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - CurveKind::Ed25519 => "ED25519", - CurveKind::Secp256k1 => "SECP256K1", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "ED25519" => Some(Self::Ed25519), - "SECP256K1" => Some(Self::Secp256k1), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ExecutionMetadata { - V1 = 0, -} -impl ExecutionMetadata { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - ExecutionMetadata::V1 => "ExecutionMetadataV1", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "ExecutionMetadataV1" => Some(Self::V1), - _ => None, - } - } -} -/// todo: add more detail? -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum FunctionCallErrorSer { - CompilationError = 0, - LinkError = 1, - MethodResolveError = 2, - WasmTrap = 3, - WasmUnknownError = 4, - HostError = 5, - EvmError = 6, - ExecutionError = 7, -} -impl FunctionCallErrorSer { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - FunctionCallErrorSer::CompilationError => "CompilationError", - FunctionCallErrorSer::LinkError => "LinkError", - FunctionCallErrorSer::MethodResolveError => "MethodResolveError", - FunctionCallErrorSer::WasmTrap => "WasmTrap", - FunctionCallErrorSer::WasmUnknownError => "WasmUnknownError", - FunctionCallErrorSer::HostError => "HostError", - FunctionCallErrorSer::EvmError => "_EVMError", - FunctionCallErrorSer::ExecutionError => "ExecutionError", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "CompilationError" => Some(Self::CompilationError), - "LinkError" => Some(Self::LinkError), - "MethodResolveError" => Some(Self::MethodResolveError), - "WasmTrap" => Some(Self::WasmTrap), - "WasmUnknownError" => Some(Self::WasmUnknownError), - "HostError" => Some(Self::HostError), - "_EVMError" => Some(Self::EvmError), - "ExecutionError" => Some(Self::ExecutionError), - _ => None, - } - } -} -/// todo: add more detail? -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ReceiptValidationError { - InvalidPredecessorId = 0, - InvalidReceiverAccountId = 1, - InvalidSignerAccountId = 2, - InvalidDataReceiverId = 3, - ReturnedValueLengthExceeded = 4, - NumberInputDataDependenciesExceeded = 5, - ActionsValidationError = 6, -} -impl ReceiptValidationError { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - ReceiptValidationError::InvalidPredecessorId => "InvalidPredecessorId", - ReceiptValidationError::InvalidReceiverAccountId => "InvalidReceiverAccountId", - ReceiptValidationError::InvalidSignerAccountId => "InvalidSignerAccountId", - ReceiptValidationError::InvalidDataReceiverId => "InvalidDataReceiverId", - ReceiptValidationError::ReturnedValueLengthExceeded => "ReturnedValueLengthExceeded", - ReceiptValidationError::NumberInputDataDependenciesExceeded => "NumberInputDataDependenciesExceeded", - ReceiptValidationError::ActionsValidationError => "ActionsValidationError", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "InvalidPredecessorId" => Some(Self::InvalidPredecessorId), - "InvalidReceiverAccountId" => Some(Self::InvalidReceiverAccountId), - "InvalidSignerAccountId" => Some(Self::InvalidSignerAccountId), - "InvalidDataReceiverId" => Some(Self::InvalidDataReceiverId), - "ReturnedValueLengthExceeded" => Some(Self::ReturnedValueLengthExceeded), - "NumberInputDataDependenciesExceeded" => Some(Self::NumberInputDataDependenciesExceeded), - "ActionsValidationError" => Some(Self::ActionsValidationError), - _ => None, - } - } -} -/// todo: add more detail? -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum InvalidTxError { - InvalidAccessKeyError = 0, - InvalidSignerId = 1, - SignerDoesNotExist = 2, - InvalidNonce = 3, - NonceTooLarge = 4, - InvalidReceiverId = 5, - InvalidSignature = 6, - NotEnoughBalance = 7, - LackBalanceForState = 8, - CostOverflow = 9, - InvalidChain = 10, - Expired = 11, - ActionsValidation = 12, - TransactionSizeExceeded = 13, -} -impl InvalidTxError { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - InvalidTxError::InvalidAccessKeyError => "InvalidAccessKeyError", - InvalidTxError::InvalidSignerId => "InvalidSignerId", - InvalidTxError::SignerDoesNotExist => "SignerDoesNotExist", - InvalidTxError::InvalidNonce => "InvalidNonce", - InvalidTxError::NonceTooLarge => "NonceTooLarge", - InvalidTxError::InvalidReceiverId => "InvalidReceiverId", - InvalidTxError::InvalidSignature => "InvalidSignature", - InvalidTxError::NotEnoughBalance => "NotEnoughBalance", - InvalidTxError::LackBalanceForState => "LackBalanceForState", - InvalidTxError::CostOverflow => "CostOverflow", - InvalidTxError::InvalidChain => "InvalidChain", - InvalidTxError::Expired => "Expired", - InvalidTxError::ActionsValidation => "ActionsValidation", - InvalidTxError::TransactionSizeExceeded => "TransactionSizeExceeded", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "InvalidAccessKeyError" => Some(Self::InvalidAccessKeyError), - "InvalidSignerId" => Some(Self::InvalidSignerId), - "SignerDoesNotExist" => Some(Self::SignerDoesNotExist), - "InvalidNonce" => Some(Self::InvalidNonce), - "NonceTooLarge" => Some(Self::NonceTooLarge), - "InvalidReceiverId" => Some(Self::InvalidReceiverId), - "InvalidSignature" => Some(Self::InvalidSignature), - "NotEnoughBalance" => Some(Self::NotEnoughBalance), - "LackBalanceForState" => Some(Self::LackBalanceForState), - "CostOverflow" => Some(Self::CostOverflow), - "InvalidChain" => Some(Self::InvalidChain), - "Expired" => Some(Self::Expired), - "ActionsValidation" => Some(Self::ActionsValidation), - "TransactionSizeExceeded" => Some(Self::TransactionSizeExceeded), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum Direction { - Left = 0, - Right = 1, -} -impl Direction { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Direction::Left => "left", - Direction::Right => "right", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "left" => Some(Self::Left), - "right" => Some(Self::Right), - _ => None, - } - } -} -// @@protoc_insertion_point(module) diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b79702ceb7f..ad4a4a9c785 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true [dependencies] anyhow = "1.0" assert-json-diff = "2.0.2" -async-stream = "0.3.5" +async-stream = "0.3.6" graph = { path = "../graph" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-substreams= {path = "../chain/substreams"} diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 9f05a680e7c..7385b4b08a2 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: ipfs: - image: docker.io/ipfs/kubo:v0.17.0 + image: docker.io/ipfs/kubo:v0.34.1 ports: - '127.0.0.1:3001:5001' postgres: @@ -20,10 +20,10 @@ services: POSTGRES_DB: graph-node POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" anvil: - image: ghcr.io/foundry-rs/foundry:latest + image: ghcr.io/foundry-rs/foundry:stable ports: - '3021:8545' - command: "'anvil --host 0.0.0.0 --gas-limit 100000000000 --base-fee 1 --block-time 5 --mnemonic \"test test test test test test test test test test test junk\"'" + command: "'anvil --host 0.0.0.0 --gas-limit 100000000000 --base-fee 1 --block-time 2 --timestamp 1743944919 --mnemonic \"test test test test test test test test test test test junk\"'" # graph-node ports: # json-rpc: 8020 diff --git a/tests/integration-tests/base/abis/Contract.abi b/tests/integration-tests/base/abis/Contract.abi new file mode 100644 index 00000000000..02da1a9e7f3 --- /dev/null +++ b/tests/integration-tests/base/abis/Contract.abi @@ -0,0 +1,33 @@ +[ + { + "inputs": [], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "Trigger", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "emitTrigger", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] diff --git a/tests/integration-tests/base/package.json b/tests/integration-tests/base/package.json new file mode 100644 index 00000000000..2cfb6b94def --- /dev/null +++ b/tests/integration-tests/base/package.json @@ -0,0 +1,25 @@ +{ + "name": "base-subgraph", + "version": "0.1.0", + "scripts": { + "build-contracts": "../../common/build-contracts.sh", + "codegen": "graph codegen --skip-migrations", + "test": "yarn build-contracts && truffle test --compile-none --network test", + "create:test": "graph create test/base-subgraph --node $GRAPH_NODE_ADMIN_URI", + "deploy:test": "graph deploy test/base-subgraph --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.69.0", + "@graphprotocol/graph-ts": "0.34.0", + "solc": "^0.8.2" + }, + "dependencies": { + "@truffle/contract": "^4.3", + "@truffle/hdwallet-provider": "^1.2", + "apollo-fetch": "^0.7.0", + "babel-polyfill": "^6.26.0", + "babel-register": "^6.26.0", + "gluegun": "^4.6.1", + "truffle": "^5.2" + } +} \ No newline at end of file diff --git a/tests/integration-tests/base/schema.graphql b/tests/integration-tests/base/schema.graphql new file mode 100644 index 00000000000..f7034353d73 --- /dev/null +++ b/tests/integration-tests/base/schema.graphql @@ -0,0 +1,5 @@ +type BaseData @entity(immutable: true) { + id: ID! + data: String! + blockNumber: BigInt! +} \ No newline at end of file diff --git a/tests/integration-tests/base/src/mapping.ts b/tests/integration-tests/base/src/mapping.ts new file mode 100644 index 00000000000..11767070a5b --- /dev/null +++ b/tests/integration-tests/base/src/mapping.ts @@ -0,0 +1,9 @@ +import { ethereum } from '@graphprotocol/graph-ts' +import { BaseData } from '../generated/schema' + +export function handleBlock(block: ethereum.Block): void { + let entity = new BaseData(block.number.toString()) + entity.data = 'from base' + entity.blockNumber = block.number + entity.save() +} \ No newline at end of file diff --git a/tests/integration-tests/base/subgraph.yaml b/tests/integration-tests/base/subgraph.yaml new file mode 100644 index 00000000000..808b446c622 --- /dev/null +++ b/tests/integration-tests/base/subgraph.yaml @@ -0,0 +1,25 @@ +specVersion: 0.0.5 +description: Base Subgraph +repository: https://github.com/graphprotocol/graph-node +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: SimpleContract + network: test + source: + address: "0x5FbDB2315678afecb367f032d93F642f64180aa3" + abi: SimpleContract + startBlock: 0 + mapping: + kind: ethereum/events + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - BaseData + abis: + - name: SimpleContract + file: ./abis/Contract.abi + blockHandlers: + - handler: handleBlock + file: ./src/mapping.ts \ No newline at end of file diff --git a/tests/integration-tests/grafted/abis/Contract.abi b/tests/integration-tests/grafted/abis/Contract.abi new file mode 100644 index 00000000000..02da1a9e7f3 --- /dev/null +++ b/tests/integration-tests/grafted/abis/Contract.abi @@ -0,0 +1,33 @@ +[ + { + "inputs": [], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "Trigger", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "emitTrigger", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] diff --git a/tests/integration-tests/grafted/package.json b/tests/integration-tests/grafted/package.json new file mode 100644 index 00000000000..d45b6fc6727 --- /dev/null +++ b/tests/integration-tests/grafted/package.json @@ -0,0 +1,25 @@ +{ + "name": "grafted-subgraph", + "version": "0.1.0", + "scripts": { + "build-contracts": "../../common/build-contracts.sh", + "codegen": "graph codegen --skip-migrations", + "test": "yarn build-contracts && truffle test --compile-none --network test", + "create:test": "graph create test/grafted-subgraph --node $GRAPH_NODE_ADMIN_URI", + "deploy:test": "graph deploy test/grafted-subgraph --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.69.0", + "@graphprotocol/graph-ts": "0.34.0", + "solc": "^0.8.2" + }, + "dependencies": { + "@truffle/contract": "^4.3", + "@truffle/hdwallet-provider": "^1.2", + "apollo-fetch": "^0.7.0", + "babel-polyfill": "^6.26.0", + "babel-register": "^6.26.0", + "gluegun": "^4.6.1", + "truffle": "^5.2" + } +} \ No newline at end of file diff --git a/tests/integration-tests/grafted/schema.graphql b/tests/integration-tests/grafted/schema.graphql new file mode 100644 index 00000000000..b83083fd466 --- /dev/null +++ b/tests/integration-tests/grafted/schema.graphql @@ -0,0 +1,5 @@ +type GraftedData @entity(immutable: true) { + id: ID! + data: String! + blockNumber: BigInt! +} \ No newline at end of file diff --git a/tests/integration-tests/grafted/src/mapping.ts b/tests/integration-tests/grafted/src/mapping.ts new file mode 100644 index 00000000000..742d5d67c54 --- /dev/null +++ b/tests/integration-tests/grafted/src/mapping.ts @@ -0,0 +1,9 @@ +import { ethereum } from '@graphprotocol/graph-ts' +import { GraftedData } from '../generated/schema' + +export function handleBlock(block: ethereum.Block): void { + let entity = new GraftedData(block.number.toString()) + entity.data = 'to grafted' + entity.blockNumber = block.number + entity.save() +} \ No newline at end of file diff --git a/tests/integration-tests/grafted/subgraph.yaml b/tests/integration-tests/grafted/subgraph.yaml new file mode 100644 index 00000000000..f946f201941 --- /dev/null +++ b/tests/integration-tests/grafted/subgraph.yaml @@ -0,0 +1,30 @@ +specVersion: 0.0.6 +description: Grafted Subgraph +repository: https://github.com/graphprotocol/graph-node +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: SimpleContract + network: test + source: + address: "0x5FbDB2315678afecb367f032d93F642f64180aa3" + abi: SimpleContract + startBlock: 0 + mapping: + kind: ethereum/events + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - GraftedData + abis: + - name: SimpleContract + file: ./abis/Contract.abi + blockHandlers: + - handler: handleBlock + file: ./src/mapping.ts +features: + - grafting +graft: + base: QmQpiC9bJGFssQfeZippfQ7rcTv7QA67X7jUejc8nV125F + block: 2 \ No newline at end of file diff --git a/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts b/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts index 649d92d3f5f..373ddd7e99e 100644 --- a/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts +++ b/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts @@ -1,28 +1,26 @@ import { dataSource, EntityTrigger, log } from '@graphprotocol/graph-ts' import { AggregatedData } from '../generated/schema' -import { SourceAData } from '../generated/subgraph-QmPWnNsD4m8T9EEF1ec5d8wetFxrMebggLj1efFHzdnZhx' -import { SourceBData } from '../generated/subgraph-Qma4Rk2D1w6mFiP15ZtHHx7eWkqFR426RWswreLiDanxej' +import { SourceAData } from '../generated/subgraph-QmYHp1bPEf7EoYBpEtJUpZv1uQHYQfWE4AhvR6frjB1Huj' +import { SourceBData } from '../generated/subgraph-QmYBEzastJi7bsa722ac78tnZa6xNnV9vvweerY4kVyJtq' -export function handleSourceAData(data: EntityTrigger): void { - let aggregated = AggregatedData.load(data.data.id) - if (!aggregated) { - aggregated = new AggregatedData(data.data.id) - aggregated.sourceA = data.data.data - aggregated.first = 'sourceA' - } else { - aggregated.sourceA = data.data.data - } + +// We know this handler will run first since its defined first in the manifest +// So we dont need to check if the Aggregated data exists +export function handleSourceAData(data: SourceAData): void { + let aggregated = new AggregatedData(data.id) + aggregated.sourceA = data.data + aggregated.first = 'sourceA' aggregated.save() } -export function handleSourceBData(data: EntityTrigger): void { - let aggregated = AggregatedData.load(data.data.id) +export function handleSourceBData(data: SourceBData): void { + let aggregated = AggregatedData.load(data.id) if (!aggregated) { - aggregated = new AggregatedData(data.data.id) - aggregated.sourceB = data.data.data + aggregated = new AggregatedData(data.id) + aggregated.sourceB = data.data aggregated.first = 'sourceB' } else { - aggregated.sourceB = data.data.data + aggregated.sourceB = data.data } aggregated.save() } diff --git a/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml b/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml index 296777c578c..4dc4fc7a9b6 100644 --- a/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml +++ b/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml @@ -6,7 +6,7 @@ dataSources: name: SourceA network: test source: - address: 'QmPWnNsD4m8T9EEF1ec5d8wetFxrMebggLj1efFHzdnZhx' + address: 'QmYHp1bPEf7EoYBpEtJUpZv1uQHYQfWE4AhvR6frjB1Huj' startBlock: 0 mapping: apiVersion: 0.0.7 @@ -22,7 +22,7 @@ dataSources: name: SourceB network: test source: - address: 'Qma4Rk2D1w6mFiP15ZtHHx7eWkqFR426RWswreLiDanxej' + address: 'QmYBEzastJi7bsa722ac78tnZa6xNnV9vvweerY4kVyJtq' startBlock: 0 mapping: apiVersion: 0.0.7 diff --git a/tests/integration-tests/source-subgraph-a/schema.graphql b/tests/integration-tests/source-subgraph-a/schema.graphql index 10be822d900..2348c9b5c57 100644 --- a/tests/integration-tests/source-subgraph-a/schema.graphql +++ b/tests/integration-tests/source-subgraph-a/schema.graphql @@ -1,4 +1,4 @@ -type SourceAData @entity { +type SourceAData @entity(immutable: true) { id: ID! data: String! blockNumber: BigInt! diff --git a/tests/integration-tests/source-subgraph-b/schema.graphql b/tests/integration-tests/source-subgraph-b/schema.graphql index 9a84bdcbba3..0b012273112 100644 --- a/tests/integration-tests/source-subgraph-b/schema.graphql +++ b/tests/integration-tests/source-subgraph-b/schema.graphql @@ -1,4 +1,4 @@ -type SourceBData @entity { +type SourceBData @entity(immutable: true) { id: ID! data: String! blockNumber: BigInt! diff --git a/tests/integration-tests/source-subgraph/schema.graphql b/tests/integration-tests/source-subgraph/schema.graphql index 15bb2a33921..4fab5be71b9 100644 --- a/tests/integration-tests/source-subgraph/schema.graphql +++ b/tests/integration-tests/source-subgraph/schema.graphql @@ -1,11 +1,10 @@ -type Block @entity { +type Block @entity(immutable: true) { id: ID! number: BigInt! hash: Bytes! - testMessage: String } -type Block2 @entity { +type Block2 @entity(immutable: true) { id: ID! number: BigInt! hash: Bytes! diff --git a/tests/integration-tests/source-subgraph/src/mapping.ts b/tests/integration-tests/source-subgraph/src/mapping.ts index ad27c43c2a3..119fb9b912b 100644 --- a/tests/integration-tests/source-subgraph/src/mapping.ts +++ b/tests/integration-tests/source-subgraph/src/mapping.ts @@ -1,6 +1,5 @@ import { ethereum, log, store } from '@graphprotocol/graph-ts'; import { Block, Block2 } from '../generated/schema'; -import { BigInt } from '@graphprotocol/graph-ts'; export function handleBlock(block: ethereum.Block): void { log.info('handleBlock {}', [block.number.toString()]); @@ -21,37 +20,6 @@ export function handleBlock(block: ethereum.Block): void { let blockEntity3 = new Block2(id3); blockEntity3.number = block.number; blockEntity3.hash = block.hash; + blockEntity3.testMessage = block.number.toString().concat('-message'); blockEntity3.save(); - - if (block.number.equals(BigInt.fromI32(1))) { - let id = 'TEST'; - let entity = new Block(id); - entity.number = block.number; - entity.hash = block.hash; - entity.testMessage = 'Created at block 1'; - log.info('Created entity at block 1', []); - entity.save(); - } - - if (block.number.equals(BigInt.fromI32(2))) { - let id = 'TEST'; - let blockEntity1 = Block.load(id); - if (blockEntity1) { - // Update the block entity - blockEntity1.testMessage = 'Updated at block 2'; - log.info('Updated entity at block 2', []); - blockEntity1.save(); - } - } - - if (block.number.equals(BigInt.fromI32(3))) { - let id = 'TEST'; - let blockEntity1 = Block.load(id); - if (blockEntity1) { - blockEntity1.testMessage = 'Deleted at block 3'; - log.info('Deleted entity at block 3', []); - blockEntity1.save(); - store.remove('Block', id); - } - } } diff --git a/tests/integration-tests/subgraph-data-sources/src/mapping.ts b/tests/integration-tests/subgraph-data-sources/src/mapping.ts index 45ecbd41076..9062970361a 100644 --- a/tests/integration-tests/subgraph-data-sources/src/mapping.ts +++ b/tests/integration-tests/subgraph-data-sources/src/mapping.ts @@ -1,26 +1,26 @@ -import { Entity, log, store, BigInt, EntityTrigger, EntityOp } from '@graphprotocol/graph-ts'; -import { Block } from '../generated/subgraph-QmVz1Pt7NhgCkz4gfavmNrMhojnMT9hW81QDqVjy56ZMUP'; +import { log, store } from '@graphprotocol/graph-ts'; +import { Block, Block2 } from '../generated/subgraph-QmWi3H11QFE2PiWx6WcQkZYZdA5UasaBptUJqGn54MFux5'; import { MirrorBlock } from '../generated/schema'; -export function handleEntity(trigger: EntityTrigger): void { - let blockEntity = trigger.data; - let id = blockEntity.id; +export function handleEntity(block: Block): void { + let id = block.id; - if (trigger.operation === EntityOp.Remove) { - log.info('Removing block entity with id: {}', [id]); - store.remove('MirrorBlock', id); - return; - } + let blockEntity = loadOrCreateMirrorBlock(id); + blockEntity.number = block.number; + blockEntity.hash = block.hash; - let block = loadOrCreateMirrorBlock(id); - block.number = blockEntity.number; - block.hash = blockEntity.hash; - - if (blockEntity.testMessage) { - block.testMessage = blockEntity.testMessage; - } + blockEntity.save(); +} + +export function handleEntity2(block: Block2): void { + let id = block.id; + + let blockEntity = loadOrCreateMirrorBlock(id); + blockEntity.number = block.number; + blockEntity.hash = block.hash; + blockEntity.testMessage = block.testMessage; - block.save(); + blockEntity.save(); } export function loadOrCreateMirrorBlock(id: string): MirrorBlock { diff --git a/tests/integration-tests/subgraph-data-sources/subgraph.yaml b/tests/integration-tests/subgraph-data-sources/subgraph.yaml index 3fdc76ac089..92dc7140514 100644 --- a/tests/integration-tests/subgraph-data-sources/subgraph.yaml +++ b/tests/integration-tests/subgraph-data-sources/subgraph.yaml @@ -6,7 +6,7 @@ dataSources: name: Contract network: test source: - address: 'QmVz1Pt7NhgCkz4gfavmNrMhojnMT9hW81QDqVjy56ZMUP' + address: 'QmWi3H11QFE2PiWx6WcQkZYZdA5UasaBptUJqGn54MFux5' startBlock: 0 mapping: apiVersion: 0.0.7 @@ -16,6 +16,6 @@ dataSources: handlers: - handler: handleEntity entity: Block - - handler: handleEntity + - handler: handleEntity2 entity: Block2 file: ./src/mapping.ts diff --git a/tests/runner-tests/yarn.lock b/tests/runner-tests/yarn.lock index 9f3bdae834d..2f9f1287bec 100644 --- a/tests/runner-tests/yarn.lock +++ b/tests/runner-tests/yarn.lock @@ -1605,7 +1605,7 @@ electron-fetch@^1.7.2: dependencies: encoding "^0.1.13" -elliptic@6.5.4, elliptic@^6.5.4: +elliptic@6.5.4: version "6.5.4" resolved "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz" integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ== @@ -1618,6 +1618,19 @@ elliptic@6.5.4, elliptic@^6.5.4: minimalistic-assert "^1.0.1" minimalistic-crypto-utils "^1.0.1" +elliptic@^6.5.7: + version "6.5.7" + resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.7.tgz#8ec4da2cb2939926a1b9a73619d768207e647c8b" + integrity sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q== + dependencies: + bn.js "^4.11.9" + brorand "^1.1.0" + hash.js "^1.0.0" + hmac-drbg "^1.0.1" + inherits "^2.0.4" + minimalistic-assert "^1.0.1" + minimalistic-crypto-utils "^1.0.1" + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" @@ -2909,9 +2922,9 @@ mustache@^4.2.0: integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ== nanoid@^3.0.2, nanoid@^3.1.20, nanoid@^3.1.23: - version "3.3.6" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz" - integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== native-abort-controller@^1.0.3, native-abort-controller@^1.0.4: version "1.0.4" @@ -2938,6 +2951,11 @@ node-addon-api@^2.0.0: resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz" integrity sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA== +node-addon-api@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.1.0.tgz#49da1ca055e109a23d537e9de43c09cca21eb762" + integrity sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA== + node-fetch@^2.6.8: version "2.6.9" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz" @@ -3394,12 +3412,12 @@ scrypt-js@^3.0.0: integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA== secp256k1@^4.0.1: - version "4.0.3" - resolved "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.3.tgz" - integrity sha512-NLZVf+ROMxwtEj3Xa562qgv2BK5e2WNmXPiOdVIPLgs6lyTzMvBq0aWTYMI5XCP9jZMVKOcqZLw/Wc4vDkuxhA== + version "4.0.4" + resolved "https://registry.yarnpkg.com/secp256k1/-/secp256k1-4.0.4.tgz#58f0bfe1830fe777d9ca1ffc7574962a8189f8ab" + integrity sha512-6JfvwvjUOn8F/jUoBY2Q1v5WY5XS+rj8qSe0v8Y4ezH4InLgTEeOOPQsRll9OV429Pvo6BCHGavIyJfr3TAhsw== dependencies: - elliptic "^6.5.4" - node-addon-api "^2.0.0" + elliptic "^6.5.7" + node-addon-api "^5.0.0" node-gyp-build "^4.2.0" semver@7.3.5: diff --git a/tests/src/config.rs b/tests/src/config.rs index 6762e542168..6cdd97a216f 100644 --- a/tests/src/config.rs +++ b/tests/src/config.rs @@ -61,7 +61,6 @@ impl EthConfig { pub struct GraphNodePorts { pub http: u16, pub index: u16, - pub ws: u16, pub admin: u16, pub metrics: u16, } @@ -70,7 +69,6 @@ impl Default for GraphNodePorts { fn default() -> Self { Self { http: 3030, - ws: 3031, admin: 3032, index: 3033, metrics: 3034, @@ -160,8 +158,6 @@ impl Config { &ports.http.to_string(), "--index-node-port", &ports.index.to_string(), - "--ws-port", - &ports.ws.to_string(), "--admin-port", &ports.admin.to_string(), "--metrics-port", @@ -179,7 +175,8 @@ impl Config { .stdout(stdout) .stderr(stderr) .args(args) - .env("GRAPH_STORE_WRITE_BATCH_DURATION", "5"); + .env("GRAPH_STORE_WRITE_BATCH_DURATION", "5") + .env("ETHEREUM_REORG_THRESHOLD", "0"); status!( "graph-node", @@ -214,7 +211,6 @@ impl Config { let setup = format!( r#" create extension pg_trgm; - create extension pg_stat_statements; create extension btree_gist; create extension postgres_fdw; grant usage on foreign data wrapper postgres_fdw to "{}"; diff --git a/tests/src/contract.rs b/tests/src/contract.rs index 4fdf767b041..05fda947839 100644 --- a/tests/src/contract.rs +++ b/tests/src/contract.rs @@ -7,7 +7,7 @@ use graph::prelude::{ api::{Eth, Namespace}, contract::{tokens::Tokenize, Contract as Web3Contract, Options}, transports::Http, - types::{Address, Bytes, TransactionReceipt}, + types::{Address, Block, BlockId, BlockNumber, Bytes, TransactionReceipt, H256}, }, }; // web3 version 0.18 does not expose this; once the graph crate updates to @@ -165,4 +165,13 @@ impl Contract { } Ok(contracts) } + + pub async fn latest_block() -> Option> { + let eth = Self::eth(); + let block = eth + .block(BlockId::Number(BlockNumber::Latest)) + .await + .unwrap_or_default(); + block + } } diff --git a/tests/src/fixture/ethereum.rs b/tests/src/fixture/ethereum.rs index fc651a512db..d93ac25c235 100644 --- a/tests/src/fixture/ethereum.rs +++ b/tests/src/fixture/ethereum.rs @@ -64,7 +64,7 @@ pub async fn chain( triggers_adapter, Arc::new(NoopRuntimeAdapterBuilder {}), eth_adapters, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), ENV_VARS.ingestor_polling_interval, // We assume the tested chain is always ingestible for now true, diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 4e8127875a0..217c7f705b6 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -80,7 +80,7 @@ pub fn test_ptr_reorged(n: BlockNumber, reorg_n: u32) -> BlockPtr { } } -type GraphQlRunner = graph_graphql::prelude::GraphQlRunner; +type GraphQlRunner = graph_graphql::prelude::GraphQlRunner; struct CommonChainConfig { logger_factory: LoggerFactory, @@ -108,6 +108,7 @@ impl CommonChainConfig { false, SubgraphLimit::Unlimited, Arc::new(EndpointMetrics::mock()), + false, ))]); Self { @@ -520,12 +521,10 @@ pub async fn setup( ); // Graphql runner - let subscription_manager = Arc::new(PanicSubscriptionManager {}); let load_manager = LoadManager::new(&logger, Vec::new(), Vec::new(), mock_registry.clone()); let graphql_runner = Arc::new(GraphQlRunner::new( &logger, stores.network_store.clone(), - subscription_manager.clone(), Arc::new(load_manager), mock_registry.clone(), )); diff --git a/tests/src/subgraph.rs b/tests/src/subgraph.rs index 810b87cbb78..92e42836b68 100644 --- a/tests/src/subgraph.rs +++ b/tests/src/subgraph.rs @@ -164,7 +164,7 @@ impl Subgraph { } /// Make a GraphQL query to the index node API - pub async fn index_with_vars(&self, text: &str, vars: Value) -> anyhow::Result { + pub async fn query_with_vars(text: &str, vars: Value) -> anyhow::Result { let endpoint = CONFIG.graph_node.index_node_uri(); graphql_query_with_vars(&endpoint, text, vars).await } diff --git a/tests/tests/integration_tests.rs b/tests/tests/integration_tests.rs index d10df25698b..9df36f7145a 100644 --- a/tests/tests/integration_tests.rs +++ b/tests/tests/integration_tests.rs @@ -11,7 +11,7 @@ use std::future::Future; use std::pin::Pin; -use std::time::{Duration, Instant}; +use std::time::{self, Duration, Instant}; use anyhow::{anyhow, bail, Context, Result}; use graph::futures03::StreamExt; @@ -25,6 +25,8 @@ use tokio::process::{Child, Command}; use tokio::task::JoinError; use tokio::time::sleep; +const SUBGRAPH_LAST_GRAFTING_BLOCK: i32 = 3; + type TestFn = Box< dyn FnOnce(TestContext) -> Pin> + Send>> + Sync @@ -110,6 +112,15 @@ impl TestCase { } } + fn new_with_grafting(name: &str, test: fn(TestContext) -> T, base_subgraph: &str) -> Self + where + T: Future> + Send + 'static, + { + let mut test_case = Self::new(name, test); + test_case.source_subgraph = Some(base_subgraph.to_string()); + test_case + } + fn new_with_source_subgraph( name: &str, test: fn(TestContext) -> T, @@ -246,7 +257,7 @@ impl TestCase { let subgraph = self.deploy_and_wait(source, contracts).await?; status!( source, - "source subgraph deployed with hash {}", + "Source subgraph deployed with hash {}", subgraph.deployment ); } @@ -456,9 +467,8 @@ async fn test_block_handlers(ctx: TestContext) -> anyhow::Result<()> { .await?; // test subgraphFeatures endpoint returns handlers correctly - let subgraph_features = subgraph - .index_with_vars( - "query GetSubgraphFeatures($deployment: String!) { + let subgraph_features = Subgraph::query_with_vars( + "query GetSubgraphFeatures($deployment: String!) { subgraphFeatures(subgraphId: $deployment) { specVersion apiVersion @@ -468,9 +478,9 @@ async fn test_block_handlers(ctx: TestContext) -> anyhow::Result<()> { handlers } }", - json!({ "deployment": subgraph.deployment }), - ) - .await?; + json!({ "deployment": subgraph.deployment }), + ) + .await?; let handlers = &subgraph_features["data"]["subgraphFeatures"]["handlers"]; assert!( handlers.is_array(), @@ -523,79 +533,34 @@ async fn subgraph_data_sources(ctx: TestContext) -> anyhow::Result<()> { assert!(subgraph.healthy); let expected_response = json!({ "mirrorBlocks": [ - { "id": "1-v1", "number": "1" }, - { "id": "1-v2", "number": "1" }, - { "id": "1-v3", "number": "1" }, - { "id": "2-v1", "number": "2" }, - { "id": "2-v2", "number": "2" }, - { "id": "2-v3", "number": "2" }, - { "id": "3-v1", "number": "3" }, - { "id": "3-v2", "number": "3" }, - { "id": "3-v3", "number": "3" }, - { "id": "4-v1", "number": "4" }, - { "id": "4-v2", "number": "4" }, - { "id": "4-v3", "number": "4" }, - { "id": "5-v1", "number": "5" }, - { "id": "5-v2", "number": "5" }, - { "id": "5-v3", "number": "5" }, - { "id": "6-v1", "number": "6" }, - { "id": "6-v2", "number": "6" }, - { "id": "6-v3", "number": "6" }, - { "id": "7-v1", "number": "7" }, - { "id": "7-v2", "number": "7" }, - { "id": "7-v3", "number": "7" }, - { "id": "8-v1", "number": "8" }, - { "id": "8-v2", "number": "8" }, - { "id": "8-v3", "number": "8" }, - { "id": "9-v1", "number": "9" }, - { "id": "9-v2", "number": "9" }, - { "id": "9-v3", "number": "9" }, - { "id": "10-v1", "number": "10" }, - { "id": "10-v2", "number": "10" }, - { "id": "10-v3", "number": "10" }, + { "id": "1-v1", "number": "1", "testMessage": null }, + { "id": "1-v2", "number": "1", "testMessage": null }, + { "id": "1-v3", "number": "1", "testMessage": "1-message" }, + { "id": "2-v1", "number": "2", "testMessage": null }, + { "id": "2-v2", "number": "2", "testMessage": null }, + { "id": "2-v3", "number": "2", "testMessage": "2-message" }, + { "id": "3-v1", "number": "3", "testMessage": null }, + { "id": "3-v2", "number": "3", "testMessage": null }, + { "id": "3-v3", "number": "3", "testMessage": "3-message" }, ] }); query_succeeds( - "Blocks should be right", - &subgraph, - "{ mirrorBlocks(where: {number_lte: 10}, orderBy: number) { id, number } }", - expected_response, - ) - .await?; - - let expected_response = json!({ - "mirrorBlock": { "id": "TEST", "number": "1", "testMessage": "Created at block 1" }, - }); - - query_succeeds( - "Blocks should be right", - &subgraph, - "{ mirrorBlock(id: \"TEST\", block: {number: 1}) { id, number, testMessage } }", - expected_response, - ) - .await?; - - let expected_response = json!({ - "mirrorBlock": { "id": "TEST", "number": "1", "testMessage": "Updated at block 2" }, - }); - - query_succeeds( - "Blocks should be right", + "Query all blocks with testMessage", &subgraph, - "{ mirrorBlock(id: \"TEST\", block: {number: 2}) { id, number, testMessage } }", + "{ mirrorBlocks(where: {number_lte: 3}, orderBy: number) { id, number, testMessage } }", expected_response, ) .await?; let expected_response = json!({ - "mirrorBlock": null, + "mirrorBlock": { "id": "1-v3", "number": "1", "testMessage": "1-message" }, }); query_succeeds( - "Blocks should be right", + "Query specific block with testMessage", &subgraph, - "{ mirrorBlock(id: \"TEST\", block: {number: 3}) { id, number, testMessage } }", + "{ mirrorBlock(id: \"1-v3\") { id, number, testMessage } }", expected_response, ) .await?; @@ -742,9 +707,8 @@ async fn test_non_fatal_errors(ctx: TestContext) -> anyhow::Result<()> { } }"; - let resp = subgraph - .index_with_vars(query, json!({ "deployment" : subgraph.deployment })) - .await?; + let resp = + Subgraph::query_with_vars(query, json!({ "deployment" : subgraph.deployment })).await?; let subgraph_features = &resp["data"]["subgraphFeatures"]; let exp = json!({ "specVersion": "0.0.4", @@ -841,6 +805,82 @@ async fn test_remove_then_update(ctx: TestContext) -> anyhow::Result<()> { Ok(()) } +async fn test_subgraph_grafting(ctx: TestContext) -> anyhow::Result<()> { + async fn get_block_hash(block_number: i32) -> Option { + const FETCH_BLOCK_HASH: &str = r#" + query blockHashFromNumber($network: String!, $blockNumber: Int!) { + hash: blockHashFromNumber( + network: $network, + blockNumber: $blockNumber, + ) } "#; + let vars = json!({ + "network": "test", + "blockNumber": block_number + }); + + let resp = Subgraph::query_with_vars(FETCH_BLOCK_HASH, vars) + .await + .unwrap(); + assert_eq!(None, resp.get("errors")); + resp["data"]["hash"].as_str().map(|s| s.to_owned()) + } + + let subgraph = ctx.subgraph; + + assert!(subgraph.healthy); + + let block_hashes: Vec<&str> = vec![ + "384c705d4d1933ae8ba89026f016f09854057a267e1143e47bb7511d772a35d4", + "b90423eead33404dae0684169d35edd494b36802b721fb8de0bb8bc036c10480", + "2a6c4b65d659e0485371a93bc1ac0f0d7bc0f25a454b5f23a842335fea0638d5", + ]; + + let pois: Vec<&str> = vec![ + "0xde9e5650e22e61def6990d3fc4bd5915a4e8e0dd54af0b6830bf064aab16cc03", + "0x5d790dca3e37bd9976345d32d437b84ba5ea720a0b6ea26231a866e9f078bd52", + "0x719c04b78e01804c86f2bd809d20f481e146327af07227960e2242da365754ef", + ]; + + for i in 1..4 { + let block_hash = get_block_hash(i).await.unwrap(); + // We need to make sure that the preconditions for POI are fulfiled + // namely that the blockchain produced the proper block hashes for the + // blocks of which we will check the POI. + assert_eq!(block_hash, block_hashes[(i - 1) as usize]); + + const FETCH_POI: &str = r#" + query proofOfIndexing($subgraph: String!, $blockNumber: Int!, $blockHash: String!, $indexer: String!) { + proofOfIndexing( + subgraph: $subgraph, + blockNumber: $blockNumber, + blockHash: $blockHash, + indexer: $indexer + ) } "#; + + let zero_addr = "0000000000000000000000000000000000000000"; + let vars = json!({ + "subgraph": subgraph.deployment, + "blockNumber": i, + "blockHash": block_hash, + "indexer": zero_addr, + }); + let resp = Subgraph::query_with_vars(FETCH_POI, vars).await?; + assert_eq!(None, resp.get("errors")); + assert!(resp["data"]["proofOfIndexing"].is_string()); + let poi = resp["data"]["proofOfIndexing"].as_str().unwrap(); + // Check the expected value of the POI. The transition from the old legacy + // hashing to the new one is done in the block #2 anything before that + // should not change as the legacy code will not be updated. Any change + // after that might indicate a change in the way new POI is now calculated. + // Change on the block #2 would mean a change in the transitioning + // from the old to the new algorithm hence would be reflected only + // subgraphs that are grafting from pre 0.0.5 to 0.0.6 or newer. + assert_eq!(poi, pois[(i - 1) as usize]); + } + + Ok(()) +} + async fn test_poi_for_failed_subgraph(ctx: TestContext) -> anyhow::Result<()> { let subgraph = ctx.subgraph; const INDEXING_STATUS: &str = r#" @@ -874,9 +914,9 @@ async fn test_poi_for_failed_subgraph(ctx: TestContext) -> anyhow::Result<()> { } async fn fetch_status(subgraph: &Subgraph) -> anyhow::Result { - let resp = subgraph - .index_with_vars(INDEXING_STATUS, json!({ "subgraphName": subgraph.name })) - .await?; + let resp = + Subgraph::query_with_vars(INDEXING_STATUS, json!({ "subgraphName": subgraph.name })) + .await?; assert_eq!(None, resp.get("errors")); let statuses = &resp["data"]["statuses"]; assert_eq!(1, statuses.as_array().unwrap().len()); @@ -922,7 +962,7 @@ async fn test_poi_for_failed_subgraph(ctx: TestContext) -> anyhow::Result<()> { "blockNumber": block_number, "blockHash": status.latest_block["hash"], }); - let resp = subgraph.index_with_vars(FETCH_POI, vars).await?; + let resp = Subgraph::query_with_vars(FETCH_POI, vars).await?; assert_eq!(None, resp.get("errors")); assert!(resp["data"]["proofOfIndexing"].is_string()); Ok(()) @@ -960,6 +1000,25 @@ async fn test_multiple_subgraph_datasources(ctx: TestContext) -> anyhow::Result< Ok(()) } +async fn wait_for_blockchain_block(block_number: i32) -> bool { + // Wait up to 5 minutes for the expected block to appear + const STATUS_WAIT: Duration = Duration::from_secs(300); + const REQUEST_REPEATING: Duration = time::Duration::from_secs(1); + let start = Instant::now(); + while start.elapsed() < STATUS_WAIT { + let latest_block = Contract::latest_block().await; + if let Some(latest_block) = latest_block { + if let Some(number) = latest_block.number { + if number >= block_number.into() { + return true; + } + } + } + tokio::time::sleep(REQUEST_REPEATING).await; + } + false +} + /// The main test entrypoint. #[tokio::test] async fn integration_tests() -> anyhow::Result<()> { @@ -981,6 +1040,7 @@ async fn integration_tests() -> anyhow::Result<()> { TestCase::new("timestamp", test_timestamp), TestCase::new("ethereum-api-tests", test_eth_api), TestCase::new("topic-filter", test_topic_filters), + TestCase::new_with_grafting("grafted", test_subgraph_grafting, "base"), TestCase::new_with_source_subgraph( "subgraph-data-sources", subgraph_data_sources, @@ -1003,6 +1063,11 @@ async fn integration_tests() -> anyhow::Result<()> { cases }; + // Here we wait for a block in the blockchain in order not to influence + // block hashes for all the blocks until the end of the grafting tests. + // Currently the last used block for grafting test is the block 3. + assert!(wait_for_blockchain_block(SUBGRAPH_LAST_GRAFTING_BLOCK).await); + let contracts = Contract::deploy_all().await?; status!("setup", "Resetting database"); diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index ac645884b5d..261c886dfea 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -12,7 +12,7 @@ use graph::data::store::scalar::Bytes; use graph::data::subgraph::schema::{SubgraphError, SubgraphHealth}; use graph::data::value::Word; use graph::data_source::CausalityRegion; -use graph::env::EnvVars; +use graph::env::{EnvVars, TEST_WITH_NO_REORG}; use graph::ipfs; use graph::ipfs::test_utils::add_files_to_local_ipfs_node_for_testing; use graph::object; @@ -109,6 +109,8 @@ fn assert_eq_ignore_backtrace(err: &SubgraphError, expected: &SubgraphError) { #[tokio::test] async fn data_source_revert() -> anyhow::Result<()> { + *TEST_WITH_NO_REORG.lock().unwrap() = true; + let RunnerTestRecipe { stores, test_info } = RunnerTestRecipe::new("data_source_revert", "data-source-revert").await; @@ -179,6 +181,8 @@ async fn data_source_revert() -> anyhow::Result<()> { // since it uses the same deployment id. data_source_long_revert().await.unwrap(); + *TEST_WITH_NO_REORG.lock().unwrap() = false; + Ok(()) }