Skip to content

Commit 4eb46d8

Browse files
bgwbgw
authored
Turbopack: Share entrypoint template logic between Turbopack/webpack (#82385)
## Goal With three different bundlers, we're often forced to duplicate next.js-specific logic and configuration across two or three different bundlers. E.g. https://github.com/SyMind/next-rspack-binding/ would cause us to duplicate our current externals logic across all three. To help, we'd like to have a shared crate that can be used in a few different ways: - Via turbopack (which is itself called through native napi bindings) - Via direct native napi bindings to JS for webpack - Via wasm bindings to JS for webpack [*(note: ideally these would be replaced with napi's recent wasm support, but it doesn't make sense to do that until after the napi v3 migration is done)*](https://napi.rs/blog/announce-v3#webassembly) - Via Rspack (we'd build a custom binary) The wasm bindings (in their current state) and Rspack cannot (or should not) depend on `turbo-tasks`. However, all of the current `next-*` crates depend on `turbo-tasks`. This introduces a `next-taskless` crate for code that can be isolated from performing IO and that does not need to depend on `turbo-tasks`. Code in this crate can be used from wasm or Rspack. ## This PR As a proof-of-concept, this deduplicates our next.js entrypoint templating logic across webpack and Turbopack. It's a large function that doesn't need to directly depend on file IO. Some path manipulation had to be moved out of `turbo-tasks-fs` to support this, so that code is in `turbo-unix-path`. I'm happy to accept different bikeshedded crate names. ## Testing Tested wasm with: ``` pnpm i pnpm swc-build-wasm --target nodejs pnpm build NODE_TEST_MODE=start NEXT_TEST_WASM=true NEXT_SKIP_NATIVE_POSTINSTALL=1 NEXT_TEST_PREFER_OFFLINE=1 node run-tests.js test/production/pages-dir/production/test/index.test.ts ``` Tested turbopack with: ``` pnpm i pnpm swc-build-native pnpm build pnpm test-start-turbo test/production/pages-dir/production/test/index.test.ts ``` Tested webpack with: ``` pnpm i pnpm swc-build-native pnpm build pnpm test-start test/production/pages-dir/production/test/index.test.ts ```
1 parent c26cb69 commit 4eb46d8

File tree

35 files changed

+821
-747
lines changed

35 files changed

+821
-747
lines changed

‎Cargo.lock

Lines changed: 26 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ members = [
1010
"crates/next-build",
1111
"crates/next-core",
1212
"crates/next-custom-transforms",
13+
"crates/next-taskless",
1314
"turbopack/crates/*",
1415
"turbopack/crates/*/fuzz",
1516
"turbopack/xtask",
@@ -252,6 +253,7 @@ next-api = { path = "crates/next-api" }
252253
next-build = { path = "crates/next-build" }
253254
next-core = { path = "crates/next-core" }
254255
next-custom-transforms = { path = "crates/next-custom-transforms" }
256+
next-taskless = { path = "crates/next-taskless" }
255257

256258
# Turbopack
257259
auto-hash-map = { path = "turbopack/crates/turbo-tasks-auto-hash-map" }
@@ -260,6 +262,7 @@ turbo-rcstr = { path = "turbopack/crates/turbo-rcstr" }
260262
turbo-dyn-eq-hash = { path = "turbopack/crates/turbo-dyn-eq-hash" }
261263
turbo-esregex = { path = "turbopack/crates/turbo-esregex" }
262264
turbo-persistence = { path = "turbopack/crates/turbo-persistence" }
265+
turbo-unix-path = { path = "turbopack/crates/turbo-unix-path" }
263266
turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
264267
turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
265268
turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }

‎crates/napi/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ owo-colors = { workspace = true }
6464
napi = { workspace = true }
6565
napi-derive = "2"
6666
next-custom-transforms = { workspace = true }
67+
next-taskless = { workspace = true }
6768
rand = { workspace = true }
6869
rustc-hash = { workspace = true }
6970
serde = "1"
@@ -105,6 +106,7 @@ turbo-rcstr = { workspace = true, features = ["napi"] }
105106
turbo-tasks = { workspace = true }
106107
turbo-tasks-backend = { workspace = true }
107108
turbo-tasks-fs = { workspace = true }
109+
turbo-unix-path = { workspace = true }
108110
next-api = { workspace = true }
109111
next-build = { workspace = true }
110112
next-core = { workspace = true }

‎crates/napi/src/next_api/project.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ use turbo_tasks::{
3939
};
4040
use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
4141
use turbo_tasks_fs::{
42-
DiskFileSystem, FileContent, FileSystem, FileSystemPath, get_relative_path_to,
43-
util::uri_from_file,
42+
DiskFileSystem, FileContent, FileSystem, FileSystemPath, util::uri_from_file,
4443
};
44+
use turbo_unix_path::get_relative_path_to;
4545
use turbopack_core::{
4646
PROJECT_FILESYSTEM_NAME, SOURCE_URL_PROTOCOL,
4747
diagnostics::PlainDiagnostic,

‎crates/napi/src/next_api/utils.rs

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use anyhow::{Context, Result, anyhow};
44
use futures_util::TryFutureExt;
55
use napi::{
66
JsFunction, JsObject, JsUnknown, NapiRaw, NapiValue, Status,
7-
bindgen_prelude::{External, ToNapiValue},
7+
bindgen_prelude::{Buffer, External, ToNapiValue},
88
threadsafe_function::{ThreadSafeCallContext, ThreadsafeFunction, ThreadsafeFunctionCallMode},
99
};
1010
use rustc_hash::FxHashMap;
@@ -380,3 +380,25 @@ pub async fn strongly_consistent_catch_collectables<R: VcValueType + Send>(
380380

381381
Ok((result, issues, diagnostics, effects))
382382
}
383+
384+
#[napi]
385+
pub fn expand_next_js_template(
386+
content: Buffer,
387+
template_path: String,
388+
next_package_dir_path: String,
389+
#[napi(ts_arg_type = "Record<string, string>")] replacements: FxHashMap<String, String>,
390+
#[napi(ts_arg_type = "Record<string, string>")] injections: FxHashMap<String, String>,
391+
#[napi(ts_arg_type = "Record<string, string | null>")] imports: FxHashMap<
392+
String,
393+
Option<String>,
394+
>,
395+
) -> napi::Result<String> {
396+
Ok(next_taskless::expand_next_js_template(
397+
str::from_utf8(&content).context("template content must be valid utf-8")?,
398+
&template_path,
399+
&next_package_dir_path,
400+
replacements.iter().map(|(k, v)| (&**k, &**v)),
401+
injections.iter().map(|(k, v)| (&**k, &**v)),
402+
imports.iter().map(|(k, v)| (&**k, v.as_deref())),
403+
)?)
404+
}

‎crates/next-api/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ turbo-rcstr = { workspace = true }
3333
turbo-tasks = { workspace = true, features = ["non_operation_vc_strongly_consistent"] }
3434
turbo-tasks-env = { workspace = true }
3535
turbo-tasks-fs = { workspace = true }
36+
turbo-unix-path = { workspace = true }
3637
turbopack = { workspace = true }
3738
turbopack-browser = { workspace = true }
3839
turbopack-core = { workspace = true }

‎crates/next-api/src/project.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,8 @@ use turbo_tasks::{
3636
trace::TraceRawVcs,
3737
};
3838
use turbo_tasks_env::{EnvMap, ProcessEnv};
39-
use turbo_tasks_fs::{
40-
DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, invalidation,
41-
util::{join_path, unix_to_sys},
42-
};
39+
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSystem, invalidation};
40+
use turbo_unix_path::{join_path, unix_to_sys};
4341
use turbopack::{
4442
ModuleAssetContext, evaluate_context::node_build_environment,
4543
global_module_ids::get_global_module_id_strategy, transition::TransitionOptions,

‎crates/next-core/Cargo.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ anyhow = { workspace = true }
1616
async-trait = { workspace = true }
1717
base64 = "0.21.0"
1818
either = { workspace = true }
19-
next-custom-transforms = { workspace = true }
2019
once_cell = { workspace = true }
2120
qstring = { workspace = true }
2221
regex = { workspace = true }
@@ -55,6 +54,9 @@ swc_core = { workspace = true, features = [
5554
] }
5655
modularize_imports = { workspace = true }
5756

57+
next-custom-transforms = { workspace = true }
58+
next-taskless = { workspace = true }
59+
5860
turbo-rcstr = { workspace = true }
5961
turbo-esregex = { workspace = true }
6062
turbo-tasks = { workspace = true }

‎crates/next-core/src/middleware.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use anyhow::Result;
22
use turbo_rcstr::{RcStr, rcstr};
3-
use turbo_tasks::{FxIndexMap, ResolvedVc, Vc, fxindexmap};
3+
use turbo_tasks::{ResolvedVc, Vc, fxindexmap};
44
use turbo_tasks_fs::FileSystemPath;
55
use turbopack_core::{context::AssetContext, module::Module, reference_type::ReferenceType};
66

@@ -27,23 +27,23 @@ pub async fn get_middleware_module(
2727
project_root: FileSystemPath,
2828
userland_module: ResolvedVc<Box<dyn Module>>,
2929
) -> Result<Vc<Box<dyn Module>>> {
30-
let inner = rcstr!("INNER_MIDDLEWARE_MODULE");
30+
const INNER: &str = "INNER_MIDDLEWARE_MODULE";
3131

3232
// Load the file from the next.js codebase.
3333
let source = load_next_js_template(
3434
"middleware.js",
3535
project_root,
36-
fxindexmap! {
37-
"VAR_USERLAND" => inner.clone(),
38-
"VAR_DEFINITION_PAGE" => rcstr!("/middleware"),
39-
},
40-
FxIndexMap::default(),
41-
FxIndexMap::default(),
36+
&[
37+
("VAR_USERLAND", INNER),
38+
("VAR_DEFINITION_PAGE", "/middleware"),
39+
],
40+
&[],
41+
&[],
4242
)
4343
.await?;
4444

4545
let inner_assets = fxindexmap! {
46-
inner => userland_module
46+
rcstr!(INNER) => userland_module
4747
};
4848

4949
let module = asset_context

‎crates/next-core/src/next_app/app_page_entry.rs

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use std::io::Write;
22

33
use anyhow::Result;
4-
use turbo_rcstr::{RcStr, rcstr};
4+
use turbo_rcstr::RcStr;
55
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc, fxindexmap};
66
use turbo_tasks_fs::{self, File, FileSystemPath, rope::RopeBuilder};
77
use turbopack::ModuleAssetContext;
@@ -86,22 +86,25 @@ pub async fn get_app_page_entry(
8686
let source = load_next_js_template(
8787
"app-page.js",
8888
project_root.clone(),
89-
fxindexmap! {
90-
"VAR_DEFINITION_PAGE" => page.to_string().into(),
91-
"VAR_DEFINITION_PATHNAME" => pathname.clone(),
92-
"VAR_MODULE_GLOBAL_ERROR" => if inner_assets.contains_key(GLOBAL_ERROR) {
93-
GLOBAL_ERROR.into()
94-
} else {
95-
rcstr!("next/dist/client/components/builtin/global-error")
96-
},
97-
},
98-
fxindexmap! {
99-
"tree" => loader_tree_code,
100-
"pages" => StringifyJs(&pages).to_string().into(),
101-
"__next_app_require__" => TURBOPACK_REQUIRE.bound().into(),
102-
"__next_app_load_chunk__" => TURBOPACK_LOAD.bound().into(),
103-
},
104-
fxindexmap! {},
89+
&[
90+
("VAR_DEFINITION_PAGE", &*page.to_string()),
91+
("VAR_DEFINITION_PATHNAME", &pathname),
92+
(
93+
"VAR_MODULE_GLOBAL_ERROR",
94+
if inner_assets.contains_key(GLOBAL_ERROR) {
95+
GLOBAL_ERROR
96+
} else {
97+
"next/dist/client/components/builtin/global-error"
98+
},
99+
),
100+
],
101+
&[
102+
("tree", &*loader_tree_code),
103+
("pages", &StringifyJs(&pages).to_string()),
104+
("__next_app_require__", &TURBOPACK_REQUIRE.bound()),
105+
("__next_app_load_chunk__", &TURBOPACK_LOAD.bound()),
106+
],
107+
&[],
105108
)
106109
.await?;
107110

@@ -158,18 +161,13 @@ async fn wrap_edge_page(
158161
let source = load_next_js_template(
159162
"edge-ssr-app.js",
160163
project_root.clone(),
161-
fxindexmap! {
162-
"VAR_USERLAND" => INNER.into(),
163-
"VAR_PAGE" => page.to_string().into(),
164-
},
165-
fxindexmap! {
164+
&[("VAR_USERLAND", INNER), ("VAR_PAGE", &page.to_string())],
165+
&[
166166
// TODO do we really need to pass the entire next config here?
167167
// This is bad for invalidation as any config change will invalidate this
168-
"nextConfig" => serde_json::to_string(next_config_val)?.into(),
169-
},
170-
fxindexmap! {
171-
"incrementalCacheHandler" => None,
172-
},
168+
("nextConfig", &*serde_json::to_string(next_config_val)?),
169+
],
170+
&[("incrementalCacheHandler", None)],
173171
)
174172
.await?;
175173

0 commit comments

Comments
 (0)