diff --git a/.evergreen.yml b/.evergreen.yml index 4f13b43..a0edaa6 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -14,7 +14,7 @@ functions: set -e set -x - export NODE_VERSION=20.5.0 + export NODE_VERSION=20.13.0 bash .evergreen/install-node.sh install: - command: shell.exec @@ -106,7 +106,7 @@ tasks: - func: install - func: test vars: - node_version: "20.5.0" + node_version: "20.13.0" - name: check commands: - func: checkout diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index eac4694..574fcf1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -36,8 +36,7 @@ jobs: matrix: language: - 'javascript' - # TODO: https://jira.mongodb.org/browse/COMPASS-7540, enable the cpp analysis. - # - 'cpp' + - 'cpp' steps: - name: Checkout repository @@ -78,7 +77,7 @@ jobs: - name: Build cpp if: matrix.language == 'cpp' run: | - node bin/boxednode.js -s test/resources/example.js -t example + ./test/compile-main-template-only.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index bc4e2ce..293c59b 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - node-version: [14.x, 16.x, 18.x, 19.x] + node-version: [14.x, 16.x, 18.x, 20.x] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index aba52b9..b1a3b11 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ tmp/ .esm-wrapper.mjs /lib/ package-lock.json +main-template-build diff --git a/.npmignore b/.npmignore index 0c615a3..376940d 100644 --- a/.npmignore +++ b/.npmignore @@ -6,3 +6,4 @@ test/ .evergreen.yml tsconfig.json .eslintrc +main-template-build/ diff --git a/bin/boxednode.js b/bin/boxednode.js index b309214..7f7804c 100755 --- a/bin/boxednode.js +++ b/bin/boxednode.js @@ -54,7 +54,8 @@ const argv = require('yargs') namespace: argv.N, useLegacyDefaultUvLoop: argv.useLegacyDefaultUvLoop, useCodeCache: argv.H, - useNodeSnapshot: argv.S + useNodeSnapshot: argv.S, + compressBlobs: argv.Z }); } catch (err) { console.error(err); diff --git a/package.json b/package.json index ca13efb..7d7f769 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "boxednode", - "version": "2.3.0", + "version": "2.4.4", "description": "Create a shippable binary from a JS file", "main": "lib/index.js", "exports": { diff --git a/resources/main-template.cc b/resources/main-template.cc index b04c19b..c63b118 100644 --- a/resources/main-template.cc +++ b/resources/main-template.cc @@ -17,6 +17,8 @@ #include #include #endif +#include // injected code may refer to std::underlying_type +#include using namespace node; using namespace v8; @@ -33,6 +35,17 @@ using namespace v8; #define NODE_VERSION_SUPPORTS_EMBEDDER_SNAPSHOT 1 #endif +// 20.13.0 has https://github.com/nodejs/node/pull/52595 for better startup snapshot +// initialization performance. +#if NODE_VERSION_AT_LEAST(20, 13, 0) +#define NODE_VERSION_SUPPORTS_STRING_VIEW_SNAPSHOT 1 +#endif + +// Snapshot config is supported since https://github.com/nodejs/node/pull/50453 +#if NODE_VERSION_AT_LEAST(20, 12, 0) && !defined(BOXEDNODE_SNAPSHOT_CONFIG_FLAGS) +#define BOXEDNODE_SNAPSHOT_CONFIG_FLAGS (SnapshotFlags::kWithoutCodeCache) +#endif + // 18.1.0 is the current minimum version that has https://github.com/nodejs/node/pull/42809, // which introduced crashes when using workers, and later 18.9.0 is the current // minimum version to contain https://github.com/nodejs/node/pull/44252, which @@ -75,6 +88,9 @@ void MarkTime(const char* category, const char* label) { Local GetBoxednodeMainScriptSource(Isolate* isolate); Local GetBoxednodeCodeCacheBuffer(Isolate* isolate); std::vector GetBoxednodeSnapshotBlobVector(); +#ifdef NODE_VERSION_SUPPORTS_STRING_VIEW_SNAPSHOT +std::optional GetBoxednodeSnapshotBlobSV(); +#endif void GetTimingData(const FunctionCallbackInfo& info) { Isolate* isolate = info.GetIsolate(); @@ -167,7 +183,15 @@ static int RunNodeInstance(MultiIsolatePlatform* platform, int exit_code = 0; std::vector errors; std::unique_ptr setup = - CommonEnvironmentSetup::CreateForSnapshotting(platform, &errors, args, exec_args); + CommonEnvironmentSetup::CreateForSnapshotting( + platform, + &errors, + args, + exec_args +#ifdef BOXEDNODE_SNAPSHOT_CONFIG_FLAGS + , SnapshotConfig { BOXEDNODE_SNAPSHOT_CONFIG_FLAGS, std::nullopt } +#endif + ); Isolate* isolate = setup->isolate(); @@ -216,11 +240,18 @@ static int RunNodeInstance(MultiIsolatePlatform* platform, ArrayBufferAllocator::Create(); #ifdef BOXEDNODE_CONSUME_SNAPSHOT - std::vector snapshot_blob_vec = boxednode::GetBoxednodeSnapshotBlobVector(); - boxednode::MarkTime("Node.js Instance", "Decoded snapshot"); assert(EmbedderSnapshotData::CanUseCustomSnapshotPerIsolate()); - node::EmbedderSnapshotData::Pointer snapshot_blob = - EmbedderSnapshotData::FromBlob(snapshot_blob_vec); + node::EmbedderSnapshotData::Pointer snapshot_blob; +#ifdef NODE_VERSION_SUPPORTS_STRING_VIEW_SNAPSHOT + if (const auto snapshot_blob_sv = boxednode::GetBoxednodeSnapshotBlobSV()) { + snapshot_blob = EmbedderSnapshotData::FromBlob(snapshot_blob_sv.value()); + } +#endif + if (!snapshot_blob) { + std::vector snapshot_blob_vec = boxednode::GetBoxednodeSnapshotBlobVector(); + boxednode::MarkTime("Node.js Instance", "Decoded snapshot"); + snapshot_blob = EmbedderSnapshotData::FromBlob(snapshot_blob_vec); + } boxednode::MarkTime("Node.js Instance", "Read snapshot"); Isolate* isolate = NewIsolate(allocator, loop, platform, snapshot_blob.get()); #elif NODE_VERSION_AT_LEAST(14, 0, 0) diff --git a/src/helpers.ts b/src/helpers.ts index 2fddc6c..2b14801 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -107,6 +107,32 @@ export function createCppJsStringDefinition (fnName: string, source: string): st `; } +export async function createUncompressedBlobDefinition (fnName: string, source: Uint8Array): Promise { + return ` + static const uint8_t ${fnName}_source_[] = { + ${Uint8Array.prototype.toString.call(source) || '0'} + }; + +#ifdef NODE_VERSION_SUPPORTS_STRING_VIEW_SNAPSHOT + std::optional ${fnName}SV() { + return { + { + reinterpret_cast(&${fnName}_source_[0]), + ${source.length} + } + }; + } +#endif + + std::vector ${fnName}Vector() { + return std::vector( + reinterpret_cast(&${fnName}_source_[0]), + reinterpret_cast(&${fnName}_source_[${source.length}])); + } + + ${blobTypedArrayAccessors(fnName, source.length)}`; +} + export async function createCompressedBlobDefinition (fnName: string, source: Uint8Array): Promise { const compressed = await promisify(zlib.brotliCompress)(source, { params: { @@ -133,13 +159,6 @@ export async function createCompressedBlobDefinition (fnName: string, source: Ui assert(decoded_size == ${source.length}); } - std::string ${fnName}() { - ${source.length === 0 ? 'return {};' : ` - std::string dst(${source.length}, 0); - ${fnName}_Read(&dst[0]); - return dst;`} - } - std::vector ${fnName}Vector() { ${source.length === 0 ? 'return {};' : ` std::vector dst(${source.length}); @@ -147,19 +166,31 @@ export async function createCompressedBlobDefinition (fnName: string, source: Ui return dst;`} } +#ifdef NODE_VERSION_SUPPORTS_STRING_VIEW_SNAPSHOT + std::optional ${fnName}SV() { + return {}; + } +#endif + + ${blobTypedArrayAccessors(fnName, source.length)} + `; +} + +function blobTypedArrayAccessors (fnName: string, sourceLength: number): string { + return ` std::shared_ptr ${fnName}BackingStore() { - std::string* str = new std::string(std::move(${fnName}())); + std::vector* str = new std::vector(std::move(${fnName}Vector())); return v8::SharedArrayBuffer::NewBackingStore( &str->front(), str->size(), [](void*, size_t, void* deleter_data) { - delete static_cast(deleter_data); + delete static_cast*>(deleter_data); }, static_cast(str)); } v8::Local ${fnName}Buffer(v8::Isolate* isolate) { - ${source.length === 0 ? ` + ${sourceLength === 0 ? ` auto array_buffer = v8::SharedArrayBuffer::New(isolate, 0); ` : ` auto array_buffer = v8::SharedArrayBuffer::New(isolate, ${fnName}BackingStore()); diff --git a/src/index.ts b/src/index.ts index b9b4d07..c7c96c7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { promisify } from 'util'; import { promises as fs, createReadStream, createWriteStream } from 'fs'; import { AddonConfig, loadGYPConfig, storeGYPConfig, modifyAddonGyp } from './native-addons'; import { ExecutableMetadata, generateRCFile } from './executable-metadata'; -import { spawnBuildCommand, ProcessEnv, pipeline, createCppJsStringDefinition, createCompressedBlobDefinition } from './helpers'; +import { spawnBuildCommand, ProcessEnv, pipeline, createCppJsStringDefinition, createCompressedBlobDefinition, createUncompressedBlobDefinition } from './helpers'; import { Readable } from 'stream'; import nv from '@pkgjs/nv'; import { fileURLToPath, URL } from 'url'; @@ -224,7 +224,7 @@ async function compileNode ( } let source = await fs.readFile(target, 'utf8'); source = source.replace(/-static/g, ''); - await fs.writeFile(target, 'utf8'); + await fs.writeFile(target, source); } } @@ -253,7 +253,7 @@ async function compileNode ( for (const module of linkedJSModules) { vcbuildArgs.push('link-module', module); } - await spawnBuildCommand(['.\\vcbuild.bat', ...vcbuildArgs], options); + await spawnBuildCommand(['cmd', '/c', '.\\vcbuild.bat', ...vcbuildArgs], options); return path.join(sourcePath, 'Release', 'node.exe'); } @@ -275,6 +275,8 @@ type CompilationOptions = { useLegacyDefaultUvLoop?: boolean; useCodeCache?: boolean, useNodeSnapshot?: boolean, + compressBlobs?: boolean, + nodeSnapshotConfigFlags?: string[], // e.g. 'WithoutCodeCache' executableMetadata?: ExecutableMetadata, preCompileHook?: (nodeSourceTree: string, options: CompilationOptions) => void | Promise } @@ -386,6 +388,10 @@ async function compileJSFileAsBinaryImpl (options: CompilationOptions, logger: L logger.stepCompleted(); } + const createBlobDefinition = options.compressBlobs + ? createCompressedBlobDefinition + : createUncompressedBlobDefinition; + async function writeMainFileAndCompile ({ codeCacheBlob = new Uint8Array(0), codeCacheMode = 'ignore', @@ -408,8 +414,8 @@ async function compileJSFileAsBinaryImpl (options: CompilationOptions, logger: L registerFunctions.map((fn) => `${fn},`).join('')); mainSource = mainSource.replace(/\bREPLACE_WITH_MAIN_SCRIPT_SOURCE_GETTER\b/g, createCppJsStringDefinition('GetBoxednodeMainScriptSource', snapshotMode !== 'consume' ? jsMainSource : '') + '\n' + - await createCompressedBlobDefinition('GetBoxednodeCodeCache', codeCacheBlob) + '\n' + - await createCompressedBlobDefinition('GetBoxednodeSnapshotBlob', snapshotBlob)); + await createBlobDefinition('GetBoxednodeCodeCache', codeCacheBlob) + '\n' + + await createBlobDefinition('GetBoxednodeSnapshotBlob', snapshotBlob)); mainSource = mainSource.replace(/\bBOXEDNODE_CODE_CACHE_MODE\b/g, JSON.stringify(codeCacheMode)); if (options.useLegacyDefaultUvLoop) { @@ -421,6 +427,14 @@ async function compileJSFileAsBinaryImpl (options: CompilationOptions, logger: L if (snapshotMode === 'consume') { mainSource = `#define BOXEDNODE_CONSUME_SNAPSHOT 1\n${mainSource}`; } + if (options.nodeSnapshotConfigFlags) { + const flags = [ + '0', + ...options.nodeSnapshotConfigFlags.map(flag => + `static_cast::type>(SnapshotFlags::k${flag})`) + ].join(' | '); + mainSource = `#define BOXEDNODE_SNAPSHOT_CONFIG_FLAGS (static_cast(${flags}))\n${mainSource}`; + } await fs.writeFile(path.join(nodeSourcePath, 'src', 'node_main.cc'), mainSource); logger.stepCompleted(); diff --git a/test/compile-main-template-only.sh b/test/compile-main-template-only.sh new file mode 100755 index 0000000..7e4316a --- /dev/null +++ b/test/compile-main-template-only.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# script to only build the code in resources/ +# for CodeQL testing + +set -e +set -x +cd "$(dirname $0)/.." +if [ ! -e main-template-build ]; then + mkdir main-template-build + pushd main-template-build + curl -O https://nodejs.org/dist/v20.12.0/node-v20.12.0.tar.xz + tar --strip-components=1 -xf node-*.tar.xz + popd +fi + +g++ \ + -Imain-template-build/src \ + -Imain-template-build/deps/v8/include \ + -Imain-template-build/deps/uv/include \ + -DREPLACE_DECLARE_LINKED_MODULES= \ + -DREPLACE_DEFINE_LINKED_MODULES= \ + -DREPLACE_WITH_ENTRY_POINT='"placeholder"' \ + -DBOXEDNODE_CODE_CACHE_MODE='"placeholder"' \ + -DREPLACE_WITH_MAIN_SCRIPT_SOURCE_GETTER= \ + -fPIC -shared \ + -o main-template-build/out.so \ + -include resources/add-node_api.h \ + -include resources/add-node.h \ + resources/main-template.cc diff --git a/test/index.ts b/test/index.ts index 2849974..f0fafbf 100644 --- a/test/index.ts +++ b/test/index.ts @@ -214,29 +214,33 @@ describe('basic functionality', () => { } }); - it('works with snapshot support', async function () { - this.timeout(2 * 60 * 60 * 1000); // 2 hours - await compileJSFileAsBinary({ - nodeVersionRange: 'v21.0.0-nightly20230801d396a041f7', - sourceFile: path.resolve(__dirname, 'resources/snapshot-echo-args.js'), - targetFile: path.resolve(__dirname, `resources/snapshot-echo-args${exeSuffix}`), - useNodeSnapshot: true, - // the nightly path name is too long for Windows... - tmpdir: process.platform === 'win32' ? path.join(os.tmpdir(), 'bn') : undefined - }); + for (const compressBlobs of [false, true]) { + it(`works with snapshot support (compressBlobs = ${compressBlobs})`, async function () { + this.timeout(2 * 60 * 60 * 1000); // 2 hours + await compileJSFileAsBinary({ + nodeVersionRange: '^20.13.0', + sourceFile: path.resolve(__dirname, 'resources/snapshot-echo-args.js'), + targetFile: path.resolve(__dirname, `resources/snapshot-echo-args${exeSuffix}`), + useNodeSnapshot: true, + compressBlobs, + nodeSnapshotConfigFlags: ['WithoutCodeCache'], + // the nightly path name is too long for Windows... + tmpdir: process.platform === 'win32' ? path.join(os.tmpdir(), 'bn') : undefined + }); - { - const { stdout } = await execFile( - path.resolve(__dirname, `resources/snapshot-echo-args${exeSuffix}`), ['a', 'b', 'c'], - { encoding: 'utf8' }); - const { currentArgv, originalArgv, timingData } = JSON.parse(stdout); - assert(currentArgv[0].includes('snapshot-echo-args')); - assert(currentArgv[1].includes('snapshot-echo-args')); - assert.deepStrictEqual(currentArgv.slice(2), ['a', 'b', 'c']); - assert.strictEqual(originalArgv.length, 2); // [execPath, execPath] - assert.strictEqual(timingData[0][0], 'Node.js Instance'); - assert.strictEqual(timingData[0][1], 'Process initialization'); - } - }); + { + const { stdout } = await execFile( + path.resolve(__dirname, `resources/snapshot-echo-args${exeSuffix}`), ['a', 'b', 'c'], + { encoding: 'utf8' }); + const { currentArgv, originalArgv, timingData } = JSON.parse(stdout); + assert(currentArgv[0].includes('snapshot-echo-args')); + assert(currentArgv[1].includes('snapshot-echo-args')); + assert.deepStrictEqual(currentArgv.slice(2), ['a', 'b', 'c']); + assert.strictEqual(originalArgv.length, 2); // [execPath, execPath] + assert.strictEqual(timingData[0][0], 'Node.js Instance'); + assert.strictEqual(timingData[0][1], 'Process initialization'); + } + }); + } }); });