From f5fac83c0829f1d08b3239ac1cffd8c38ea79158 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 17 Jun 2021 10:09:48 +0200 Subject: [PATCH 1/4] clear compilation queues to reduce memory usage --- lib/Compilation.js | 25 +++++++++++++++++-------- lib/util/ArrayQueue.js | 8 ++++++++ lib/util/AsyncQueue.js | 9 +++++++++ types.d.ts | 1 + 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index 124c4732749..e3cdc21afaf 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -1999,6 +1999,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si } finish(callback) { + this.factorizeQueue.clear(); if (this.profile) { this.logger.time("finish module profiles"); const ParallelismFactorCalculator = require("./util/ParallelismFactorCalculator"); @@ -2233,6 +2234,14 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si * @returns {void} */ seal(callback) { + const finalCallback = err => { + this.factorizeQueue.clear(); + this.buildQueue.clear(); + this.rebuildQueue.clear(); + this.processDependenciesQueue.clear(); + this.addModuleQueue.clear(); + return callback(err); + }; const chunkGraph = new ChunkGraph(this.moduleGraph); this.chunkGraph = chunkGraph; @@ -2397,7 +2406,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o this.hooks.optimizeTree.callAsync(this.chunks, this.modules, err => { if (err) { - return callback( + return finalCallback( makeWebpackError(err, "Compilation.hooks.optimizeTree") ); } @@ -2409,7 +2418,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o this.modules, err => { if (err) { - return callback( + return finalCallback( makeWebpackError(err, "Compilation.hooks.optimizeChunkModules") ); } @@ -2452,7 +2461,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o this.hooks.beforeCodeGeneration.call(); this.codeGeneration(err => { if (err) { - return callback(err); + return finalCallback(err); } this.hooks.afterCodeGeneration.call(); this.logger.timeEnd("code generation"); @@ -2471,7 +2480,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o this._runCodeGenerationJobs(codeGenerationJobs, err => { if (err) { - return callback(err); + return finalCallback(err); } if (shouldRecord) { @@ -2491,7 +2500,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o this.logger.time("process assets"); this.hooks.processAssets.callAsync(this.assets, err => { if (err) { - return callback( + return finalCallback( makeWebpackError(err, "Compilation.hooks.processAssets") ); } @@ -2517,12 +2526,12 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o } return this.hooks.afterSeal.callAsync(err => { if (err) { - return callback( + return finalCallback( makeWebpackError(err, "Compilation.hooks.afterSeal") ); } this.fileSystemInfo.logStatistics(); - callback(); + finalCallback(); }); }); }; @@ -2533,7 +2542,7 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o this.createChunkAssets(err => { this.logger.timeEnd("create chunk assets"); if (err) { - return callback(err); + return finalCallback(err); } cont(); }); diff --git a/lib/util/ArrayQueue.js b/lib/util/ArrayQueue.js index 60a20208f6b..321baf3dcbf 100644 --- a/lib/util/ArrayQueue.js +++ b/lib/util/ArrayQueue.js @@ -27,6 +27,14 @@ class ArrayQueue { return this._list.length + this._listReversed.length; } + /** + * Empties the queue. + */ + clear() { + this._list.length = 0; + this._listReversed.length = 0; + } + /** * Appends the specified element to this queue. * @param {T} item The element to add. diff --git a/lib/util/AsyncQueue.js b/lib/util/AsyncQueue.js index 4b6f3e9bb43..9191b2e6c2b 100644 --- a/lib/util/AsyncQueue.js +++ b/lib/util/AsyncQueue.js @@ -359,6 +359,15 @@ class AsyncQueue { inHandleResult--; }); } + + clear() { + this._entries.clear(); + this._queued.clear(); + this._activeTasks = 0; + this._willEnsureProcessing = false; + this._needProcessing = false; + this._stopped = false; + } } module.exports = AsyncQueue; diff --git a/types.d.ts b/types.d.ts index 997c21b0bf2..9c3e7db22dc 100644 --- a/types.d.ts +++ b/types.d.ts @@ -358,6 +358,7 @@ declare abstract class AsyncQueue { isProcessing(item: T): boolean; isQueued(item: T): boolean; isDone(item: T): boolean; + clear(): void; } declare class AsyncWebAssemblyModulesPlugin { constructor(options?: any); From 273bd8f5754f65839b81ae5022b89b89a4673d0a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 17 Jun 2021 10:11:24 +0200 Subject: [PATCH 2/4] move defineRemovedModuleTemplates into separate function as it leaks memory in the feedback vectors --- lib/Compilation.js | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index e3cdc21afaf..637c5aa4409 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -337,6 +337,31 @@ const deprecatedNormalModuleLoaderHook = util.deprecate( "DEP_WEBPACK_COMPILATION_NORMAL_MODULE_LOADER_HOOK" ); +// TODO webpack 6: remove +const defineRemovedModuleTemplates = moduleTemplates => { + Object.defineProperties(moduleTemplates, { + asset: { + enumerable: false, + configurable: false, + get: () => { + throw new WebpackError( + "Compilation.moduleTemplates.asset has been removed" + ); + } + }, + webassembly: { + enumerable: false, + configurable: false, + get: () => { + throw new WebpackError( + "Compilation.moduleTemplates.webassembly has been removed" + ); + } + } + }); + moduleTemplates = undefined; +}; + const byId = compareSelect( /** * @param {Chunk} c chunk @@ -861,26 +886,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si this.moduleTemplates = { javascript: new ModuleTemplate(this.runtimeTemplate, this) }; - Object.defineProperties(this.moduleTemplates, { - asset: { - enumerable: false, - configurable: false, - get() { - throw new WebpackError( - "Compilation.moduleTemplates.asset has been removed" - ); - } - }, - webassembly: { - enumerable: false, - configurable: false, - get() { - throw new WebpackError( - "Compilation.moduleTemplates.webassembly has been removed" - ); - } - } - }); + defineRemovedModuleTemplates(this.moduleTemplates); this.moduleGraph = new ModuleGraph(); /** @type {ChunkGraph} */ From 3b555175e6808128cbe05f701dbc9bdaab8e5d0a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 17 Jun 2021 11:25:49 +0200 Subject: [PATCH 3/4] unreference some of the intermediate data when unserializing --- lib/cache/PackFileCacheStrategy.js | 3 +++ lib/serialization/SerializerMiddleware.js | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/cache/PackFileCacheStrategy.js b/lib/cache/PackFileCacheStrategy.js index a5047321e95..75191854af8 100644 --- a/lib/cache/PackFileCacheStrategy.js +++ b/lib/cache/PackFileCacheStrategy.js @@ -8,6 +8,7 @@ const FileSystemInfo = require("../FileSystemInfo"); const ProgressPlugin = require("../ProgressPlugin"); const { formatSize } = require("../SizeFormatHelpers"); +const SerializerMiddleware = require("../serialization/SerializerMiddleware"); const LazySet = require("../util/LazySet"); const makeSerializable = require("../util/makeSerializable"); const memoize = require("../util/memoize"); @@ -715,6 +716,7 @@ class PackContent { this.logger.timeEnd(timeMessage); } this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); return map.get(identifier); }); } else { @@ -723,6 +725,7 @@ class PackContent { this.logger.timeEnd(timeMessage); } this.content = map; + this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy); return map.get(identifier); } } diff --git a/lib/serialization/SerializerMiddleware.js b/lib/serialization/SerializerMiddleware.js index f9d79ed833f..3b8e2d0dd36 100644 --- a/lib/serialization/SerializerMiddleware.js +++ b/lib/serialization/SerializerMiddleware.js @@ -126,6 +126,25 @@ class SerializerMiddleware { fn[LAZY_SERIALIZED_VALUE] = lazy; return fn; } + + /** + * @param {function(): Promise | any} lazy lazy function + * @returns {function(): Promise | any} new lazy + */ + static unMemoizeLazy(lazy) { + if (!SerializerMiddleware.isLazy(lazy)) return lazy; + const fn = () => { + throw new Error( + "A lazy value that has been unmemorized can't be called again" + ); + }; + fn[LAZY_SERIALIZED_VALUE] = SerializerMiddleware.unMemoizeLazy( + lazy[LAZY_SERIALIZED_VALUE] + ); + fn[LAZY_TARGET] = lazy[LAZY_TARGET]; + fn.options = /** @type {any} */ (lazy).options; + return fn; + } } module.exports = SerializerMiddleware; From e5ccd070f6c18794173d3f2e60445de5bc2f52a2 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 17 Jun 2021 12:21:18 +0200 Subject: [PATCH 4/4] 5.39.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e6dcf9ed85e..45b6721ae70 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webpack", - "version": "5.39.0", + "version": "5.39.1", "author": "Tobias Koppers @sokra", "description": "Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.", "license": "MIT",