From a82ab66fc1dbfa841c59439b1efe8e2a990b6014 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 28 Sep 2021 23:52:25 +0200 Subject: [PATCH 1/4] fix accidentically shared mem caches --- lib/Compilation.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index 5431aab3e40..f8ede7d6c3a 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -2261,7 +2261,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si affectedModules.add(referencingModule); } const memCache = new WeakTupleMap(); - const cache = moduleMemCacheCache.get(module); + const cache = moduleMemCacheCache.get(referencingModule); cache.memCache = memCache; moduleMemCaches.set(referencingModule, memCache); } From 6c6e48c3826216e055c3ee4f99b3cf160d21a47a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 29 Sep 2021 00:06:44 +0200 Subject: [PATCH 2/4] avoid RuntimeSpecMap in favor of directly setting on memCache --- lib/Compilation.js | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index f8ede7d6c3a..353cc57efe8 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -78,7 +78,7 @@ const { createFakeHook } = require("./util/deprecation"); const processAsyncTree = require("./util/processAsyncTree"); -const { getRuntimeKey, RuntimeSpecMap } = require("./util/runtime"); +const { getRuntimeKey } = require("./util/runtime"); const { isSourceEqual } = require("./util/source"); /** @template T @typedef {import("tapable").AsArray} AsArray */ @@ -3095,16 +3095,11 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o // modules with async blocks depend on the chunk graph and can't be cached that way module.blocks.length === 0 && moduleMemCaches.get(module); - /** @type {RuntimeSpecMap>} */ - const moduleRuntimeRequirementsMemCache = - memCache && - memCache.provide( - "moduleRuntimeRequirements", - () => new RuntimeSpecMap() - ); for (const runtime of chunkGraph.getModuleRuntimes(module)) { - if (moduleRuntimeRequirementsMemCache) { - const cached = moduleRuntimeRequirementsMemCache.get(runtime); + if (memCache) { + const cached = memCache.get( + `moduleRuntimeRequirements-${getRuntimeKey(runtime)}` + ); if (cached !== undefined) { if (cached !== null) { chunkGraph.addModuleRuntimeRequirements( @@ -3125,8 +3120,11 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o } else if (additionalModuleRuntimeRequirements.isUsed()) { set = new Set(); } else { - if (moduleRuntimeRequirementsMemCache) { - moduleRuntimeRequirementsMemCache.set(runtime, null); + if (memCache) { + memCache.set( + `moduleRuntimeRequirements-${getRuntimeKey(runtime)}`, + null + ); } continue; } @@ -3137,12 +3135,18 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o if (hook !== undefined) hook.call(module, set, context); } if (set.size === 0) { - if (moduleRuntimeRequirementsMemCache) { - moduleRuntimeRequirementsMemCache.set(runtime, null); + if (memCache) { + memCache.set( + `moduleRuntimeRequirements-${getRuntimeKey(runtime)}`, + null + ); } } else { - if (moduleRuntimeRequirementsMemCache) { - moduleRuntimeRequirementsMemCache.set(runtime, set); + if (memCache) { + memCache.set( + `moduleRuntimeRequirements-${getRuntimeKey(runtime)}`, + set + ); chunkGraph.addModuleRuntimeRequirements( module, runtime, @@ -3555,13 +3559,9 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o // modules with async blocks depend on the chunk graph and can't be cached that way module.blocks.length === 0 && moduleMemCaches.get(module); - /** @type {RuntimeSpecMap} */ - const moduleHashesMemCache = - memCache && - memCache.provide("moduleHashes", () => new RuntimeSpecMap()); for (const runtime of chunkGraph.getModuleRuntimes(module)) { - if (moduleHashesMemCache) { - const digest = moduleHashesMemCache.get(runtime); + if (memCache) { + const digest = memCache.get(`moduleHash-${getRuntimeKey(runtime)}`); if (digest !== undefined) { chunkGraph.setModuleHashes( module, @@ -3583,8 +3583,8 @@ Or do you want to use the entrypoints '${name}' and '${runtime}' independently o hashDigest, hashDigestLength ); - if (moduleHashesMemCache) { - moduleHashesMemCache.set(runtime, digest); + if (memCache) { + memCache.set(`moduleHash-${getRuntimeKey(runtime)}`, digest); } } } From 61569e26b9f3783e72756e0c5104cb245f4e6191 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 29 Sep 2021 09:09:42 +0200 Subject: [PATCH 3/4] compare references modules when restoring mem cache --- lib/Compilation.js | 58 ++++++++++++++++++++++++++++++++++++++-------- lib/Compiler.js | 3 ++- types.d.ts | 6 ++++- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/lib/Compilation.js b/lib/Compilation.js index 353cc57efe8..68fef1e9eb5 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -2178,7 +2178,39 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si let statNew = 0; let statChanged = 0; let statUnchanged = 0; + let statReferencesChanged = 0; let statWithoutHash = 0; + + const computeReferences = module => { + /** @type {WeakMap} */ + let references = undefined; + for (const connection of moduleGraph.getOutgoingConnections(module)) { + const d = connection.dependency; + const m = connection.module; + if (!d || !m || unsafeCacheDependencies.has(d)) continue; + if (references === undefined) references = new WeakMap(); + references.set(d, m); + } + return references; + }; + + /** + * @param {Module} module the module + * @param {WeakMap} references references + * @returns {boolean} true, when the references differ + */ + const compareReferences = (module, references) => { + if (references === undefined) return true; + for (const connection of moduleGraph.getOutgoingConnections(module)) { + const d = connection.dependency; + if (!d) continue; + const entry = references.get(d); + if (entry === undefined) continue; + if (entry !== connection.module) return false; + } + return true; + }; + for (const module of modules) { const hash = module.buildInfo && module.buildInfo.hash; if (typeof hash === "string") { @@ -2188,27 +2220,33 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si const memCache = new WeakTupleMap(); moduleMemCacheCache.set(module, { hash: hash, + references: computeReferences(module), memCache }); moduleMemCaches.set(module, memCache); affectedModules.add(module); statNew++; - } else if (cachedMemCache.hash === hash) { - // keep the old mem cache - moduleMemCaches.set(module, cachedMemCache.memCache); - statUnchanged++; - } else { + } else if (cachedMemCache.hash !== hash) { // use a new one const memCache = new WeakTupleMap(); - moduleMemCacheCache.set(module, { - hash: hash, - memCache - }); moduleMemCaches.set(module, memCache); affectedModules.add(module); cachedMemCache.hash = hash; + cachedMemCache.references = computeReferences(module); cachedMemCache.memCache = memCache; statChanged++; + } else if (!compareReferences(module, cachedMemCache.references)) { + // use a new one + const memCache = new WeakTupleMap(); + moduleMemCaches.set(module, memCache); + affectedModules.add(module); + cachedMemCache.references = computeReferences(module); + cachedMemCache.memCache = memCache; + statReferencesChanged++; + } else { + // keep the old mem cache + moduleMemCaches.set(module, cachedMemCache.memCache); + statUnchanged++; } } else { infectedModules.add(module); @@ -2275,7 +2313,7 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si infectedModules.size } infected of ${ this.modules.size - }) modules flagged as affected (${statNew} new modules, ${statChanged} changed, ${statUnchanged} unchanged, ${statWithoutHash} without hash)` + }) modules flagged as affected (${statNew} new modules, ${statChanged} changed, ${statReferencesChanged} references changed, ${statUnchanged} unchanged, ${statWithoutHash} without hash)` ); } diff --git a/lib/Compiler.js b/lib/Compiler.js index eade25766d6..a8e5cfff043 100644 --- a/lib/Compiler.js +++ b/lib/Compiler.js @@ -40,6 +40,7 @@ const { isSourceEqual } = require("./util/source"); /** @typedef {import("../declarations/WebpackOptions").WebpackOptionsNormalized} WebpackOptions */ /** @typedef {import("../declarations/WebpackOptions").WebpackPluginInstance} WebpackPluginInstance */ /** @typedef {import("./Chunk")} Chunk */ +/** @typedef {import("./Dependency")} Dependency */ /** @typedef {import("./FileSystemInfo").FileSystemInfoEntry} FileSystemInfoEntry */ /** @typedef {import("./Module")} Module */ /** @typedef {import("./util/WeakTupleMap")} WeakTupleMap */ @@ -248,7 +249,7 @@ class Compiler { this.cache = new Cache(); - /** @type {WeakMap | undefined} */ + /** @type {WeakMap, memCache: WeakTupleMap }> | undefined} */ this.moduleMemCaches = undefined; this.compilerPath = ""; diff --git a/types.d.ts b/types.d.ts index d00da846c9c..bd3d43b32a0 100644 --- a/types.d.ts +++ b/types.d.ts @@ -1909,7 +1909,11 @@ declare class Compiler { cache: Cache; moduleMemCaches?: WeakMap< Module, - { hash: string; memCache: WeakTupleMap } + { + hash: string; + references: WeakMap; + memCache: WeakTupleMap; + } >; compilerPath: string; running: boolean; From e78403d6a8d4e682fbd4a0eb58f042866cfde42a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 29 Sep 2021 10:56:21 +0200 Subject: [PATCH 4/4] 5.55.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 89f1b834b5a..280dcfda0f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webpack", - "version": "5.55.0", + "version": "5.55.1", "author": "Tobias Koppers @sokra", "description": "Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.", "license": "MIT",