Skip to content

Commit 05275e6

Browse files
sokrasokra
committed
improve SplitChunksPlugin performance
avoid looping better complexity for finding possible combinations avoid creating a chunks array caching
1 parent 1c0d4f7 commit 05275e6

File tree

2 files changed

+159
-50
lines changed

2 files changed

+159
-50
lines changed

‎lib/optimize/RemoveEmptyChunksPlugin.js

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,18 @@ class RemoveEmptyChunksPlugin {
2222
"RemoveEmptyChunksPlugin",
2323
handler
2424
);
25+
compilation.hooks.optimizeChunksAdvanced.tap(
26+
"RemoveEmptyChunksPlugin",
27+
handler
28+
);
2529
compilation.hooks.optimizeExtractedChunksBasic.tap(
2630
"RemoveEmptyChunksPlugin",
2731
handler
2832
);
33+
compilation.hooks.optimizeExtractedChunksAdvanced.tap(
34+
"RemoveEmptyChunksPlugin",
35+
handler
36+
);
2937
});
3038
}
3139
}

‎lib/optimize/SplitChunksPlugin.js

Lines changed: 151 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ const SortableSet = require("../util/SortableSet");
99
const GraphHelpers = require("../GraphHelpers");
1010
const { isSubset } = require("../util/SetHelpers");
1111

12+
/** @typedef {import("../Chunk")} Chunk */
13+
1214
const hashFilename = name => {
1315
return crypto
1416
.createHash("md4")
@@ -78,6 +80,10 @@ const compareEntries = (a, b) => {
7880
}
7981
};
8082

83+
const INITIAL_CHUNK_FILTER = chunk => chunk.canBeInitial();
84+
const ASYNC_CHUNK_FILTER = chunk => !chunk.canBeInitial();
85+
const ALL_CHUNK_FILTER = chunk => true;
86+
8187
module.exports = class SplitChunksPlugin {
8288
constructor(options) {
8389
this.options = SplitChunksPlugin.normalizeOptions(options);
@@ -107,9 +113,20 @@ module.exports = class SplitChunksPlugin {
107113

108114
static normalizeName({ name, automaticNameDelimiter }) {
109115
if (name === true) {
116+
const cache = new Map();
110117
const fn = (module, chunks, cacheGroup) => {
118+
let cacheEntry = cache.get(chunks);
119+
if (cacheEntry === undefined) {
120+
cacheEntry = {};
121+
cache.set(chunks, cacheEntry);
122+
} else if (cacheGroup in cacheEntry) {
123+
return cacheEntry[cacheGroup];
124+
}
111125
const names = chunks.map(c => c.name);
112-
if (!names.every(Boolean)) return;
126+
if (!names.every(Boolean)) {
127+
cacheEntry[cacheGroup] = undefined;
128+
return;
129+
}
113130
names.sort();
114131
let name =
115132
(cacheGroup && cacheGroup !== "default"
@@ -124,6 +141,7 @@ module.exports = class SplitChunksPlugin {
124141
name =
125142
name.slice(0, 100) + automaticNameDelimiter + hashFilename(name);
126143
}
144+
cacheEntry[cacheGroup] = name;
127145
return name;
128146
};
129147
return fn;
@@ -139,23 +157,26 @@ module.exports = class SplitChunksPlugin {
139157

140158
static normalizeChunksFilter(chunks) {
141159
if (chunks === "initial") {
142-
return chunk => chunk.canBeInitial();
160+
return INITIAL_CHUNK_FILTER;
143161
}
144162
if (chunks === "async") {
145-
return chunk => !chunk.canBeInitial();
163+
return ASYNC_CHUNK_FILTER;
146164
}
147165
if (chunks === "all") {
148-
return () => true;
166+
return ALL_CHUNK_FILTER;
149167
}
150168
if (typeof chunks === "function") return chunks;
151169
}
152170

153171
static normalizeCacheGroups({ cacheGroups, automaticNameDelimiter }) {
154172
if (typeof cacheGroups === "function") {
173+
// TODO webpack 5 remove this
174+
if (cacheGroups.length !== 1)
175+
return module => cacheGroups(module, module.getChunks());
155176
return cacheGroups;
156177
}
157178
if (cacheGroups && typeof cacheGroups === "object") {
158-
const fn = (module, chunks) => {
179+
const fn = module => {
159180
let results;
160181
for (const key of Object.keys(cacheGroups)) {
161182
let option = cacheGroups[key];
@@ -185,7 +206,7 @@ module.exports = class SplitChunksPlugin {
185206
results.push(result);
186207
}
187208
}
188-
} else if (SplitChunksPlugin.checkTest(option.test, module, chunks)) {
209+
} else if (SplitChunksPlugin.checkTest(option.test, module)) {
189210
if (results === undefined) results = [];
190211
results.push({
191212
key: key,
@@ -215,20 +236,27 @@ module.exports = class SplitChunksPlugin {
215236
return fn;
216237
}
217238

218-
static checkTest(test, module, chunks) {
239+
static checkTest(test, module) {
219240
if (test === undefined) return true;
220-
if (typeof test === "function") return test(module, chunks);
241+
if (typeof test === "function") {
242+
if (test.length !== 1) return test(module, module.getChunks());
243+
return test(module);
244+
}
221245
if (typeof test === "boolean") return test;
222-
const names = chunks
223-
.map(c => c.name)
224-
.concat(module.nameForCondition ? [module.nameForCondition()] : [])
225-
.filter(Boolean);
226246
if (typeof test === "string") {
227-
for (const name of names) if (name.startsWith(test)) return true;
247+
if (module.nameForCondition && module.nameForCondition().startsWith(test))
248+
return true;
249+
for (const chunk of module.chunksIterable) {
250+
if (chunk.name && chunk.name.startsWith(test)) return true;
251+
}
228252
return false;
229253
}
230254
if (test instanceof RegExp) {
231-
for (const name of names) if (test.test(name)) return true;
255+
if (module.nameForCondition && test.test(module.nameForCondition()))
256+
return true;
257+
for (const chunk of module.chunksIterable) {
258+
if (chunk.name && test.test(chunk.name)) return true;
259+
}
232260
return false;
233261
}
234262
return false;
@@ -256,32 +284,92 @@ module.exports = class SplitChunksPlugin {
256284
.sort()
257285
.join();
258286
};
259-
// Create a list of possible combinations
260-
const chunkSetsInGraph = new Map(); // Map<string, Set<Chunk>>
287+
/** @type {Map<string, Set<Chunk>>} */
288+
const chunkSetsInGraph = new Map();
261289
for (const module of compilation.modules) {
262-
const chunkIndices = getKey(module.chunksIterable);
263-
chunkSetsInGraph.set(chunkIndices, new Set(module.chunksIterable));
290+
const chunksKey = getKey(module.chunksIterable);
291+
if (!chunkSetsInGraph.has(chunksKey)) {
292+
chunkSetsInGraph.set(chunksKey, new Set(module.chunksIterable));
293+
}
264294
}
265-
const combinations = new Map(); // Map<string, Set<Chunk>[]>
266-
for (const [key, chunksSet] of chunkSetsInGraph) {
267-
var array = [];
268-
for (const set of chunkSetsInGraph.values()) {
269-
if (isSubset(chunksSet, set)) {
270-
array.push(set);
271-
}
295+
296+
// group these set of chunks by count
297+
// to allow to check less sets via isSubset
298+
// (only smaller sets can be subset)
299+
/** @type {Map<number, Array<Set<Chunk>>>} */
300+
const chunkSetsByCount = new Map();
301+
for (const chunksSet of chunkSetsInGraph.values()) {
302+
const count = chunksSet.size;
303+
let array = chunkSetsByCount.get(count);
304+
if (array === undefined) {
305+
array = [];
306+
chunkSetsByCount.set(count, array);
272307
}
273-
combinations.set(key, array);
308+
array.push(chunksSet);
274309
}
310+
311+
// Create a list of possible combinations
312+
const combinationsCache = new Map(); // Map<string, Set<Chunk>[]>
313+
const selectedChunksCacheByChunksSet = new WeakMap(); // WeakMap<Set<Chunk>, WeakMap<Function, {chunks: Chunk[], key: string}>>
314+
315+
const getCombinations = key => {
316+
const chunksSet = chunkSetsInGraph.get(key);
317+
var array = [chunksSet];
318+
if (chunksSet.size > 1) {
319+
for (const [count, setArray] of chunkSetsByCount) {
320+
// "equal" is not needed because they would have been merge in the first step
321+
if (count < chunksSet.size) {
322+
for (const set of setArray) {
323+
if (isSubset(chunksSet, set)) {
324+
array.push(set);
325+
}
326+
}
327+
}
328+
}
329+
}
330+
return array;
331+
};
332+
333+
const getSelectedChunks = (chunks, chunkFilter) => {
334+
let entry = selectedChunksCacheByChunksSet.get(chunks);
335+
if (entry === undefined) {
336+
entry = new Map();
337+
selectedChunksCacheByChunksSet.set(chunks, entry);
338+
}
339+
let entry2 = entry.get(chunkFilter);
340+
if (entry2 === undefined) {
341+
const selectedChunks = [];
342+
for (const chunk of chunks) {
343+
if (chunkFilter(chunk)) selectedChunks.push(chunk);
344+
}
345+
entry2 = {
346+
chunks: selectedChunks,
347+
key: getKey(selectedChunks)
348+
};
349+
entry.set(chunkFilter, entry2);
350+
}
351+
return entry2;
352+
};
353+
275354
// Map a list of chunks to a list of modules
276355
// For the key the chunk "index" is used, the value is a SortableSet of modules
277356
const chunksInfoMap = new Map();
357+
278358
// Walk through all modules
279359
for (const module of compilation.modules) {
280-
// Get array of chunks
281-
const chunks = module.getChunks();
282360
// Get cache group
283-
let cacheGroups = this.options.getCacheGroups(module, chunks);
284-
if (!Array.isArray(cacheGroups)) continue;
361+
let cacheGroups = this.options.getCacheGroups(module);
362+
if (!Array.isArray(cacheGroups) || cacheGroups.length === 0)
363+
continue;
364+
365+
// Prepare some values
366+
const chunksKey = getKey(module.chunksIterable);
367+
let combs = combinationsCache.get(chunksKey);
368+
if (combs === undefined) {
369+
combs = getCombinations(chunksKey);
370+
combinationsCache.set(chunksKey, combs);
371+
}
372+
285373
for (const cacheGroupSource of cacheGroups) {
286374
const cacheGroup = {
287375
key: cacheGroupSource.key,
@@ -323,15 +411,15 @@ module.exports = class SplitChunksPlugin {
323411
reuseExistingChunk: cacheGroupSource.reuseExistingChunk
324412
};
325413
// For all combination of chunk selection
326-
for (const chunkCombination of combinations.get(getKey(chunks))) {
327-
// Get indices of chunks in which this module occurs
328-
const chunkIndices = Array.from(chunkCombination, chunk =>
329-
indexMap.get(chunk)
330-
);
414+
for (const chunkCombination of combs) {
331415
// Break if minimum number of chunks is not reached
332-
if (chunkIndices.length < cacheGroup.minChunks) continue;
416+
if (chunkCombination.size < cacheGroup.minChunks) continue;
333417
// Select chunks by configuration
334-
const selectedChunks = Array.from(chunkCombination).filter(
418+
const {
419+
chunks: selectedChunks,
420+
key: selectedChunksKey
421+
} = getSelectedChunks(
422+
chunkCombination,
335423
cacheGroup.chunksFilter
336424
);
337425
// Break if minimum number of chunks is not reached
@@ -346,10 +434,9 @@ module.exports = class SplitChunksPlugin {
346434
// When it has a name we use the name as key
347435
// Elsewise we create the key from chunks and cache group key
348436
// This automatically merges equal names
349-
const chunksKey = getKey(selectedChunks);
350437
const key =
351438
(name && `name:${name}`) ||
352-
`chunks:${chunksKey} key:${cacheGroup.key}`;
439+
`chunks:${selectedChunksKey} key:${cacheGroup.key}`;
353440
// Add module to maps
354441
let info = chunksInfoMap.get(key);
355442
if (info === undefined) {
@@ -359,30 +446,31 @@ module.exports = class SplitChunksPlugin {
359446
modules: new SortableSet(undefined, sortByIdentifier),
360447
cacheGroup,
361448
name,
449+
size: 0,
362450
chunks: new Map(),
363451
reusedableChunks: new Set(),
364452
chunksKeys: new Set()
365453
})
366454
);
367455
}
368456
info.modules.add(module);
369-
if (!info.chunksKeys.has(chunksKey)) {
370-
info.chunksKeys.add(chunksKey);
457+
info.size += module.size();
458+
if (!info.chunksKeys.has(selectedChunksKey)) {
459+
info.chunksKeys.add(selectedChunksKey);
371460
for (const chunk of selectedChunks) {
372461
info.chunks.set(chunk, chunk.getNumberOfModules());
373462
}
374463
}
375464
}
376465
}
377466
}
467+
378468
for (const [key, info] of chunksInfoMap) {
379469
// Get size of module lists
380-
info.size = getModulesSize(info.modules);
381470
if (info.size < info.cacheGroup.minSize) {
382471
chunksInfoMap.delete(key);
383472
}
384473
}
385-
let changed = false;
386474
while (chunksInfoMap.size > 0) {
387475
// Find best matching entry
388476
let bestEntryKey;
@@ -429,6 +517,7 @@ module.exports = class SplitChunksPlugin {
429517
}
430518
}
431519
}
520+
const usedChunks = [];
432521
// Walk through all chunks
433522
for (const chunk of item.chunks.keys()) {
434523
// skip if we address ourself
@@ -450,12 +539,9 @@ module.exports = class SplitChunksPlugin {
450539
}
451540
// Add graph connections for splitted chunk
452541
chunk.split(newChunk);
453-
// Remove all selected modules from the chunk
454-
for (const module of item.modules) {
455-
chunk.removeModule(module);
456-
module.rewriteChunkInReasons(chunk, [newChunk]);
457-
}
542+
usedChunks.push(chunk);
458543
}
544+
459545
// If we successfully created a new chunk or reused one
460546
if (newChunk) {
461547
// Add a note to the chunk
@@ -491,7 +577,24 @@ module.exports = class SplitChunksPlugin {
491577
if (!isReused) {
492578
// Add all modules to the new chunk
493579
for (const module of item.modules) {
580+
if (typeof module.chunkCondition === "function") {
581+
if (!module.chunkCondition(newChunk)) continue;
582+
}
583+
// Add module to new chunk
494584
GraphHelpers.connectChunkAndModule(newChunk, module);
585+
// Remove module from used chunks
586+
for (const chunk of usedChunks) {
587+
chunk.removeModule(module);
588+
module.rewriteChunkInReasons(chunk, [newChunk]);
589+
}
590+
}
591+
} else {
592+
// Remove all modules from used chunks
593+
for (const module of item.modules) {
594+
for (const chunk of usedChunks) {
595+
chunk.removeModule(module);
596+
module.rewriteChunkInReasons(chunk, [newChunk]);
597+
}
495598
}
496599
}
497600
// remove all modules from other entries and update size
@@ -512,10 +615,8 @@ module.exports = class SplitChunksPlugin {
512615
}
513616
}
514617
}
515-
changed = true;
516618
}
517619
}
518-
if (changed) return true;
519620
}
520621
);
521622
});

0 commit comments

Comments
 (0)