Skip to content

Commit

Permalink
chunk graph performance optimization
Browse files Browse the repository at this point in the history
  • Loading branch information
@sokra
sokra committed Aug 14, 2020
1 parent a3bef27 commit 84fd99c
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 2 deletions.
132 changes: 130 additions & 2 deletions lib/buildChunkGraph.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,20 @@ const visitModules = (
logger.time("visitModules: prepare");
const blockModulesMap = extractBlockModulesMap(compilation);

let statProcessedQueueItems = 0;
let statProcessedBlocks = 0;
let statConnectedChunkGroups = 0;
let statProcessedChunkGroupsForMerging = 0;
let statMergedAvailableModuleSets = 0;
let statForkedAvailableModules = 0;
let statForkedAvailableModulesCount = 0;
let statForkedAvailableModulesCountPlus = 0;
let statForkedMergedModulesCount = 0;
let statForkedMergedModulesCountPlus = 0;
let statForkedResultModulesCount = 0;
let statChunkGroupInfoUpdated = 0;
let statChildChunkGroupsReconnected = 0;

let nextChunkGroupIndex = 0;
let nextFreeModulePreOrderIndex = 0;
let nextFreeModulePostOrderIndex = 0;
Expand Down Expand Up @@ -357,6 +371,7 @@ const visitModules = (
* @returns {void}
*/
const processBlock = block => {
statProcessedBlocks++;
// get prepared block info
const blockModules = blockModulesMap.get(block);

Expand Down Expand Up @@ -423,6 +438,7 @@ const visitModules = (

const processQueue = () => {
while (queue.length) {
statProcessedQueueItems++;
const queueItem = queue.pop();
module = queueItem.module;
block = queueItem.block;
Expand Down Expand Up @@ -552,16 +568,22 @@ const visitModules = (
target.availableModulesToBeMerged.push(resultingAvailableModules);
chunkGroupsForMerging.add(target);
}

statConnectedChunkGroups += targets.size;
}
queueConnect.clear();
};

const processChunkGroupsForMerging = () => {
statProcessedChunkGroupsForMerging += chunkGroupsForMerging.size;

// Execute the merge
for (const info of chunkGroupsForMerging) {
const availableModulesToBeMerged = info.availableModulesToBeMerged;
let cachedMinAvailableModules = info.minAvailableModules;

statMergedAvailableModuleSets += availableModulesToBeMerged.length;

// 1. Get minimal available modules
// It doesn't make sense to traverse a chunk again with more available modules.
// This step calculates the minimal available modules and skips traversal when
Expand All @@ -579,15 +601,98 @@ const visitModules = (
} else {
if (info.minAvailableModulesOwned) {
// We own it and can modify it
if (cachedMinAvailableModules.plus === availableModules.plus) {
for (const m of cachedMinAvailableModules) {
if (!availableModules.has(m)) {
cachedMinAvailableModules.delete(m);
changed = true;
}
}
} else {
for (const m of cachedMinAvailableModules) {
if (!availableModules.has(m) && !availableModules.plus.has(m)) {
cachedMinAvailableModules.delete(m);
changed = true;
}
}
for (const m of cachedMinAvailableModules.plus) {
if (!availableModules.has(m) && !availableModules.plus.has(m)) {
// We can't remove modules from the plus part
// so we need to merge plus into the normal part to allow modifying it
const iterator = cachedMinAvailableModules.plus[
Symbol.iterator
]();
// fast forward add all modules until m
/** @type {IteratorResult<Module>} */
let it;
while (!(it = iterator.next()).done) {
const module = it.value;
if (module === m) break;
cachedMinAvailableModules.add(module);
}
// check the remaining modules before adding
while (!(it = iterator.next()).done) {
const module = it.value;
if (
availableModules.has(module) ||
availableModules.plus.has(m)
) {
cachedMinAvailableModules.add(module);
}
}
cachedMinAvailableModules.plus = EMPTY_SET;
changed = true;
continue merge;
}
}
}
} else if (cachedMinAvailableModules.plus === availableModules.plus) {
for (const m of cachedMinAvailableModules) {
if (!availableModules.has(m) && !availableModules.plus.has(m)) {
cachedMinAvailableModules.delete(m);
if (!availableModules.has(m)) {
statForkedAvailableModules++;
statForkedAvailableModulesCount +=
cachedMinAvailableModules.size;
statForkedMergedModulesCount += availableModules.size;
// cachedMinAvailableModules need to be modified
// but we don't own it
// construct a new Set as intersection of cachedMinAvailableModules and availableModules
// as the plus part is equal we can just take over this one
const newSet = /** @type {ModuleSetPlus} */ (new Set());
newSet.plus = availableModules.plus;
const iterator = cachedMinAvailableModules[Symbol.iterator]();
// fast forward add all modules until m
/** @type {IteratorResult<Module>} */
let it;
while (!(it = iterator.next()).done) {
const module = it.value;
if (module === m) break;
newSet.add(module);
}
// check the remaining modules before adding
while (!(it = iterator.next()).done) {
const module = it.value;
if (availableModules.has(module)) {
newSet.add(module);
}
}
statForkedResultModulesCount += newSet.size;
cachedMinAvailableModules = newSet;
info.minAvailableModulesOwned = true;
info.minAvailableModules = newSet;
changed = true;
continue merge;
}
}
} else {
for (const m of cachedMinAvailableModules) {
if (!availableModules.has(m) && !availableModules.plus.has(m)) {
statForkedAvailableModules++;
statForkedAvailableModulesCount +=
cachedMinAvailableModules.size;
statForkedAvailableModulesCountPlus +=
cachedMinAvailableModules.plus.size;
statForkedMergedModulesCount += availableModules.size;
statForkedMergedModulesCountPlus += availableModules.plus.size;
// cachedMinAvailableModules need to be modified
// but we don't own it
// construct a new Set as intersection of cachedMinAvailableModules and availableModules
Expand Down Expand Up @@ -621,6 +726,7 @@ const visitModules = (
newSet.add(module);
}
}
statForkedResultModulesCount += newSet.size;
cachedMinAvailableModules = newSet;
info.minAvailableModulesOwned = true;
info.minAvailableModules = newSet;
Expand All @@ -630,6 +736,13 @@ const visitModules = (
}
for (const m of cachedMinAvailableModules.plus) {
if (!availableModules.has(m) && !availableModules.plus.has(m)) {
statForkedAvailableModules++;
statForkedAvailableModulesCount +=
cachedMinAvailableModules.size;
statForkedAvailableModulesCountPlus +=
cachedMinAvailableModules.plus.size;
statForkedMergedModulesCount += availableModules.size;
statForkedMergedModulesCountPlus += availableModules.plus.size;
// cachedMinAvailableModules need to be modified
// but we don't own it
// construct a new Set as intersection of cachedMinAvailableModules and availableModules
Expand Down Expand Up @@ -659,6 +772,7 @@ const visitModules = (
newSet.add(module);
}
}
statForkedResultModulesCount += newSet.size;
cachedMinAvailableModules = newSet;
info.minAvailableModulesOwned = true;
info.minAvailableModules = newSet;
Expand All @@ -675,6 +789,7 @@ const visitModules = (
outdatedChunkGroupInfo.add(info);
}
}
chunkGroupsForMerging.clear();
};

const processChunkGroupsForCombining = () => {
Expand Down Expand Up @@ -709,6 +824,7 @@ const visitModules = (
};

const processOutdatedChunkGroupInfo = () => {
statChunkGroupInfoUpdated += outdatedChunkGroupInfo.size;
// Revisit skipped elements
for (const info of outdatedChunkGroupInfo) {
// 1. Reconsider skipped items
Expand All @@ -734,6 +850,7 @@ const visitModules = (

// 2. Reconsider children chunk groups
if (info.children !== undefined) {
statChildChunkGroupsReconnected += info.children.size;
for (const cgi of info.children) {
let connectList = queueConnect.get(info);
if (connectList === undefined) {
Expand Down Expand Up @@ -794,6 +911,17 @@ const visitModules = (
queueDelayed = tempQueue;
}
}

logger.log(
`Statistics: ${statProcessedQueueItems} queue items processed (${statProcessedBlocks} blocks)`
);
logger.log(`Statistics: ${statConnectedChunkGroups} chunk groups connected`);
logger.log(
`Statistics: ${statProcessedChunkGroupsForMerging} chunk groups processed for merging (${statMergedAvailableModuleSets} module sets, ${statForkedAvailableModules} forked, ${statForkedAvailableModulesCount} + ${statForkedAvailableModulesCountPlus} modules forked, ${statForkedMergedModulesCount} + ${statForkedMergedModulesCountPlus} modules merged into fork, ${statForkedResultModulesCount} resulting modules)`
);
logger.log(
`Statistics: ${statChunkGroupInfoUpdated} chunk group info updated (${statChildChunkGroupsReconnected} already connected chunk groups reconnected)`
);
};

/**
Expand Down
18 changes: 18 additions & 0 deletions test/__snapshots__/StatsTestCases.test.js.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1361,6 +1361,13 @@ Entrypoint <CLR=BOLD>main</CLR> = <CLR=32,BOLD>main.js</CLR>
<CLR=31,BOLD>DEBUG</CLR> <CLR=BOLD>LOG from ./node_modules/custom-loader/index.js Named Logger ./node_modules/custom-loader/index.js!./index.js</CLR>
Message with named logger
<CLR=BOLD>LOG from webpack.buildChunkGraph</CLR>
<CLR=BOLD>Statistics: 2 queue items processed (1 blocks)</CLR>
<CLR=BOLD>Statistics: 0 chunk groups connected</CLR>
<CLR=BOLD>Statistics: 0 chunk groups processed for merging (0 module sets, 0 forked, 0 + 0 modules forked, 0 + 0 modules merged into fork, 0 resulting modules)</CLR>
<CLR=BOLD>Statistics: 0 chunk group info updated (0 already connected chunk groups reconnected)</CLR>
+ 5 hidden lines
"
`;

Expand Down Expand Up @@ -2140,6 +2147,13 @@ LOG from LogTestPlugin
Log
End
+ 6 hidden lines
LOG from webpack.buildChunkGraph
Statistics: 15 queue items processed (9 blocks)
Statistics: 3 chunk groups connected
Statistics: 3 chunk groups processed for merging (3 module sets, 0 forked, 0 + 0 modules forked, 0 + 0 modules merged into fork, 0 resulting modules)
Statistics: 3 chunk group info updated (0 already connected chunk groups reconnected)
+ 13 hidden lines
"
`;
Expand Down Expand Up @@ -2439,6 +2453,10 @@ LOG from webpack.buildChunkGraph
<t> visitModules: merging available modules: X ms
<t> visitModules: check modules for revisit: X ms
<t> visitModules: visiting: X ms
Statistics: 15 queue items processed (9 blocks)
Statistics: 3 chunk groups connected
Statistics: 3 chunk groups processed for merging (3 module sets, 0 forked, 0 + 0 modules forked, 0 + 0 modules merged into fork, 0 resulting modules)
Statistics: 3 chunk group info updated (0 already connected chunk groups reconnected)
<t> visitModules: X ms
<t> connectChunkGroups: X ms
<t> cleanup: X ms
Expand Down

0 comments on commit 84fd99c

Please sign in to comment.