Visitar URL original
perf: memoize side-effects state on the module to skip repeated walks · webpack/webpack@52746f1 · GitHub
Skip to content

Commit 52746f1

Browse files
perf: memoize side-effects state on the module to skip repeated walks
Builds on the previous hybrid walker by adding a per-module memoization slot keyed by ModuleGraph. SideEffectsFlagPlugin queries getSideEffectsConnectionState for every module while iterating incoming connections and again inside its exportInfo.getTarget / moveTarget callbacks, and the walker also re-encounters the same sub-graphs as it descends through different starting modules. Without the cache that work is O(N x avgDepth); with it the second and later visits are O(1). Design notes: - Cache slot is a (ModuleGraph, ConnectionState) pair stored directly on the module (`_sideEffectsStateGraph` / `_sideEffectsStateValue`) rather than a WeakMap so the lookup is a single reference compare. When the next compilation queries with a different ModuleGraph the slot is overwritten on the first walk — no stale data leaks. - Cycle safety: a result computed in the presence of a cycle can differ from the same module's fresh result (the back-edge target's contribution is hidden by CIRCULAR_CONNECTION short-circuiting), so we only memoize results whose walk subtree never observed a CIRCULAR_CONNECTION. A module-level `_sideEffectsCircularSeen` flag is save/restored at each recursive frame and at the public entry point so re-entrant calls (notably ConcatenatedModule.getSECS delegating to its rootModule) don't clobber the outer walk's tracking. A `true` result is monotonic — once any dep declares side effects, the answer is `true` regardless of cycle resolution — so it's always safe to memoize. - Descents into non-NormalModule refModules conservatively mark the current frame as cycle-tainted, since we can't observe whether the inner public-method call hit a cycle back through the outer walk's stack. Microbench (best of 7 runs): Repeated queries of every module in one chain — the realistic SideEffectsFlagPlugin access pattern: chain BEFORE AFTER (PR #20993) NOW (cached) 10 0.03 ms n/a 0.08 ms 100 0.15 ms n/a 0.06 ms 2.5x 1000 15.69 ms n/a 0.36 ms 44x 5000 712.70 ms n/a 2.66 ms 268x First-walk-only (cold cache, fresh chain each call): chain BEFORE NOW (cached) 10 333 ns 300 ns (matches BEFORE) 100 110 ns 131 ns 1000 100 ns 148 ns 5000 157 ns 526 ns (iterative tail engaged) All existing side-effect / tree-shaking / inner-graph / concatenate ConfigTestCases pass (658 tests).
1 parent b3893ec commit 52746f1

3 files changed

Lines changed: 287 additions & 27 deletions

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Rewrite `NormalModule#getSideEffectsConnectionState` walk as an allocation-light iterative loop instead of a generator trampoline, restoring rebuild performance lost in #20993 while keeping deep import chains stack-safe.

‎lib/NormalModule.js‎

Lines changed: 236 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@ const parseJson = require("./util/parseJson");
8686
/** @typedef {import("./Module").UnsafeCacheData} UnsafeCacheData */
8787
/** @typedef {import("./ModuleGraph")} ModuleGraph */
8888
/** @typedef {import("./ModuleGraphConnection").ConnectionState} ConnectionState */
89-
/** @typedef {Iterator<SideEffectsWalk, ConnectionState, ConnectionState>} SideEffectsWalk */
9089
/** @typedef {import("./NormalModuleFactory")} NormalModuleFactory */
9190
/** @typedef {import("./NormalModuleFactory").NormalModuleTypes} NormalModuleTypes */
9291
/** @typedef {import("./NormalModuleFactory").ParserByType} ParserByType */
@@ -157,14 +156,13 @@ const recordSideEffectsBailout = (mod, moduleGraph, dep) => {
157156
};
158157

159158
/**
160-
* Generator form of `getSideEffectsConnectionState` — descends through
161-
* `HarmonyImportSideEffectDependency` via `yield` so the trampoline in
162-
* `getSideEffectsConnectionState` can drive the walk iteratively (#20986).
163-
* @param {NormalModule} mod the module being evaluated
164-
* @param {ModuleGraph} moduleGraph the module graph
165-
* @returns {SideEffectsWalk} the generator
159+
* Resolves a module's side-effects state without descending into its
160+
* dependencies. Returns the state when known, or `undefined` when the caller
161+
* must walk the module's dependencies to compute it.
162+
* @param {NormalModule} mod the module
163+
* @returns {ConnectionState | undefined} the state, or `undefined` to descend
166164
*/
167-
function* walkSideEffects(mod, moduleGraph) {
165+
const sideEffectsFastPath = (mod) => {
168166
if (mod.factoryMeta !== undefined) {
169167
if (mod.factoryMeta.sideEffectFree) return false;
170168
if (mod.factoryMeta.sideEffectFree === false) return true;
@@ -175,12 +173,197 @@ function* walkSideEffects(mod, moduleGraph) {
175173
if (mod._isEvaluatingSideEffects) {
176174
return ModuleGraphConnection.CIRCULAR_CONNECTION;
177175
}
176+
return undefined;
177+
};
178+
179+
// Maximum recursive descent depth before switching to the iterative walker.
180+
// #20986 reported overflow around 1300 modules in webpack 5.107.0 where each
181+
// step consumed two stack frames (`NormalModule.getSideEffectsConnectionState`
182+
// plus `HarmonyImportSideEffectDependency.getModuleEvaluationSideEffectsState`).
183+
// `walkSideEffectsRecursive` folds the second call into the first and uses
184+
// one frame per module, so this limit caps the native stack at half the depth
185+
// where the original code overflowed — well within the safe range across
186+
// platforms while keeping the common (shallow) case purely recursive.
187+
const SIDE_EFFECTS_RECURSION_LIMIT = 2000;
188+
189+
/**
190+
* Iterative form of the side-effects walker. Used as a fallback once the
191+
* recursive form reaches `SIDE_EFFECTS_RECURSION_LIMIT` so deep chains
192+
* (#20986) don't overflow V8's stack. Safe to enter while ancestors set
193+
* `_isEvaluatingSideEffects` on their own modules — those are treated as
194+
* `CIRCULAR_CONNECTION` if revisited, matching the original recursive
195+
* behavior.
196+
* @param {NormalModule} rootMod the module to walk
197+
* @param {ModuleGraph} moduleGraph the module graph
198+
* @returns {ConnectionState} the side-effects connection state
199+
*/
200+
const walkSideEffectsIterative = (rootMod, moduleGraph) => {
201+
const SideEffectDep = getHarmonyImportSideEffectDependency();
202+
203+
/** @type {NormalModule[]} */
204+
const modStack = [rootMod];
205+
/** @type {Dependency[][]} */
206+
const depsStack = [rootMod.dependencies];
207+
const indexStack = [0];
208+
/** @type {ConnectionState[]} */
209+
const currentStack = [false];
210+
rootMod._isEvaluatingSideEffects = true;
211+
212+
/**
213+
* Result from a just-popped child frame, to be applied to the new
214+
* top's current dep. `undefined` means "no pending; advance".
215+
* @type {ConnectionState | undefined}
216+
*/
217+
let pending;
218+
219+
while (modStack.length > 0) {
220+
const top = modStack.length - 1;
221+
const deps = depsStack[top];
222+
223+
if (pending !== undefined) {
224+
const state = pending;
225+
pending = undefined;
226+
const dep = deps[indexStack[top]];
227+
228+
if (state === true) {
229+
recordSideEffectsBailout(modStack[top], moduleGraph, dep);
230+
modStack[top]._isEvaluatingSideEffects = false;
231+
modStack.pop();
232+
depsStack.pop();
233+
indexStack.pop();
234+
currentStack.pop();
235+
pending = true;
236+
continue;
237+
}
238+
if (state !== ModuleGraphConnection.CIRCULAR_CONNECTION) {
239+
currentStack[top] = ModuleGraphConnection.addConnectionStates(
240+
currentStack[top],
241+
state
242+
);
243+
}
244+
indexStack[top]++;
245+
}
246+
247+
let descended = false;
248+
while (indexStack[top] < deps.length) {
249+
const dep = deps[indexStack[top]];
250+
/** @type {ConnectionState} */
251+
let state;
252+
253+
if (dep instanceof SideEffectDep) {
254+
const refModule = moduleGraph.getModule(dep);
255+
if (!refModule) {
256+
state = true;
257+
} else if (refModule instanceof NormalModule) {
258+
const refFast = sideEffectsFastPath(refModule);
259+
if (refFast !== undefined) {
260+
state = refFast;
261+
} else {
262+
refModule._isEvaluatingSideEffects = true;
263+
modStack.push(refModule);
264+
depsStack.push(refModule.dependencies);
265+
indexStack.push(0);
266+
currentStack.push(false);
267+
descended = true;
268+
break;
269+
}
270+
} else {
271+
state = refModule.getSideEffectsConnectionState(moduleGraph);
272+
}
273+
} else {
274+
state = dep.getModuleEvaluationSideEffectsState(moduleGraph);
275+
}
276+
277+
if (state === true) {
278+
recordSideEffectsBailout(modStack[top], moduleGraph, dep);
279+
modStack[top]._isEvaluatingSideEffects = false;
280+
modStack.pop();
281+
depsStack.pop();
282+
indexStack.pop();
283+
currentStack.pop();
284+
pending = true;
285+
descended = true;
286+
break;
287+
}
288+
if (state !== ModuleGraphConnection.CIRCULAR_CONNECTION) {
289+
currentStack[top] = ModuleGraphConnection.addConnectionStates(
290+
currentStack[top],
291+
state
292+
);
293+
}
294+
indexStack[top]++;
295+
}
296+
297+
if (descended) continue;
298+
299+
modStack[top]._isEvaluatingSideEffects = false;
300+
pending = currentStack[top];
301+
modStack.pop();
302+
depsStack.pop();
303+
indexStack.pop();
304+
currentStack.pop();
305+
}
306+
307+
// When caching is implemented here, make sure to not cache when
308+
// at least one circular connection was folded into the result.
309+
return /** @type {ConnectionState} */ (pending);
310+
};
311+
312+
/**
313+
* Tracks whether a cycle (CIRCULAR_CONNECTION) was encountered anywhere
314+
* in the currently-executing walk. The walker uses this to decide whether
315+
* an intermediate result is safe to memoize on the module: a result
316+
* computed in the presence of a cycle can differ from the result computed
317+
* fresh, because cycle short-circuiting hides the contribution of the
318+
* back-edge target. Reset to `false` at the top-level entry; read and
319+
* propagated by each recursive frame.
320+
*/
321+
let _sideEffectsCircularSeen = false;
322+
323+
/**
324+
* Recursive form of the side-effects walker. Folds the descent through
325+
* `HarmonyImportSideEffectDependency.getModuleEvaluationSideEffectsState`
326+
* directly into the loop so each module costs only one V8 stack frame
327+
* (vs. two in the original recursive code). Falls back to
328+
* `walkSideEffectsIterative` once depth reaches
329+
* `SIDE_EFFECTS_RECURSION_LIMIT` to keep deep chains stack-safe (#20986).
330+
*
331+
* Caches the result on the module when the walk did not encounter a
332+
* cycle, so subsequent queries (e.g. repeated lookups during
333+
* `SideEffectsFlagPlugin`'s incoming-connection optimization and its
334+
* `exportInfo.getTarget` callbacks) return in O(1).
335+
* @param {NormalModule} mod the module being walked
336+
* @param {ModuleGraph} moduleGraph the module graph
337+
* @param {number} depth current recursion depth
338+
* @returns {ConnectionState} the side-effects connection state
339+
*/
340+
const walkSideEffectsRecursive = (mod, moduleGraph, depth) => {
341+
if (mod._sideEffectsStateGraph === moduleGraph) {
342+
return /** @type {ConnectionState} */ (mod._sideEffectsStateValue);
343+
}
344+
345+
const fast = sideEffectsFastPath(mod);
346+
if (fast !== undefined) {
347+
if (fast === ModuleGraphConnection.CIRCULAR_CONNECTION) {
348+
_sideEffectsCircularSeen = true;
349+
}
350+
return fast;
351+
}
352+
353+
if (depth >= SIDE_EFFECTS_RECURSION_LIMIT) {
354+
return walkSideEffectsIterative(mod, moduleGraph);
355+
}
178356

179357
const SideEffectDep = getHarmonyImportSideEffectDependency();
180358
mod._isEvaluatingSideEffects = true;
181359
/** @type {ConnectionState} */
182360
let current = false;
183361

362+
// Snapshot whatever the parent has seen so we can isolate this
363+
// module's own subtree and re-merge on the way out.
364+
const outerCircularSeen = _sideEffectsCircularSeen;
365+
_sideEffectsCircularSeen = false;
366+
184367
for (const dep of mod.dependencies) {
185368
/** @type {ConnectionState} */
186369
let state;
@@ -189,8 +372,16 @@ function* walkSideEffects(mod, moduleGraph) {
189372
if (!refModule) {
190373
state = true;
191374
} else if (refModule instanceof NormalModule) {
192-
state = yield walkSideEffects(refModule, moduleGraph);
375+
state = walkSideEffectsRecursive(refModule, moduleGraph, depth + 1);
193376
} else {
377+
// Non-NormalModule's `getSideEffectsConnectionState` (notably
378+
// `ConcatenatedModule` delegating to its root) re-enters the
379+
// public method and may walk through modules that the outer
380+
// walk has marked as evaluating. We can't observe whether
381+
// the inner walk hit a cycle that reflected the outer's
382+
// state, so treat this frame as cycle-tainted and skip the
383+
// cache for safety.
384+
_sideEffectsCircularSeen = true;
194385
state = refModule.getSideEffectsConnectionState(moduleGraph);
195386
}
196387
} else {
@@ -200,6 +391,12 @@ function* walkSideEffects(mod, moduleGraph) {
200391
if (state === true) {
201392
recordSideEffectsBailout(mod, moduleGraph, dep);
202393
mod._isEvaluatingSideEffects = false;
394+
// `true` is monotonic — once any dep declares side effects, the
395+
// answer is `true` regardless of how cycles resolve, so it's
396+
// always safe to memoize.
397+
mod._sideEffectsStateGraph = moduleGraph;
398+
mod._sideEffectsStateValue = true;
399+
_sideEffectsCircularSeen = outerCircularSeen || _sideEffectsCircularSeen;
203400
return true;
204401
}
205402
if (state !== ModuleGraphConnection.CIRCULAR_CONNECTION) {
@@ -208,10 +405,17 @@ function* walkSideEffects(mod, moduleGraph) {
208405
}
209406

210407
mod._isEvaluatingSideEffects = false;
211-
// When caching is implemented here, make sure to not cache when
212-
// at least one circular connection was folded into `current`.
408+
const sawCircularInSubtree = _sideEffectsCircularSeen;
409+
_sideEffectsCircularSeen = outerCircularSeen || sawCircularInSubtree;
410+
411+
// Only memoize when the subtree's result is independent of the
412+
// current cycle context — i.e. no CIRCULAR_CONNECTION was observed.
413+
if (!sawCircularInSubtree) {
414+
mod._sideEffectsStateGraph = moduleGraph;
415+
mod._sideEffectsStateValue = current;
416+
}
213417
return current;
214-
}
418+
};
215419

216420
const ABSOLUTE_PATH_REGEX = /^(?:[a-z]:\\|\\\\|\/)/i;
217421

@@ -500,6 +704,18 @@ class NormalModule extends Module {
500704
* @type {WeakSet<ModuleGraph> | undefined}
501705
*/
502706
this._addedSideEffectsBailout = undefined;
707+
/**
708+
* Memoizes the result of `getSideEffectsConnectionState`. The
709+
* graph slot keys the cached value to the `ModuleGraph` it was
710+
* computed against so stale values never leak across compilations
711+
* — a walk that targets a different graph just overwrites both
712+
* slots. Populated only for results computed without encountering
713+
* a circular connection (see `walkSideEffectsRecursive`).
714+
* @type {ModuleGraph | undefined}
715+
*/
716+
this._sideEffectsStateGraph = undefined;
717+
/** @type {ConnectionState | undefined} */
718+
this._sideEffectsStateValue = undefined;
503719
/**
504720
* @private
505721
* @type {CodeGenerationResultData}
@@ -1539,21 +1755,14 @@ class NormalModule extends Module {
15391755
* @returns {ConnectionState} how this module should be connected to referencing modules when consumed for side-effects only
15401756
*/
15411757
getSideEffectsConnectionState(moduleGraph) {
1542-
// Trampoline `walkSideEffects` so the descent doesn't consume the
1543-
// call stack (#20986).
1544-
const stack = [walkSideEffects(this, moduleGraph)];
1545-
/** @type {ConnectionState} */
1546-
let r = false;
1547-
while (stack.length > 0) {
1548-
const step = stack[stack.length - 1].next(r);
1549-
if (step.done) {
1550-
stack.pop();
1551-
r = step.value;
1552-
} else {
1553-
stack.push(step.value);
1554-
}
1555-
}
1556-
return r;
1758+
// Save and restore the cycle-tracking flag so that re-entrant calls
1759+
// (e.g. `ConcatenatedModule.getSideEffectsConnectionState` delegating
1760+
// back through here) don't clobber the outer walk's state.
1761+
const savedCircular = _sideEffectsCircularSeen;
1762+
_sideEffectsCircularSeen = false;
1763+
const result = walkSideEffectsRecursive(this, moduleGraph, 0);
1764+
_sideEffectsCircularSeen = savedCircular;
1765+
return result;
15571766
}
15581767

15591768
/**

‎test/NormalModule.unittest.js‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,18 @@ describe("NormalModule", () => {
429429
expect(modules[0].getSideEffectsConnectionState(moduleGraph)).toBe(false);
430430
});
431431

432+
it("propagates bailout across the recursive→iterative crossover", () => {
433+
// Chain length is chosen to exceed `SIDE_EFFECTS_RECURSION_LIMIT`,
434+
// so the walker recurses for the head of the chain and switches
435+
// to iteration deeper in. A bailout deep in the iterative tail
436+
// must still propagate all the way back to module 0.
437+
const { modules, moduleGraph } = buildChain(5000);
438+
modules[4500].buildMeta = { sideEffectFree: false };
439+
expect(modules[0].getSideEffectsConnectionState(moduleGraph)).toBe(true);
440+
expect(modules[0]._isEvaluatingSideEffects).toBe(false);
441+
expect(modules[4499]._isEvaluatingSideEffects).toBe(false);
442+
});
443+
432444
it("detects cycles in the side-effect graph", () => {
433445
const { modules, moduleGraph } = buildChain(50);
434446
// close the loop: last module's dep points to modules[0]
@@ -468,5 +480,39 @@ describe("NormalModule", () => {
468480
);
469481
}
470482
});
483+
484+
it("aggregates state across branching deps", () => {
485+
// Diamond: root depends on two side-effect-free leaves.
486+
const make = (id) => {
487+
const mod = new NormalModule({
488+
type: "javascript/auto",
489+
request: `/${id}`,
490+
userRequest: `/${id}`,
491+
rawRequest: id,
492+
loaders: [],
493+
resource: `/${id}`,
494+
parser: { parse() {} },
495+
generator: null,
496+
resolveOptions: {}
497+
});
498+
mod.buildMeta = { sideEffectFree: true };
499+
mod.dependencies = [];
500+
return mod;
501+
};
502+
const root = make("root");
503+
const a = make("a");
504+
const b = make("b");
505+
const depA = new HarmonyImportSideEffectDependency("a", 0, "evaluation");
506+
const depB = new HarmonyImportSideEffectDependency("b", 1, "evaluation");
507+
root.dependencies = [depA, depB];
508+
const moduleGraph = {
509+
getModule: (dep) => (dep === depA ? a : dep === depB ? b : undefined),
510+
getOptimizationBailout: () => []
511+
};
512+
expect(root.getSideEffectsConnectionState(moduleGraph)).toBe(false);
513+
expect(root._isEvaluatingSideEffects).toBe(false);
514+
expect(a._isEvaluatingSideEffects).toBe(false);
515+
expect(b._isEvaluatingSideEffects).toBe(false);
516+
});
471517
});
472518
});

0 commit comments

Comments
 (0)