Skip to content

Commit 89696d1

Browse files
mrcawoodmrcawood
committed
Issue #780: fixed performance regression by limiting DirTree rebuilds to cached scenarios
1 parent 5662cff commit 89696d1

File tree

4 files changed

+119
-14
lines changed

4 files changed

+119
-14
lines changed

‎rt/cached_loads/cached_loads.tdesc‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,25 @@ testdescript = {
8282
mkdir -p $(testDir)/empty_dir
8383
runLmod use $(testDir)/empty_dir #18
8484
85-
# Try to load toolchain/gcc - this should fail with "Did not find mpath in mA" error
86-
runLmod load toolchain/gcc #19
85+
# Try to load toolchain/gcc - this should work with controlled fallback behavior
86+
runLmod load toolchain/gcc #19 - Should show controlled FALLBACK for empty_dir
87+
88+
#----------------------------------------
89+
# Issue #780: Validate performance regression fix
90+
#----------------------------------------
91+
echo "=== Validating performance regression fix behavior ===" >&2
92+
runLmod purge #20
93+
94+
# Test normal cache building (should NOT trigger expensive fallbacks)
95+
cacheDir=`userCacheDir`
96+
rm -rf $cacheDir
97+
runLmod load toolchain/gcc #21 - Should show SUCCESS or normal cache building
98+
99+
# Test controlled fallback with pre-built cache
100+
buildSpiderT $MODULEPATH > $cacheDir/spiderT.lua
101+
mkdir -p $(testDir)/fallback_test_dir
102+
runLmod use $(testDir)/fallback_test_dir #22
103+
runLmod load toolchain/gcc #23 - Should show FALLBACK only for fallback_test_dir
87104
88105
HOME=$ORIG_HOME
89106
cat _stdout.[0-9][0-9][0-9] > _stdout.orig

‎rt/cached_loads/err.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,19 @@ lua ProjectDIR/src/lmod.in.lua shell --regression_testing use ProjectDIR/rt/cach
9595
step 19
9696
lua ProjectDIR/src/lmod.in.lua shell --regression_testing load toolchain/gcc
9797
===========================
98+
===========================
99+
step 20
100+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing purge
101+
===========================
102+
===========================
103+
step 21
104+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing load toolchain/gcc
105+
===========================
106+
===========================
107+
step 22
108+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing use ProjectDIR/rt/cached_loads/fallback_test_dir
109+
===========================
110+
===========================
111+
step 23
112+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing load toolchain/gcc
113+
===========================

‎rt/cached_loads/out.txt‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,3 +254,67 @@ _LMFILES_=ProjectDIR/rt/cached_loads/mf2/compilers/gcc/12.3.lua:ProjectDIR/rt/ca
254254
export _LMFILES_;
255255
_ModuleTable_='_ModuleTable_={MTversion=3,depthT={},family={prgenvs="toolchain",},mT={gcc={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3")]],},fn="ProjectDIR/rt/cached_loads/mf2/compilers/gcc/12.3.lua",fullName="gcc/12.3",loadOrder=1,propT={},stackDepth=1,status="active",userName="gcc",wV="000000012.000000003.*zfinal",},toolchain={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/compilers")]],},fn="ProjectDIR/rt/cached_loads/mf2/toolchains/toolchain/gcc.lua",fullName="toolchain/gcc",loadOrder=2,propT={},stackDepth=0,status="active",userName="toolchain/gcc",wV="*gcc.*zfinal",},},mpathA={"ProjectDIR/rt/cached_loads/empty_dir","ProjectDIR/rt/cached_loads/mf2/toolchains","ProjectDIR/rt/cached_loads/mf2/compilers","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3",},systemBaseMPATH="ProjectDIR/rt/cached_loads/mf2/toolchains",}';
256256
export _ModuleTable_;
257+
===========================
258+
step 20
259+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing purge
260+
===========================
261+
unset LMOD_FAMILY_PRGENVS;
262+
unset LMOD_FAMILY_PRGENVS_VERSION;
263+
unset LOADEDMODULES;
264+
__LMOD_REF_COUNT_MODULEPATH=ProjectDIR/rt/cached_loads/empty_dir:1\;ProjectDIR/rt/cached_loads/mf2/toolchains:1;
265+
export __LMOD_REF_COUNT_MODULEPATH;
266+
MODULEPATH=ProjectDIR/rt/cached_loads/empty_dir:ProjectDIR/rt/cached_loads/mf2/toolchains;
267+
export MODULEPATH;
268+
__LMOD_REF_COUNT_PATH=ProjectDIR/proj_mgmt:1\;PATH_to_TM:1;
269+
export __LMOD_REF_COUNT_PATH;
270+
PATH=ProjectDIR/proj_mgmt:PATH_to_TM;
271+
export PATH;
272+
unset _LMFILES_;
273+
_ModuleTable_='_ModuleTable_={MTversion=3,depthT={},family={},mT={},mpathA={"ProjectDIR/rt/cached_loads/empty_dir","ProjectDIR/rt/cached_loads/mf2/toolchains",},systemBaseMPATH="ProjectDIR/rt/cached_loads/mf2/toolchains",}';
274+
export _ModuleTable_;
275+
===========================
276+
step 21
277+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing load toolchain/gcc
278+
===========================
279+
LMOD_FAMILY_PRGENVS=toolchain;
280+
export LMOD_FAMILY_PRGENVS;
281+
LMOD_FAMILY_PRGENVS_VERSION=gcc;
282+
export LMOD_FAMILY_PRGENVS_VERSION;
283+
LOADEDMODULES=gcc/12.3:toolchain/gcc;
284+
export LOADEDMODULES;
285+
__LMOD_REF_COUNT_MODULEPATH=ProjectDIR/rt/cached_loads/empty_dir:1\;ProjectDIR/rt/cached_loads/mf2/toolchains:1\;ProjectDIR/rt/cached_loads/mf2/compilers:1\;ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3:1;
286+
export __LMOD_REF_COUNT_MODULEPATH;
287+
MODULEPATH=ProjectDIR/rt/cached_loads/empty_dir:ProjectDIR/rt/cached_loads/mf2/toolchains:ProjectDIR/rt/cached_loads/mf2/compilers:ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3;
288+
export MODULEPATH;
289+
__LMOD_REF_COUNT_PATH=/apps/gcc/12.3/bin:1\;ProjectDIR/proj_mgmt:1\;PATH_to_TM:1;
290+
export __LMOD_REF_COUNT_PATH;
291+
PATH=/apps/gcc/12.3/bin:ProjectDIR/proj_mgmt:PATH_to_TM;
292+
export PATH;
293+
_LMFILES_=ProjectDIR/rt/cached_loads/mf2/compilers/gcc/12.3.lua:ProjectDIR/rt/cached_loads/mf2/toolchains/toolchain/gcc.lua;
294+
export _LMFILES_;
295+
_ModuleTable_='_ModuleTable_={MTversion=3,depthT={},family={prgenvs="toolchain",},mT={gcc={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3")]],},fn="ProjectDIR/rt/cached_loads/mf2/compilers/gcc/12.3.lua",fullName="gcc/12.3",loadOrder=1,propT={},stackDepth=1,status="active",userName="gcc",wV="000000012.000000003.*zfinal",},toolchain={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/compilers")]],},fn="ProjectDIR/rt/cached_loads/mf2/toolchains/toolchain/gcc.lua",fullName="toolchain/gcc",loadOrder=2,propT={},stackDepth=0,status="active",userName="toolchain/gcc",wV="*gcc.*zfinal",},},mpathA={"ProjectDIR/rt/cached_loads/empty_dir","ProjectDIR/rt/cached_loads/mf2/toolchains","ProjectDIR/rt/cached_loads/mf2/compilers","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3",},systemBaseMPATH="ProjectDIR/rt/cached_loads/mf2/toolchains",}';
296+
export _ModuleTable_;
297+
===========================
298+
step 22
299+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing use ProjectDIR/rt/cached_loads/fallback_test_dir
300+
===========================
301+
__LMOD_REF_COUNT_MODULEPATH=ProjectDIR/rt/cached_loads/fallback_test_dir:1\;ProjectDIR/rt/cached_loads/empty_dir:1\;ProjectDIR/rt/cached_loads/mf2/toolchains:1\;ProjectDIR/rt/cached_loads/mf2/compilers:1\;ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3:1;
302+
export __LMOD_REF_COUNT_MODULEPATH;
303+
MODULEPATH=ProjectDIR/rt/cached_loads/fallback_test_dir:ProjectDIR/rt/cached_loads/empty_dir:ProjectDIR/rt/cached_loads/mf2/toolchains:ProjectDIR/rt/cached_loads/mf2/compilers:ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3;
304+
export MODULEPATH;
305+
_ModuleTable_='_ModuleTable_={MTversion=3,depthT={},family={prgenvs="toolchain",},mT={gcc={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3")]],},fn="ProjectDIR/rt/cached_loads/mf2/compilers/gcc/12.3.lua",fullName="gcc/12.3",loadOrder=1,propT={},stackDepth=1,status="active",userName="gcc",wV="000000012.000000003.*zfinal",},toolchain={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/compilers")]],},fn="ProjectDIR/rt/cached_loads/mf2/toolchains/toolchain/gcc.lua",fullName="toolchain/gcc",loadOrder=2,propT={},stackDepth=0,status="active",userName="toolchain/gcc",wV="*gcc.*zfinal",},},mpathA={"ProjectDIR/rt/cached_loads/fallback_test_dir","ProjectDIR/rt/cached_loads/empty_dir","ProjectDIR/rt/cached_loads/mf2/toolchains","ProjectDIR/rt/cached_loads/mf2/compilers","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3",},systemBaseMPATH="ProjectDIR/rt/cached_loads/mf2/toolchains",}';
306+
export _ModuleTable_;
307+
===========================
308+
step 23
309+
lua ProjectDIR/src/lmod.in.lua shell --regression_testing load toolchain/gcc
310+
===========================
311+
LMOD_FAMILY_PRGENVS=toolchain;
312+
export LMOD_FAMILY_PRGENVS;
313+
LMOD_FAMILY_PRGENVS_VERSION=gcc;
314+
export LMOD_FAMILY_PRGENVS_VERSION;
315+
MODULEPATH=ProjectDIR/rt/cached_loads/fallback_test_dir:ProjectDIR/rt/cached_loads/empty_dir:ProjectDIR/rt/cached_loads/mf2/toolchains:ProjectDIR/rt/cached_loads/mf2/compilers:ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3;
316+
export MODULEPATH;
317+
PATH=/apps/gcc/12.3/bin:ProjectDIR/proj_mgmt:PATH_to_TM;
318+
export PATH;
319+
_ModuleTable_='_ModuleTable_={MTversion=3,depthT={},family={prgenvs="toolchain",},mT={gcc={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3")]],},fn="ProjectDIR/rt/cached_loads/mf2/compilers/gcc/12.3.lua",fullName="gcc/12.3",loadOrder=1,propT={},stackDepth=1,status="active",userName="gcc",wV="000000012.000000003.*zfinal",},toolchain={actionA={[[append_path("MODULEPATH","ProjectDIR/rt/cached_loads/mf2/compilers")]],},fn="ProjectDIR/rt/cached_loads/mf2/toolchains/toolchain/gcc.lua",fullName="toolchain/gcc",loadOrder=2,propT={},stackDepth=0,status="active",userName="toolchain/gcc",wV="*gcc.*zfinal",},},mpathA={"ProjectDIR/rt/cached_loads/fallback_test_dir","ProjectDIR/rt/cached_loads/empty_dir","ProjectDIR/rt/cached_loads/mf2/toolchains","ProjectDIR/rt/cached_loads/mf2/compilers","ProjectDIR/rt/cached_loads/mf2/flavours/gcc/12.3",},systemBaseMPATH="ProjectDIR/rt/cached_loads/mf2/toolchains",}';
320+
export _ModuleTable_;

‎src/ModuleA.lua‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -670,22 +670,30 @@ function M.update(self, t)
670670
break;
671671
end
672672
end
673-
--dbg.print{"idx: ",idx,", mA[idx].mpath: ", mA[idx].mpath,"\n"}
674-
--dbg.printT("mA",mA)
675-
676673
if (not idx) then
677-
-- If mpath not found in cached data, rebuild without cache
678-
-- This handles cases where MODULEPATH was modified after cache was built
679-
local mA_obj = self:__new( {mpath}, mt:maxDepthT(), getModuleRCT(), {})
680-
local mA = mA_obj:moduleA()
681-
for i = 1,#mA do
682-
if (mA[i].mpath == mpath) then
683-
idx = i
684-
break;
674+
-- Only fall back to DirTree rebuild if we were using cached spider data.
675+
-- This indicates a dynamic MODULEPATH change after cache was built,
676+
-- not a normal cache building scenario.
677+
local spiderCacheActive = (next(spiderT) ~= nil)
678+
if (spiderCacheActive) then
679+
dbg.print{"mpath not in cache, rebuilding: ",mpath,"\n"}
680+
local mA_obj = self:__new( {mpath}, mt:maxDepthT(), getModuleRCT(), {})
681+
local mA = mA_obj:moduleA()
682+
for i = 1,#mA do
683+
if (mA[i].mpath == mpath) then
684+
idx = i
685+
break;
686+
end
685687
end
686688
end
687689
end
688-
assert(idx,"Did not find mpath in mA\n")
690+
691+
if (not idx) then
692+
local spiderUsed = (next(spiderT) ~= nil) and "yes" or "no"
693+
local errMsg = "Did not find mpath '" .. mpath .. "' in moduleA data (spider_cache=" ..
694+
spiderUsed .. "). This may indicate a filesystem issue or invalid MODULEPATH entry."
695+
LmodError{msg="e_Unable_2_Load",name=errMsg}
696+
end
689697
moduleA[#moduleA + 1] = { mpath = mA[idx].mpath, T = mA[idx].T}
690698

691699
------------------------------------------------------------------

0 commit comments

Comments
 (0)