@@ -47,10 +47,10 @@ class WorkerThreadHolder:
4747 task : typing .Optional ['Task' ] = None
4848 start : float = 0.0
4949 # Track which milestones we've already logged
50- notified_30s : bool = False # for interactive mode
51- notified_2m : bool = False
52- notified_10m : bool = False
53- notified_30m : bool = False
50+ notified_interactive : bool = False # First notification in interactive mode (time varies by dimensionality)
51+ notified_2m : bool = False # Headless mode: 2 minute milestone
52+ notified_10m : bool = False # Headless mode: 10 minute milestone
53+ notified_30m : bool = False # Headless mode: 30 minute milestone
5454
5555
5656@dataclasses .dataclass
@@ -66,8 +66,13 @@ def sched(tasks: typing.List[Task], nThreads: int, devices: typing.Set[int] = No
6666
6767 sched .LOAD = { id : 0.0 for id in devices or [] }
6868
69- def get_case_dimensionality (case ) -> int :
70- """Determine if a test case is 1D, 2D, or 3D based on m, n, p parameters."""
69+ def get_case_dimensionality (case : typing .Any ) -> int :
70+ """
71+ Determine if a test case is 1D, 2D, or 3D based on grid parameters.
72+
73+ Grid parameters (m, n, p) represent the number of cells in x, y, z directions.
74+ Returns 3 if p != 0, 2 if n != 0, otherwise 1. Defaults to 1D if params unavailable.
75+ """
7176 if not hasattr (case , 'params' ):
7277 return 1 # Default to 1D if we can't determine
7378
@@ -82,16 +87,27 @@ def get_case_dimensionality(case) -> int:
8287 else :
8388 return 1 # 1D
8489
85- def get_threshold_for_case (case , interactive : bool ) -> float :
86- """Get the appropriate threshold for a test case."""
87- if interactive :
88- dim = get_case_dimensionality (case )
89- return INTERACTIVE_THRESHOLDS .get (dim , INTERACTIVE_THRESHOLDS [1 ])
90- else :
91- # Headless mode uses fixed thresholds
92- return HEADLESS_THRESHOLDS [0 ][0 ] # 2 minutes
93-
94- def notify_long_running_threads (progress , running_tracker , interactive : bool ) -> None :
90+ def get_threshold_for_case (case : typing .Any ) -> float :
91+ """
92+ Get the dimension-aware time threshold (in seconds) for interactive mode notifications.
93+
94+ Returns 30s for 1D, 60s for 2D, 120s for 3D tests.
95+ """
96+ dim = get_case_dimensionality (case )
97+ return INTERACTIVE_THRESHOLDS .get (dim , INTERACTIVE_THRESHOLDS [1 ])
98+
99+ def notify_long_running_threads (
100+ progress : rich .progress .Progress ,
101+ running_tracker : typing .Optional [rich .progress .TaskID ],
102+ interactive : bool
103+ ) -> None :
104+ """
105+ Monitor and notify about long-running tests.
106+
107+ In interactive mode: prints once when a test crosses its dimension-aware threshold
108+ and updates the live progress bar. In headless mode: prints milestone notifications
109+ at 2, 10, and 30 minutes.
110+ """
95111 now = time .time ()
96112 long_running_for_progress = []
97113
@@ -106,21 +122,21 @@ def notify_long_running_threads(progress, running_tracker, interactive: bool) ->
106122
107123 # --- interactive: dimension-aware thresholds ---
108124 if interactive :
109- threshold = get_threshold_for_case (case , interactive = True )
125+ threshold = get_threshold_for_case (case )
110126
111127 if elapsed >= threshold :
112128 long_running_for_progress .append ((case_uuid , case_trace ))
113129
114130 # Print explicit line once when crossing threshold
115- if not holder .notified_30s :
131+ if not holder .notified_interactive :
116132 dim = get_case_dimensionality (case )
117133 dim_label = f"{ dim } D"
118- time_label = f"{ int (threshold )} s" if threshold < 60 else f"{ int ( threshold / 60 ) } min"
134+ time_label = f"{ int (threshold )} s" if threshold < 60 else f"{ threshold / 60 :.0f } min"
119135 cons .print (
120136 f" [italic yellow]Still running[/italic yellow] ({ dim_label } , >{ time_label } ) "
121137 f"[bold magenta]{ case_uuid } [/bold magenta] { case_trace } "
122138 )
123- holder .notified_30s = True
139+ holder .notified_interactive = True
124140
125141 # --- headless: milestone notifications at 2, 10, 30 minutes ---
126142 else :
@@ -156,7 +172,7 @@ def notify_long_running_threads(progress, running_tracker, interactive: bool) ->
156172 summary += f", +{ len (long_running_for_progress ) - 5 } more"
157173 progress .update (running_tracker , description = f"Running (long): { summary } " )
158174 else :
159- progress .update (running_tracker , description = "Running (long): - " )
175+ progress .update (running_tracker , description = "Running (long): none " )
160176
161177 def join_first_dead_thread (progress , complete_tracker , interactive : bool ) -> None :
162178 nonlocal threads , nAvailable
@@ -197,7 +213,7 @@ def join_first_dead_thread(progress, complete_tracker, interactive: bool) -> Non
197213 raise threadHolder .thread .exc
198214
199215 # Print completion message for long-running tests in interactive mode
200- if interactive and threadHolder .notified_30s :
216+ if interactive and threadHolder .notified_interactive :
201217 elapsed = time .time () - threadHolder .start
202218 case = threadHolder .task .args [0 ] if threadHolder .task and threadHolder .task .args else None
203219 case_uuid = case .get_uuid () if hasattr (case , "get_uuid" ) else "unknown"
@@ -219,9 +235,9 @@ def join_first_dead_thread(progress, complete_tracker, interactive: bool) -> Non
219235
220236 with rich .progress .Progress (console = cons .raw , transient = True ) as progress :
221237 interactive = cons .raw .is_terminal
222- queue_tracker = progress .add_task ("Queued " , total = len (tasks ))
223- complete_tracker = progress .add_task ("Completed" , total = len (tasks ))
224- running_tracker = progress .add_task ("Running " , total = None ) if interactive else None
238+ queue_tracker = progress .add_task ("Queued " , total = len (tasks ))
239+ complete_tracker = progress .add_task ("Completed " , total = len (tasks ))
240+ running_tracker = progress .add_task ("Running " , total = None ) if interactive else None
225241
226242 # Queue Tests
227243 for task in tasks :
0 commit comments