From b1e1acfa853bd9be53d5e4a7926e827a68592c0d Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 29 Jan 2026 14:31:46 -0800 Subject: [PATCH 1/9] Add start/end date selectors for experiment filtering --- dashboard/app.py | 20 +++++++++++++++++++- dashboard/state_manager.py | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dashboard/app.py b/dashboard/app.py index f6ae483f..a621b2ad 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -361,6 +361,7 @@ def gui_setup(): # add toolbar components with layout.toolbar: vuetify.VSpacer() + # experiment selector vuetify.VSelect( v_model=("experiment",), label="Experiments", @@ -368,7 +369,24 @@ def gui_setup(): dense=True, hide_details=True, prepend_icon="mdi-atom", - style="max-width: 250px", + style="max-width: 250px; margin-right: 14px;", + ) + # date range selector for experiment filtering + vuetify.VTextField( + v_model=("experiment_start_date",), + label="Start", + type="date", + dense=True, + hide_details=True, + style="max-width: 180px; margin-right: 14px;", + ) + vuetify.VTextField( + v_model=("experiment_end_date",), + label="End", + type="date", + dense=True, + hide_details=True, + style="max-width: 180px; margin-right: 14px;", ) # set up router view with layout.content: diff --git a/dashboard/state_manager.py b/dashboard/state_manager.py index 1ece270b..49668fc0 100644 --- a/dashboard/state_manager.py +++ b/dashboard/state_manager.py @@ -23,6 +23,8 @@ def initialize_state(): ][0] print(f"Setting default experiment to {default_experiment}...") state.experiment = default_experiment + state.experiment_start_date = None + state.experiment_end_date = None # ML model state.model_type = "Neural Network (single)" state.model_training = False From ca767ab36e1159ca9016df37508b574cc7ad936e Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Thu, 29 Jan 2026 17:15:08 -0800 Subject: [PATCH 2/9] Implement date filter for experimental data --- dashboard/app.py | 6 +++++- dashboard/utils.py | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/dashboard/app.py b/dashboard/app.py index a621b2ad..06f203fe 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -105,7 +105,11 @@ def update( ctrl.figure_update(fig) -@state.change("experiment") +@state.change( + "experiment", + "experiment_start_date", + "experiment_end_date", +) def update_on_change_experiment(**kwargs): # skip if triggered on server ready (all state variables marked as modified) if len(state.modified_keys) == 1: diff --git a/dashboard/utils.py b/dashboard/utils.py index 2e822bff..cae62744 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -73,8 +73,29 @@ def load_variables(experiment): @timer def load_data(db): print("Loading data from database...") + # build date filter if start/end dates are set + date_filter = {} + if ( + state.experiment_start_date is not None + and state.experiment_end_date is not None + ): + # convert to datetime objects + filter_start_date = pd.to_datetime(state.experiment_start_date) + filter_end_date = pd.to_datetime(state.experiment_end_date).replace( + hour=23, minute=59, second=59 + ) + # build date filter + date_filter = { + "date": { + "$gte": filter_start_date, + "$lte": filter_end_date, + } + } + print(f"Filtering data between {filter_start_date} and {filter_end_date}...") # load experiment and simulation data points in dataframes - exp_data = pd.DataFrame(db[state.experiment].find({"experiment_flag": 1})) + exp_data = pd.DataFrame( + db[state.experiment].find({**{"experiment_flag": 1}, **date_filter}) + ) sim_data = pd.DataFrame(db[state.experiment].find({"experiment_flag": 0})) # Store '_id', 'date' as string for key in ["_id", "date"]: From 8569361e8f5626d667ba7ec785ab78def4727645 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 30 Jan 2026 17:02:46 -0800 Subject: [PATCH 3/9] Use VDateInput component --- dashboard/app.py | 21 ++++++--------------- dashboard/state_manager.py | 5 +++-- dashboard/utils.py | 18 ++++++------------ 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/dashboard/app.py b/dashboard/app.py index 06f203fe..7b9df87e 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -107,8 +107,7 @@ def update( @state.change( "experiment", - "experiment_start_date", - "experiment_end_date", + "experiment_date_range", ) def update_on_change_experiment(**kwargs): # skip if triggered on server ready (all state variables marked as modified) @@ -376,21 +375,13 @@ def gui_setup(): style="max-width: 250px; margin-right: 14px;", ) # date range selector for experiment filtering - vuetify.VTextField( - v_model=("experiment_start_date",), - label="Start", - type="date", + vuetify.VDateInput( + v_model=("experiment_date_range",), + label="Date range", + multiple="range", dense=True, hide_details=True, - style="max-width: 180px; margin-right: 14px;", - ) - vuetify.VTextField( - v_model=("experiment_end_date",), - label="End", - type="date", - dense=True, - hide_details=True, - style="max-width: 180px; margin-right: 14px;", + style="max-width: 250px; margin-right: 14px;", ) # set up router view with layout.content: diff --git a/dashboard/state_manager.py b/dashboard/state_manager.py index 49668fc0..8b9ad87b 100644 --- a/dashboard/state_manager.py +++ b/dashboard/state_manager.py @@ -1,5 +1,6 @@ from pathlib import Path from trame.app import get_server +from trame.widgets import vuetify3 as vuetify EXPERIMENTS_PATH = Path.cwd().parent / "experiments/" @@ -8,6 +9,7 @@ server = get_server(client_type="vue3") state = server.state ctrl = server.controller +vuetify.enable_lab() # Enable Labs components def initialize_state(): @@ -23,8 +25,7 @@ def initialize_state(): ][0] print(f"Setting default experiment to {default_experiment}...") state.experiment = default_experiment - state.experiment_start_date = None - state.experiment_end_date = None + state.experiment_date_range = [] # ML model state.model_type = "Neural Network (single)" state.model_training = False diff --git a/dashboard/utils.py b/dashboard/utils.py index cae62744..7a6fb20a 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -75,23 +75,17 @@ def load_data(db): print("Loading data from database...") # build date filter if start/end dates are set date_filter = {} - if ( - state.experiment_start_date is not None - and state.experiment_end_date is not None - ): - # convert to datetime objects - filter_start_date = pd.to_datetime(state.experiment_start_date) - filter_end_date = pd.to_datetime(state.experiment_end_date).replace( - hour=23, minute=59, second=59 - ) + if state.experiment_date_range: # build date filter date_filter = { "date": { - "$gte": filter_start_date, - "$lte": filter_end_date, + "$gte": pd.to_datetime(state.experiment_date_range[0].to_datetime()), + "$lte": pd.to_datetime(state.experiment_date_range[1].to_datetime()), } } - print(f"Filtering data between {filter_start_date} and {filter_end_date}...") + print( + f"Filtering data between {state.experiment_date_range[0].to_datetime().date()} and {state.experiment_date_range[-1].to_datetime().date()}..." + ) # load experiment and simulation data points in dataframes exp_data = pd.DataFrame( db[state.experiment].find({**{"experiment_flag": 1}, **date_filter}) From 27b12b547191908055a515751f5b7fdce7f3be92 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 30 Jan 2026 17:29:31 -0800 Subject: [PATCH 4/9] Fix date filter math --- dashboard/utils.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/dashboard/utils.py b/dashboard/utils.py index 7a6fb20a..52c756f0 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -73,19 +73,40 @@ def load_variables(experiment): @timer def load_data(db): print("Loading data from database...") - # build date filter if start/end dates are set + # build date filter if date range is set date_filter = {} if state.experiment_date_range: - # build date filter + # convert to naive datetime to match database format + start_date = ( + pd.to_datetime(state.experiment_date_range[0].to_datetime()) + .to_pydatetime() + .replace(hour=0, minute=0, second=0) + ) + end_date = pd.to_datetime(state.experiment_date_range[-1].to_datetime()) + # VDateInput returns exclusive end date for date ranges, but single-date selection has len=1 + if len(state.experiment_date_range) == 1: + # single date selection: use the end date as-is + end_date = end_date.replace(hour=23, minute=59, second=59) + else: + # date range selection: subtract 1 day from exclusive end date + end_date = ( + (end_date - pd.Timedelta(days=1)) + .to_pydatetime() + .replace(hour=23, minute=59, second=59) + ) + # remove timezone info to match naive datetime in database + start_date = ( + start_date.replace(tzinfo=None) if start_date.tzinfo else start_date + ) + end_date = end_date.replace(tzinfo=None) if end_date.tzinfo else end_date date_filter = { "date": { - "$gte": pd.to_datetime(state.experiment_date_range[0].to_datetime()), - "$lte": pd.to_datetime(state.experiment_date_range[1].to_datetime()), + "$gte": start_date, + "$lte": end_date, } } - print( - f"Filtering data between {state.experiment_date_range[0].to_datetime().date()} and {state.experiment_date_range[-1].to_datetime().date()}..." - ) + print(date_filter) + print(f"Filtering data between {start_date.date()} and {end_date.date()}...") # load experiment and simulation data points in dataframes exp_data = pd.DataFrame( db[state.experiment].find({**{"experiment_flag": 1}, **date_filter}) From 06264f02e631ba3f97d0c8d7ffc2447f7ae500aa Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Mon, 2 Feb 2026 13:05:25 -0800 Subject: [PATCH 5/9] Improve code to compute the end date correction --- dashboard/utils.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/dashboard/utils.py b/dashboard/utils.py index 52c756f0..99c043c6 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -76,24 +76,17 @@ def load_data(db): # build date filter if date range is set date_filter = {} if state.experiment_date_range: - # convert to naive datetime to match database format - start_date = ( - pd.to_datetime(state.experiment_date_range[0].to_datetime()) - .to_pydatetime() - .replace(hour=0, minute=0, second=0) - ) + start_date = pd.to_datetime(state.experiment_date_range[0].to_datetime()) + start_date = start_date.to_pydatetime().replace(hour=0, minute=0, second=0) + # VDateInput returns exclusive end date for date ranges, so we need to subtract 1 day end_date = pd.to_datetime(state.experiment_date_range[-1].to_datetime()) - # VDateInput returns exclusive end date for date ranges, but single-date selection has len=1 - if len(state.experiment_date_range) == 1: - # single date selection: use the end date as-is - end_date = end_date.replace(hour=23, minute=59, second=59) - else: - # date range selection: subtract 1 day from exclusive end date - end_date = ( - (end_date - pd.Timedelta(days=1)) - .to_pydatetime() - .replace(hour=23, minute=59, second=59) - ) + end_date_correction = ( + pd.Timedelta(days=0) + if len(state.experiment_date_range) == 1 + else pd.Timedelta(days=1) + ) + end_date = end_date - end_date_correction + end_date = end_date.to_pydatetime().replace(hour=23, minute=59, second=59) # remove timezone info to match naive datetime in database start_date = ( start_date.replace(tzinfo=None) if start_date.tzinfo else start_date @@ -105,7 +98,6 @@ def load_data(db): "$lte": end_date, } } - print(date_filter) print(f"Filtering data between {start_date.date()} and {end_date.date()}...") # load experiment and simulation data points in dataframes exp_data = pd.DataFrame( From 7f4aa19bdbe2abc4d25174258942d3eeaee93726 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Mon, 2 Feb 2026 16:14:23 -0800 Subject: [PATCH 6/9] Simplify dictionary unpacking --- dashboard/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/utils.py b/dashboard/utils.py index 99c043c6..1527ab73 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -101,7 +101,7 @@ def load_data(db): print(f"Filtering data between {start_date.date()} and {end_date.date()}...") # load experiment and simulation data points in dataframes exp_data = pd.DataFrame( - db[state.experiment].find({**{"experiment_flag": 1}, **date_filter}) + db[state.experiment].find({"experiment_flag": 1, **date_filter}) ) sim_data = pd.DataFrame(db[state.experiment].find({"experiment_flag": 0})) # Store '_id', 'date' as string From 2077a1c48351cc08c54d4a0950c11f0208cc1a53 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Mon, 2 Feb 2026 16:19:48 -0800 Subject: [PATCH 7/9] Explain logic for calculation of end_date_correction --- dashboard/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dashboard/utils.py b/dashboard/utils.py index 1527ab73..fa522beb 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -78,7 +78,9 @@ def load_data(db): if state.experiment_date_range: start_date = pd.to_datetime(state.experiment_date_range[0].to_datetime()) start_date = start_date.to_pydatetime().replace(hour=0, minute=0, second=0) - # VDateInput returns exclusive end date for date ranges, so we need to subtract 1 day + # VDateInput returns exclusive end date for date ranges: + # - subtract 1 day for multi-date ranges with different start/end dates + # - do not subtract anything (use end date as is) for single-date ranges end_date = pd.to_datetime(state.experiment_date_range[-1].to_datetime()) end_date_correction = ( pd.Timedelta(days=0) From c2c92e43a8a9eb04bdba79d5c9eb5d1e71bf223e Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Mon, 2 Feb 2026 17:39:17 -0800 Subject: [PATCH 8/9] Use date filter in the ML training code --- dashboard/model_manager.py | 8 ++++++-- dashboard/utils.py | 20 +++++++++++++------- ml/train_model.py | 3 ++- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/dashboard/model_manager.py b/dashboard/model_manager.py index c9ed20e3..60b820e8 100644 --- a/dashboard/model_manager.py +++ b/dashboard/model_manager.py @@ -11,7 +11,7 @@ from lume_model.models.ensemble import NNEnsemble from lume_model.models.gp_model import GPModel from trame.widgets import vuetify3 as vuetify -from utils import verify_input_variables, timer, load_config_dict +from utils import verify_input_variables, timer, load_config_dict, create_date_filter from error_manager import add_error from sfapi_manager import monitor_sfapi_job from state_manager import state @@ -190,9 +190,13 @@ async def training_kernel(self): client_id=state.sfapi_client_id, secret=state.sfapi_key ) as client: perlmutter = await client.compute(Machine.perlmutter) - # Upload the config.yaml to nersc + # upload the configuration file to NERSC config_dict = load_config_dict(state.experiment) config_dict["simulation_calibration"] = state.simulation_calibration + # add date range filter to the configuration dictionary + date_filter = create_date_filter(state.experiment_date_range) + config_dict["date_filter"] = date_filter + # define the target path on NERSC target_path = "/global/cfs/cdirs/m558/superfacility/model_training" [target_path] = await perlmutter.ls(target_path, directory=True) with tempfile.TemporaryDirectory() as temp_dir: diff --git a/dashboard/utils.py b/dashboard/utils.py index fa522beb..174ab1df 100644 --- a/dashboard/utils.py +++ b/dashboard/utils.py @@ -70,21 +70,19 @@ def load_variables(experiment): return (input_variables, output_variables, simulation_calibration) -@timer -def load_data(db): - print("Loading data from database...") +def create_date_filter(experiment_date_range): # build date filter if date range is set date_filter = {} - if state.experiment_date_range: - start_date = pd.to_datetime(state.experiment_date_range[0].to_datetime()) + if experiment_date_range: + start_date = pd.to_datetime(experiment_date_range[0].to_datetime()) start_date = start_date.to_pydatetime().replace(hour=0, minute=0, second=0) # VDateInput returns exclusive end date for date ranges: # - subtract 1 day for multi-date ranges with different start/end dates # - do not subtract anything (use end date as is) for single-date ranges - end_date = pd.to_datetime(state.experiment_date_range[-1].to_datetime()) + end_date = pd.to_datetime(experiment_date_range[-1].to_datetime()) end_date_correction = ( pd.Timedelta(days=0) - if len(state.experiment_date_range) == 1 + if len(experiment_date_range) == 1 else pd.Timedelta(days=1) ) end_date = end_date - end_date_correction @@ -101,6 +99,14 @@ def load_data(db): } } print(f"Filtering data between {start_date.date()} and {end_date.date()}...") + return date_filter + + +@timer +def load_data(db): + print("Loading data from database...") + # create date filter if date range is set + date_filter = create_date_filter(state.experiment_date_range) # load experiment and simulation data points in dataframes exp_data = pd.DataFrame( db[state.experiment].find({"experiment_flag": 1, **date_filter}) diff --git a/ml/train_model.py b/ml/train_model.py index 65c27324..1c2113e6 100644 --- a/ml/train_model.py +++ b/ml/train_model.py @@ -439,7 +439,8 @@ def write_model(model, model_type, experiment, db): # Extract experimental and simulation data from the database as pandas dataframe db = connect_to_db(config_dict) - df_exp = pd.DataFrame(db[experiment].find({"experiment_flag": 1})) + date_filter = config_dict["date_filter"] + df_exp = pd.DataFrame(db[experiment].find({"experiment_flag": 1, **date_filter})) df_sim = pd.DataFrame(db[experiment].find({"experiment_flag": 0})) # Apply simulation calibration to the simulation data From 7fc0054e0ec70c798916e05d2babbccea2c3200b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 11 Feb 2026 11:05:52 -0800 Subject: [PATCH 9/9] Apply suggestions from code review --- ml/train_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml/train_model.py b/ml/train_model.py index 1c2113e6..7f4cc58d 100644 --- a/ml/train_model.py +++ b/ml/train_model.py @@ -439,7 +439,7 @@ def write_model(model, model_type, experiment, db): # Extract experimental and simulation data from the database as pandas dataframe db = connect_to_db(config_dict) - date_filter = config_dict["date_filter"] + date_filter = config_dict.get("date_filter", {}) df_exp = pd.DataFrame(db[experiment].find({"experiment_flag": 1, **date_filter})) df_sim = pd.DataFrame(db[experiment].find({"experiment_flag": 0}))