Skip to content

Commit 3eecd59

Browse files
authored
feat: save original user-provided config in experiment's artifacts (#637)
The user-provided config is now stored with no overrides applied and pushed to MLFlow: <img width="1484" height="974" alt="image" src="https://github.com/user-attachments/assets/26b58a10-fea3-414f-9a5a-75a5b5ce993d" /> Additionally this PR introduces: * Refactored command building to use list-based construction instead of string concatenation for better maintainability * Introduced constant for container results directory path to reduce hard-coded values * Enhanced debug output to include unresolved configuration details Signed-off-by: Marta Stepniewska-Dziubinska <martas@nvidia.com>
1 parent 72aa762 commit 3eecd59

File tree

2 files changed

+30
-7
lines changed

2 files changed

+30
-7
lines changed

packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/types.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ def from_hydra(
8181
version_base=None,
8282
)
8383
else:
84+
config_path = None
8485
hydra.initialize_config_module(
8586
config_module="nemo_evaluator_launcher.configs",
8687
version_base=None,
@@ -93,6 +94,11 @@ def from_hydra(
9394
# Merge dict_overrides if provided
9495
if dict_overrides:
9596
cfg = OmegaConf.merge(cfg, dict_overrides)
97+
if config_path:
98+
is_struct = OmegaConf.is_struct(cfg)
99+
OmegaConf.set_struct(cfg, False)
100+
cfg.user_config_path = str(config_path)
101+
OmegaConf.set_struct(cfg, is_struct)
96102

97103
logger.debug(
98104
"Loaded run config from hydra",

packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/helpers.py

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
from nemo_evaluator_launcher.cli.version import get_versions
2525
from nemo_evaluator_launcher.common.logging_utils import logger
2626

27+
CONTAINER_RESULTS_DIR = "/results"
28+
2729

2830
@dataclass(frozen=True)
2931
class CmdAndReadableComment:
@@ -195,7 +197,7 @@ def get_eval_factory_command(
195197
_set_nested_optionally_overriding(
196198
merged_nemo_evaluator_config,
197199
["config", "output_dir"],
198-
"/results",
200+
CONTAINER_RESULTS_DIR,
199201
)
200202
# FIXME(martas): update to api_key_name after 25.12 is released
201203
_set_nested_optionally_overriding(
@@ -237,11 +239,28 @@ def get_eval_factory_command(
237239
pre_cmd,
238240
)
239241

242+
commands = []
243+
debug = []
244+
240245
create_pre_script_cmd = _str_to_echo_command(pre_cmd, filename="pre_cmd.sh")
246+
commands.append(create_pre_script_cmd.cmd)
247+
debug.append(create_pre_script_cmd.debug)
241248

242249
create_yaml_cmd = _str_to_echo_command(
243250
yaml.safe_dump(merged_nemo_evaluator_config), "config_ef.yaml"
244251
)
252+
commands.append(create_yaml_cmd.cmd)
253+
debug.append(create_yaml_cmd.debug)
254+
255+
# Store the original unresolved config if available
256+
config_path = cfg.get("user_config_path", None)
257+
if config_path:
258+
create_unresolved_config_cmd = _str_to_echo_command(
259+
open(config_path, "r").read(),
260+
filename=f"{CONTAINER_RESULTS_DIR}/launcher_unresolved_config.yaml",
261+
)
262+
commands.append(create_unresolved_config_cmd.cmd)
263+
debug.append(create_unresolved_config_cmd.debug)
245264

246265
# NOTE: we use `source` to allow tricks like exports etc (if needed) -- it runs in the same
247266
# shell as the command.
@@ -251,15 +270,13 @@ def get_eval_factory_command(
251270
+ "&& $cmd run_eval --run_config config_ef.yaml"
252271
)
253272

273+
commands.append(eval_command)
274+
254275
# We return both the command and the debugging base64-decoded strings, useful
255276
# for exposing when building scripts.
256277
return CmdAndReadableComment(
257-
cmd=create_pre_script_cmd.cmd
258-
+ " && "
259-
+ create_yaml_cmd.cmd
260-
+ " && "
261-
+ eval_command,
262-
debug=create_pre_script_cmd.debug + "\n\n" + create_yaml_cmd.debug,
278+
cmd=" && ".join(commands),
279+
debug="\n\n".join(debug),
263280
is_potentially_unsafe=is_potentially_unsafe,
264281
)
265282

0 commit comments

Comments
 (0)