From 073d8c0a4dccc693ca8c062e2e7db45e0b9b1a42 Mon Sep 17 00:00:00 2001 From: Darina Rustamova Date: Tue, 18 Mar 2025 23:44:49 +0300 Subject: [PATCH 1/6] feat: change log time for wandb --- autointent/_callbacks/wandb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autointent/_callbacks/wandb.py b/autointent/_callbacks/wandb.py index 3349e02c9..9ac36282b 100644 --- a/autointent/_callbacks/wandb.py +++ b/autointent/_callbacks/wandb.py @@ -63,6 +63,7 @@ def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any] group=self.group, name=f"{module_name}_{num}", config=module_kwargs, + settings=self.wandb.Settings(x_stats_sampling_interval=1), ) def log_value(self, **kwargs: dict[str, Any]) -> None: From 2a16754c85817fe88d2f5def84f938cdf7161543 Mon Sep 17 00:00:00 2001 From: Darina Rustamova Date: Wed, 19 Mar 2025 19:40:21 +0300 Subject: [PATCH 2/6] feat: added log_interval_time to loggerconfig --- autointent/_callbacks/base.py | 3 ++- autointent/_callbacks/callback_handler.py | 5 +++-- autointent/_callbacks/tensorboard.py | 5 ++++- autointent/_callbacks/wandb.py | 7 +++++-- autointent/_pipeline/_pipeline.py | 1 + autointent/configs/_optimization.py | 3 +++ 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/autointent/_callbacks/base.py b/autointent/_callbacks/base.py index b021ad472..5bb04c52e 100644 --- a/autointent/_callbacks/base.py +++ b/autointent/_callbacks/base.py @@ -16,12 +16,13 @@ def __init__(self) -> None: pass @abstractmethod - def start_run(self, run_name: str, dirpath: Path) -> None: + def start_run(self, run_name: str, dirpath: Path, log_interval_time: float) -> None: """Start a new run. Args: run_name: Name of the run. dirpath: Path to the directory where the logs will be saved. + log_interval_time: Sampling interval for the system monitor in seconds. """ @abstractmethod diff --git a/autointent/_callbacks/callback_handler.py b/autointent/_callbacks/callback_handler.py index aff8d1a23..d89429ccc 100644 --- a/autointent/_callbacks/callback_handler.py +++ b/autointent/_callbacks/callback_handler.py @@ -21,14 +21,15 @@ def __init__(self, callbacks: list[type[OptimizerCallback]] | None = None) -> No self.callbacks = [cb() for cb in callbacks] - def start_run(self, run_name: str, dirpath: Path) -> None: + def start_run(self, run_name: str, dirpath: Path, log_interval_time: float) -> None: """Start a new run. Args: run_name: Name of the run. dirpath: Path to the directory where the logs will be saved. + log_interval_time: Sampling interval for the system monitor in seconds. """ - self.call_events("start_run", run_name=run_name, dirpath=dirpath) + self.call_events("start_run", run_name=run_name, dirpath=dirpath, log_interval_time=log_interval_time) def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any]) -> None: """Start a new module. diff --git a/autointent/_callbacks/tensorboard.py b/autointent/_callbacks/tensorboard.py index 1639d6e45..daa5fb249 100644 --- a/autointent/_callbacks/tensorboard.py +++ b/autointent/_callbacks/tensorboard.py @@ -31,15 +31,18 @@ def __init__(self) -> None: ) raise ImportError(msg) from None - def start_run(self, run_name: str, dirpath: Path) -> None: + def start_run(self, run_name: str, dirpath: Path, log_interval_time: float) -> None: """Starts a new run and sets the directory for storing logs. Args: run_name: Name of the run. dirpath: Path to the directory where logs will be saved. + log_interval_time: Sampling interval for the system monitor in seconds. + (not utilized in Tensorboard logging). """ self.run_name = run_name self.dirpath = dirpath + self.log_interval_time = log_interval_time def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any]) -> None: """Starts a new module and initializes a TensorBoard writer for it. diff --git a/autointent/_callbacks/wandb.py b/autointent/_callbacks/wandb.py index 9ac36282b..dd2b12f9d 100644 --- a/autointent/_callbacks/wandb.py +++ b/autointent/_callbacks/wandb.py @@ -33,7 +33,7 @@ def __init__(self) -> None: self.wandb = wandb - def start_run(self, run_name: str, dirpath: Path) -> None: + def start_run(self, run_name: str, dirpath: Path, log_interval_time: float) -> None: """Starts a new W&B run. Initializes the project name and run group. The directory path argument is not @@ -42,10 +42,12 @@ def start_run(self, run_name: str, dirpath: Path) -> None: Args: run_name: Name of the run (used as a W&B group). dirpath: Path to store logs (not utilized in W&B logging). + log_interval_time: Sampling interval for the system monitor in seconds. """ self.project_name = os.getenv("WANDB_PROJECT", "autointent") self.group = run_name self.dirpath = dirpath + self.log_interval_time = log_interval_time def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any]) -> None: """Starts a new module within the W&B logging system. @@ -63,7 +65,7 @@ def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any] group=self.group, name=f"{module_name}_{num}", config=module_kwargs, - settings=self.wandb.Settings(x_stats_sampling_interval=1), + settings=self.wandb.Settings(x_stats_sampling_interval=self.log_interval_time), ) def log_value(self, **kwargs: dict[str, Any]) -> None: @@ -97,6 +99,7 @@ def log_final_metrics(self, metrics: dict[str, Any]) -> None: group=self.group, name="final_metrics", config=metrics, + settings=self.wandb.Settings(x_stats_sampling_interval=self.log_interval_time), ) self.wandb.log(metrics.get("pipeline_metrics", {})) diff --git a/autointent/_pipeline/_pipeline.py b/autointent/_pipeline/_pipeline.py index cec6851fc..73a68bb06 100644 --- a/autointent/_pipeline/_pipeline.py +++ b/autointent/_pipeline/_pipeline.py @@ -147,6 +147,7 @@ def _fit(self, context: Context, sampler: SamplerType) -> None: self.context.callback_handler.start_run( run_name=self.context.logging_config.get_run_name(), dirpath=self.context.logging_config.dirpath, + log_interval_time=self.context.logging_config.log_interval_time, ) for node_type in NodeType: node_optimizer = self.nodes.get(node_type, None) diff --git a/autointent/configs/_optimization.py b/autointent/configs/_optimization.py index 990dbf334..c47eb6e91 100644 --- a/autointent/configs/_optimization.py +++ b/autointent/configs/_optimization.py @@ -51,6 +51,9 @@ class LoggingConfig(BaseModel): report_to: list[REPORTERS_NAMES] | None = Field( # type: ignore[valid-type] None, description="List of callbacks to report to. If None, no callbacks will be used" ) + log_interval_time: float = Field( + 0.1, description="Sampling interval for the system monitor in seconds for Wandb logger." + ) """List of callbacks to report to. If None, no callbacks will be used""" @property From df289f1947d418e81c3dc4e2247a4a936d3c12a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 19 Mar 2025 16:43:20 +0000 Subject: [PATCH 3/6] Update optimizer_config.schema.json --- docs/optimizer_config.schema.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/optimizer_config.schema.json b/docs/optimizer_config.schema.json index aabd685ac..d478ecc41 100644 --- a/docs/optimizer_config.schema.json +++ b/docs/optimizer_config.schema.json @@ -297,6 +297,12 @@ "default": null, "description": "List of callbacks to report to. If None, no callbacks will be used", "title": "Report To" + }, + "log_interval_time": { + "default": 0.1, + "description": "Sampling interval for the system monitor in seconds for Wandb logger.", + "title": "Log Interval Time", + "type": "number" } }, "title": "LoggingConfig", @@ -328,7 +334,8 @@ "run_name": null, "dump_modules": false, "clear_ram": false, - "report_to": null + "report_to": null, + "log_interval_time": 0.1 } }, "embedder_config": { From 9ef3ea833cc9401f5ab09bbacde7bc77fd0108e4 Mon Sep 17 00:00:00 2001 From: Darina Rustamova Date: Fri, 21 Mar 2025 15:10:36 +0300 Subject: [PATCH 4/6] chore: for checks --- autointent/_callbacks/tensorboard.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/autointent/_callbacks/tensorboard.py b/autointent/_callbacks/tensorboard.py index daa5fb249..bc946250e 100644 --- a/autointent/_callbacks/tensorboard.py +++ b/autointent/_callbacks/tensorboard.py @@ -31,18 +31,15 @@ def __init__(self) -> None: ) raise ImportError(msg) from None - def start_run(self, run_name: str, dirpath: Path, log_interval_time: float) -> None: + def start_run(self, run_name: str, dirpath: Path) -> None: """Starts a new run and sets the directory for storing logs. Args: run_name: Name of the run. dirpath: Path to the directory where logs will be saved. - log_interval_time: Sampling interval for the system monitor in seconds. - (not utilized in Tensorboard logging). """ self.run_name = run_name self.dirpath = dirpath - self.log_interval_time = log_interval_time def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any]) -> None: """Starts a new module and initializes a TensorBoard writer for it. @@ -64,7 +61,7 @@ def log_value(self, **kwargs: dict[str, int | float | Any]) -> None: """Logs scalar or text values. Args: - **kwargs: Key-value pairs of data to log. Scalars will be logged as numerical values, others as text. + **kwargs: Key-value pairs of data to log. Scalars wil be logged as numerical values, others as text. """ for key, value in kwargs.items(): if isinstance(value, int | float): From c2f586c4d6eef70fd9ebf6c3add9458497b715c1 Mon Sep 17 00:00:00 2001 From: Darina Rustamova Date: Fri, 21 Mar 2025 15:10:42 +0300 Subject: [PATCH 5/6] chore: for checks --- autointent/_callbacks/tensorboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autointent/_callbacks/tensorboard.py b/autointent/_callbacks/tensorboard.py index bc946250e..1639d6e45 100644 --- a/autointent/_callbacks/tensorboard.py +++ b/autointent/_callbacks/tensorboard.py @@ -61,7 +61,7 @@ def log_value(self, **kwargs: dict[str, int | float | Any]) -> None: """Logs scalar or text values. Args: - **kwargs: Key-value pairs of data to log. Scalars wil be logged as numerical values, others as text. + **kwargs: Key-value pairs of data to log. Scalars will be logged as numerical values, others as text. """ for key, value in kwargs.items(): if isinstance(value, int | float): From 7da326a71db7996da0be2720411e07c1da613926 Mon Sep 17 00:00:00 2001 From: Darina Rustamova Date: Fri, 21 Mar 2025 15:22:49 +0300 Subject: [PATCH 6/6] fix: added log_interval_time to the tensorboard --- autointent/_callbacks/tensorboard.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autointent/_callbacks/tensorboard.py b/autointent/_callbacks/tensorboard.py index 1639d6e45..0da46d6cf 100644 --- a/autointent/_callbacks/tensorboard.py +++ b/autointent/_callbacks/tensorboard.py @@ -31,15 +31,17 @@ def __init__(self) -> None: ) raise ImportError(msg) from None - def start_run(self, run_name: str, dirpath: Path) -> None: + def start_run(self, run_name: str, dirpath: Path, log_interval_time: float) -> None: """Starts a new run and sets the directory for storing logs. Args: run_name: Name of the run. dirpath: Path to the directory where logs will be saved. + log_interval_time: Sampling interval for the system monitor in seconds. """ self.run_name = run_name self.dirpath = dirpath + self.log_interval_time = log_interval_time def start_module(self, module_name: str, num: int, module_kwargs: dict[str, Any]) -> None: """Starts a new module and initializes a TensorBoard writer for it.