From 6651313fcd570cf4a5e696872ec4c77b4e647d52 Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Wed, 15 Jan 2025 13:19:39 +0100
Subject: [PATCH 01/11] Environment recorder

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvInterface.py | 450 ++++++++++++++++++++++++++++
 grid2op/Environment/EnvRecorder.py  |  97 ++++++
 grid2op/Environment/baseEnv.py      |  38 +--
 grid2op/Environment/environment.py  | 299 ------------------
 grid2op/tests/test_EnvRecorder.py   |  43 +++
 setup.py                            |   3 +-
 6 files changed, 595 insertions(+), 335 deletions(-)
 create mode 100644 grid2op/Environment/EnvInterface.py
 create mode 100644 grid2op/Environment/EnvRecorder.py
 create mode 100644 grid2op/tests/test_EnvRecorder.py

diff --git a/grid2op/Environment/EnvInterface.py b/grid2op/Environment/EnvInterface.py
new file mode 100644
index 00000000..ca306ac7
--- /dev/null
+++ b/grid2op/Environment/EnvInterface.py
@@ -0,0 +1,450 @@
+from abc import ABC, abstractmethod
+from typing import Tuple, Union
+
+from grid2op.Action import BaseAction
+from grid2op.Observation import BaseObservation
+from grid2op.typing_variables import STEP_INFO_TYPING, RESET_OPTIONS_TYPING
+
+
+class EnvInterface(ABC):
+
+    @abstractmethod
+    def reset(self,
+              *,
+              seed: Union[int, None] = None,
+              options: RESET_OPTIONS_TYPING = None) -> BaseObservation:
+        """
+        Reset the environment to a clean state.
+        It will reload the next chronics if any. And reset the grid to a clean state.
+
+        This triggers a full reloading of both the chronics (if they are stored as files) and of the powergrid,
+        to ensure the episode is fully over.
+
+        This method should be called only at the end of an episode.
+
+        Parameters
+        ----------
+        seed: int
+            The seed to used (new in version 1.9.8), see examples for more details. Ignored if not set (meaning no seeds will
+            be used, experiments might not be reproducible)
+
+        options: dict
+            Some options to "customize" the reset call. For example (see detailed example bellow) :
+
+            - "time serie id" (grid2op >= 1.9.8) to use a given time serie from the input data
+            - "init state" that allows you to apply a given "action" when generating the
+              initial observation (grid2op >= 1.10.2)
+            - "init ts" (grid2op >= 1.10.3) to specify to which "steps" of the time series
+              the episode will start
+            - "max step" (grid2op >= 1.10.3) : maximum number of steps allowed for the episode
+            - "thermal limit" (grid2op >= 1.11.0): which thermal limit to use for this episode
+              (and the next ones, until they are changed)
+            - "init datetime": which time stamp is used in the first observation of the episode.
+
+            See examples for more information about this. Ignored if
+            not set.
+
+        Examples
+        --------
+        The standard "gym loop" can be done with the following code:
+
+        .. code-block:: python
+
+            import grid2op
+
+            # create the environment
+            env_name = "l2rpn_case14_sandbox"
+            env = grid2op.make(env_name)
+
+            # start a new episode
+            obs = env.reset()
+            done = False
+            reward = env.reward_range[0]
+            while not done:
+                action = agent.act(obs, reward, done)
+                obs, reward, done, info = env.step(action)
+
+        .. versionadded:: 1.9.8
+            It is now possible to set the seed and the time series you want to use at the new
+            episode by calling `env.reset(seed=..., options={"time serie id": ...})`
+
+        Before version 1.9.8, if you wanted to use a fixed seed, you would need to (see
+        doc of :func:`grid2op.Environment.BaseEnv.seed` ):
+
+        .. code-block:: python
+
+            seed = ...
+            env.seed(seed)
+            obs = env.reset()
+            ...
+
+        Starting from version 1.9.8 you can do this in one call:
+
+        .. code-block:: python
+
+            seed = ...
+            obs = env.reset(seed=seed)
+
+        For the "time series id" it is the same concept. Before you would need to do (see
+        doc of :func:`Environment.set_id` for more information ):
+
+        .. code-block:: python
+
+            time_serie_id = ...
+            env.set_id(time_serie_id)
+            obs = env.reset()
+            ...
+
+        And now (from version 1.9.8) you can more simply do:
+
+        .. code-block:: python
+
+            time_serie_id = ...
+            obs = env.reset(options={"time serie id": time_serie_id})
+            ...
+
+        .. versionadded:: 1.10.2
+
+        Another feature has been added in version 1.10.2, which is the possibility to set the
+        grid to a given "topological" state at the first observation (before this version,
+        you could only retrieve an observation with everything connected together).
+
+        In grid2op 1.10.2, you can do that by using the keys `"init state"` in the "options" kwargs of
+        the reset function. The value associated to this key should be dictionnary that can be
+        converted to a non ambiguous grid2op action using an "action space".
+
+        .. note::
+            The "action space" used here is not the action space of the agent. It's an "action
+            space" that uses a :func:`grid2op.Action.Action.BaseAction` class meaning you can do any
+            type of action, on shunts, on topology, on line status etc. even if the agent is not
+            allowed to.
+
+            Likewise, nothing check if this action is legal or not.
+
+        You can use it like this:
+
+        .. code-block:: python
+
+            # to start an episode with a line disconnected, you can do:
+            init_state_dict = {"set_line_status": [(0, -1)]}
+            obs = env.reset(options={"init state": init_state_dict})
+            obs.line_status[0] is False
+
+            # to start an episode with a different topolovy
+            init_state_dict = {"set_bus": {"lines_or_id": [(0, 2)], "lines_ex_id": [(3, 2)]}}
+            obs = env.reset(options={"init state": init_state_dict})
+
+        .. note::
+            Since grid2op version 1.10.2, there is also the possibility to set the "initial state"
+            of the grid directly in the time series. The priority is always given to the
+            argument passed in the "options" value.
+
+            Concretely if, in the "time series" (formelly called "chronics") provides an action would change
+            the topology of substation 1 and 2 (for example) and you provide an action that disable the
+            line 6, then the initial state will see substation 1 and 2 changed (as in the time series)
+            and line 6 disconnected.
+
+            Another example in this case: if the action you provide would change topology of substation 2 and 4
+            then the initial state (after `env.reset`) will give:
+
+            - substation 1 as in the time serie
+            - substation 2 as in "options"
+            - substation 4 as in "options"
+
+        .. note::
+            Concerning the previously described behaviour, if you want to ignore the data in the
+            time series, you can add : `"method": "ignore"` in the dictionary describing the action.
+            In this case the action in the time series will be totally ignored and the initial
+            state will be fully set by the action passed in the "options" dict.
+
+            An example is:
+
+            .. code-block:: python
+
+                init_state_dict = {"set_line_status": [(0, -1)], "method": "force"}
+                obs = env.reset(options={"init state": init_state_dict})
+                obs.line_status[0] is False
+
+        .. versionadded:: 1.10.3
+
+        Another feature has been added in version 1.10.3, the possibility to skip the
+        some steps of the time series and starts at some given steps.
+
+        The time series often always start at a given day of the week (*eg* Monday)
+        and at a given time (*eg* midnight). But for some reason you notice that your
+        agent performs poorly on other day of the week or time of the day. This might be
+        because it has seen much more data from Monday at midnight that from any other
+        day and hour of the day.
+
+        To alleviate this issue, you can now easily reset an episode and ask grid2op
+        to start this episode after xxx steps have "passed".
+
+        Concretely, you can do it with:
+
+        .. code-block:: python
+
+            import grid2op
+            env_name = "l2rpn_case14_sandbox"
+            env = grid2op.make(env_name)
+
+            obs = env.reset(options={"init ts": 1})
+
+        Doing that your agent will start its episode not at midnight (which
+        is the case for this environment), but at 00:05
+
+        If you do:
+
+        .. code-block:: python
+
+            obs = env.reset(options={"init ts": 12})
+
+        In this case, you start the episode at 01:00 and not at midnight (you
+        start at what would have been the 12th steps)
+
+        If you want to start the "next day", you can do:
+
+        .. code-block:: python
+
+            obs = env.reset(options={"init ts": 288})
+
+        etc.
+
+        .. note::
+            On this feature, if a powerline is on soft overflow (meaning its flow is above
+            the limit but below the :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`)
+            then it is still connected (of course) and the counter
+            :attr:`grid2op.Observation.BaseObservation.timestep_overflow` is at 0.
+
+            If a powerline is on "hard overflow" (meaning its flow would be above
+            :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`), then, as it is
+            the case for a "normal" (without options) reset, this line is disconnected, but can be reconnected
+            directly (:attr:`grid2op.Observation.BaseObservation.time_before_cooldown_line` == 0)
+
+        .. seealso::
+            The function :func:`Environment.fast_forward_chronics` for an alternative usage (that will be
+            deprecated at some point)
+
+        Yet another feature has been added in grid2op version 1.10.3 in this `env.reset` function. It is
+        the capacity to limit the duration of an episode.
+
+        .. code-block:: python
+
+            import grid2op
+            env_name = "l2rpn_case14_sandbox"
+            env = grid2op.make(env_name)
+
+            obs = env.reset(options={"max step": 288})
+
+        This will limit the duration to 288 steps (1 day), meaning your agent
+        will have successfully managed the entire episode if it manages to keep
+        the grid in a safe state for a whole day (depending on the environment you are
+        using the default duration is either one week - roughly 2016 steps or 4 weeks)
+
+        .. note::
+            This option only affect the current episode. It will have no impact on the
+            next episode (after reset)
+
+        For example:
+
+        .. code-block:: python
+
+            obs = env.reset()
+            obs.max_step == 8064  # default for this environment
+
+            obs = env.reset(options={"max step": 288})
+            obs.max_step == 288  # specified by the option
+
+            obs = env.reset()
+            obs.max_step == 8064  # retrieve the default behaviour
+
+        .. seealso::
+            The function :func:`Environment.set_max_iter` for an alternative usage with the different
+            that `set_max_iter` is permenanent: it impacts all the future episodes and not only
+            the next one.
+
+        If you want your environment to start at a given time stamp you can do:
+
+        .. code-block:: python
+
+            import grid2op
+            env_name = "l2rpn_case14_sandbox"
+
+            env = grid2op.make(env_name)
+            obs = env.reset(options={"init datetime": "2024-12-06 00:00"})
+            obs.year == 2024
+            obs.month == 12
+            obs.day == 6
+
+        .. seealso::
+            If you specify "init datetime" then the observation resulting to the
+            `env.reset` call will have this datetime. If you specify also `"skip ts"`
+            option the behaviour does not change: the first observation will
+            have the date time attributes you specified.
+
+            In other words, the "init datetime" refers to the initial observation of the
+            episode and NOT the initial time present in the time series.
+
+        """
+        pass
+
+    @abstractmethod
+    def step(self, action: BaseAction) -> Tuple[BaseObservation,
+                                                float,
+                                                bool,
+                                                STEP_INFO_TYPING]:
+        """
+                Run one timestep of the environment's dynamics. When end of
+                episode is reached, you are responsible for calling `reset()`
+                to reset this environment's state.
+                Accepts an action and returns a tuple (observation, reward, done, info).
+
+                If the :class:`grid2op.BaseAction.BaseAction` is illegal or ambiguous, the step is performed, but the action is
+                replaced with a "do nothing" action.
+
+                Parameters
+                ----------
+                    action: :class:`grid2op.Action.Action`
+                        an action provided by the agent that is applied on the underlying through the backend.
+
+                Returns
+                -------
+                    observation: :class:`grid2op.Observation.Observation`
+                        agent's observation of the current environment
+
+                    reward: ``float``
+                        amount of reward returned after previous action
+
+                    done: ``bool``
+                        whether the episode has ended, in which case further step() calls will return undefined results
+
+                    info: ``dict``
+                        contains auxiliary diagnostic information (helpful for debugging, and sometimes learning). It is a
+                        dictionary with keys:
+
+                            - "disc_lines": a numpy array (or ``None``) saying, for each powerline if it has been disconnected
+                              due to overflow (if not disconnected it will be -1, otherwise it will be a
+                              positive integer: 0 meaning that is one of the cause of the cascading failure, 1 means
+                              that it is disconnected just after, 2 that it's disconnected just after etc.)
+                            - "is_illegal" (``bool``) whether the action given as input was illegal
+                            - "is_ambiguous" (``bool``) whether the action given as input was ambiguous.
+                            - "is_dispatching_illegal" (``bool``) was the action illegal due to redispatching
+                            - "is_illegal_reco" (``bool``) was the action illegal due to a powerline reconnection
+                            - "reason_alarm_illegal" (``None`` or ``Exception``) reason for which the alarm is illegal
+                              (it's None if no alarm are raised or if the alarm feature is not used)
+                            - "reason_alert_illegal" (``None`` or ``Exception``) reason for which the alert is illegal
+                              (it's None if no alert are raised or if the alert feature is not used)
+                            - "opponent_attack_line" (``np.ndarray``, ``bool``) for each powerline, say if the opponent
+                              attacked it (``True``) or not (``False``).
+                            - "opponent_attack_sub" (``np.ndarray``, ``bool``) for each substation, say if the opponent
+                              attacked it (``True``) or not (``False``).
+                            - "opponent_attack_duration" (``int``) the duration of the current attack (if any)
+                            - "exception" (``list`` of :class:`Exceptions.Exceptions.Grid2OpException` if an exception was
+                              raised  or ``[]`` if everything was fine.)
+                            - "detailed_infos_for_cascading_failures" (optional, only if the backend has been create with
+                              `detailed_infos_for_cascading_failures=True`) the list of the intermediate steps computed during
+                              the simulation of the "cascading failures".
+                            - "rewards": dictionary of all "other_rewards" provided when the env was built.
+                            - "time_series_id": id of the time series used (if any, similar to a call to `env.chronics_handler.get_id()`)
+                Examples
+                ---------
+
+                This is used like:
+
+                .. code-block:: python
+
+                    import grid2op
+                    from grid2op.Agent import RandomAgent
+
+                    # I create an environment
+                    env = grid2op.make("l2rpn_case14_sandbox")
+
+                    # define an agent here, this is an example
+                    agent = RandomAgent(env.action_space)
+
+                    # environment need to be "reset" before usage:
+                    obs = env.reset()
+                    reward = env.reward_range[0]
+                    done = False
+
+                    # now run through each steps like this
+                    while not done:
+                        action = agent.act(obs, reward, done)
+                        obs, reward, done, info = env.step(action)
+
+                Notes
+                -----
+
+                If the flag `done=True` is raised (*ie* this is the end of the episode) then the observation is NOT properly
+                updated and should not be used at all.
+
+                Actually, it will be in a "game over" state (see :class:`grid2op.Observation.BaseObservation.set_game_over`).
+
+                """
+        pass
+
+    def render(self, mode="rgb_array"):
+        """
+        Render the state of the environment on the screen, using matplotlib
+        Also returns the Matplotlib figure
+
+        Examples
+        --------
+        Rendering need first to define a "renderer" which can be done with the following code:
+
+        .. code-block:: python
+
+            import grid2op
+
+            # create the environment
+            env = grid2op.make("l2rpn_case14_sandbox")
+
+            # if you want to use the renderer
+            env.attach_renderer()
+
+            # and now you can "render" (plot) the state of the grid
+            obs = env.reset()
+            done = False
+            reward = env.reward_range[0]
+            while not done:
+                env.render()  # this piece of code plot the grid
+                action = agent.act(obs, reward, done)
+                obs, reward, done, info = env.step(action)
+        """
+        pass
+
+    def close(self):
+        """close an environment: this will attempt to free as much memory as possible.
+        Note that after an environment is closed, you will not be able to use anymore.
+
+        Any attempt to use a closed environment might result in non deterministic behaviour.
+        """
+        pass
+
+    def __enter__(self):
+        """
+        Support *with-statement* for the environment.
+
+        Examples
+        --------
+
+        .. code-block:: python
+
+            import grid2op
+            import grid2op.BaseAgent
+            with grid2op.make("l2rpn_case14_sandbox") as env:
+                agent = grid2op.BaseAgent.DoNothingAgent(env.action_space)
+                act = env.action_space()
+                obs, r, done, info = env.step(act)
+                act = agent.act(obs, r, info)
+                obs, r, done, info = env.step(act)
+
+        """
+        return self
+
+    def __exit__(self, *args):
+        """
+        Support *with-statement* for the environment.
+        """
+        self.close()
+        # propagate exception
+        return False
diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
new file mode 100644
index 00000000..c7c0a27a
--- /dev/null
+++ b/grid2op/Environment/EnvRecorder.py
@@ -0,0 +1,97 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Tuple, Union, Callable, Dict, List
+
+from grid2op.Action import BaseAction
+from grid2op.Environment.EnvInterface import EnvInterface
+from grid2op.Observation import BaseObservation
+from grid2op.typing_variables import STEP_INFO_TYPING, RESET_OPTIONS_TYPING
+import pyarrow as pa
+import pyarrow.parquet
+
+ObservationVectorGetter = Callable[[BaseObservation], List[float]]
+
+class ObservationTable:
+    def __init__(self, columns: List[str], getter: ObservationVectorGetter, directory: Path, table_name: str, write_chunk_size: int):
+        self._columns = columns
+        self._getter = getter
+        self._directory = directory
+        self._table_name = table_name
+        self._write_chunk_size = write_chunk_size
+        self._buffer = [[] for _ in range(len(columns) + 1)]
+        self._writer = None
+
+    def append(self, obs: BaseObservation):
+        time = obs.get_time_stamp()
+        self._buffer[0].append(int(time.timestamp()))
+        vec = self._getter(obs)
+        for i in range(len(self._columns)):
+            self._buffer[i + 1].append(vec[i])
+
+        if len(self._buffer[0]) >= self._write_chunk_size:
+            table = pa.table(self._buffer, ['time'] + list(self._columns))
+            if self._writer is None:
+                parquet_file = self._directory / f"{self._table_name}.parquet"
+                self._writer = pa.parquet.ParquetWriter(parquet_file, schema=table.schema)
+            self._writer.write_table(table)
+            self._buffer = [[] for _ in range(len(self._columns) + 1)] # reset buffer
+
+    def close(self):
+        if self._writer is not None:
+            self._writer.close()
+            self._writer = None
+
+class EnvRecorder(EnvInterface):
+
+    def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
+        super().__init__()
+        self._env = env
+
+        self.write_element_table([env.name_gen, env.gen_type], ['name', 'type'], directory, 'gen')
+        self.write_element_table([env.name_load], ['name'], directory, 'load')
+
+        self._gen_p_before_curtail_table = ObservationTable(self._env.name_gen, lambda obs: obs.gen_p_before_curtail, directory, 'gen_p_before_curtail', write_chunk_size)
+        self._gen_p_table = ObservationTable(self._env.name_gen, lambda obs: obs.gen_p, directory, 'gen_p', write_chunk_size)
+        self._gen_v_table = ObservationTable(self._env.name_gen, lambda obs: obs.gen_v, directory, 'gen_v', write_chunk_size)
+        self._load_p_table = ObservationTable(self._env.name_load, lambda obs: obs.load_p, directory, 'load_p', write_chunk_size)
+        self._load_q_table = ObservationTable(self._env.name_load, lambda obs: obs.load_q, directory, 'load_q', write_chunk_size)
+
+    @staticmethod
+    def write_element_table(data, column_names, directory: Path, table_name: str):
+        element_table = pa.table({col: data[i] for i, col in enumerate(column_names)})
+        pa.parquet.write_table(element_table, directory / f"{table_name}.parquet")
+
+    @property
+    def env(self):
+        return self._env
+
+    def reset(self,
+              *,
+              seed: Union[int, None] = None,
+              options: RESET_OPTIONS_TYPING = None) -> BaseObservation:
+        return self._env.reset(seed=seed, options=options)
+
+    def step(self, action: BaseAction) -> Tuple[BaseObservation,
+                                                float,
+                                                bool,
+                                                STEP_INFO_TYPING]:
+        result = self._env.step(action)
+        obs = result[0]
+#        print(result[3]['time_series_id'])
+        self._gen_p_before_curtail_table.append(obs)
+        self._gen_p_table.append(obs)
+        self._gen_v_table.append(obs)
+        self._load_p_table.append(obs)
+        self._load_q_table.append(obs)
+        return result
+
+    def render(self, mode="rgb_array"):
+        self._env.render(mode=mode)
+
+    def close(self):
+        self._gen_p_before_curtail_table.close()
+        self._gen_p_table.close()
+        self._gen_v_table.close()
+        self._load_p_table.close()
+        self._load_q_table.close()
+        self._env.close()
diff --git a/grid2op/Environment/baseEnv.py b/grid2op/Environment/baseEnv.py
index e3c32323..0877f89d 100644
--- a/grid2op/Environment/baseEnv.py
+++ b/grid2op/Environment/baseEnv.py
@@ -23,6 +23,8 @@
 from scipy.optimize import (minimize, LinearConstraint)
 
 from abc import ABC, abstractmethod
+
+from grid2op.Environment.EnvInterface import EnvInterface
 from grid2op.Observation import (BaseObservation,
                                  ObservationSpace,
                                  HighResSimCounter)
@@ -89,7 +91,7 @@
 # WE DO NOT RECOMMEND TO ALTER IT IN ANY WAY
 """
 
-class BaseEnv(GridObjects, RandomObject, ABC):
+class BaseEnv(EnvInterface, GridObjects, RandomObject, ABC):
     """
     INTERNAL
 
@@ -3720,41 +3722,7 @@ def _reset_maintenance(self):
         self._time_next_maintenance[:] = -1
         self._duration_next_maintenance[:] = 0
 
-    def __enter__(self):
-        """
-        Support *with-statement* for the environment.
-
-        Examples
-        --------
-
-        .. code-block:: python
-
-            import grid2op
-            import grid2op.BaseAgent
-            with grid2op.make("l2rpn_case14_sandbox") as env:
-                agent = grid2op.BaseAgent.DoNothingAgent(env.action_space)
-                act = env.action_space()
-                obs, r, done, info = env.step(act)
-                act = agent.act(obs, r, info)
-                obs, r, done, info = env.step(act)
-
-        """
-        return self
-
-    def __exit__(self, *args):
-        """
-        Support *with-statement* for the environment.
-        """
-        self.close()
-        # propagate exception
-        return False
-
     def close(self):
-        """close an environment: this will attempt to free as much memory as possible.
-        Note that after an environment is closed, you will not be able to use anymore.
-
-        Any attempt to use a closed environment might result in non deterministic behaviour.
-        """
         if self.__closed:
             raise EnvError(
                 f"This environment {id(self)} {self} is closed already, you cannot close it a second time."
diff --git a/grid2op/Environment/environment.py b/grid2op/Environment/environment.py
index 6f13d926..eb50fd43 100644
--- a/grid2op/Environment/environment.py
+++ b/grid2op/Environment/environment.py
@@ -1035,278 +1035,6 @@ def reset(self,
               *,
               seed: Union[int, None] = None,
               options: RESET_OPTIONS_TYPING = None) -> BaseObservation:
-        """
-        Reset the environment to a clean state.
-        It will reload the next chronics if any. And reset the grid to a clean state.
-
-        This triggers a full reloading of both the chronics (if they are stored as files) and of the powergrid,
-        to ensure the episode is fully over.
-
-        This method should be called only at the end of an episode.
-        
-        Parameters
-        ----------
-        seed: int
-            The seed to used (new in version 1.9.8), see examples for more details. Ignored if not set (meaning no seeds will 
-            be used, experiments might not be reproducible)
-            
-        options: dict
-            Some options to "customize" the reset call. For example (see detailed example bellow) :
-            
-            - "time serie id" (grid2op >= 1.9.8) to use a given time serie from the input data
-            - "init state" that allows you to apply a given "action" when generating the 
-              initial observation (grid2op >= 1.10.2)
-            - "init ts" (grid2op >= 1.10.3) to specify to which "steps" of the time series 
-              the episode will start
-            - "max step" (grid2op >= 1.10.3) : maximum number of steps allowed for the episode
-            - "thermal limit" (grid2op >= 1.11.0): which thermal limit to use for this episode 
-              (and the next ones, until they are changed)
-            - "init datetime": which time stamp is used in the first observation of the episode.
-            
-            See examples for more information about this. Ignored if 
-            not set.
-        
-        Examples
-        --------
-        The standard "gym loop" can be done with the following code:
-
-        .. code-block:: python
-
-            import grid2op
-
-            # create the environment
-            env_name = "l2rpn_case14_sandbox"
-            env = grid2op.make(env_name)
-
-            # start a new episode
-            obs = env.reset()
-            done = False
-            reward = env.reward_range[0]
-            while not done:
-                action = agent.act(obs, reward, done)
-                obs, reward, done, info = env.step(action)
-                
-        .. versionadded:: 1.9.8
-            It is now possible to set the seed and the time series you want to use at the new
-            episode by calling `env.reset(seed=..., options={"time serie id": ...})`
-
-        Before version 1.9.8, if you wanted to use a fixed seed, you would need to (see 
-        doc of :func:`grid2op.Environment.BaseEnv.seed` ):
-        
-        .. code-block:: python
-
-            seed = ...
-            env.seed(seed)
-            obs = env.reset()
-            ...
-            
-        Starting from version 1.9.8 you can do this in one call:
-        
-        .. code-block:: python
-
-            seed = ...
-            obs = env.reset(seed=seed)  
-            
-        For the "time series id" it is the same concept. Before you would need to do (see
-        doc of :func:`Environment.set_id` for more information ):
-        
-        .. code-block:: python
-
-            time_serie_id = ...
-            env.set_id(time_serie_id)
-            obs = env.reset()
-            ...        
-            
-        And now (from version 1.9.8) you can more simply do:
-        
-        .. code-block:: python
-
-            time_serie_id = ...
-            obs = env.reset(options={"time serie id": time_serie_id})
-            ... 
-        
-        .. versionadded:: 1.10.2
-        
-        Another feature has been added in version 1.10.2, which is the possibility to set the 
-        grid to a given "topological" state at the first observation (before this version, 
-        you could only retrieve an observation with everything connected together). 
-        
-        In grid2op 1.10.2, you can do that by using the keys `"init state"` in the "options" kwargs of 
-        the reset function. The value associated to this key should be dictionnary that can be
-        converted to a non ambiguous grid2op action using an "action space".
-        
-        .. note::
-            The "action space" used here is not the action space of the agent. It's an "action
-            space" that uses a :func:`grid2op.Action.Action.BaseAction` class meaning you can do any
-            type of action, on shunts, on topology, on line status etc. even if the agent is not
-            allowed to.
-            
-            Likewise, nothing check if this action is legal or not.
-            
-        You can use it like this:
-        
-        .. code-block:: python
-
-            # to start an episode with a line disconnected, you can do:
-            init_state_dict = {"set_line_status": [(0, -1)]}
-            obs = env.reset(options={"init state": init_state_dict})
-            obs.line_status[0] is False
-            
-            # to start an episode with a different topolovy
-            init_state_dict = {"set_bus": {"lines_or_id": [(0, 2)], "lines_ex_id": [(3, 2)]}}
-            obs = env.reset(options={"init state": init_state_dict})
-            
-        .. note::
-            Since grid2op version 1.10.2, there is also the possibility to set the "initial state"
-            of the grid directly in the time series. The priority is always given to the 
-            argument passed in the "options" value. 
-            
-            Concretely if, in the "time series" (formelly called "chronics") provides an action would change
-            the topology of substation 1 and 2 (for example) and you provide an action that disable the
-            line 6, then the initial state will see substation 1 and 2 changed (as in the time series)
-            and line 6 disconnected. 
-            
-            Another example in this case: if the action you provide would change topology of substation 2 and 4
-            then the initial state (after `env.reset`) will give:
-            
-            - substation 1 as in the time serie
-            - substation 2 as in "options"
-            - substation 4 as in "options"
-        
-        .. note::
-            Concerning the previously described behaviour, if you want to ignore the data in the
-            time series, you can add : `"method": "ignore"` in the dictionary describing the action.
-            In this case the action in the time series will be totally ignored and the initial
-            state will be fully set by the action passed in the "options" dict.
-            
-            An example is:
-            
-            .. code-block:: python
-
-                init_state_dict = {"set_line_status": [(0, -1)], "method": "force"}
-                obs = env.reset(options={"init state": init_state_dict})
-                obs.line_status[0] is False
-
-        .. versionadded:: 1.10.3
-        
-        Another feature has been added in version 1.10.3, the possibility to skip the
-        some steps of the time series and starts at some given steps.
-        
-        The time series often always start at a given day of the week (*eg* Monday)
-        and at a given time (*eg* midnight). But for some reason you notice that your
-        agent performs poorly on other day of the week or time of the day. This might be
-        because it has seen much more data from Monday at midnight that from any other 
-        day and hour of the day.
-        
-        To alleviate this issue, you can now easily reset an episode and ask grid2op
-        to start this episode after xxx steps have "passed".
-        
-        Concretely, you can do it with:
-                    
-        .. code-block:: python
-
-            import grid2op
-            env_name = "l2rpn_case14_sandbox"
-            env = grid2op.make(env_name)
-            
-            obs = env.reset(options={"init ts": 1})
-        
-        Doing that your agent will start its episode not at midnight (which
-        is the case for this environment), but at 00:05
-        
-        If you do:
-        
-        .. code-block:: python
-        
-            obs = env.reset(options={"init ts": 12})
-            
-        In this case, you start the episode at 01:00 and not at midnight (you
-        start at what would have been the 12th steps)
-        
-        If you want to start the "next day", you can do:
-        
-        .. code-block:: python
-        
-            obs = env.reset(options={"init ts": 288})
-            
-        etc.
-        
-        .. note::
-            On this feature, if a powerline is on soft overflow (meaning its flow is above 
-            the limit but below the :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`)
-            then it is still connected (of course) and the counter 
-            :attr:`grid2op.Observation.BaseObservation.timestep_overflow` is at 0.
-            
-            If a powerline is on "hard overflow" (meaning its flow would be above 
-            :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`), then, as it is 
-            the case for a "normal" (without options) reset, this line is disconnected, but can be reconnected
-            directly (:attr:`grid2op.Observation.BaseObservation.time_before_cooldown_line` == 0)
-        
-        .. seealso::
-            The function :func:`Environment.fast_forward_chronics` for an alternative usage (that will be
-            deprecated at some point)
-            
-        Yet another feature has been added in grid2op version 1.10.3 in this `env.reset` function. It is
-        the capacity to limit the duration of an episode.
-                    
-        .. code-block:: python
-
-            import grid2op
-            env_name = "l2rpn_case14_sandbox"
-            env = grid2op.make(env_name)
-            
-            obs = env.reset(options={"max step": 288})
-
-        This will limit the duration to 288 steps (1 day), meaning your agent
-        will have successfully managed the entire episode if it manages to keep
-        the grid in a safe state for a whole day (depending on the environment you are
-        using the default duration is either one week - roughly 2016 steps or 4 weeks)
-        
-        .. note::
-            This option only affect the current episode. It will have no impact on the 
-            next episode (after reset)
-            
-        For example:
-        
-        .. code-block:: python
-        
-            obs = env.reset()
-            obs.max_step == 8064  # default for this environment
-            
-            obs = env.reset(options={"max step": 288})
-            obs.max_step == 288  # specified by the option
-            
-            obs = env.reset()
-            obs.max_step == 8064  # retrieve the default behaviour
-
-        .. seealso::
-            The function :func:`Environment.set_max_iter` for an alternative usage with the different
-            that `set_max_iter` is permenanent: it impacts all the future episodes and not only
-            the next one.
-            
-        If you want your environment to start at a given time stamp you can do:
-        
-        .. code-block:: python
-        
-            import grid2op
-            env_name = "l2rpn_case14_sandbox"
-            
-            env = grid2op.make(env_name)
-            obs = env.reset(options={"init datetime": "2024-12-06 00:00"})
-            obs.year == 2024
-            obs.month == 12
-            obs.day == 6
-            
-        .. seealso::
-            If you specify "init datetime" then the observation resulting to the 
-            `env.reset` call will have this datetime. If you specify also `"skip ts"`
-            option the behaviour does not change: the first observation will 
-            have the date time attributes you specified. 
-            
-            In other words, the "init datetime" refers to the initial observation of the
-            episode and NOT the initial time present in the time series.
-            
-        """
         # process the "options" kwargs
         # (if there is an init state then I need to process it to remove the 
         # some keys)
@@ -1417,33 +1145,6 @@ def reset(self,
         return self.get_obs()
 
     def render(self, mode="rgb_array"):
-        """
-        Render the state of the environment on the screen, using matplotlib
-        Also returns the Matplotlib figure
-
-        Examples
-        --------
-        Rendering need first to define a "renderer" which can be done with the following code:
-
-        .. code-block:: python
-
-            import grid2op
-
-            # create the environment
-            env = grid2op.make("l2rpn_case14_sandbox")
-
-            # if you want to use the renderer
-            env.attach_renderer()
-
-            # and now you can "render" (plot) the state of the grid
-            obs = env.reset()
-            done = False
-            reward = env.reward_range[0]
-            while not done:
-                env.render()  # this piece of code plot the grid
-                action = agent.act(obs, reward, done)
-                obs, reward, done, info = env.step(action)
-        """
         # Try to create a plotter instance
         # Does nothing if viewer exists
         # Raises if matplot is not installed
diff --git a/grid2op/tests/test_EnvRecorder.py b/grid2op/tests/test_EnvRecorder.py
new file mode 100644
index 00000000..eaa6cf4e
--- /dev/null
+++ b/grid2op/tests/test_EnvRecorder.py
@@ -0,0 +1,43 @@
+import unittest
+import warnings
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import pandas as pd
+
+from getting_started import grid2op
+from grid2op.Backend import PandaPowerBackend
+from grid2op.Environment.EnvRecorder import EnvRecorder
+
+
+class TestEnvRecorder(unittest.TestCase):
+
+    @staticmethod
+    def make_backend(detailed_infos_for_cascading_failures=False):
+        return PandaPowerBackend(detailed_infos_for_cascading_failures)
+
+    def test_recording(self):
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore")
+            env = grid2op.make(
+                "rte_case5_example",
+                test=True,
+                backend=self.make_backend(),
+                _add_to_name=type(self).__name__
+            )
+            with TemporaryDirectory() as tmp_dir_name:
+                tmp_dir_path = Path(tmp_dir_name)
+                with EnvRecorder(env, tmp_dir_path, 3) as env_rec:
+                    env_rec.reset()
+                    do_nothing = env.action_space()
+                    done = False
+                    while not done:
+                        _, _, done, _ = env_rec.step(do_nothing)
+
+                for file_name in ['gen_p_before_curtail', 'gen_p', 'gen_v', 'load_p', 'load_q']:
+                    pq_file = tmp_dir_path / f"{file_name}.parquet"
+                    assert pq_file.is_file()
+
+                gen_p_pq = pd.read_parquet(tmp_dir_path / "gen_p.parquet")
+                assert gen_p_pq.shape == (93, 3)
+                assert gen_p_pq.columns.tolist() == ['time', 'gen_0_0', 'gen_1_1']
diff --git a/setup.py b/setup.py
index dc880c67..a514be04 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,8 @@ def my_test_suite():
         "networkx>=2.4",
         "requests>=2.23.0",
         "packaging",  # because gym changes the way it uses numpy prng in version 0.26 and i need both gym before and after...
-        "typing_extensions"
+        "typing_extensions",
+        "pyarrow>=18.1.0"
     ],
     "extras": {
         "optional": [

From d0965e34ad34c8ae8aa1dbe48e0ed6e631806b9f Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Wed, 15 Jan 2025 14:55:59 +0100
Subject: [PATCH 02/11] Refactoring + add doc

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvInterface.py | 13 ++++-
 grid2op/Environment/EnvRecorder.py  | 89 +++++++++++++++++++++++------
 grid2op/tests/test_EnvRecorder.py   |  7 +++
 3 files changed, 90 insertions(+), 19 deletions(-)

diff --git a/grid2op/Environment/EnvInterface.py b/grid2op/Environment/EnvInterface.py
index ca306ac7..e53d79e4 100644
--- a/grid2op/Environment/EnvInterface.py
+++ b/grid2op/Environment/EnvInterface.py
@@ -1,3 +1,10 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0.
+# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file,
+# you can obtain one at http://mozilla.org/MPL/2.0/.
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
 from abc import ABC, abstractmethod
 from typing import Tuple, Union
 
@@ -7,7 +14,11 @@
 
 
 class EnvInterface(ABC):
-
+    """
+    This is an interface for Grid2op environments designed to ensure that all implementations (except for multi-environments,
+    which have the same methods but slightly different signatures) define the minimum methods required to interact with
+    an environment.
+    """
     @abstractmethod
     def reset(self,
               *,
diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index c7c0a27a..61666c3d 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -1,6 +1,12 @@
-from datetime import datetime
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0.
+# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file,
+# you can obtain one at http://mozilla.org/MPL/2.0/.
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
 from pathlib import Path
-from typing import Tuple, Union, Callable, Dict, List
+from typing import Tuple, Union, Callable, List
 
 from grid2op.Action import BaseAction
 from grid2op.Environment.EnvInterface import EnvInterface
@@ -12,6 +18,39 @@
 ObservationVectorGetter = Callable[[BaseObservation], List[float]]
 
 class ObservationTable:
+    """
+    A class to accumulate, organize, and write observation data in a columnar format
+    to Parquet files. Designed to handle large-scale data efficiently, buffering
+    observations before writing in chunks.
+
+    This class is intended to facilitate data management for time-stamped observations,
+    appending new observation vectors, and exporting them to disk efficiently to
+    reduce memory usage and improve disk I/O performance over time.
+
+    Attributes
+    ----------
+    _columns : List[str]
+        List of column names representing the structure of the observation vector.
+
+    _getter : ObservationVectorGetter
+        Callable to extract observation vector from a BaseObservation instance.
+
+    _directory : Path
+        Path to the directory where the Parquet file will be stored.
+
+    _table_name : str
+        Name of the output Parquet file (without the extension).
+
+    _write_chunk_size : int
+        Number of rows to buffer before writing to the Parquet file.
+
+    _buffer : List[List]
+        Internal buffer to temporarily store observation data before writing.
+
+    _writer : Optional[pa.parquet.ParquetWriter]
+        Writer object to manage Parquet file I/O operations, lazy initialized.
+
+    """
     def __init__(self, columns: List[str], getter: ObservationVectorGetter, directory: Path, table_name: str, write_chunk_size: int):
         self._columns = columns
         self._getter = getter
@@ -42,19 +81,40 @@ def close(self):
             self._writer = None
 
 class EnvRecorder(EnvInterface):
+    """
+    An environment recorder for capturing and storing environment data.
+
+    This class serves as a wrapper for a given environment and records its
+    observations into Parquet files for later analysis. It ensures that environment
+    data such as observations are properly stored in a structured format.
+
+    Attributes
+    ----------
+
+    _env : EnvInterface
+        The underlying environment to be wrapped and recorded.
+
+    _tables : list of ObservationTable
+        A list of observation tables used to record specific environment
+        observations, such as generator power or load power.
 
+    """
     def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
         super().__init__()
         self._env = env
 
+        # one table for each kind of element
         self.write_element_table([env.name_gen, env.gen_type], ['name', 'type'], directory, 'gen')
         self.write_element_table([env.name_load], ['name'], directory, 'load')
 
-        self._gen_p_before_curtail_table = ObservationTable(self._env.name_gen, lambda obs: obs.gen_p_before_curtail, directory, 'gen_p_before_curtail', write_chunk_size)
-        self._gen_p_table = ObservationTable(self._env.name_gen, lambda obs: obs.gen_p, directory, 'gen_p', write_chunk_size)
-        self._gen_v_table = ObservationTable(self._env.name_gen, lambda obs: obs.gen_v, directory, 'gen_v', write_chunk_size)
-        self._load_p_table = ObservationTable(self._env.name_load, lambda obs: obs.load_p, directory, 'load_p', write_chunk_size)
-        self._load_q_table = ObservationTable(self._env.name_load, lambda obs: obs.load_q, directory, 'load_q', write_chunk_size)
+        # one table per element attributs.
+        self._tables = [
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_p_before_curtail, directory, 'gen_p_before_curtail', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_p, directory, 'gen_p', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_v, directory, 'gen_v', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_p, directory, 'load_p', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_q, directory, 'load_q', write_chunk_size)
+        ]
 
     @staticmethod
     def write_element_table(data, column_names, directory: Path, table_name: str):
@@ -77,21 +137,14 @@ def step(self, action: BaseAction) -> Tuple[BaseObservation,
                                                 STEP_INFO_TYPING]:
         result = self._env.step(action)
         obs = result[0]
-#        print(result[3]['time_series_id'])
-        self._gen_p_before_curtail_table.append(obs)
-        self._gen_p_table.append(obs)
-        self._gen_v_table.append(obs)
-        self._load_p_table.append(obs)
-        self._load_q_table.append(obs)
+        for table in self._tables:
+            table.append(obs)
         return result
 
     def render(self, mode="rgb_array"):
         self._env.render(mode=mode)
 
     def close(self):
-        self._gen_p_before_curtail_table.close()
-        self._gen_p_table.close()
-        self._gen_v_table.close()
-        self._load_p_table.close()
-        self._load_q_table.close()
+        for table in self._tables:
+            table.close()
         self._env.close()
diff --git a/grid2op/tests/test_EnvRecorder.py b/grid2op/tests/test_EnvRecorder.py
index eaa6cf4e..64bbb166 100644
--- a/grid2op/tests/test_EnvRecorder.py
+++ b/grid2op/tests/test_EnvRecorder.py
@@ -1,3 +1,10 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0.
+# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file,
+# you can obtain one at http://mozilla.org/MPL/2.0/.
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
 import unittest
 import warnings
 from pathlib import Path

From e22f9db3cd5b6e7929b7ec2829b3b00db1436a53 Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Wed, 22 Jan 2025 09:21:37 +0100
Subject: [PATCH 03/11] Record actions

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvRecorder.py | 82 ++++++++++++++++++++++--------
 grid2op/tests/test_EnvRecorder.py  |  2 +-
 2 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index 61666c3d..c08b9771 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -5,35 +5,33 @@
 # you can obtain one at http://mozilla.org/MPL/2.0/.
 # SPDX-License-Identifier: MPL-2.0
 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
+from abc import ABC
+from datetime import datetime
 from pathlib import Path
 from typing import Tuple, Union, Callable, List
 
-from grid2op.Action import BaseAction
+from grid2op.Action import BaseAction, baseAction
 from grid2op.Environment.EnvInterface import EnvInterface
 from grid2op.Observation import BaseObservation
 from grid2op.typing_variables import STEP_INFO_TYPING, RESET_OPTIONS_TYPING
 import pyarrow as pa
 import pyarrow.parquet
 
-ObservationVectorGetter = Callable[[BaseObservation], List[float]]
 
-class ObservationTable:
+class AbstractTable(ABC):
     """
-    A class to accumulate, organize, and write observation data in a columnar format
+    A class to accumulate, organize, and write objects data in a columnar format
     to Parquet files. Designed to handle large-scale data efficiently, buffering
-    observations before writing in chunks.
+    objects before writing in chunks.
 
-    This class is intended to facilitate data management for time-stamped observations,
-    appending new observation vectors, and exporting them to disk efficiently to
+    This class is intended to facilitate data management for time-stamped objects,
+    appending new objects vectors, and exporting them to disk efficiently to
     reduce memory usage and improve disk I/O performance over time.
 
     Attributes
     ----------
     _columns : List[str]
-        List of column names representing the structure of the observation vector.
-
-    _getter : ObservationVectorGetter
-        Callable to extract observation vector from a BaseObservation instance.
+        List of column names representing the structure of the object vector.
 
     _directory : Path
         Path to the directory where the Parquet file will be stored.
@@ -51,23 +49,19 @@ class ObservationTable:
         Writer object to manage Parquet file I/O operations, lazy initialized.
 
     """
-    def __init__(self, columns: List[str], getter: ObservationVectorGetter, directory: Path, table_name: str, write_chunk_size: int):
+    def __init__(self, columns: List[str], directory: Path, table_name: str, write_chunk_size: int):
         self._columns = columns
-        self._getter = getter
         self._directory = directory
         self._table_name = table_name
         self._write_chunk_size = write_chunk_size
         self._buffer = [[] for _ in range(len(columns) + 1)]
         self._writer = None
 
-    def append(self, obs: BaseObservation):
-        time = obs.get_time_stamp()
-        self._buffer[0].append(int(time.timestamp()))
-        vec = self._getter(obs)
-        for i in range(len(self._columns)):
-            self._buffer[i + 1].append(vec[i])
+    def reset(self):
+        self.close() # or with discard buffered data ?
 
-        if len(self._buffer[0]) >= self._write_chunk_size:
+    def _flush(self, force: bool):
+        if force or len(self._buffer[0]) >= self._write_chunk_size:
             table = pa.table(self._buffer, ['time'] + list(self._columns))
             if self._writer is None:
                 parquet_file = self._directory / f"{self._table_name}.parquet"
@@ -76,10 +70,43 @@ def append(self, obs: BaseObservation):
             self._buffer = [[] for _ in range(len(self._columns) + 1)] # reset buffer
 
     def close(self):
+        self._flush(True)
         if self._writer is not None:
             self._writer.close()
             self._writer = None
 
+
+ObservationVectorGetter = Callable[[BaseObservation], List[float]]
+
+class ObservationTable(AbstractTable):
+
+    def __init__(self, columns: List[str], getter: ObservationVectorGetter, directory: Path, table_name: str,
+                 write_chunk_size: int):
+        super().__init__(columns, directory, table_name, write_chunk_size)
+        self._getter = getter
+
+    def append(self, obs: BaseObservation):
+        time = obs.get_time_stamp()
+        self._buffer[0].append(int(time.timestamp()))
+
+        vec = self._getter(obs)
+        for i in range(len(self._columns)):
+            self._buffer[i + 1].append(vec[i])
+
+        self._flush(False)
+
+
+class ActionTable(AbstractTable):
+
+    def __init__(self, directory: Path, table_name: str, write_chunk_size: int):
+        super().__init__(['action'], directory, table_name, write_chunk_size)
+
+    def append(self, time: datetime, act: BaseAction):
+        self._buffer[0].append(int(time.timestamp()))
+        self._buffer[1].append(str(act.as_dict()))
+        self._flush(False)
+
+
 class EnvRecorder(EnvInterface):
     """
     An environment recorder for capturing and storing environment data.
@@ -102,6 +129,7 @@ class EnvRecorder(EnvInterface):
     def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
         super().__init__()
         self._env = env
+        self._directory = directory
 
         # one table for each kind of element
         self.write_element_table([env.name_gen, env.gen_type], ['name', 'type'], directory, 'gen')
@@ -116,6 +144,8 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
             ObservationTable(self._env.name_load, lambda obs: obs.load_q, directory, 'load_q', write_chunk_size)
         ]
 
+        self._actions_table = ActionTable(directory, 'actions', write_chunk_size)
+
     @staticmethod
     def write_element_table(data, column_names, directory: Path, table_name: str):
         element_table = pa.table({col: data[i] for i, col in enumerate(column_names)})
@@ -129,6 +159,11 @@ def reset(self,
               *,
               seed: Union[int, None] = None,
               options: RESET_OPTIONS_TYPING = None) -> BaseObservation:
+        for table in self._tables:
+            table.reset()
+
+        self._actions_table.reset()
+
         return self._env.reset(seed=seed, options=options)
 
     def step(self, action: BaseAction) -> Tuple[BaseObservation,
@@ -137,8 +172,12 @@ def step(self, action: BaseAction) -> Tuple[BaseObservation,
                                                 STEP_INFO_TYPING]:
         result = self._env.step(action)
         obs = result[0]
+
         for table in self._tables:
             table.append(obs)
+
+        self._actions_table.append(obs.get_time_stamp(), action)
+
         return result
 
     def render(self, mode="rgb_array"):
@@ -147,4 +186,7 @@ def render(self, mode="rgb_array"):
     def close(self):
         for table in self._tables:
             table.close()
+
+        self._actions_table.close()
+
         self._env.close()
diff --git a/grid2op/tests/test_EnvRecorder.py b/grid2op/tests/test_EnvRecorder.py
index 64bbb166..7067e363 100644
--- a/grid2op/tests/test_EnvRecorder.py
+++ b/grid2op/tests/test_EnvRecorder.py
@@ -46,5 +46,5 @@ def test_recording(self):
                     assert pq_file.is_file()
 
                 gen_p_pq = pd.read_parquet(tmp_dir_path / "gen_p.parquet")
-                assert gen_p_pq.shape == (93, 3)
+                assert gen_p_pq.shape == (95, 3)
                 assert gen_p_pq.columns.tolist() == ['time', 'gen_0_0', 'gen_1_1']

From a75ab258fc7acea1e9603bd754150455904fbed2 Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Wed, 14 May 2025 09:17:34 +0200
Subject: [PATCH 04/11] Add more attributes

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvRecorder.py | 75 ++++++++++++++++++++++++------
 1 file changed, 61 insertions(+), 14 deletions(-)

diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index c08b9771..083bf505 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -5,17 +5,19 @@
 # you can obtain one at http://mozilla.org/MPL/2.0/.
 # SPDX-License-Identifier: MPL-2.0
 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
+import json
 from abc import ABC
 from datetime import datetime
 from pathlib import Path
 from typing import Tuple, Union, Callable, List
 
-from grid2op.Action import BaseAction, baseAction
+import pyarrow as pa
+import pyarrow.parquet
+
+from grid2op.Action import BaseAction
 from grid2op.Environment.EnvInterface import EnvInterface
 from grid2op.Observation import BaseObservation
 from grid2op.typing_variables import STEP_INFO_TYPING, RESET_OPTIONS_TYPING
-import pyarrow as pa
-import pyarrow.parquet
 
 
 class AbstractTable(ABC):
@@ -103,7 +105,8 @@ def __init__(self, directory: Path, table_name: str, write_chunk_size: int):
 
     def append(self, time: datetime, act: BaseAction):
         self._buffer[0].append(int(time.timestamp()))
-        self._buffer[1].append(str(act.as_dict()))
+        json_str = json.dumps(act.as_serializable_dict())
+        self._buffer[1].append(json_str)
         self._flush(False)
 
 
@@ -131,17 +134,56 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
         self._env = env
         self._directory = directory
 
+        # save the grid
+        grid_path = Path(self._env._init_grid_path)
+        (directory / grid_path.name).write_bytes(grid_path.read_bytes())
+
+        # env general data
+        env_data = {
+            "n_sub": env.n_sub,
+            "n_busbar_per_sub": env.n_busbar_per_sub
+        }
+        with open(directory / "env.json", "w") as f:
+            json.dump(env_data, f, indent=4)
+
         # one table for each kind of element
-        self.write_element_table([env.name_gen, env.gen_type], ['name', 'type'], directory, 'gen')
-        self.write_element_table([env.name_load], ['name'], directory, 'load')
+        self.write_element_table([env.name_gen, env.gen_type, env.gen_to_subid], ['name', 'type', 'gen_to_subid'], directory, 'gen')
+        self.write_element_table([env.name_load, env.load_to_subid], ['name', 'load_to_subid'], directory, 'load')
+        self.write_element_table([env.name_shunt, env.shunt_to_subid], ['name', 'shunt_to_subid'], directory, 'shunt')
+        self.write_element_table([env.name_line, env.line_or_to_subid, env.line_ex_to_subid], ['name', 'line_or_to_subid', 'line_ex_to_subid'], directory, 'line')
 
         # one table per element attributs.
         self._tables = [
             ObservationTable(self._env.name_gen, lambda obs: obs.gen_p_before_curtail, directory, 'gen_p_before_curtail', write_chunk_size),
             ObservationTable(self._env.name_gen, lambda obs: obs.gen_p, directory, 'gen_p', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_p_detached, directory, 'gen_p_detached', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_q, directory, 'gen_q', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_bus, directory, 'gen_bus', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_detached, directory, 'gen_detached', write_chunk_size),
             ObservationTable(self._env.name_gen, lambda obs: obs.gen_v, directory, 'gen_v', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.gen_theta, directory, 'gen_theta', write_chunk_size),
+
             ObservationTable(self._env.name_load, lambda obs: obs.load_p, directory, 'load_p', write_chunk_size),
-            ObservationTable(self._env.name_load, lambda obs: obs.load_q, directory, 'load_q', write_chunk_size)
+            ObservationTable(self._env.name_load, lambda obs: obs.load_p_detached, directory, 'load_p_detached', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_q, directory, 'load_q', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_q_detached, directory, 'load_q_detached', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_bus, directory, 'load_bus', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_v, directory, 'load_v', write_chunk_size),
+            ObservationTable(self._env.name_load, lambda obs: obs.load_theta, directory, 'load_theta', write_chunk_size),
+
+            ObservationTable(self._env.name_line, lambda obs: obs.line_or_bus, directory, 'line_or_bus', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.line_ex_bus, directory, 'line_ex_bus', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.line_ex_bus, directory, 'line_status', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.p_or, directory, 'line_or_p', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.p_ex, directory, 'line_ex_p', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.q_or, directory, 'line_or_q', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.q_ex, directory, 'line_ex_q', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.v_or, directory, 'line_or_v', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.v_ex, directory, 'line_ex_v', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.theta_or, directory, 'line_or_theta', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.theta_ex, directory, 'line_ex_theta', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.rho, directory, 'line_rho', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.thermal_limit, directory, 'line_thermal_limit', write_chunk_size)
         ]
 
         self._actions_table = ActionTable(directory, 'actions', write_chunk_size)
@@ -164,19 +206,24 @@ def reset(self,
 
         self._actions_table.reset()
 
-        return self._env.reset(seed=seed, options=options)
+        obs = self._env.reset(seed=seed, options=options)
+        self._append_obs(obs)
+        self._actions_table.append(obs.get_time_stamp(), self._env.action_space())
+        return obs
+
+    def _append_obs(self, obs: BaseObservation):
+        for table in self._tables:
+            table.append(obs)
 
     def step(self, action: BaseAction) -> Tuple[BaseObservation,
                                                 float,
                                                 bool,
                                                 STEP_INFO_TYPING]:
         result = self._env.step(action)
-        obs = result[0]
-
-        for table in self._tables:
-            table.append(obs)
-
-        self._actions_table.append(obs.get_time_stamp(), action)
+        if not result[2]:
+            obs = result[0]
+            self._append_obs(obs)
+            self._actions_table.append(obs.get_time_stamp(), action)
 
         return result
 

From 1fceedfddfbc79da3495d0b5e72834e871af2c96 Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Tue, 20 May 2025 15:43:49 +0200
Subject: [PATCH 05/11] Add more attributes

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvRecorder.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index 083bf505..a08b8277 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -101,12 +101,13 @@ def append(self, obs: BaseObservation):
 class ActionTable(AbstractTable):
 
     def __init__(self, directory: Path, table_name: str, write_chunk_size: int):
-        super().__init__(['action'], directory, table_name, write_chunk_size)
+        super().__init__(['action', 'done'], directory, table_name, write_chunk_size)
 
-    def append(self, time: datetime, act: BaseAction):
+    def append(self, time: datetime, act: BaseAction, done: bool):
         self._buffer[0].append(int(time.timestamp()))
         json_str = json.dumps(act.as_serializable_dict())
         self._buffer[1].append(json_str)
+        self._buffer[2].append(done)
         self._flush(False)
 
 
@@ -162,6 +163,8 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
             ObservationTable(self._env.name_gen, lambda obs: obs.gen_detached, directory, 'gen_detached', write_chunk_size),
             ObservationTable(self._env.name_gen, lambda obs: obs.gen_v, directory, 'gen_v', write_chunk_size),
             ObservationTable(self._env.name_gen, lambda obs: obs.gen_theta, directory, 'gen_theta', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.actual_dispatch, directory, 'gen_actual_dispatch', write_chunk_size),
+            ObservationTable(self._env.name_gen, lambda obs: obs.target_dispatch, directory, 'gen_target_dispatch', write_chunk_size),
 
             ObservationTable(self._env.name_load, lambda obs: obs.load_p, directory, 'load_p', write_chunk_size),
             ObservationTable(self._env.name_load, lambda obs: obs.load_p_detached, directory, 'load_p_detached', write_chunk_size),
@@ -208,7 +211,7 @@ def reset(self,
 
         obs = self._env.reset(seed=seed, options=options)
         self._append_obs(obs)
-        self._actions_table.append(obs.get_time_stamp(), self._env.action_space())
+        self._actions_table.append(obs.get_time_stamp(), self._env.action_space(), False)
         return obs
 
     def _append_obs(self, obs: BaseObservation):
@@ -220,11 +223,10 @@ def step(self, action: BaseAction) -> Tuple[BaseObservation,
                                                 bool,
                                                 STEP_INFO_TYPING]:
         result = self._env.step(action)
-        if not result[2]:
-            obs = result[0]
-            self._append_obs(obs)
-            self._actions_table.append(obs.get_time_stamp(), action)
-
+        done = result[2]
+        obs = result[0]
+        self._append_obs(obs)
+        self._actions_table.append(obs.get_time_stamp(), action, done)
         return result
 
     def render(self, mode="rgb_array"):

From b98c8c4f0e5a7ff5e776a37705528537a7589131 Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Tue, 20 May 2025 16:30:23 +0200
Subject: [PATCH 06/11] Add more attributes

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvRecorder.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index a08b8277..6bf6e25a 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -135,12 +135,11 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
         self._env = env
         self._directory = directory
 
-        # save the grid
-        grid_path = Path(self._env._init_grid_path)
-        (directory / grid_path.name).write_bytes(grid_path.read_bytes())
-
         # env general data
         env_data = {
+            "name": env.name,
+            "path": self._env._init_env_path,
+            "backend": self._env.backend.__class__.__name__,
             "n_sub": env.n_sub,
             "n_busbar_per_sub": env.n_busbar_per_sub
         }
@@ -151,6 +150,7 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
         self.write_element_table([env.name_gen, env.gen_type, env.gen_to_subid], ['name', 'type', 'gen_to_subid'], directory, 'gen')
         self.write_element_table([env.name_load, env.load_to_subid], ['name', 'load_to_subid'], directory, 'load')
         self.write_element_table([env.name_shunt, env.shunt_to_subid], ['name', 'shunt_to_subid'], directory, 'shunt')
+        self.write_element_table([env.name_storage, env.storage_to_subid], ['name', 'storage_to_subid'], directory, 'storage')
         self.write_element_table([env.name_line, env.line_or_to_subid, env.line_ex_to_subid], ['name', 'line_or_to_subid', 'line_ex_to_subid'], directory, 'line')
 
         # one table per element attributs.
@@ -174,13 +174,28 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
             ObservationTable(self._env.name_load, lambda obs: obs.load_v, directory, 'load_v', write_chunk_size),
             ObservationTable(self._env.name_load, lambda obs: obs.load_theta, directory, 'load_theta', write_chunk_size),
 
+            ObservationTable(self._env.name_shunt, lambda obs: obs._shunt_p, directory, 'shunt_p', write_chunk_size),
+            ObservationTable(self._env.name_shunt, lambda obs: obs._shunt_q, directory, 'shunt_q', write_chunk_size),
+            ObservationTable(self._env.name_shunt, lambda obs: obs._shunt_v, directory, 'shunt_v', write_chunk_size),
+            ObservationTable(self._env.name_shunt, lambda obs: obs._shunt_bus, directory, 'shunt_bus', write_chunk_size),
+
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_power_target, directory, 'storage_power_target', write_chunk_size),
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_power, directory, 'storage_power', write_chunk_size),
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_charge, directory, 'storage_charge', write_chunk_size),
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_theta, directory, 'storage_theta', write_chunk_size),
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_detached, directory, 'storage_detached', write_chunk_size),
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_p_detached, directory, 'storage_p_detached', write_chunk_size),
+            ObservationTable(self._env.name_storage, lambda obs: obs.storage_bus, directory, 'storage_bus', write_chunk_size),
+
             ObservationTable(self._env.name_line, lambda obs: obs.line_or_bus, directory, 'line_or_bus', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.line_ex_bus, directory, 'line_ex_bus', write_chunk_size),
-            ObservationTable(self._env.name_line, lambda obs: obs.line_ex_bus, directory, 'line_status', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.line_status, directory, 'line_status', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.p_or, directory, 'line_or_p', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.p_ex, directory, 'line_ex_p', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.q_or, directory, 'line_or_q', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.q_ex, directory, 'line_ex_q', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.a_or, directory, 'line_or_a', write_chunk_size),
+            ObservationTable(self._env.name_line, lambda obs: obs.a_ex, directory, 'line_ex_a', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.v_or, directory, 'line_or_v', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.v_ex, directory, 'line_ex_v', write_chunk_size),
             ObservationTable(self._env.name_line, lambda obs: obs.theta_or, directory, 'line_or_theta', write_chunk_size),

From e440939790f36920a5fc71f82308c5516f88dd9d Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Tue, 20 May 2025 16:33:57 +0200
Subject: [PATCH 07/11] Fix setup

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 1e9d3f50..5b8124a6 100644
--- a/setup.py
+++ b/setup.py
@@ -32,7 +32,7 @@ def my_test_suite():
         "requests>=2.23.0",
         "packaging",  # because gym changes the way it uses numpy prng in version 0.26 and i need both gym before and after...
         "typing_extensions",
-        "orderly_set<5.4.0; python_version<='3.8'"
+        "orderly_set<5.4.0; python_version<='3.8'",
         "pyarrow>=18.1.0"
     ],
     "extras": {

From 1ca62e854bf0d553c68baeab4b4a2844a922191a Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Tue, 20 May 2025 16:39:45 +0200
Subject: [PATCH 08/11] Downgrade pyarrow version to be compatible with 1.7.0

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5b8124a6..4952b366 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,7 @@ def my_test_suite():
         "packaging",  # because gym changes the way it uses numpy prng in version 0.26 and i need both gym before and after...
         "typing_extensions",
         "orderly_set<5.4.0; python_version<='3.8'",
-        "pyarrow>=18.1.0"
+        "pyarrow>=17.0.0"
     ],
     "extras": {
         "optional": [

From 647472dd6af123371dca16eacc1de586660b6895 Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Wed, 21 May 2025 09:23:52 +0200
Subject: [PATCH 09/11] Fix unit test

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/Environment/EnvRecorder.py |  4 +-
 grid2op/tests/test_EnvRecorder.py  | 59 ++++++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index 6bf6e25a..1bd33bca 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -17,6 +17,7 @@
 from grid2op.Action import BaseAction
 from grid2op.Environment.EnvInterface import EnvInterface
 from grid2op.Observation import BaseObservation
+from grid2op.Space import GRID2OP_CURRENT_VERSION_STR
 from grid2op.typing_variables import STEP_INFO_TYPING, RESET_OPTIONS_TYPING
 
 
@@ -63,7 +64,7 @@ def reset(self):
         self.close() # or with discard buffered data ?
 
     def _flush(self, force: bool):
-        if force or len(self._buffer[0]) >= self._write_chunk_size:
+        if len(self._buffer[0]) > 0 and (force or len(self._buffer[0]) >= self._write_chunk_size):
             table = pa.table(self._buffer, ['time'] + list(self._columns))
             if self._writer is None:
                 parquet_file = self._directory / f"{self._table_name}.parquet"
@@ -137,6 +138,7 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
 
         # env general data
         env_data = {
+            "grid2op_version": GRID2OP_CURRENT_VERSION_STR,
             "name": env.name,
             "path": self._env._init_env_path,
             "backend": self._env.backend.__class__.__name__,
diff --git a/grid2op/tests/test_EnvRecorder.py b/grid2op/tests/test_EnvRecorder.py
index 7067e363..d212c2d6 100644
--- a/grid2op/tests/test_EnvRecorder.py
+++ b/grid2op/tests/test_EnvRecorder.py
@@ -12,7 +12,7 @@
 
 import pandas as pd
 
-from getting_started import grid2op
+import grid2op
 from grid2op.Backend import PandaPowerBackend
 from grid2op.Environment.EnvRecorder import EnvRecorder
 
@@ -41,10 +41,61 @@ def test_recording(self):
                     while not done:
                         _, _, done, _ = env_rec.step(do_nothing)
 
-                for file_name in ['gen_p_before_curtail', 'gen_p', 'gen_v', 'load_p', 'load_q']:
-                    pq_file = tmp_dir_path / f"{file_name}.parquet"
+                # check all files have been generated
+                for file_name in ['gen_detached.parquet',
+                                  'line_ex_q.parquet',
+                                  'storage.parquet',
+                                  'storage_detached.parquet',
+                                  'line_rho.parquet',
+                                  'gen_p_before_curtail.parquet',
+                                  'line_or_bus.parquet',
+                                  'line_ex_theta.parquet',
+                                  'load_q.parquet',
+                                  'line_ex_a.parquet',
+                                  'line_or_v.parquet',
+                                  'line_or_q.parquet',
+                                  'line_or_theta.parquet',
+                                  'line_or_p.parquet',
+                                  'storage_power.parquet',
+                                  'load_p.parquet',
+                                  'gen_theta.parquet',
+                                  'line_ex_v.parquet',
+                                  'shunt_bus.parquet',
+                                  'line_ex_p.parquet',
+                                  'storage_p_detached.parquet',
+                                  'gen_bus.parquet',
+                                  'line_thermal_limit.parquet',
+                                  'load_p_detached.parquet',
+                                  'gen_actual_dispatch.parquet',
+                                  'shunt_q.parquet',
+                                  'line_or_a.parquet',
+                                  'gen_q.parquet',
+                                  'storage_theta.parquet',
+                                  'gen.parquet',
+                                  'load_v.parquet',
+                                  'gen_p.parquet',
+                                  'load.parquet',
+                                  'storage_power_target.parquet',
+                                  'load_q_detached.parquet',
+                                  'shunt_v.parquet',
+                                  'line_status.parquet',
+                                  'gen_target_dispatch.parquet',
+                                  'shunt.parquet',
+                                  'storage_charge.parquet',
+                                  'env.json',
+                                  'line.parquet',
+                                  'load_theta.parquet',
+                                  'storage_bus.parquet',
+                                  'load_bus.parquet',
+                                  'gen_v.parquet',
+                                  'line_ex_bus.parquet',
+                                  'gen_p_detached.parquet',
+                                  'shunt_p.parquet',
+                                  'actions.parquet']:
+                    pq_file = tmp_dir_path / f"{file_name}"
                     assert pq_file.is_file()
 
+                # check one of the file content
                 gen_p_pq = pd.read_parquet(tmp_dir_path / "gen_p.parquet")
-                assert gen_p_pq.shape == (95, 3)
+                assert gen_p_pq.shape == (96, 3)
                 assert gen_p_pq.columns.tolist() == ['time', 'gen_0_0', 'gen_1_1']

From f1e92ee8b2273bc32bb53ee122fd8b3bb6d5841a Mon Sep 17 00:00:00 2001
From: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
Date: Wed, 21 May 2025 09:29:48 +0200
Subject: [PATCH 10/11] Improve unit test

Signed-off-by: Geoffroy Jamgotchian <geoffroy.jamgotchian@rte-france.com>
---
 grid2op/tests/test_EnvRecorder.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/grid2op/tests/test_EnvRecorder.py b/grid2op/tests/test_EnvRecorder.py
index d212c2d6..7ba41cb8 100644
--- a/grid2op/tests/test_EnvRecorder.py
+++ b/grid2op/tests/test_EnvRecorder.py
@@ -5,6 +5,7 @@
 # you can obtain one at http://mozilla.org/MPL/2.0/.
 # SPDX-License-Identifier: MPL-2.0
 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
+import json
 import unittest
 import warnings
 from pathlib import Path
@@ -95,7 +96,17 @@ def test_recording(self):
                     pq_file = tmp_dir_path / f"{file_name}"
                     assert pq_file.is_file()
 
-                # check one of the file content
+                # check one of the table file content
                 gen_p_pq = pd.read_parquet(tmp_dir_path / "gen_p.parquet")
                 assert gen_p_pq.shape == (96, 3)
                 assert gen_p_pq.columns.tolist() == ['time', 'gen_0_0', 'gen_1_1']
+
+                # check the environment infos file content
+                with open(tmp_dir_path / "env.json", "r", encoding="utf-8") as f:
+                    env_infos = json.load(f)
+                    assert env_infos['grid2op_version']
+                    assert env_infos['name'] == 'rte_case5_examplePandaPowerBackendTestEnvRecorder'
+                    assert env_infos['path']
+                    assert env_infos['backend'] == 'PandaPowerBackend_rte_case5_examplePandaPowerBackendTestEnvRecorder'
+                    assert env_infos['n_sub'] == 5
+                    assert env_infos['n_busbar_per_sub'] == 2

From 857542d4dd4ca3d8cf8985d1ff8ab4065b3615f7 Mon Sep 17 00:00:00 2001
From: Giovanni Ferrari <giovanni.ferrari@soft.it>
Date: Mon, 11 Aug 2025 16:49:28 +0200
Subject: [PATCH 11/11] Apply changes after review

Signed-off-by: Giovanni Ferrari <giovanni.ferrari@soft.it>
---
 grid2op/Environment/EnvInterface.py | 272 ----------------------------
 grid2op/Environment/EnvRecorder.py  |  20 +-
 grid2op/Environment/baseEnv.py      |   2 +-
 grid2op/Environment/environment.py  | 272 ++++++++++++++++++++++++++++
 setup.py                            |   4 +-
 5 files changed, 289 insertions(+), 281 deletions(-)

diff --git a/grid2op/Environment/EnvInterface.py b/grid2op/Environment/EnvInterface.py
index e53d79e4..468e0339 100644
--- a/grid2op/Environment/EnvInterface.py
+++ b/grid2op/Environment/EnvInterface.py
@@ -24,278 +24,6 @@ def reset(self,
               *,
               seed: Union[int, None] = None,
               options: RESET_OPTIONS_TYPING = None) -> BaseObservation:
-        """
-        Reset the environment to a clean state.
-        It will reload the next chronics if any. And reset the grid to a clean state.
-
-        This triggers a full reloading of both the chronics (if they are stored as files) and of the powergrid,
-        to ensure the episode is fully over.
-
-        This method should be called only at the end of an episode.
-
-        Parameters
-        ----------
-        seed: int
-            The seed to used (new in version 1.9.8), see examples for more details. Ignored if not set (meaning no seeds will
-            be used, experiments might not be reproducible)
-
-        options: dict
-            Some options to "customize" the reset call. For example (see detailed example bellow) :
-
-            - "time serie id" (grid2op >= 1.9.8) to use a given time serie from the input data
-            - "init state" that allows you to apply a given "action" when generating the
-              initial observation (grid2op >= 1.10.2)
-            - "init ts" (grid2op >= 1.10.3) to specify to which "steps" of the time series
-              the episode will start
-            - "max step" (grid2op >= 1.10.3) : maximum number of steps allowed for the episode
-            - "thermal limit" (grid2op >= 1.11.0): which thermal limit to use for this episode
-              (and the next ones, until they are changed)
-            - "init datetime": which time stamp is used in the first observation of the episode.
-
-            See examples for more information about this. Ignored if
-            not set.
-
-        Examples
-        --------
-        The standard "gym loop" can be done with the following code:
-
-        .. code-block:: python
-
-            import grid2op
-
-            # create the environment
-            env_name = "l2rpn_case14_sandbox"
-            env = grid2op.make(env_name)
-
-            # start a new episode
-            obs = env.reset()
-            done = False
-            reward = env.reward_range[0]
-            while not done:
-                action = agent.act(obs, reward, done)
-                obs, reward, done, info = env.step(action)
-
-        .. versionadded:: 1.9.8
-            It is now possible to set the seed and the time series you want to use at the new
-            episode by calling `env.reset(seed=..., options={"time serie id": ...})`
-
-        Before version 1.9.8, if you wanted to use a fixed seed, you would need to (see
-        doc of :func:`grid2op.Environment.BaseEnv.seed` ):
-
-        .. code-block:: python
-
-            seed = ...
-            env.seed(seed)
-            obs = env.reset()
-            ...
-
-        Starting from version 1.9.8 you can do this in one call:
-
-        .. code-block:: python
-
-            seed = ...
-            obs = env.reset(seed=seed)
-
-        For the "time series id" it is the same concept. Before you would need to do (see
-        doc of :func:`Environment.set_id` for more information ):
-
-        .. code-block:: python
-
-            time_serie_id = ...
-            env.set_id(time_serie_id)
-            obs = env.reset()
-            ...
-
-        And now (from version 1.9.8) you can more simply do:
-
-        .. code-block:: python
-
-            time_serie_id = ...
-            obs = env.reset(options={"time serie id": time_serie_id})
-            ...
-
-        .. versionadded:: 1.10.2
-
-        Another feature has been added in version 1.10.2, which is the possibility to set the
-        grid to a given "topological" state at the first observation (before this version,
-        you could only retrieve an observation with everything connected together).
-
-        In grid2op 1.10.2, you can do that by using the keys `"init state"` in the "options" kwargs of
-        the reset function. The value associated to this key should be dictionnary that can be
-        converted to a non ambiguous grid2op action using an "action space".
-
-        .. note::
-            The "action space" used here is not the action space of the agent. It's an "action
-            space" that uses a :func:`grid2op.Action.Action.BaseAction` class meaning you can do any
-            type of action, on shunts, on topology, on line status etc. even if the agent is not
-            allowed to.
-
-            Likewise, nothing check if this action is legal or not.
-
-        You can use it like this:
-
-        .. code-block:: python
-
-            # to start an episode with a line disconnected, you can do:
-            init_state_dict = {"set_line_status": [(0, -1)]}
-            obs = env.reset(options={"init state": init_state_dict})
-            obs.line_status[0] is False
-
-            # to start an episode with a different topolovy
-            init_state_dict = {"set_bus": {"lines_or_id": [(0, 2)], "lines_ex_id": [(3, 2)]}}
-            obs = env.reset(options={"init state": init_state_dict})
-
-        .. note::
-            Since grid2op version 1.10.2, there is also the possibility to set the "initial state"
-            of the grid directly in the time series. The priority is always given to the
-            argument passed in the "options" value.
-
-            Concretely if, in the "time series" (formelly called "chronics") provides an action would change
-            the topology of substation 1 and 2 (for example) and you provide an action that disable the
-            line 6, then the initial state will see substation 1 and 2 changed (as in the time series)
-            and line 6 disconnected.
-
-            Another example in this case: if the action you provide would change topology of substation 2 and 4
-            then the initial state (after `env.reset`) will give:
-
-            - substation 1 as in the time serie
-            - substation 2 as in "options"
-            - substation 4 as in "options"
-
-        .. note::
-            Concerning the previously described behaviour, if you want to ignore the data in the
-            time series, you can add : `"method": "ignore"` in the dictionary describing the action.
-            In this case the action in the time series will be totally ignored and the initial
-            state will be fully set by the action passed in the "options" dict.
-
-            An example is:
-
-            .. code-block:: python
-
-                init_state_dict = {"set_line_status": [(0, -1)], "method": "force"}
-                obs = env.reset(options={"init state": init_state_dict})
-                obs.line_status[0] is False
-
-        .. versionadded:: 1.10.3
-
-        Another feature has been added in version 1.10.3, the possibility to skip the
-        some steps of the time series and starts at some given steps.
-
-        The time series often always start at a given day of the week (*eg* Monday)
-        and at a given time (*eg* midnight). But for some reason you notice that your
-        agent performs poorly on other day of the week or time of the day. This might be
-        because it has seen much more data from Monday at midnight that from any other
-        day and hour of the day.
-
-        To alleviate this issue, you can now easily reset an episode and ask grid2op
-        to start this episode after xxx steps have "passed".
-
-        Concretely, you can do it with:
-
-        .. code-block:: python
-
-            import grid2op
-            env_name = "l2rpn_case14_sandbox"
-            env = grid2op.make(env_name)
-
-            obs = env.reset(options={"init ts": 1})
-
-        Doing that your agent will start its episode not at midnight (which
-        is the case for this environment), but at 00:05
-
-        If you do:
-
-        .. code-block:: python
-
-            obs = env.reset(options={"init ts": 12})
-
-        In this case, you start the episode at 01:00 and not at midnight (you
-        start at what would have been the 12th steps)
-
-        If you want to start the "next day", you can do:
-
-        .. code-block:: python
-
-            obs = env.reset(options={"init ts": 288})
-
-        etc.
-
-        .. note::
-            On this feature, if a powerline is on soft overflow (meaning its flow is above
-            the limit but below the :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`)
-            then it is still connected (of course) and the counter
-            :attr:`grid2op.Observation.BaseObservation.timestep_overflow` is at 0.
-
-            If a powerline is on "hard overflow" (meaning its flow would be above
-            :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`), then, as it is
-            the case for a "normal" (without options) reset, this line is disconnected, but can be reconnected
-            directly (:attr:`grid2op.Observation.BaseObservation.time_before_cooldown_line` == 0)
-
-        .. seealso::
-            The function :func:`Environment.fast_forward_chronics` for an alternative usage (that will be
-            deprecated at some point)
-
-        Yet another feature has been added in grid2op version 1.10.3 in this `env.reset` function. It is
-        the capacity to limit the duration of an episode.
-
-        .. code-block:: python
-
-            import grid2op
-            env_name = "l2rpn_case14_sandbox"
-            env = grid2op.make(env_name)
-
-            obs = env.reset(options={"max step": 288})
-
-        This will limit the duration to 288 steps (1 day), meaning your agent
-        will have successfully managed the entire episode if it manages to keep
-        the grid in a safe state for a whole day (depending on the environment you are
-        using the default duration is either one week - roughly 2016 steps or 4 weeks)
-
-        .. note::
-            This option only affect the current episode. It will have no impact on the
-            next episode (after reset)
-
-        For example:
-
-        .. code-block:: python
-
-            obs = env.reset()
-            obs.max_step == 8064  # default for this environment
-
-            obs = env.reset(options={"max step": 288})
-            obs.max_step == 288  # specified by the option
-
-            obs = env.reset()
-            obs.max_step == 8064  # retrieve the default behaviour
-
-        .. seealso::
-            The function :func:`Environment.set_max_iter` for an alternative usage with the different
-            that `set_max_iter` is permenanent: it impacts all the future episodes and not only
-            the next one.
-
-        If you want your environment to start at a given time stamp you can do:
-
-        .. code-block:: python
-
-            import grid2op
-            env_name = "l2rpn_case14_sandbox"
-
-            env = grid2op.make(env_name)
-            obs = env.reset(options={"init datetime": "2024-12-06 00:00"})
-            obs.year == 2024
-            obs.month == 12
-            obs.day == 6
-
-        .. seealso::
-            If you specify "init datetime" then the observation resulting to the
-            `env.reset` call will have this datetime. If you specify also `"skip ts"`
-            option the behaviour does not change: the first observation will
-            have the date time attributes you specified.
-
-            In other words, the "init datetime" refers to the initial observation of the
-            episode and NOT the initial time present in the time series.
-
-        """
         pass
 
     @abstractmethod
diff --git a/grid2op/Environment/EnvRecorder.py b/grid2op/Environment/EnvRecorder.py
index 1bd33bca..6c8124b2 100644
--- a/grid2op/Environment/EnvRecorder.py
+++ b/grid2op/Environment/EnvRecorder.py
@@ -10,9 +10,14 @@
 from datetime import datetime
 from pathlib import Path
 from typing import Tuple, Union, Callable, List
+try:
+    import pyarrow as pa
+    import pyarrow.parquet
+except ImportError:
+    print("pyarrow is not installed. Please install it to use EnvRecorder.")
+    import sys
+    sys.exit(1)
 
-import pyarrow as pa
-import pyarrow.parquet
 
 from grid2op.Action import BaseAction
 from grid2op.Environment.EnvInterface import EnvInterface
@@ -119,6 +124,8 @@ class EnvRecorder(EnvInterface):
     This class serves as a wrapper for a given environment and records its
     observations into Parquet files for later analysis. It ensures that environment
     data such as observations are properly stored in a structured format.
+    All the genrated Parquet files are stored in the configured output directory.
+    This class does not save simulation (obs.simulate) or forecast (obs.get_forecast_env) data.
 
     Attributes
     ----------
@@ -139,6 +146,7 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
         # env general data
         env_data = {
             "grid2op_version": GRID2OP_CURRENT_VERSION_STR,
+            "env_grid2op_version": type(env).glop_version,
             "name": env.name,
             "path": self._env._init_env_path,
             "backend": self._env.backend.__class__.__name__,
@@ -149,11 +157,11 @@ def __init__(self, env, directory: Path, write_chunk_size: int = 1000):
             json.dump(env_data, f, indent=4)
 
         # one table for each kind of element
-        self.write_element_table([env.name_gen, env.gen_type, env.gen_to_subid], ['name', 'type', 'gen_to_subid'], directory, 'gen')
-        self.write_element_table([env.name_load, env.load_to_subid], ['name', 'load_to_subid'], directory, 'load')
+        self.write_element_table([env.name_gen, env.gen_type, env.gen_to_subid, env.gen_pos_topo_vect], ['name', 'type', 'gen_to_subid', 'gen_pos_topo_vect'], directory, 'gen')
+        self.write_element_table([env.name_load, env.load_to_subid, env.load_pos_topo_vect], ['name', 'load_to_subid', 'load_pos_topo_vect'], directory, 'load')
         self.write_element_table([env.name_shunt, env.shunt_to_subid], ['name', 'shunt_to_subid'], directory, 'shunt')
-        self.write_element_table([env.name_storage, env.storage_to_subid], ['name', 'storage_to_subid'], directory, 'storage')
-        self.write_element_table([env.name_line, env.line_or_to_subid, env.line_ex_to_subid], ['name', 'line_or_to_subid', 'line_ex_to_subid'], directory, 'line')
+        self.write_element_table([env.name_storage, env.storage_to_subid, env.storage_pos_topo_vect], ['name', 'storage_to_subid', 'storage_pos_topo_vect'], directory, 'storage')
+        self.write_element_table([env.name_line, env.line_or_to_subid, env.line_ex_to_subid, env.line_or_pos_topo_vect, env.line_ex_pos_topo_vect], ['name', 'line_or_to_subid', 'line_ex_to_subid', 'line_or_pos_topo_vect', 'line_ex_pos_topo_vect'], directory, 'line')
 
         # one table per element attributs.
         self._tables = [
diff --git a/grid2op/Environment/baseEnv.py b/grid2op/Environment/baseEnv.py
index 8281d96b..7e8039ba 100644
--- a/grid2op/Environment/baseEnv.py
+++ b/grid2op/Environment/baseEnv.py
@@ -91,7 +91,7 @@
 # WE DO NOT RECOMMEND TO ALTER IT IN ANY WAY
 """
 
-class BaseEnv(EnvInterface, GridObjects, RandomObject, ABC):
+class BaseEnv(GridObjects, EnvInterface, RandomObject, ABC):
     """
     INTERNAL
 
diff --git a/grid2op/Environment/environment.py b/grid2op/Environment/environment.py
index 6871197f..096dcb79 100644
--- a/grid2op/Environment/environment.py
+++ b/grid2op/Environment/environment.py
@@ -1066,6 +1066,278 @@ def reset(self,
               *,
               seed: Union[int, None] = None,
               options: RESET_OPTIONS_TYPING = None) -> BaseObservation:
+        """
+        Reset the environment to a clean state.
+        It will reload the next chronics if any. And reset the grid to a clean state.
+
+        This triggers a full reloading of both the chronics (if they are stored as files) and of the powergrid,
+        to ensure the episode is fully over.
+
+        This method should be called only at the end of an episode.
+
+        Parameters
+        ----------
+        seed: int
+            The seed to used (new in version 1.9.8), see examples for more details. Ignored if not set (meaning no seeds will
+            be used, experiments might not be reproducible)
+
+        options: dict
+            Some options to "customize" the reset call. For example (see detailed example bellow) :
+
+            - "time serie id" (grid2op >= 1.9.8) to use a given time serie from the input data
+            - "init state" that allows you to apply a given "action" when generating the
+              initial observation (grid2op >= 1.10.2)
+            - "init ts" (grid2op >= 1.10.3) to specify to which "steps" of the time series
+              the episode will start
+            - "max step" (grid2op >= 1.10.3) : maximum number of steps allowed for the episode
+            - "thermal limit" (grid2op >= 1.11.0): which thermal limit to use for this episode
+              (and the next ones, until they are changed)
+            - "init datetime": which time stamp is used in the first observation of the episode.
+
+            See examples for more information about this. Ignored if
+            not set.
+
+        Examples
+        --------
+        The standard "gym loop" can be done with the following code:
+
+        .. code-block:: python
+
+            import grid2op
+
+            # create the environment
+            env_name = "l2rpn_case14_sandbox"
+            env = grid2op.make(env_name)
+
+            # start a new episode
+            obs = env.reset()
+            done = False
+            reward = env.reward_range[0]
+            while not done:
+                action = agent.act(obs, reward, done)
+                obs, reward, done, info = env.step(action)
+
+        .. versionadded:: 1.9.8
+            It is now possible to set the seed and the time series you want to use at the new
+            episode by calling `env.reset(seed=..., options={"time serie id": ...})`
+
+        Before version 1.9.8, if you wanted to use a fixed seed, you would need to (see
+        doc of :func:`grid2op.Environment.BaseEnv.seed` ):
+
+        .. code-block:: python
+
+            seed = ...
+            env.seed(seed)
+            obs = env.reset()
+            ...
+
+        Starting from version 1.9.8 you can do this in one call:
+
+        .. code-block:: python
+
+            seed = ...
+            obs = env.reset(seed=seed)
+
+        For the "time series id" it is the same concept. Before you would need to do (see
+        doc of :func:`Environment.set_id` for more information ):
+
+        .. code-block:: python
+
+            time_serie_id = ...
+            env.set_id(time_serie_id)
+            obs = env.reset()
+            ...
+
+        And now (from version 1.9.8) you can more simply do:
+
+        .. code-block:: python
+
+            time_serie_id = ...
+            obs = env.reset(options={"time serie id": time_serie_id})
+            ...
+
+        .. versionadded:: 1.10.2
+
+        Another feature has been added in version 1.10.2, which is the possibility to set the
+        grid to a given "topological" state at the first observation (before this version,
+        you could only retrieve an observation with everything connected together).
+
+        In grid2op 1.10.2, you can do that by using the keys `"init state"` in the "options" kwargs of
+        the reset function. The value associated to this key should be dictionnary that can be
+        converted to a non ambiguous grid2op action using an "action space".
+
+        .. note::
+            The "action space" used here is not the action space of the agent. It's an "action
+            space" that uses a :func:`grid2op.Action.Action.BaseAction` class meaning you can do any
+            type of action, on shunts, on topology, on line status etc. even if the agent is not
+            allowed to.
+
+            Likewise, nothing check if this action is legal or not.
+
+        You can use it like this:
+
+        .. code-block:: python
+
+            # to start an episode with a line disconnected, you can do:
+            init_state_dict = {"set_line_status": [(0, -1)]}
+            obs = env.reset(options={"init state": init_state_dict})
+            obs.line_status[0] is False
+
+            # to start an episode with a different topolovy
+            init_state_dict = {"set_bus": {"lines_or_id": [(0, 2)], "lines_ex_id": [(3, 2)]}}
+            obs = env.reset(options={"init state": init_state_dict})
+
+        .. note::
+            Since grid2op version 1.10.2, there is also the possibility to set the "initial state"
+            of the grid directly in the time series. The priority is always given to the
+            argument passed in the "options" value.
+
+            Concretely if, in the "time series" (formelly called "chronics") provides an action would change
+            the topology of substation 1 and 2 (for example) and you provide an action that disable the
+            line 6, then the initial state will see substation 1 and 2 changed (as in the time series)
+            and line 6 disconnected.
+
+            Another example in this case: if the action you provide would change topology of substation 2 and 4
+            then the initial state (after `env.reset`) will give:
+
+            - substation 1 as in the time serie
+            - substation 2 as in "options"
+            - substation 4 as in "options"
+
+        .. note::
+            Concerning the previously described behaviour, if you want to ignore the data in the
+            time series, you can add : `"method": "ignore"` in the dictionary describing the action.
+            In this case the action in the time series will be totally ignored and the initial
+            state will be fully set by the action passed in the "options" dict.
+
+            An example is:
+
+            .. code-block:: python
+
+                init_state_dict = {"set_line_status": [(0, -1)], "method": "force"}
+                obs = env.reset(options={"init state": init_state_dict})
+                obs.line_status[0] is False
+
+        .. versionadded:: 1.10.3
+
+        Another feature has been added in version 1.10.3, the possibility to skip the
+        some steps of the time series and starts at some given steps.
+
+        The time series often always start at a given day of the week (*eg* Monday)
+        and at a given time (*eg* midnight). But for some reason you notice that your
+        agent performs poorly on other day of the week or time of the day. This might be
+        because it has seen much more data from Monday at midnight that from any other
+        day and hour of the day.
+
+        To alleviate this issue, you can now easily reset an episode and ask grid2op
+        to start this episode after xxx steps have "passed".
+
+        Concretely, you can do it with:
+
+        .. code-block:: python
+
+            import grid2op
+            env_name = "l2rpn_case14_sandbox"
+            env = grid2op.make(env_name)
+
+            obs = env.reset(options={"init ts": 1})
+
+        Doing that your agent will start its episode not at midnight (which
+        is the case for this environment), but at 00:05
+
+        If you do:
+
+        .. code-block:: python
+
+            obs = env.reset(options={"init ts": 12})
+
+        In this case, you start the episode at 01:00 and not at midnight (you
+        start at what would have been the 12th steps)
+
+        If you want to start the "next day", you can do:
+
+        .. code-block:: python
+
+            obs = env.reset(options={"init ts": 288})
+
+        etc.
+
+        .. note::
+            On this feature, if a powerline is on soft overflow (meaning its flow is above
+            the limit but below the :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`)
+            then it is still connected (of course) and the counter
+            :attr:`grid2op.Observation.BaseObservation.timestep_overflow` is at 0.
+
+            If a powerline is on "hard overflow" (meaning its flow would be above
+            :attr:`grid2op.Parameters.Parameters.HARD_OVERFLOW_THRESHOLD` * `the limit`), then, as it is
+            the case for a "normal" (without options) reset, this line is disconnected, but can be reconnected
+            directly (:attr:`grid2op.Observation.BaseObservation.time_before_cooldown_line` == 0)
+
+        .. seealso::
+            The function :func:`Environment.fast_forward_chronics` for an alternative usage (that will be
+            deprecated at some point)
+
+        Yet another feature has been added in grid2op version 1.10.3 in this `env.reset` function. It is
+        the capacity to limit the duration of an episode.
+
+        .. code-block:: python
+
+            import grid2op
+            env_name = "l2rpn_case14_sandbox"
+            env = grid2op.make(env_name)
+
+            obs = env.reset(options={"max step": 288})
+
+        This will limit the duration to 288 steps (1 day), meaning your agent
+        will have successfully managed the entire episode if it manages to keep
+        the grid in a safe state for a whole day (depending on the environment you are
+        using the default duration is either one week - roughly 2016 steps or 4 weeks)
+
+        .. note::
+            This option only affect the current episode. It will have no impact on the
+            next episode (after reset)
+
+        For example:
+
+        .. code-block:: python
+
+            obs = env.reset()
+            obs.max_step == 8064  # default for this environment
+
+            obs = env.reset(options={"max step": 288})
+            obs.max_step == 288  # specified by the option
+
+            obs = env.reset()
+            obs.max_step == 8064  # retrieve the default behaviour
+
+        .. seealso::
+            The function :func:`Environment.set_max_iter` for an alternative usage with the different
+            that `set_max_iter` is permenanent: it impacts all the future episodes and not only
+            the next one.
+
+        If you want your environment to start at a given time stamp you can do:
+
+        .. code-block:: python
+
+            import grid2op
+            env_name = "l2rpn_case14_sandbox"
+
+            env = grid2op.make(env_name)
+            obs = env.reset(options={"init datetime": "2024-12-06 00:00"})
+            obs.year == 2024
+            obs.month == 12
+            obs.day == 6
+
+        .. seealso::
+            If you specify "init datetime" then the observation resulting to the
+            `env.reset` call will have this datetime. If you specify also `"skip ts"`
+            option the behaviour does not change: the first observation will
+            have the date time attributes you specified.
+
+            In other words, the "init datetime" refers to the initial observation of the
+            episode and NOT the initial time present in the time series.
+
+        """
         # process the "options" kwargs
         # (if there is an init state then I need to process it to remove the 
         # some keys)
diff --git a/setup.py b/setup.py
index 4952b366..3b79244f 100644
--- a/setup.py
+++ b/setup.py
@@ -32,8 +32,7 @@ def my_test_suite():
         "requests>=2.23.0",
         "packaging",  # because gym changes the way it uses numpy prng in version 0.26 and i need both gym before and after...
         "typing_extensions",
-        "orderly_set<5.4.0; python_version<='3.8'",
-        "pyarrow>=17.0.0"
+        "orderly_set<5.4.0; python_version<='3.8'"
     ],
     "extras": {
         "optional": [
@@ -49,6 +48,7 @@ def my_test_suite():
             "psutil>=5.7.0",
             "gymnasium",
             "lightsim2grid",
+            "pyarrow>=17.0.0",
         ],
         "gym": [
             "gym>=0.17.2",