Merge pull request #702 from Grid2op/dev_1.11.0

Dev 1.11.0

Author: BDonnot

.circleci/config.yml

@@ -39,14 +39,15 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   pip install -U pip setuptools wheel
+                   pip install -U pip setuptools wheel coverage
             - run:
                 command: |
                   source venv_test/bin/activate
                   pip install -e .[test]
                   export _GRID2OP_FORCE_TEST=1
                   cd grid2op/tests/
-                  python3 helper_list_test.py | circleci tests split > /tmp/tests_run
+                  python3 helper_list_test.py > li_test
+                  cat li_test | circleci tests split > /tmp/tests_run
             - run:
                 command: |
                   source venv_test/bin/activate
@@ -57,8 +58,42 @@ jobs:
                   source venv_test/bin/activate
                   cd grid2op/tests/
                   export _GRID2OP_FORCE_TEST=1
-                  python3 -m unittest -v $(cat /tmp/tests_run)
-                  
+                  coverage run -m unittest -v $(cat /tmp/tests_run)
+                #   ls -lah | grep .coverage
+            # - run:
+            #     command: |
+            #       ls -lah /Grid2Op/grid2op/tests/.coverage.*
+
+            # - store_artifacts:
+            #     path: "/Grid2Op/grid2op/tests/test_Action.py"
+            #     destination: coverage_artifacts/
+
+    gen_coverage:
+        executor: grid2op-executor
+        resource_class: small
+        steps:
+            - checkout
+            - run: python -m pip install virtualenv
+            - run: python -m virtualenv venv_test
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   pip install -U pip setuptools wheel coverage
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   cd grid2op/tests/
+                   coverage combine ../../coverage_artifacts/ --keep
+                   coverage report -m -i
+                   coverage html -i
+                   coverage xml -i
+            - store_artifacts:
+                path: grid2op/tests/htmlcov
+                destination: htmlcov
+            - store_artifacts:
+                path: grid2op/tests/coverage.xml
+                destination: coverage.xml
+            
     install36:
         executor: python36
         resource_class: small
@@ -403,3 +438,19 @@ workflows:
           - install310
           - install311
           - install312
+
+    # gather_test_report:
+    #     requires:
+    #         - test
+    #     jobs:
+    #         - gen_coverage
+
+    # send_tests_report:
+    #     requires:
+    #         - gather_test_report
+    #     orbs:
+    #         coverage-reporter: codacy/[email protected]
+    #     jobs:
+    #         coverage-reporter/send_report:
+    #             coverage-reports: coverage.xml
+    #             project-token: $CODACY_PROJECT_TOKEN

.github/workflows/main.yml

@@ -68,7 +68,6 @@ jobs:
       - name: Build wheel
         run: |
           python setup.py bdist_wheel
-          # auditwheel repair dist/*.whl  # only for compiled code !
 
       - name: Install wheel
         run: |
@@ -80,7 +79,23 @@ jobs:
           python -c "import grid2op"
           python -c "from grid2op import *"
           python -c "from grid2op.Action._backendAction import _BackendAction"
+          
+      # - name: Run full tests + code coverage
+      #   if: matrix.python.name == 'cp312'
+      #   run: |
+      #     python -m pip install --upgrade coverage
+      #     python -m pip install --find-links=dist/ grid2op[test]
+      #     cd grid2op/tests
+      #     coverage run -m unittest discover
+      #     coverage xml -i 
 
+      # - name: Upload coverage
+      #   if: matrix.python.name == 'cp312'
+      #   uses: actions/upload-artifact@v4
+      #   with: 
+      #     name: coverage.xml
+      #     path: grid2op/tests/*.xml
+
       - name: List wheel
         run: 
           ls ./dist/*.whl

CHANGELOG.rst

@@ -3,6 +3,7 @@ Work "in progress"
 
 General grid2op improvments:
 
+- ill formed docstring in the BaseAction module
 - remove pandapower dependency (have a way to install grid2op without pandapower)
 - better logging
 - have functions that automatically computes topo_vect and switch_state in the backend 
@@ -23,6 +24,8 @@ General grid2op improvments:
 - in parallel distribute the loading of the time series if using a `MultifolderWithCache`
 - Code and test the "load from disk" method
 - add a "plot action" method
+- does not read every data of the backend if not used
+- backend converter: now test it properly with pandapower / lightsim2grid and pypowsybl2grid
 
 Better multi processing support: 
 
@@ -94,6 +97,128 @@ Native multi agents support:
 
 - cf ad-hoc branch (dev-multiagents)
 - properly model interconnecting powerlines
+- add detachment
+- add change_bus / set_bus
+
+[1.11.0] - 2025-04-14
+-----------------------
+- [BREAKING] Change for `FromMultiEpisodeData` that disables the caching by default
+  when creating the data.
+- [BREAKING] deprecation of `backend.check_kirchoff` in favor of `backend.check_kirchhoff` 
+  (fix the typo in the name)
+- [BREAKING] change the name of the generated classes: now by default the backend class
+  name is added. This behaviour can be turned off by passing `_add_cls_nm_bk=False`
+  when calling `grid2op.make(...)`. If you develop a new Backend, you can also
+  customize the added name by overloading the `get_class_added_name` class method.
+- [BREAKING] it is now forbidden to create environment with arguments.
+  Only key-word arguments are allowed.
+- [BREAKING] the way actions is serialized has been changed with respect to the `from_vect` /
+  `to_vect` method. This might introduce some issues when loading previously saved actions
+  with this methods.
+- [BREAKING] first kwargs of `backend.apply_action` method is now spelled `backend_action` 
+  (instead of backendAction)
+- [BREAKING] (not yet) rationalization of the backend public / private API part. The 
+  environment (and simulator, forecast env etc.) will always call the method `_public` 
+  for example `load_grid_public`, `reset_public`, `copy_public` and `apply_action_public`. 
+  These function of the base `Backend` should NOT be overriden, and will internally call 
+  the functions `load_grid`, `reset`, `copy` and `apply_action` which were part of the public
+  API. These last member functions will be renamed (in a later version) `_load_grid`,
+  `_reset`, `_copy` and `_apply_action` to reflect this change. NOT for this version however !
+- [BREAKING] removal of the `rest_server` grid2op module (it will be release as a separate package instead)
+  It has been removed from grid2op core package for securtiy reasons.
+- [FIXED] issue https://github.com/Grid2op/grid2op/issues/657
+- [FIXED] missing an import on the `MaskedEnvironment` class
+- [FIXED] a bug when trying to set the load_p, load_q, gen_p, gen_v by names.
+- [FIXED] the `obs.get_forecast_env` : in some cases the resulting first
+  observation (obtained from `for_env.reset()`) did not have the correct
+  topology.
+- [FIXED] issue https://github.com/Grid2op/grid2op/issues/665 (`obs.reset()`
+  was not correctly implemented: some attributes were forgotten)
+- [FIXED] issue https://github.com/Grid2op/grid2op/issues/667 (`act.as_serializable_dict()`
+  was not correctly implemented AND the `_aux_affect_object_int` and `_aux_affect_object_float`
+  have been also fixed - weird behaviour when you give them a list with the exact length of the
+  object you tried to modified (for example a list with a size of `n_load` that affected the loads))
+- [FIXED] a bug when using the `DoNothingHandler` for the maintenance and the 
+  environment data
+- [FIXED] an issue preventing to set the thermal limit in the options
+  if the last simulated action lead to a game over
+- [FIXED] some bugs in `act.from_json(...)` due to the handling of the injection modifications.
+- [FIXED] logos now have the correct URL
+- [FIXED] deprecated call to `tostring_rgb` (replaced `tostring_argb`) in the env.render function.
+- [FIXED] warnings not properly issued in the AAA test when backend failed to call
+  `can_handle_XXX` functions (*eg* `can_handle_more_than_2_busbar()` or `can_handle_detachment()`)
+- [FIXED] an issue with `obs.get_forecast_env` with changeNothing and DoNothingHandler time series
+- [FIXED] a bug in updating the shunt in PandaPowerBackend (depdending on pandas version)
+- [FIXED] a bug when action that reconnect loads, storage units or shunts are done
+  in the "obs.simulate" (results could depend from previous "obs.simulate" calls)
+- [FIXED] a bug in "obs.simulate" and "obs.get_forecast_env" : when a line was disconnected
+  and the user tried to reconnect it (without specifying on which bus) it could do something
+  different than "env.step" (with the same action)
+- [FIXED] a powerflow is run when the environment is first created even before the initial "env.step"
+  function is called. This is to ensure proper behaviour if env is used without being reset.
+- [FIXED] no error was catched if the backend could not properly apply the action sent by the environment.
+- [FIXED] an issue in the AAA tests: when backend does not support storages, some tests were skipped not correctly
+- [FIXED] an issue when computing the cascading failure routine, in case multiple iterations were performed, 
+  the cooldowns were not updated correctly.
+- [FIXED] cascading failure could be started at the first observation (t=0, just after a reset).
+- [FIXED] a bug when "SOFT_OVERFLOW_THRESHOLD" was not 1.: it also impacted "instantaneous overcurrent protections" 
+  (it was triggered when `flow > SOFT_OVERFLOW_THRESHOLD * HARD_OVERFLOW_THRESHOLD * th_lim`)
+- [FIXED] a bug when "SOFT_OVERFLOW_THRESHOLD" was not 1.: the backend routine to compute the protections 
+  disconnected the lines with a counter based on `flow > th_lim` and not `flow > th_lim * SOFT_OVERFLOW_THRESHOLD`
+- [ADDED] Possibility to disconnect loads, generators and storage units (if proper flag set in the environment).
+  See documentation.
+- [ADDED] possibility to set the "thermal limits" when calling `env.reset(..., options={"thermal limit": xxx})`
+- [ADDED] possibility to retrieve some structural information about elements with
+  with `gridobj.get_line_info(...)`, `gridobj.get_load_info(...)`, `gridobj.get_gen_info(...)` 
+  or , `gridobj.get_storage_info(...)` 
+- [ADDED] codacy badge on the readme
+- [ADDED] a method to check the KCL (`obs.check_kirchhoff`) directly from the observation
+  (previously it was only possible to do it from the backend). This should 
+  be used for testing purpose only
+- [ADDED] parameters to disable the "redispatching routine" of the environment 
+  (see `params.ENV_DOES_REDISPATCHING`)
+- [ADDED] parameters to stop the episode when one of the constraints of one of the 
+  generators is not met (see `params.STOP_EP_IF_SLACK_BREAK_CONSTRAINTS`)
+- [ADDED] possibility to set the initial time stamp of the observation in the `env.reset`
+  kwargs by using `env.reset(..., options={"init datetime": XXX})`
+- [ADDED] the `ChangeNothing` time series class now supports forecast
+- [ADDED] test coverage on the CI
+- [ADDED] the `obs.timestep_protection_triggered` counter which counts whether or not the 
+  "time overcurrent protection" (soft overflow) will be triggered: lines will be disconnected
+  if `time overcurrent protection > parameters.NB_TIMESTEP_POWERFLOW_ALLOWED`
+- [IMPROVED] possibility to set the injections values with names
+  to be consistent with other way to set the actions (*eg* set_bus)
+- [IMPROVED] error messages when creating an action which changes the injections
+- [IMPROVED] (linked to https://github.com/Grid2op/grid2op/issues/657) the way the 
+  "chronics_hander" in the ObsEnv behaves (it now fully implements the public interface of 
+  a "real" chronic_handler)
+- [IMPROVED] error message in the `FromNPY` class when the backend is checked
+- [IMRPOVED] the `FromMultiEpisodeData` class with the addition of the `caching` 
+  kwargs to allow / disable caching (which was default behavior in previous version) 
+- [IMPROVED] the `FromMultiEpisodeData` class that now returns also the path of the data
+- [IMPROVED] the classes inherited from `GreedyAgent` with the added possibility to 
+  do the `obs.simulate` on a different time horizon (kwarg `simulated_time_step`)
+- [IMPROVED] some type hints for some agent class
+- [IMPROVED] the `backend.update_from_obs` function to work even when observation
+  does not have shunt information but there are not shunts on the grid.
+- [IMPROVED] consistency of `MultiMixEnv` in case of automatic_classes (only one
+  class is generated for all mixes)
+- [IMRPOVED] handling of disconnected elements in the backend no more
+  raise error. The base `Backend` class does that.
+- [IMPROVED] the `act.as_serializable_dict()` to be more 'backend agnostic'as
+  it nows tries to use the name of the elements in the json output
+- [IMPROVED] the way shunt data are digested in the `BaseAction` class (it is now 
+  possible to use the same things as for the other types of element)
+- [IMPROVED] grid2op does not require the `chronics` folder when using the `FromHandlers`
+  class
+- [IMPROVED] the function `action.get_topological_impact(...)` has now a "caching" mechanism
+  that allows not to recompute it over and over again (this is internal API please do not change 
+  it... unless you know what you are doing)
+- [IMPROVED] `ForecastEnv` is now part of the public API.
+- [IMPROVED] no need to call `self._compute_pos_big_top()` at the end of the implementation of `backend.load_grid()`
+- [IMPROVED] type hints in various files.
+- [IMPROVED] documentation of the backend
+- [IMRPOVED] `SOFT_OVERFLOW_THRESHOLD` can now be lower than 1
 
 [1.10.5] - 2025-03-07
 ------------------------

README.md

@@ -3,8 +3,10 @@
 [![Downloads](https://pepy.tech/badge/grid2op)](https://pepy.tech/project/grid2op)
 [![PyPi_Version](https://img.shields.io/pypi/v/grid2op.svg)](https://pypi.org/project/Grid2Op/)
 [![PyPi_Compat](https://img.shields.io/pypi/pyversions/grid2op.svg)](https://pypi.org/project/Grid2Op/)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/10367/badge)](https://www.bestpractices.dev/projects/10367)
 [![LICENSE](https://img.shields.io/pypi/l/grid2op.svg)](https://www.mozilla.org/en-US/MPL/2.0/)
 [![Documentation Status](https://readthedocs.org/projects/grid2op/badge/?version=latest)](https://grid2op.readthedocs.io/en/latest/?badge=latest)
+[![Codacy Badge](https://app.codacy.com/project/badge/Grade/3a4e666ba20f4f20b9131e9a6081622c)](https://app.codacy.com/gh/Grid2op/grid2op/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
 [![CircleCI](https://dl.circleci.com/status-badge/img/gh/Grid2op/grid2op/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/Grid2op/grid2op/tree/master)
 [![discord](https://discord.com/api/guilds/698080905209577513/embed.png)](https://discord.gg/cYsYrPT)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/grid2op/grid2op.git/master)
@@ -200,6 +202,9 @@ interesting part of this framework:
 * [11_IntegrationWithExistingRLFrameworks](getting_started/11_IntegrationWithExistingRLFrameworks.ipynb)
   [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Grid2Op/grid2op/blob/master/getting_started/11_IntegrationWithExistingRLFrameworks.ipynb)
   explains how to use grid2op with other reinforcement learning framework. TODO: this needs to be redone
+* [12_DetachmentOfLoadsAndGenerators.ipynb](getting_started/12_DetachmentOfLoadsAndGenerators.ipynb.ipynb)
+  [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Grid2Op/grid2op/blob/master/getting_started/12_DetachmentOfLoadsAndGenerators.ipynb.ipynb)
+  explains briefly what is detachment.
 
 Try them out in your own browser without installing
 anything with the help of mybinder:

binder/environment.yml

@@ -8,7 +8,6 @@ dependencies:
   - pip
   - pip:
     - grid2op
-    - jyquickhelper
     - numpy
     - numba
     - seaborn

docs/conf.py

@@ -22,8 +22,8 @@
 author = 'Benjamin Donnot'
 
 # The full version, including alpha/beta/rc tags
-release = '1.10.5.post1'
-version = '1.10'
+release = '1.11.0.dev6'
+version = '1.11'
 
 
 # -- General configuration ---------------------------------------------------