Merge pull request #625 from rte-france/dev_1.10.3

Ready for version 1.10.3

Author: BDonnot

.circleci/config.yml

@@ -140,7 +140,7 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.20,<1.21" "pandas<2.2" "scipy<1.12" numba .[test]
+                   python -m pip install -U "numpy>=1.20,<1.21" "pandas<2.2" "scipy<1.12" numba "pillow<10.4.0" .[test]
                    pip freeze
             - run:
                 command: |
@@ -151,7 +151,7 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.24,<1.25" "pandas<2.2" "scipy<1.12" numba .[test]
+                   python -m pip install -U "numpy>=1.24,<1.25" "pandas<2.2" "scipy<1.12" numba "pillow<10.4.0" .[test]
             - run:
                 command: |
                    source venv_test/bin/activate
@@ -163,7 +163,7 @@ jobs:
                    cd /tmp
                    grid2op.testinstall
 
-    legacy_lightsim:               
+    legacy_lightsim_old_pp:               
         executor: python38  # needs to be 38: whl of lightsim were not released for 3.10 at the time
         resource_class: small
         steps:
@@ -190,6 +190,59 @@ jobs:
                    export _GRID2OP_FORCE_TEST=1
                    python -m unittest grid2op/tests/test_basic_env_ls.py
 
+    legacy_lightsim:               
+        executor: python38  # needs to be 38: whl of lightsim were not released for 3.10 at the time
+        resource_class: small
+        steps:
+            - checkout
+            - run: 
+                command: |
+                    apt-get update
+                    apt-get install -y coinor-cbc
+            - run: python -m pip install virtualenv
+            - run: python -m virtualenv venv_test
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U pip setuptools wheel
+                   python -m pip install -U lightsim2grid==0.6.0 gymnasium "numpy<1.22"
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -e . 
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
+                   export _GRID2OP_FORCE_TEST=1
+                   python -m unittest grid2op/tests/test_basic_env_ls.py
+
+    test_chronix2grid:               
+        executor: python310  # needs to be 38: whl of lightsim were not released for 3.10 at the time
+        resource_class: small
+        steps:
+            - checkout
+            - run: 
+                command: |
+                    apt-get update
+                    apt-get install -y coinor-cbc
+            - run: python -m pip install virtualenv
+            - run: python -m virtualenv venv_test
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U pip setuptools wheel "numpy==1.26.4"
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -e .[chronix2grid] "linopy==0.3.8" "scs==3.2.4.post1" "ecos==2.0.13" "pillow==10.3.0" "numpy==1.26.4" "xarray==2024.3.0"
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
+                   export _GRID2OP_FORCE_TEST=1
+                   python -m unittest grid2op/tests/fromChronix2grid.py
+
     install39:
         executor: python39
         resource_class: small
@@ -205,8 +258,7 @@ jobs:
                command: |
                    export _GRID2OP_FORCE_TEST=1
                    source venv_test/bin/activate
-                   python -m pip install -U pip setuptools wheel "numpy>=1.20,<1.21" "pandas<2.2" "scipy==1.10.1" numba
-                   python -m pip install "chronix2grid>=1.1.0.post1" "gymnasium==0.26.3" "matplotlib==3.7.5" "xarray==2023.10.0" "scs==3.0.0" "ecos==2.0.0"
+                   python -m pip install -U pip setuptools wheel "numpy>=1.20,<1.21" "pandas<2.2" "scipy==1.10.1" "pillow<10.4.0" numba
                    python -m pip uninstall -y grid2op
             - run:
                command: |  # issue with previous more simple install, so I fix some versions
@@ -222,7 +274,7 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install "numpy>=1.26,<1.27" "pandas<2.2" "scipy<1.12" numba
+                   python -m pip install "numpy>=1.26,<1.27" "pandas<2.2" "scipy<1.12" numba "pillow<10.4.0"
                    pip freeze
             - run:
                 command: |
@@ -340,7 +392,10 @@ workflows:
     test:
         jobs:
           - test
+          - legacy_lightsim_old_pp
           - legacy_lightsim
+          - test_chronix2grid
+
     install:
         jobs:
           - install38

.github/workflows/main.yml

@@ -12,6 +12,8 @@ jobs:
     name: Build linux ${{ matrix.python.name }} wheel
     runs-on: ubuntu-latest
     container: quay.io/pypa/manylinux2014_x86_64
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     strategy:
       matrix:
         python:
@@ -59,7 +61,7 @@ jobs:
           
       - name: Build wheel
         run: |
-          python3 setup.py bdist_wheel
+          python setup.py bdist_wheel
           # auditwheel repair dist/*.whl  # only for compiled code !
 
       - name: Install wheel
@@ -69,12 +71,16 @@ jobs:
 
       - name: Check package can be imported
         run: |
-          python3 -c "import grid2op"
-          python3 -c "from grid2op import *"
-          python3 -c "from grid2op.Action._backendAction import _BackendAction"
-          
+          python -c "import grid2op"
+          python -c "from grid2op import *"
+          python -c "from grid2op.Action._backendAction import _BackendAction"
+      
+      - name: List wheel
+        run: 
+          ls ./dist/*.whl
+
       - name: Upload wheel
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: grid2op-wheel-${{ matrix.config.name }}-${{ matrix.python.name }}
           path: dist/*.whl
@@ -165,10 +171,70 @@ jobs:
           name: grid2op-sources
           path: dist/*.tar.gz
 
+  auto_class_in_file:
+    name: Test ${{ matrix.config.name }} OS can handle automatic class generation
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      matrix:
+        config:
+          - {
+            name: darwin,
+            os: macos-latest,
+          }
+          # - {
+          #   name: windows,
+          #   os: windows-2019,
+          # }
+          - { 
+            name: ubuntu,
+            os: ubuntu-latest,
+          }
+        python:
+          - {
+            name: cp39,
+            version: '3.9',
+          }  
+          - {
+            name: cp312,
+            version: '3.12',
+          }  
+
+    steps:
+
+      - name: Checkout sources
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python.version }}
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade wheel
+          python -m pip install --upgrade setuptools
+          python -m pip install --upgrade gymnasium "numpy<2"
+
+      - name: Build wheel
+        run: python setup.py bdist_wheel
+
+      - name: Install wheel
+        shell: bash
+        run: |
+          python -m pip install dist/*.whl --user
+          pip freeze
+
+      - name: Test the automatic generation of classes in the env folder
+        run: |
+          python -m unittest grid2op/tests/automatic_classes.py -f
+  
   package:
     name: Test install
     runs-on: ubuntu-latest
-    needs: [manylinux_build, macos_windows_build]
+    needs: [manylinux_build, macos_windows_build, auto_class_in_file]
 
     steps:
       - name: Download wheels

.gitignore

@@ -410,6 +410,12 @@ grid2op/tests/req_38_np121
 test_make_2_envs.py
 getting_started/env_py38_grid2op110_ray110.ipynb
 getting_started/env_py38_grid2op110_ray210.ipynb
+grid2op/tests/req_chronix2grid
+grid2op/tests/venv_test_chronix2grid/
+getting_started/venv_310_ray/
+grid2op/tests/venv_test_autoclass/
+test_eduardo.py
+grid2op/tests/failed_test*
 
 # profiling files
 **.prof

CHANGELOG.rst

@@ -31,9 +31,113 @@ Change Log
 - [???] "asynch" multienv
 - [???] properly model interconnecting powerlines
 
+Work kind of in progress
+----------------------------------
+- TODO A number of max buses per sub
+- TODO in the runner, save multiple times the same sceanrio
+- TODO in the gym env, make the action_space and observation_space attribute
+  filled automatically (see ray integration, it's boring to have to copy paste...)
+
+Next release
+---------------------------------
+- numpy 2 compat (need pandapower for that)
+- automatic read from local dir also on windows !
+- TODO doc for the "new" feature of automatic "experimental_read_from_local_dir"
+- TODO bug on maintenance starting at midnight (they are not correctly handled in the observation)
+  => cf script test_issue_616
+- TODO put the Grid2opEnvWrapper directly in grid2op as GymEnv
+- TODO faster gym_compat (especially for DiscreteActSpace and BoxGymObsSpace)
+- TODO Notebook for tf_agents
+- TODO Notebook for acme
+- TODO Notebook using "keras rl" (see https://keras.io/examples/rl/ppo_cartpole/)
+- TODO example for MCTS https://github.com/bwfbowen/muax et https://github.com/google-deepmind/mctx
+- TODO jax everything that can be: create a simple env based on jax for topology manipulation, without
+  redispatching or rules
+- TODO backend in jax, maybe ?
+- TODO done and truncated properly handled in gym_compat module (when game over
+  before the end it's probably truncated and not done) 
+- TODO when reset, have an attribute "reset_infos" with some infos about the
+  way reset was called.
+- TODO ForecastEnv in MaskedEnv ! (and obs.simulate there too !)
+- TODO finish the test in automatic_classes
+- TODO in multi-mix increase the reset options with the mix the user wants
+- TODO L2RPN scores as reward (sum loads after the game over and have it in the final reward)
+- TODO on CI: test only gym, only gymnasium and keep current test for both gym and gymnasium
+
+[1.10.3] - 2024-07-yy
+-------------------------
+- TODO Automatic "experimental_read_from_local_dir"
+
+- [BREAKING] `env.chronics_hander.set_max_iter(xxx)` is now a private function. Use 
+  `env.set_max_iter(xxx)` or even better `env.reset(options={"max step": xxx})`. 
+  Indeed, `env.chronics_hander.set_max_iter()` will likely have
+  no effect at all on your environment.
+- [BREAKING] for all the `Handler` (*eg* `CSVForecastHandler`) the method `set_max_iter` is 
+  now private (for the same reason as the `env.chronics_handler`). We do not recommend to
+  use it (will likely have no effect). Prefer using `env.set_max_iter` instead.
+- [BREAKING] now the `runner.run()` method only accept kwargs argument 
+  (because  it should always have been like this)
+- [BREAKING] to improve pickle support and multi processing capabilities, the attribute
+  `gym_env.observation_space._init_env` and `gym_env.observation_space.initial_obs_space`
+  have been deleted (for the `Dict` space only, for the other spaces like the `Box` they
+  were not present in the first place)
+- [BREAKING] in the `GymEnv` class now by default the underlying grid2op environment has no
+  forecast anymore in an attempt to make this wrapper faster AND more easily pickle-able. You can
+  retrieve the old behaviour by passing `gym_env = GymEnv(grid2op_env, with_forecast=True)`
+- [FIXED] a bug in the `MultiFolder` and `MultifolderWithCache` leading to the wrong 
+  computation of `max_iter` on some corner cases
+- [FIXED] the function `cleanup_action_space()` did not work correctly when the "chronics_hander"
+  was not initialized for some classes
+- [FIXED] the `_observationClass` attribute of the "observation env" (used for simulate and forecasted env)
+  is now an Observation and not an Action.
+- [FIXED] a bug when deep copying an "observation environment" (it changes its class)
+- [FIXED] issue on `seed` and `MultifolderWithCache` which caused 
+  https://github.com/rte-france/Grid2Op/issues/616 
+- [FIXED] another issue with the seeding of `MultifolderWithCache`: the seed was not used
+  correctly on the cache data when calling `chronics_handler.reset` multiple times without 
+  any changes
+- [FIXED] `Backend` now properly raise EnvError (grid2op exception) instead of previously 
+  `EnvironmentError` (python default exception)
+- [FIXED] a bug in `PandaPowerBackend` (missing attribute) causing directly 
+  https://github.com/rte-france/Grid2Op/issues/617
+- [FIXED] a bug in `Environment`: the thermal limit were used when loading the environment 
+  even before the "time series" are applied (and before the user defined thermal limits were set) 
+  which could lead to disconnected powerlines even before the initial step (t=0, when time 
+  series are loaded)
+- [FIXED] an issue with the "max_iter" for `FromNPY` time series generator
+- [FIXED] a bug in `MultiMixEnvironment` : a multi-mix could be created even if the underlying 
+  powergrids (for each mix) where not the same.
+- [FIXED] a bug in `generate_classes` (experimental_read_from_local_dir) with alert data.
+- [FIXED] a bug in the `Runner` when using multi processing on macos and windows OS: some non default
+  parameters where not propagated in the "child" process (bug in `runner._ger_params`)
+- [ADDED] possibility to skip some step when calling `env.reset(..., options={"init ts": ...})`
+- [ADDED] possibility to limit the duration of an episode with `env.reset(..., options={"max step": ...})`
+- [ADDED] possibility to specify the "reset_options" used in `env.reset` when
+  using the runner with `runner.run(..., reset_options=xxx)`
+- [ADDED] the argument `mp_context` when building the runner to help pass a multiprocessing context in the
+  grid2op `Runner` 
+- [ADDED] the time series are now able to regenerate their "random" part 
+  even when "cached" thanks to the addition of the `regenerate_with_new_seed` of the 
+  `GridValue` class (in public API)
+- [ADDED] `MultifolderWithCache` now supports `FromHandlers` time series generator
+- [IMPROVED] more consistency in the way the classes are initialized at the creation of an environment
+- [IMPROVED] more consistency when an environment is copied (some attributes of the copied env were 
+  deep copied incorrectly)
+- [IMPROVED] Doc about the runner
+- [IMPROVED] the documentation on the `time series` folder.
+- [IMPROVED] now the "maintenance from json" (*eg* the `JSONMaintenanceHandler` or the 
+  `GridStateFromFileWithForecastsWithMaintenance`) can be customized with the day 
+  of the week where the maintenance happens (key `maintenance_day_of_week`)
+- [IMPROVED] in case of "`MultiMixEnvironment`" there is now only class generated for 
+  all the underlying mixes (instead of having one class per mixes)
+- [IMPROVED] the `EpisodeData` have now explicitely a mode where they can be shared accross 
+  processes (using `fork` at least), see `ep_data.make_serializable`
+- [IMPROVED] chronix2grid tests are now done independantly on the CI
+
+
 [1.10.2] - 2024-05-27
 -------------------------
-- [BREAKING] the `runner.run_one_episode` now returns an extra first argument: 
+- [BREAKING] the `runner.run_one_episode` now returns an extra argument (first position): 
   `chron_id, chron_name, cum_reward, timestep, max_ts = runner.run_one_episode()` which 
   is consistant with `runner.run(...)` (previously it returned only 
   `chron_name, cum_reward, timestep, max_ts = runner.run_one_episode()`)
@@ -822,7 +926,7 @@ Change Log
   `Issue#185 <https://github.com/rte-france/Grid2Op/issues/185>`_ )
 - [IMPROVED] the seed of openAI gym for composed action space (see issue `https://github.com/openai/gym/issues/2166`):
   in waiting for an official fix, grid2op will use the solution proposed there
-  https://github.com/openai/gym/issues/2166#issuecomment-803984619 )
+  https://github.com/openai/gym/issues/2166#issuecomment-803984619
 
 [1.5.1] - 2021-04-15
 -----------------------