Merge pull request #162 from pockerman/bug/159-fix-envs-for-new-api

Refactor rest rl env client

Author: pockerman

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build rlenvscpp
+name: Build bitrl
 
 on:
   push:
@@ -21,7 +21,39 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install dependencies
       run: |
-        sudo apt-get install -y g++ cmake libboost-all-dev libgtest-dev libeigen3-dev libblas-dev
+        sudo apt-get install -y \
+          build-essential \
+          cmake \
+          git \
+          libboost-all-dev \
+          libgtest-dev \ 
+          libeigen3-dev \ 
+          libblas-dev \ 
+          libopencv-dev
+      # g++ cmake libboost-all-dev libgtest-dev libeigen3-dev libblas-dev
+    - name: Build and install Paho MQTT C
+      run: |
+        git clone https://github.com/eclipse/paho.mqtt.c.git
+        cd paho.mqtt.c
+        cmake -Bbuild -H. \
+          -DPAHO_WITH_SSL=ON \
+          -DPAHO_BUILD_SHARED=ON \
+          -DPAHO_BUILD_STATIC=OFF
+        cmake --build build
+        sudo cmake --install build
+        sudo ldconfig
+    - name: Build and install Paho MQTT C++
+      run: |
+        git clone https://github.com/eclipse/paho.mqtt.cpp.git
+        cd paho.mqtt.cpp
+        cmake -Bbuild -H. \
+          -DPAHO_WITH_SSL=ON \
+          -DPAHO_BUILD_SHARED=ON \
+          -DPAHO_BUILD_STATIC=OFF
+        cmake --build build
+        sudo cmake --install build
+        sudo ldconfig
+
     - name: Configure CMake
       # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
       # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type

CMakeLists.txt

@@ -3,7 +3,7 @@ MESSAGE(STATUS "Using CMake ${CMAKE_VERSION}")
 
 
 SET(BITRL_VERSION_MAJOR 1)
-SET(BITRL_VERSION_MINOR 2)
+SET(BITRL_VERSION_MINOR 8)
 SET(BITRL_VERSION_PATCH 0)
 
 SET(BITRL_VERSION "${BITRL_VERSION_MAJOR}.${BITRL_VERSION_MINOR}.${BITRL_VERSION_PATCH}")

README.md

@@ -2,23 +2,16 @@
 # bitrl
 
 ```bitrl``` is an effort to provide implementations and wrappers of environments suitable for training reinforcement learning agents
-using  C++. 
+using  C++.  The documentation for the library can be found <a href="https://pockerman.github.io/bitrl/index.html">here</a>.
 
-Furthermore, there is some minimal support for working with Arduino UNO boards over USB or WiFi.
-See also <a href="https://rlenvscpp.readthedocs.io/en/latest/working_with_webots.html">Working with Webots</a>
-for how to integrate ```bitrl``` with <a href="https://cyberbotics.com/doc/guide/installing-webots">Webots</a>.
-
-Various RL algorithms using the environments can be found at <a href="https://github.com/pockerman/cuberl/tree/master">cuberl</a>.
-
-The documentation for the library can be found <a href="https://pockerman.github.io/bitrl/index.html">here</a>.
 The following is an example how to use the 
 ``FrozenLake``   environment from <a href="https://github.com/Farama-Foundation/Gymnasium/tree/main">Gymnasium</a>.
 
 ```
 
 #include "bitrl/bitrl_types.h"
 #include "bitrl/envs/gymnasium/toy_text/frozen_lake_env.h"
-#include "bitrl/envs/api_server/apiserver.h"
+#include "bitrl/network/rest_rl_env_client.h"
 
 #include <iostream>
 #include <string>
@@ -32,29 +25,33 @@ const std::string SERVER_URL = "http://0.0.0.0:8001/api";
 using bitrl::envs::gymnasium::FrozenLake;
 using bitrl::envs::RESTApiServerWrapper;
 
-
 void test_frozen_lake(const RESTApiServerWrapper& server){
 
     FrozenLake<4> env(server);
 
     std::cout<<"Environame URL: "<<env.get_url()<<std::endl;
 
     // make the environment
-    std::unordered_map<std::string, std::any> options;
-    options.insert({"is_slippery", false});
-    env.make("v1", options);
+    std::unordered_map<std::string, std::any> make_ops;
+    make_ops.insert({"is_slippery", false});
+
+	std::unordered_map<std::string, std::any> reset_ops;
+	reset_ops.insert({"seed", static_cast<uint_t>(42)});
+    env.make("v1", make_ops, reset_ops);
 
     std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
     std::cout<<"Is environment alive? "<<env.is_alive()<<std::endl;
     std::cout<<"Number of valid actions? "<<env.n_actions()<<std::endl;
     std::cout<<"Number of states? "<<env.n_states()<<std::endl;
+	std::cout<<"Env idx: "<<env.idx()<<std::endl;
 
     // reset the environment
-    auto time_step = env.reset(42, std::unordered_map<std::string, std::any>());
+    auto time_step = env.reset();
 
     std::cout<<"Reward on reset: "<<time_step.reward()<<std::endl;
     std::cout<<"Observation on reset: "<<time_step.observation()<<std::endl;
     std::cout<<"Is terminal state: "<<time_step.done()<<std::endl;
+	std::cout<<"Env idx: "<<env.idx()<<std::endl;
 
     //...print the time_step
     std::cout<<time_step<<std::endl;
@@ -73,7 +70,6 @@ void test_frozen_lake(const RESTApiServerWrapper& server){
     std::cout<<"Dynamics for state="<<state<<" and action="<<action<<std::endl;
 
     for(auto item:dynamics){
-
         std::cout<<std::get<0>(item)<<std::endl;
         std::cout<<std::get<1>(item)<<std::endl;
         std::cout<<std::get<2>(item)<<std::endl;
@@ -85,13 +81,10 @@ void test_frozen_lake(const RESTApiServerWrapper& server){
 	
 	new_time_step = env.step(action);
     std::cout<<new_time_step<<std::endl;
-	
-	std::cout<<"env cidx: "<<env.cidx()<<std::endl;
 
     // close the environment
     env.close();
 }
-
 }
 
 
@@ -109,3 +102,11 @@ int main(){
 }
 
 ```
+
+Gymnasium environments exposed over a REST like API can be found at: <href="https://github.com/pockerman/bitrl-rest-api">bitrl-rest-api</a>
+Various RL algorithms using the environments can be found at <a href="https://github.com/pockerman/cuberl/tree/master">cuberl</a>.
+
+Furthermore, there is some minimal support for working with Arduino UNO boards over USB or WiFi.
+See also <a href="https://rlenvscpp.readthedocs.io/en/latest/working_with_webots.html">Working with Webots</a>
+for how to integrate ```bitrl``` with <a href="https://cyberbotics.com/doc/guide/installing-webots">Webots</a>.
+

examples/CMakeLists.txt

@@ -25,8 +25,8 @@ IF(BITRL_WEBOTS)
 ENDIF()
 
 ADD_SUBDIRECTORY(example_1)
-ADD_SUBDIRECTORY(example_2)
-ADD_SUBDIRECTORY(example_3)
+#ADD_SUBDIRECTORY(example_2)
+#ADD_SUBDIRECTORY(example_3)
 #ADD_SUBDIRECTORY(example_4)
 ADD_SUBDIRECTORY(example_5)
 ADD_SUBDIRECTORY(example_6)

examples/box2d/box2d_example.cpp

@@ -3,17 +3,19 @@
 //
 
 #include "bitrl/envs/gymnasium/box2d/lunar_lander_env.h"
-#include "bitrl/envs/api_server/apiserver.h"
+#include "bitrl/network/rest_rl_env_client.h"
 
 #include <unordered_map>
 #include <vector>
 #include <iostream>
 
+#include "../../src/bitrl/sensors/ekf_sensor_fusion.h"
+
 namespace box2d_example
 {
+    using namespace bitrl;
     const std::string SERVER_URL = "http://0.0.0.0:8001/api";
     using bitrl::real_t;
-    using bitrl::envs::RESTApiServerWrapper;
     using bitrl::envs::gymnasium::LunarLanderDiscreteEnv;
     using bitrl::envs::gymnasium::LunarLanderContinuousEnv;
 }
@@ -22,7 +24,7 @@ int main()
 {
     using namespace box2d_example;
 
-    RESTApiServerWrapper server(SERVER_URL, true);
+    bitrl::network::RESTRLEnvClient server(SERVER_URL, true);
 
     std::unordered_map<std::string, std::any> options;
     options["wind_power"] = std::any(static_cast<bitrl::real_t>(10.0));
@@ -57,7 +59,9 @@ int main()
         std::cout<<"Working with LunarLanderContinuousEnv..."<<std::endl;
 
         LunarLanderContinuousEnv env(server);
-        env.make("v3", options);
+
+        std::unordered_map<std::string, std::any> reset_options;
+        env.make("v3", options, reset_options);
 
         std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
         std::cout<<"Is environment alive? "<<env.is_alive()<<std::endl;
@@ -70,13 +74,8 @@ int main()
         std::vector<real_t> action = {0.8, 0.9};
         time_step = env.step(action);
         std::cout<<"Time step: "<<time_step<<std::endl;
-
-        auto copy = env.make_copy(1);
-        time_step = copy.reset();
-        std::cout<<"Time step: "<<time_step<<std::endl;
-
         env.close();
-        copy.close();
+
     }
 
 }