Merge pull request #173 from pockerman/feat/169_refactor_ctrl_lib_from_cuberl

Update documentation

Author: pockerman

examples/example_1/example_1.md

@@ -1,11 +1,11 @@
-\page bitrl_example_1 Example 1 Using Gymnasium environments
+\page bitrl_example_1 BitRL Example 1 Using Gymnasium environments (Part 1)
 
 In this example we will see how to interact with  <a href="https://gymnasium.farama.org/index.html">Gymnasium</a> environments and 
 specifically how to create an interact with <a href="https://gymnasium.farama.org/environments/toy_text/frozen_lake/">FroznLake</a> environment.
 
-In bitrl, Gymnasium-based environment are interacted over a REST-like API maintained here: <a href="https://github.com/pockerman/bitrl-rest-api">bitrl-envs-api</a>.
-bitrl itself implements classes that hide this interaction from the client code. 
-In general, environment classes in bitrl, have to implement the \ref bitrl::envs::EnvBase "bitrl::envs::EnvBase" API.
+In _bitrl_, Gymnasium-based environments are interacted over a REST-like API maintained here: <a href="https://github.com/pockerman/bitrl-rest-api">bitrl-envs-api</a>.
+_bitrl_ itself implements classes that hide this interaction from the client code. 
+In general, environment classes in _bitrl_, have to implement the \ref bitrl::envs::EnvBase "bitrl::envs::EnvBase" API.
 
 In this example we will use the \ref bitrl::envs::gymnasium::FrozenLake "bitrl::envs::gymnasium::FrozenLake"  
 class. This is a template class, see the example below, that itself inherits from  \ref bitrl::envs::gymnasium::GymnasiumEnvBase "bitrl::envs::gymnasium::GymnasiumEnvBase"
@@ -132,6 +132,6 @@ int main()
 @endcode
 
 In order to run the example you will need an instance of the <a href="https://github.com/pockerman/bitrl-rest-api">bitrl-envs-api</a> server running
-on your machine listening at por 8001. Note the actual example also shows how to use \ref bitrl::envs::gymnasium::Taxi "bitrl::envs::gymnasium::Taxi",
+on your machine listening at port 8001. Note the actual example also shows how to use \ref bitrl::envs::gymnasium::Taxi "bitrl::envs::gymnasium::Taxi",
 \ref bitrl::envs::gymnasium::CliffWorld "bitrl::envs::gymnasium::CliffWorld" and \ref bitrl::envs::gymnasium::BlackJack "bitrl::envs::gymnasium::BlackJack"
 environments.

examples/example_2/example_2.cpp

@@ -43,7 +43,7 @@ void test_gymwalk(RESTRLEnvClient &server)
     // once the env is created we can get it's id
     std::cout << "Environment idx is: " << env.idx() << std::endl;
 
-    // the create flag should be true
+    // the created flag should be true
     std::cout << "Is environment created? " << env.is_created() << std::endl;
 
     // environment should be alive on the server

examples/example_2/example_2.md

@@ -0,0 +1,122 @@
+\page bitrl_example_2 BitRL Example 2 Using GymWalk Environment
+
+The \ref bitrl::envs::gdrl::GymWalk "bitrl::envs::gdrl::GymWalk" is an implementation of the
+environment from https://github.com/mimoralea/gym-walk. Just like with \ref bitrl_example_1, you will need
+an instance of the  <a href="https://github.com/pockerman/bitrl-rest-api">bitrl-envs-api</a> server running in order 
+to actually use the environment. Other than that there isn't much to say for this example.
+Below is the driver code.
+
+@code{.cpp}
+#include "bitrl/bitrl_types.h"
+#include "bitrl/envs/gdrl/gym_walk.h"
+#include "bitrl/network/rest_rl_env_client.h"
+
+#include <any>
+#include <iostream>
+#include <string>
+#include <unordered_map>
+
+namespace example_1
+{
+using namespace bitrl;
+
+const std::string SERVER_URL = "http://0.0.0.0:8001/api";
+
+using bitrl::envs::gdrl::GymWalk;
+using bitrl::network::RESTRLEnvClient;
+
+void test_gymwalk(RESTRLEnvClient &server)
+{
+// the environment is not registered with the server
+std::cout << "Is environment registered: " << server.is_env_registered(GymWalk<4>::name)
+<< std::endl;
+
+    // when the environment is created we register it with the REST client
+    GymWalk<4> env(server);
+
+    // environment name can also be accessed via env.env_name()
+    std::cout << "Is environment registered: " << server.is_env_registered(env.env_name())
+              << std::endl;
+    std::cout << "Environment URL: " << env.get_url() << std::endl;
+
+    // make the environment we pass both make options
+    // and reset options
+    std::unordered_map<std::string, std::any> make_ops;
+    std::unordered_map<std::string, std::any> reset_ops;
+    reset_ops.insert({"seed", static_cast<uint_t>(42)});
+    env.make("v1", make_ops, reset_ops);
+
+    // query the environemnt version
+    std::cout << "Environment version: " << env.version() << std::endl;
+
+    // once the env is created we can get it's id
+    std::cout << "Environment idx is: " << env.idx() << std::endl;
+
+    // the created flag should be true
+    std::cout << "Is environment created? " << env.is_created() << std::endl;
+
+    // environment should be alive on the server
+    std::cout << "Is environment alive? " << env.is_alive() << std::endl;
+
+    // FrozenLake is a discrete state-action env so we can
+    // query number of actions and states
+    std::cout << "Number of valid actions? " << env.n_actions() << std::endl;
+    std::cout << "Number of states? " << env.n_states() << std::endl;
+
+    // reset the environment
+    auto time_step = env.reset();
+
+    std::cout << "Reward on reset: " << time_step.reward() << std::endl;
+    std::cout << "Observation on reset: " << time_step.observation() << std::endl;
+    std::cout << "Is terminal state: " << time_step.done() << std::endl;
+
+    //...print the time_step
+    std::cout << time_step << std::endl;
+
+    // take an action in the environment
+    // 2 = RIGHT
+    auto new_time_step = env.step(1);
+    std::cout << new_time_step << std::endl;
+
+    // get the dynamics of the environment for the given state and action
+    auto state = 0;
+    auto action = 1;
+    auto dynamics = env.p(state, action);
+
+    std::cout << "Dynamics for state=" << state << " and action=" << action << std::endl;
+    for (auto item : dynamics)
+    {
+        std::cout << std::get<0>(item) << std::endl;
+        std::cout << std::get<1>(item) << std::endl;
+        std::cout << std::get<2>(item) << std::endl;
+        std::cout << std::get<3>(item) << std::endl;
+    }
+
+    // discrete action environments can sample
+    // actions
+    action = env.sample_action();
+    std::cout << "Action sampled: " << action << std::endl;
+
+    new_time_step = env.step(action);
+    std::cout << new_time_step << std::endl;
+
+    // close the environment
+    env.close();
+}
+
+} // namespace example_1
+
+int main()
+{
+
+    using namespace example_1;
+
+    RESTRLEnvClient server(SERVER_URL, false);
+
+    std::cout << "Testing GymWalk..." << std::endl;
+    example_1::test_gymwalk(server);
+    std::cout << "====================" << std::endl;
+    return 0;
+}
+
+@endcode

examples/example_3/example_3.md

@@ -0,0 +1,88 @@
+\page bitrl_example_3 BitRL Example 3 Using Gymnasium environments (Part 2)
+
+In this example we will see how to interact with  <a href="https://gymnasium.farama.org/index.html">Gymnasium</a> environments and
+specifically how to create an interact with <a href="https://gymnasium.farama.org/environments/classic_control/cart_pole/">CartPole</a> environment.
+
+As already mentioned in  \ref bitrl_example_1, Gymnasium-based environments are interacted over a REST-like API maintained here: <a href="https://github.com/pockerman/bitrl-rest-api">bitrl-envs-api</a>.
+_bitrl_ itself implements classes that hide this interaction from the client code.
+In general, environment classes in _bitrl_, have to implement the \ref bitrl::envs::EnvBase "bitrl::envs::EnvBase" API.
+
+In this example we will use the \ref bitrl::envs::gymnasium::CartPole "bitrl::envs::gymnasium::CartPole"  
+class. This is a template class, see the example below, that itself inherits from  \ref bitrl::envs::gymnasium::GymnasiumEnvBase "bitrl::envs::gymnasium::GymnasiumEnvBase"
+class.
+
+Below is the driver code. 
+
+@code{.cpp}
+
+#include "bitrl/bitrl_consts.h"
+#include "bitrl/bitrl_types.h"
+#include "bitrl/envs/gymnasium/classic_control/cart_pole_env.h"
+#include "bitrl/network/rest_rl_env_client.h"
+
+#ifdef BITRL_DEBUG
+#include <cassert>
+#endif
+
+#include <iostream>
+#include <random>
+#include <string>
+#include <unordered_map>
+
+namespace example
+{
+using namespace bitrl;
+using namespace bitrl::envs::gymnasium;
+using bitrl::network::RESTRLEnvClient;
+
+void test_cart_pole(RESTRLEnvClient &server)
+{
+std::cout << "Is environment registered: " << server.is_env_registered(CartPole::name)
+<< std::endl;
+
+    // create the environment
+    CartPole env(server);
+
+    std::cout << "Name: " << env.name << std::endl;
+    std::cout << "Number of actions: " << env.n_actions() << std::endl;
+
+    // make the environment
+    std::unordered_map<std::string, std::any> options;
+    std::unordered_map<std::string, std::any> reset_ops;
+    reset_ops.insert({"seed", static_cast<uint_t>(42)});
+    env.make("v1", options, reset_ops);
+
+    auto time_step = env.reset();
+    std::cout << "Time step: " << time_step << std::endl;
+
+    // step in the environment
+    time_step = env.step(0);
+    std::cout << "Time step after action: " << time_step << std::endl;
+
+    env.close();
+}
+
+} // namespace example
+
+int main()
+{
+
+    using namespace example;
+
+    const std::string SERVER_URL = "http://0.0.0.0:8001/api";
+
+    RESTRLEnvClient server(SERVER_URL, true);
+
+    std::cout << "Testing CartPole..." << std::endl;
+    example::test_cart_pole(server);
+    std::cout << "====================" << std::endl;
+
+    return 0;
+}
+@endcode
+
+
+In order to run the example you will need an instance of the <a href="https://github.com/pockerman/bitrl-rest-api">bitrl-envs-api</a> server running
+on your machine listening at port 8001. Note the actual example also shows how to use \ref bitrl::envs::gymnasium::Pendulum "bitrl::envs::gymnasium::Pendulum",
+\ref bitrl::envs::gymnasium::Acrobot "bitrl::envs::gymnasium::Acrobot" and \ref bitrl::envs::gymnasium::MountainCar "bitrl::envs::gymnasium::MountainCar"
+environments.

examples/example_4/example_4.md

@@ -0,0 +1,84 @@
+\page bitrl_example_4 BitRL Example 4 Using Gymnasium environments (Part 3)
+
+This example follows \ref bitrl_example_1 and \ref bitrl_example_3 and shows how to use
+\ref bitrl::envs::gymnasium::LunarLanderDiscreteEnv "bitrl::envs::gymnasium::LunarLanderDiscreteEnv"  and
+\ref bitrl::envs::gymnasium::LunarLanderContinuousEnv "bitrl::envs::gymnasium::LunarLanderContinuousEnv".
+
+
+@code{.cpp}
+
+#include "bitrl/envs/gymnasium/box2d/lunar_lander_env.h"
+#include "bitrl/network/rest_rl_env_client.h"
+
+#include <iostream>
+#include <unordered_map>
+#include <vector>
+
+namespace box2d_example
+{
+using namespace bitrl;
+const std::string SERVER_URL = "http://0.0.0.0:8001/api";
+using bitrl::real_t;
+using bitrl::envs::gymnasium::LunarLanderContinuousEnv;
+using bitrl::envs::gymnasium::LunarLanderDiscreteEnv;
+} // namespace box2d_example
+
+int main()
+{
+using namespace box2d_example;
+
+    bitrl::network::RESTRLEnvClient server(SERVER_URL, true);
+
+    std::unordered_map<std::string, std::any> options;
+    options["wind_power"] = std::any(static_cast<bitrl::real_t>(10.0));
+    options["enable_wind"] = std::any(static_cast<bool>(true));
+    options["gravity"] = std::any(static_cast<bitrl::real_t>(-9.86));
+    options["turbulence_power"] = std::any(static_cast<bitrl::real_t>(1.5));
+
+    std::unordered_map<std::string, std::any> reset_ops;
+    reset_ops.insert({"seed", static_cast<uint_t>(42)});
+
+    {
+        std::cout<<"Working with LunarLanderDiscreteEnv..."<<std::endl;
+        std::cout << "Is environment registered: " << server.is_env_registered(LunarLanderDiscreteEnv::name)
+              << std::endl;
+        LunarLanderDiscreteEnv env(server);
+
+        env.make("v3", options, reset_ops);
+
+        std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
+        std::cout<<"Is environment alive? "<<env.is_alive()<<std::endl;
+        std::cout<<"Number of valid actions? "<<env.n_actions()<<std::endl;
+
+        auto time_step = env.reset();
+        std::cout<<"Time step: "<<time_step<<std::endl;
+
+        time_step = env.step(1);
+        std::cout<<"Time step: "<<time_step<<std::endl;
+        env.close();
+
+    }
+    {
+        std::cout << "Working with LunarLanderContinuousEnv..." << std::endl;
+
+        LunarLanderContinuousEnv env(server);
+        env.make("v3", options, reset_ops);
+
+        std::cout << "Is environment created? " << env.is_created() << std::endl;
+        std::cout << "Is environment alive? " << env.is_alive() << std::endl;
+        std::cout << "Action space size " << env.n_actions() << std::endl;
+        std::cout << "Environment URI: " << server.get_uri(env.env_name()) << std::endl;
+
+        auto time_step = env.reset();
+        std::cout << "Time step: " << time_step << std::endl;
+
+        std::vector<real_t> action = {0.8, 0.9};
+        time_step = env.step(action);
+        std::cout << "Time step: " << time_step << std::endl;
+        env.close();
+    }
+
+    return 0;
+}
+
+@endcode