Merge pull request #145 from pockerman/feature_142_lunar_lander_environment

Feature 142 lunar lander environment

Author: pockerman

CMakeLists.txt

@@ -147,7 +147,10 @@ FILE(GLOB SRCS src/rlenvs/*.cpp
 			   src/rlenvs/utils/trajectory/*.cpp
    )
 
-ADD_LIBRARY(rlenvscpplib SHARED ${SRCS})
+ADD_LIBRARY(rlenvscpplib SHARED ${SRCS}
+		src/rlenvs/envs/gymnasium/box2d/lunar_lander_env.h
+		src/rlenvs/envs/gymnasium/box2d/lunar_lander_env.cpp
+		examples/box2d/box2d_example.cpp)
 
 SET_TARGET_PROPERTIES(rlenvscpplib PROPERTIES LINKER_LANGUAGE CXX)
 INSTALL(TARGETS rlenvscpplib DESTINATION ${CMAKE_INSTALL_PREFIX})

docs/requirements.txt

@@ -2,3 +2,5 @@ sphinx==5.0.2
 breathe==4.35.0
 exhale==0.3.7
 sphinx_rtd_theme==3.0.2
+
+

examples/CMakeLists.txt

@@ -45,14 +45,15 @@ ADD_SUBDIRECTORY(example_10)
 ADD_SUBDIRECTORY(example_11)
 ADD_SUBDIRECTORY(example_12)
 ADD_SUBDIRECTORY(example_13)
+ADD_SUBDIRECTORY(box2d)
 
-#IF(RLENVSCPP_WEBOTS)
-#	ADD_SUBDIRECTORY(webots/world_1)
-#ENDIF()
+IF(RLENVSCPP_WEBOTS)
+	ADD_SUBDIRECTORY(webots/world_1)
+ENDIF()
 
 
 # We follow a different approach when working with
 # Ray so we don't compile the example with CMake
-#IF(RLENVSCPP_RAY)
-#	ADD_SUBDIRECTORY(ray/ray_example_2)
-#ENDIF()
+IF(RLENVSCPP_RAY)
+	ADD_SUBDIRECTORY(ray/ray_example_2)
+ENDIF()

examples/box2d/CMakeLists.txt

@@ -0,0 +1,11 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.20)
+
+SET(EXECUTABLE  box2d_example)
+SET(SOURCE ${EXECUTABLE}.cpp)
+
+ADD_EXECUTABLE(${EXECUTABLE} ${SOURCE})
+TARGET_LINK_LIBRARIES(${EXECUTABLE} rlenvscpplib)
+
+IF(RLENVSCPP_WEBOTS)
+	TARGET_LINK_LIBRARIES(${EXECUTABLE} CppController)
+ENDIF()

examples/box2d/box2d_example.cpp

@@ -0,0 +1,50 @@
+//
+// Created by alex on 7/5/25.
+//
+
+#include "rlenvs/envs/gymnasium/box2d/lunar_lander_env.h"
+#include "rlenvs/envs/api_server/apiserver.h"
+
+#include <unordered_map>
+#include <iostream>
+
+namespace box2d_example
+{
+    const std::string SERVER_URL = "http://0.0.0.0:8001/api";
+    using rlenvscpp::envs::RESTApiServerWrapper;
+    using rlenvscpp::envs::gymnasium::LunarLanderDiscreteEnv;
+}
+
+int main()
+{
+    using namespace box2d_example;
+
+    RESTApiServerWrapper server(SERVER_URL, true);
+
+    LunarLanderDiscreteEnv env(server);
+
+    std::unordered_map<std::string, std::any> options;
+    options["wind_power"] = std::any(static_cast<rlenvscpp::real_t>(10.0));
+    options["enable_wind"] = std::any(static_cast<bool>(true));
+    options["gravity"] = std::any(static_cast<rlenvscpp::real_t>(-9.86));
+    options["turbulence_power"] = std::any(static_cast<rlenvscpp::real_t>(1.5));
+    env.make("v3", options);
+
+    std::cout<<"Is environment created? "<<env.is_created()<<std::endl;
+    std::cout<<"Is environment alive? "<<env.is_alive()<<std::endl;
+    std::cout<<"Number of valid actions? "<<env.n_actions()<<std::endl;
+
+    auto time_step = env.reset();
+    std::cout<<"Time step: "<<time_step<<std::endl;
+
+    time_step = env.step(1);
+    std::cout<<"Time step: "<<time_step<<std::endl;
+
+    auto copy = env.make_copy(1);
+    time_step = copy.reset();
+    std::cout<<"Time step: "<<time_step<<std::endl;
+
+    env.close();
+    copy.close();
+
+}

examples/example_13/example_13.cpp

@@ -1,20 +1,3 @@
-/**
- * This example utilises the ```TensorboardServer``` class to log values of interest when running
- * an experiment. We can monitor the experimet  using <a href="https://www.tensorflow.org/tensorboard">tensorboard</a>.
- * The   ```TensorboardServer``` class is a simple wrapper that exposes three functions
- *
- *	- ```add_scalar```
- *	- ```add_scalars```
- *	- ```add_text```
- *
- *	We will use ```add_scalar``` and ```add_text```. In order to run this example, fire up the server using the ```torchboard_server/start_uvicorn.sh```.
- *	The server listens at port 8002. You can change this however you want just make sure that the port is not used and also update the
- *	variable ```TORCH_SERVER_HOST``` in the code below accordingly. Note that the implementation uses
- *	<a href="https://pytorch.org/docs/stable/_modules/torch/utils/tensorboard/writer.html#SummaryWriter">SummaryWriter</a> class.
- *	Thus you will need to have PyTorch installed on the machine that you run the server.
- */ 
-
-
 #include "rlenvs/rlenvs_types_v2.h"
 #include "rlenvs/utils/geometry/geom_point.h"
 #include "rlenvs/utils/geometry/mesh/mesh.h"

rest_api/gymnasium_envs/box2d/__init__.py

@@ -0,0 +1,182 @@
+"""The Acrobot environment from gymnasium:
+for more information check: https://gymnasium.farama.org/environments/classic_control/acrobot/
+
+"""
+import gymnasium as gym
+from typing import Any
+from loguru import logger
+from fastapi import APIRouter, Body, status
+from fastapi.responses import JSONResponse
+from fastapi import HTTPException
+from time_step_response import TimeStep, TimeStepType
+
+lunar_lander_discrete_router = APIRouter(prefix="/gymnasium/lunar-lander-discrete-env",
+                                         tags=["Lunar Lander Discrete API"])
+
+ENV_NAME = "LunarLander"
+
+# the environments to create
+envs = {
+    0: None
+}
+
+# actions that the environment accepts
+ACTIONS_SPACE = {0: "do nothing",
+                 1: "fire left orientation engine",
+                 2: "fire main engine",
+                 3: "fire right orientation engine"
+                 }
+
+
+@lunar_lander_discrete_router.get("/action-space")
+async def get_action_space() -> JSONResponse:
+    return JSONResponse(status_code=status.HTTP_200_OK,
+                        content={"action_space": ACTIONS_SPACE})
+
+
+@lunar_lander_discrete_router.get("/is-alive")
+async def get_is_alive(cidx: int) -> JSONResponse:
+    global envs
+    if cidx in envs:
+        env = envs[cidx]
+
+        if env is None:
+            return JSONResponse(status_code=status.HTTP_200_OK,
+                                content={"result": False})
+        else:
+            return JSONResponse(status_code=status.HTTP_200_OK,
+                                content={"result": True})
+    else:
+        return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST,
+                            content={"message": f"Environment {ENV_NAME} and index {cidx} has not been created"})
+
+
+@lunar_lander_discrete_router.post("/close")
+async def close(cidx: int) -> JSONResponse:
+    global envs
+    if cidx in envs:
+        env = envs[cidx]
+        if env is not None:
+            envs[cidx].close()
+            envs[cidx] = None
+            logger.info(f'Closed environment {ENV_NAME}  and index {cidx}')
+            return JSONResponse(status_code=status.HTTP_202_ACCEPTED,
+                                content={"message": f"Environment {ENV_NAME} and index {cidx} is closed"})
+
+    return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST,
+                        content={"message": f"Environment {ENV_NAME} and index {cidx} has not been created"})
+
+
+@lunar_lander_discrete_router.post("/make")
+async def make(version: str = Body(default="v3"), cidx: int = Body(...),
+               options: dict[str, Any] = Body(default={'gravity': -10.0, 'enable_wind': False,
+                                                       'wind_power': 15.0, 'turbulence_power': 1.5})) -> JSONResponse:
+    if version == 'v1' or version == 'v2':
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
+                            detail='Environment version v1 for `LunarLander` '
+                                   'is deprecated. Please use `LunarLander-v3` instead.')
+
+    global envs
+    env_type = f"{ENV_NAME}-{version}"
+    if cidx in envs:
+        env = envs[cidx]
+
+        if env is not None:
+            envs[cidx].close()
+
+        try:
+            env = gym.make(env_type, continuous=False, **options)
+            envs[cidx] = env
+        except Exception as e:
+            logger.error('An exception was raised')
+            logger.opt(exception=e).info("Logging exception traceback")
+            raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                                detail=str(e))
+    else:
+        try:
+            env = gym.make(env_type)
+            envs[cidx] = env
+        except Exception as e:
+            logger.error('An exception was raised')
+            logger.opt(exception=e).info("Logging exception traceback")
+            raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                                detail=str(e))
+
+    logger.info(f'Created environment {ENV_NAME} and index {cidx}')
+    return JSONResponse(status_code=status.HTTP_201_CREATED,
+                        content={"result": True})
+
+
+@lunar_lander_discrete_router.post("/reset")
+async def reset(seed: int = Body(default=42), cidx: int = Body(...),
+                options: dict[str, Any] = Body(default={})) -> JSONResponse:
+    """Reset the environment
+
+    :return:
+    """
+
+    global envs
+    if cidx in envs:
+        env = envs[cidx]
+
+        if env is not None:
+
+            if len(options) != 0:
+                observation, info = env.reset(seed=seed, options=options)
+            else:
+                observation, info = env.reset(seed=seed)
+            observation = [float(val) for val in observation]
+            step = TimeStep(observation=observation,
+                            reward=0.0,
+                            step_type=TimeStepType.FIRST,
+                            info=info,
+                            discount=1.0)
+            logger.info(f'Reset environment {ENV_NAME}  and index {cidx}')
+            return JSONResponse(status_code=status.HTTP_202_ACCEPTED,
+                                content={"time_step": step.model_dump()})
+
+    raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
+                        detail={"message": f"Environment {ENV_NAME} is not initialized."
+                                           " Have you called make()?"})
+
+
+@lunar_lander_discrete_router.post("/step")
+async def step(action: int = Body(...), cidx: int = Body(...)) -> JSONResponse:
+    if action not in ACTIONS_SPACE:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
+                            detail=f"Action {action} not in {list(ACTIONS_SPACE.keys())}")
+
+    global envs
+    if cidx in envs:
+        env = envs[cidx]
+
+        if env is not None:
+            logger.info(f"Stepping in environment {ENV_NAME} with action={action}")
+            observation, reward, terminated, truncated, info = env.step(action)
+            observation = [float(val) for val in observation]
+
+            step_type = TimeStepType.MID
+            if terminated:
+                step_type = TimeStepType.LAST
+
+            if info is not None:
+                info['truncated'] = truncated
+
+            step = TimeStep(observation=observation,
+                            reward=reward,
+                            step_type=step_type,
+                            info=info,
+                            discount=1.0)
+
+            logger.info(f'Step in environment {ENV_NAME} and index {cidx}')
+            return JSONResponse(status_code=status.HTTP_202_ACCEPTED,
+                                content={"time_step": step.model_dump()})
+
+    raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST,
+                        detail=f"Environment {ENV_NAME} is not initialized. Have you called make()?")
+
+
+@lunar_lander_discrete_router.post("/sync")
+async def sync(cidx: int = Body(...), options: dict[str, Any] = Body(default={})) -> JSONResponse:
+    return JSONResponse(status_code=status.HTTP_202_ACCEPTED,
+                        content={"message": "OK"})

rest_api/gymnasium_envs/box2d/lunar_lander_env_discrete_api.py

@@ -0,0 +1,5 @@
+Box2D==2.3.10
+fastapi==0.115.9
+fastapi-cli==0.0.7
+gymnasium==1.0.0
+swig==4.3.1

rest_api/requirements.txt

@@ -12,6 +12,7 @@
 from gymnasium_envs.classic_control.pendulum_env_api import pendulum_router
 from gymnasium_envs.classic_control.acrobot_env_api import acrobot_router
 from gymnasium_envs.classic_control.v.acrobot_v_env_api import acrobot_v_router
+from gymnasium_envs.box2d.lunar_lander_env_discrete_api import lunar_lander_discrete_router
 from gdrl.gym_walk_env.gym_walk_env_api import gym_walk_env_router
 
 from gym_pybullet_drones_env.quadrotor_sim_env_api import FOUND_GYM_PYBULLET
@@ -49,6 +50,8 @@
 app.include_router(pendulum_router, prefix=BASE_URL)
 app.include_router(acrobot_router, prefix=BASE_URL)
 app.include_router(acrobot_v_router, prefix=BASE_URL)
+app.include_router(lunar_lander_discrete_router, prefix=BASE_URL)
+
 if FOUND_GYM_PYBULLET:
     app.include_router(quadrotor_sim_router, prefix=BASE_URL)