Merge pull request #134 from pockerman/integrate_ray_for_distributed_computation

Update documentation

Author: pockerman

CMakeLists.txt

@@ -26,9 +26,10 @@ IF(COMMAND cmake_policy)
 ENDIF(COMMAND cmake_policy)
 
 OPTION(ENABLE_TESTS_FLAG OFF)
-OPTION(ENABLE_EXAMPLES_FLAG OFF)
+OPTION(ENABLE_EXAMPLES OFF)
 OPTION(ENABLE_DOC_FLAG OFF)
 OPTION(ENABLE_WEBOTS OFF)
+OPTION(ENABLE_RAY OFF)
 
 SET(CMAKE_BUILD_TYPE "Debug")
 
@@ -74,16 +75,26 @@ IF(ENABLE_WEBOTS)
 	# we have webots include the directories
     SET(WEBOTS_LIB_DIRS ${PROJECT_SOURCE_DIR}/external/webots/lib/controller)    
     SET(WEBOTS_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/external/webots/include/controller/cpp)
+	SET(RLENVSCPP_WEBOTS ON)
 
 	INCLUDE_DIRECTORIES(${WEBOTS_INCLUDE_DIRS})
 	LINK_DIRECTORIES(${WEBOTS_LIB_DIRS})
-	#MESSAGE( STATUS  "${WEBOTS_INCLUDE_DIRS}")
-	
-	SET(RLENVSCPP_WEBOTS ON)
 ELSE()
 	MESSAGE( WARNING  "Building without Webots.")
 ENDIF()
 
+IF(ENABLE_RAY)
+
+	SET(RLENVSCPP_RAY ON)
+    SET(RAY_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/external/ray/thirdparty/include)
+	SET(RAY_LIB_DIRS ${PROJECT_SOURCE_DIR}/external/ray/thirdparty/lib)
+	
+	INCLUDE_DIRECTORIES(${RAY_INCLUDE_DIRS})
+	LINK_DIRECTORIES(${RAY_LIB_DIRS})
+ELSE()
+	MESSAGE( WARNING  "Building without Ray.")
+ENDIF()
+
 configure_file(config.h.in ${PROJECT_SOURCE_DIR}/src/rlenvs/rlenvscpp_config.h @ONLY)
 configure_file(version.h.in ${PROJECT_SOURCE_DIR}/src/rlenvs/rlenvscpp_version.h @ONLY)
 
@@ -141,7 +152,7 @@ SET_TARGET_PROPERTIES(rlenvscpplib PROPERTIES LINKER_LANGUAGE CXX)
 INSTALL(TARGETS rlenvscpplib DESTINATION ${CMAKE_INSTALL_PREFIX})
 MESSAGE(STATUS "Installation destination at: ${CMAKE_INSTALL_PREFIX}")
 
-IF(ENABLE_EXAMPLES_FLAG)
+IF(ENABLE_EXAMPLES)
 	# Add the examples
 	ADD_SUBDIRECTORY(examples)
 ELSE()

config.h.in

@@ -7,7 +7,11 @@
 /*Use PyTorch */
 #cmakedefine USE_PYTORCH
 
+/*Use Webots*/
 #cmakedefine RLENVSCPP_WEBOTS
 
+/*Use Ray*/
+#cmakedefine RLENVSCPP_RAY
+
 #endif
 

docs/examples.rst

@@ -0,0 +1,10 @@
+Examples
+============
+
+
+.. toctree::
+   :maxdepth: 4
+
+   examples/ray/ray_example_1.rst
+   
+   

docs/examples/ray/ray_example_1.rst

@@ -0,0 +1,175 @@
+Using Ray core Tasks
+===========================
+
+This example will show you how to use one of Ray's core abstractions.
+Namely how to create Ray `Tasks <a href="https://docs.ray.io/en/latest/ray-core/tasks.html">`_, 
+
+With Ray we can execute arbitrary functions on separate workers.
+These functions are called Ray tasks. The code snippet below shows 
+you how to create a Ray Task:
+
+.. code-block::
+
+	// we need to include the Ray API
+	#include <ray/api.h>
+	
+	// This is the C++ function we want to execute
+	int do_sth() {
+	  return 1;
+	}
+	
+	// In order to be able to execute this function
+	// we need to register it using `RAY_REMOTE`.
+	RAY_REMOTE(do_sth);
+	
+	int main(int argc, char **argv) {
+		
+		// We need to initialize Ray before using it
+		ray::Init();
+
+		// Invoke the above method as a Ray task.
+		// This will immediately return an object ref (a future) and then create
+		// a task that will be executed on a worker process.
+		auto res = ray::Task(do_sth).Remote();
+	
+		// The result can be retrieved with ``ray::ObjectRef::Get``.
+		auto result = *res.Get();
+		
+		// shutodown Ray
+		ray::Shutdown();
+		
+		std::cout<<"Result is: "<<result<<std::endl;
+		std::cout<<"Running normal C++ stuff here..."<<std::endl;
+		
+		result = do_sth();
+		std::cout<<"Result is: "<<result<<std::endl;
+		
+		
+		return 0;
+	}
+
+You can build the code using the ``cmake`` toolchain, but Ray favors `Bazel <https://bazel.build/>`_ 
+for building and running jobs. Let's see how we can use this to build the example.
+Copy the following files and directories from ``external/ray`` directory:
+
+-	``run.sh``
+-  ``BUILD.bazel``
+-  ``WORKSPACE``
+- ``thirdparty`` 
+
+into the directory where the ``ray_example_1`` is located.
+Edit the ``run.sh`` file and change the line where the ``bazel`` executable is called to this:
+
+.. code-block::
+
+	bazel build //:ray_example_1
+	
+In addition you need to change where the Ray library is located.
+Edit the line where the ``LD_LIBRARY_PATH`` variable is declared. This should
+point to ``external/ray/thirdparty/lib`` path.
+
+.. code-block::
+
+	LD_LIBRARY_PATH="thirdparty/lib" "${ROOT_DIR}"/bazel-bin/ray_example_1
+
+Edit the ``BUILD.bazel`` file and change accordingly the variables:
+
+.. code-block::
+
+	cc_binary(
+		name = "ray_example_1",
+		srcs = glob([
+			"*.cpp",
+		]),
+		data = [
+			"ray_example_1.so",
+		],
+		linkstatic = True,
+		deps = [
+			":ray_api",
+		],
+	)
+
+	cc_binary(
+		name = "ray_example_1.so",
+		srcs = glob([
+			"*.cpp",
+		]),
+		linkopts = ["-shared"],
+		linkstatic = True,
+		deps = [
+			":ray_api",
+		],
+	)
+
+	cc_library(
+		name = "ray_api",
+		srcs = [
+			"thirdparty/lib/libray_api.so",
+		],
+		hdrs = glob([
+			"thirdparty/include/**/*.h",
+			"thirdparty/include/**/*.hpp",
+		]),
+		linkopts = ["-Wl,-rpath,./"],
+		strip_include_prefix = "thirdparty/include",
+		visibility = ["//visibility:public"],
+	)
+
+In order to build and run the example execute the ``run.sh`` script. This should
+produce the following output:
+
+.. code-block::
+
+	INFO: Analyzed target //:ray_example_2 (1 packages loaded, 3440 targets configured).
+	INFO: Found 1 target...
+	Target //:ray_example_2 up-to-date:
+	bazel-bin/ray_example_2
+	INFO: Elapsed time: 2.509s, Critical Path: 2.15s
+	INFO: 4 processes: 4 linux-sandbox.
+	INFO: Build completed successfully, 6 total actions
+	[2025-04-05 12:13:21,602 I 24601 24601] config_internal.cc:216: No code search path found yet. The program location path "/home/alex/.cache/bazel/_bazel_alex/25a018d10ef2129864cd574bc2dbc5b9/execroot/__main__/bazel-out/k8-fastbuild/bin" will be added for searching dynamic libraries by default. And you can add some search paths by '--ray_code_search_path'
+	[2025-04-05 12:13:21,603 I 24601 24601] process_helper.cc:51: ray start --head --port 6379 --redis-username default --redis-password 5241590000000000 --node-ip-address '192.168.0.129'
+	2025-04-05 12:13:22,376 - INFO - NumExpr defaulting to 8 threads.
+	Usage stats collection is enabled. To disable this, add `--disable-usage-stats` to the command that starts the cluster, or run the following command: `ray disable-usage-stats` before starting the cluster. See https://docs.ray.io/en/master/cluster/usage-stats.html for more details.
+	
+	Local node IP: 192.168.0.129
+
+	--------------------
+	Ray runtime started.
+	--------------------
+
+	Next steps
+	To add another node to this Ray cluster, run
+		ray start --address='192.168.0.129:6379'
+	
+	To connect to this Ray cluster:
+		import ray
+		ray.init(_node_ip_address='192.168.0.129')
+	
+	To submit a Ray job using the Ray Jobs CLI:
+		RAY_ADDRESS='http://127.0.0.1:8265' ray job submit --working-dir . -- python my_script.py
+	
+	See https://docs.ray.io/en/latest/cluster/running-applications/job-submission/index.html 
+	for more information on submitting Ray jobs to the Ray cluster.
+	
+	To terminate the Ray runtime, run
+		ray stop
+	
+	To view the status of the cluster, use
+		ray status
+	
+	To monitor and debug Ray, view the dashboard at 
+		127.0.0.1:8265
+	
+	If connection to the dashboard fails, check your firewall settings and network configuration.
+	[2025-04-05 12:13:25,278 I 24601 24601] gcs_client.cc:98: GcsClient has no Cluster ID set, and won't fetch from GCS.
+	[2025-04-05 12:13:25,292 I 24601 24601] gcs_client.cc:98: GcsClient has no Cluster ID set, and won't fetch from GCS.
+	2025-04-05 12:13:26,854 - INFO - NumExpr defaulting to 8 threads.
+	Stopped all 5 Ray processes.
+	Result is: 1
+	Running normal C++ stuff here...
+	Result is: 1
+
+
+

docs/images/ray_libs.png

@@ -12,8 +12,10 @@
 
    overview
    how_to_use
-   working_with_webots
    installation
+   working_with_webots
+   working_with_ray
+   examples
    api/library_root
 
 

docs/index.rst

@@ -13,32 +13,27 @@ The library has the following general dependencies
 - `CMake >= 3.10 <https://cmake.org/>`_
 - `Gtest  <https://github.com/google/googletest>`_  (if configured with tests)
 - `Eigen3 <https://eigen.tuxfamily.org/index.php?title=Main_Page>`_
+- `Ray Ray <https://docs.ray.io/en/master/index.html>`_ (if configured with Ray)
 
 
-``rlenvscpp`` also incorporates, see ``(src/extern)``, the following libraries
+``rlenvscpp`` also incorporates, see ``(src/extern)``, the following libraries:
 
 - `HTTPRequest <https://github.com/elnormous/HTTPRequest>`_
 - `nlohmann/json <https://github.com/nlohmann/json>`_
 
+.. note::
 
-Using the Gymnasium environments requires `Gymnasium <https://github.com/Farama-Foundation/Gymnasium/tree/main>`_ 
-installed on your machine. In addition, you need to install
+	Using the Gymnasium environments requires `Gymnasium <https://github.com/Farama-Foundation/Gymnasium/tree/main>`_ 
+	installed on your machine. In addition, you need to install
 
-- `FastAPI <https://fastapi.tiangolo.com/>`_
-- `Uvicorn <https://www.uvicorn.org/>`_
-- `Pydantic <a href="https://docs.pydantic.dev/latest/>`_
+	- `FastAPI <https://fastapi.tiangolo.com/>`_
+	- `Uvicorn <https://www.uvicorn.org/>`_
+	- `Pydantic <a href="https://docs.pydantic.dev/latest/>`_
 
-By installing the requirement under ``requirements.txt`` should set your Python environment  up correctly.
+	By installing the requirement under ``requirements.txt`` should set your Python environment  up correctly.
 
-There are extra dependencies if you want to generate the documentation. Namely,
 
-- Doxygen
-- Sphinx
-- sphinx_rtd_theme
-- breathe
-- m2r2
-
-Building the code
+Build the code
 -----------------
 
 The usual CMake based installation process is used. Namely
@@ -55,24 +50,56 @@ You can toggle the following variables
 - ENABLE_TESTS_FLAG (default is OFF)
 - ENABLE_EXAMPLES_FLAG (default is OFF)
 - ENABLE_DOC_FLAG (default is OFF)
+- ENABLE_WEBOTS (default OFF)
+- ENABLE_RAY (default OFF)
 
-For example enbling the examples 
+For example enabling the examples: 
 
 .. code-block::
 
 	cmake -DENABLE_EXAMPLES_FLAG=ON ..
 	make install
+	
 
+.. note::
 
-Run the tests
--------------
+	If you choose to enable Ray make sure you follow the instructions before building the library.
 
-You can execute all the tests by running the helper script ``execute_tests.sh``.
+Run the unit tests
+-------------------
+
+You can execute all the tests by running the helper script 
+
+.. code-block::
+
+	./execute_tests.sh
 
 
 Build the documentation
 -----------------------
 
+Building the documentation is done via `Sphinx <https://www.sphinx-doc.org/en/master/>`_ and there are 
+extra dependencies you need to install. Namely,
+
+- `Doxygen <https://www.doxygen.nl/>`_
+- breathe
+- sphinx==5.0.2
+- breathe==4.35.0
+- exhale==0.3.7
+- sphinx_rtd_theme==3.0.2
+
+In order to build the documentation locally execute:
+
+.. code-block::
+	
+	cd docs
+	sphinx-build -M html . outputdir
+
+
+where ``outputdir`` indicates a path in your system where you want to install the documentation files.
+Check the official Sphinx `Getting started <https://www.sphinx-doc.org/en/master/usage/quickstart.html>`_ guide
+for more options.
+
 
 Issues
 -------