Merge branch 'main' into Add-c++23-kernel

Author: mcbarton

.github/workflows/deploy-github-page.yml

@@ -57,15 +57,14 @@ jobs:
 
           export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-cpp-wasm-host 
           echo "PREFIX=$PREFIX" >> $GITHUB_ENV
-          export CMAKE_PREFIX_PATH=$PREFIX
-          export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
+          export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot
 
           emcmake cmake \
             -DCMAKE_BUILD_TYPE=Release                        \
-            -DCMAKE_PREFIX_PATH=$PREFIX                       \
             -DCMAKE_INSTALL_PREFIX=$PREFIX                    \
             -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON               \
-            -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON            \
+            -DCMAKE_FIND_ROOT_PATH=$PREFIX                    \
+            -DSYSROOT_PATH=$SYSROOT_PATH                      \
             ..
           emmake make -j  ${{ env.ncpus }} install
 
@@ -74,11 +73,10 @@ jobs:
         run: |
           micromamba create -n xeus-lite-host jupyterlite-core
           micromamba activate xeus-lite-host
-          python -m pip install jupyterlite-xeus
-          jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} --output-dir dist
-          mkdir -p dist/files
-          mv notebooks dist/files
-          mv README.md dist/files
+          python -m pip install jupyterlite-xeus jupyter_server notebook
+          jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} --contents notebooks/xeus-cpp-lite-demo.ipynb --output-dir dist
+          cp $PREFIX/bin/xcpp.data dist/extensions/@jupyterlite/xeus/static
+          cp $PREFIX/lib/libclangCppInterOp.so dist/extensions/@jupyterlite/xeus/static
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/main.yml

@@ -255,15 +255,14 @@ jobs:
 
           export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-cpp-wasm-host 
           echo "PREFIX=$PREFIX" >> $GITHUB_ENV
-          export CMAKE_PREFIX_PATH=$PREFIX
-          export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
+          export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot
 
           emcmake cmake \
             -DCMAKE_BUILD_TYPE=Release                        \
-            -DCMAKE_PREFIX_PATH=$PREFIX                       \
             -DCMAKE_INSTALL_PREFIX=$PREFIX                    \
             -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON               \
-            -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON            \
+            -DCMAKE_FIND_ROOT_PATH=$PREFIX                    \
+            -DSYSROOT_PATH=$SYSROOT_PATH                      \
             ..
           emmake make -j  ${{ env.ncpus }} install
 
@@ -274,6 +273,8 @@ jobs:
           micromamba activate xeus-lite-host
           python -m pip install jupyterlite-xeus
           jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }}
+          cp $PREFIX/bin/xcpp.data _output/extensions/@jupyterlite/xeus/static
+          cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static
 
       - name: Setup tmate session
         if: ${{ failure() && runner.debug }}

CMakeLists.txt

@@ -84,15 +84,15 @@ if(EMSCRIPTEN)
     set(XEUS_CPP_USE_SHARED_XEUS_CPP OFF)
     set(XEUS_CPP_BUILD_TESTS OFF)
     # ENV (https://github.com/emscripten-core/emscripten/commit/6d9681ad04f60b41ef6345ab06c29bbc9eeb84e0)
-    set(EMSCRIPTEN_FEATURES "${EMSCRIPTEN_FEATURES} -s \"EXTRA_EXPORTED_RUNTIME_METHODS=[ENV']\"")
+    set(EMSCRIPTEN_FEATURES "${EMSCRIPTEN_FEATURES} -s \"EXPORTED_RUNTIME_METHODS=[ENV']\"")
 endif()
 
 # Dependencies
 # ============
 
 set(xeus_REQUIRED_VERSION 5.0.0)
 set(xeus_zmq_REQUIRED_VERSION 3.0.0)
-set(CppInterOp_REQUIRED_VERSION 1.4.0)
+set(CppInterOp_REQUIRED_VERSION 1.5.0)
 
 if (NOT TARGET xeus AND NOT TARGET xeus-static)
     find_package(xeus ${xeus_REQUIRED_VERSION} REQUIRED)
@@ -103,10 +103,7 @@ endif ()
 
 find_package(CppInterOp ${CppInterOp_REQUIRED_VERSION} REQUIRED)
 if(CppInterOp_FOUND)
-  message(STATUS "Found CppInterOp: compatible with Clang ${CPPINTEROP_LLVM_VERSION_MAJOR}.x")
-  # TODO : Fix after variables are set correctly
-  # Check https://github.com/compiler-research/CppInterOp/pull/336
-  # message(STATUS "Found CppInterOp: config=${CPPINTEROP_CONFIG} dir=${CPPINTEROP_DIR} (found version=${CPPINTEROP_VERSION} compatible with Clang ${CPPINTEROP_LLVM_VERSION_MAJOR}.x)")
+  message(STATUS "Found CppInterOp: config=${CPPINTEROP_CMAKE_DIR} dir=${CPPINTEROP_INSTALL_PREFIX} (found version=${CPPINTEROP_VERSION} compatible with Clang ${CPPINTEROP_LLVM_VERSION_MAJOR}.x)")
 endif()
 
 find_package(argparse REQUIRED)
@@ -152,9 +149,23 @@ function(configure_kernel kernel)
 endfunction()
 
 message("Configure kernels: ...")
-configure_kernel("/share/jupyter/kernels/xcpp17/")
-configure_kernel("/share/jupyter/kernels/xcpp20/")
-configure_kernel("/share/jupyter/kernels/xcpp23/")
+if(EMSCRIPTEN)
+    # TODO: Currently jupyterlite-xeus and xeus-lite do not provide
+    # methods to fetch information from the arguments present in the
+    # generated emscripten kernel.
+    # The following needs to be done here :
+    # 1) We need to configure the kernel properly
+    #    Check issue https://github.com/compiler-research/xeus-cpp/issues/185.
+    # 2) Once the above is done we need to add support in jupyterlite-xeus & xeus-lite
+    #    to be able to deal with arguments present in kernel.json
+    # 3) Finally we should fetch the C++ version from the kernel.json file and
+    #    be able to pass it to our wasm interpreter rather than forcing a version.
+    configure_kernel("/share/jupyter/kernels/xcpp20/")
+else()
+    configure_kernel("/share/jupyter/kernels/xcpp17/")
+    configure_kernel("/share/jupyter/kernels/xcpp20/")
+    configure_kernel("/share/jupyter/kernels/xcpp23/")
+endif()
 
 # Source files
 # ============
@@ -402,8 +413,24 @@ if(EMSCRIPTEN)
     xeus_cpp_set_kernel_options(xcpp)
     xeus_wasm_compile_options(xcpp)
     xeus_wasm_link_options(xcpp "web,worker")
+    # TODO: Remove the exported runtime methods
+    # after the next xeus release.
+    target_link_options(xcpp PUBLIC
+        -sEXPORTED_RUNTIME_METHODS=FS,PATH,ERRNO_CODES
+        # add sysroot location here
+        --preload-file ${SYSROOT_PATH}/include@/include
+    )
+    # TODO: Emscripten supports preloading files just once before it generates
+    # the xcpp.data file (containing the binary representation of the file(s) we
+    # want to include in our application).
+    # Hence although we are adding support for Standard Headers, Libraries etc
+    # through emscripten's sysroot for now, we need to do the following:
+    # 1) Enable CppInterOp to provide us with a resource dir.
+    # 2) If the above cannot be done, we can use the resource dir provided
+    #    by llvm on emscripten-forge but would involve adding a dependency.
+    # 3) Shift the resource dir and the sysroot to a common location.
+    # 4) Preload everything required together.
 endif()
-
 # Tests
 # =====
 
@@ -493,6 +520,7 @@ if(EMSCRIPTEN)
     install(FILES
             "$<TARGET_FILE_DIR:xcpp>/xcpp.js"
             "$<TARGET_FILE_DIR:xcpp>/xcpp.wasm"
+            "$<TARGET_FILE_DIR:xcpp>/xcpp.data"
             DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif ()
 

README.md

@@ -85,15 +85,14 @@ micromamba create -f environment-wasm-host.yml --platform=emscripten-wasm32
 mkdir build
 pushd build
 export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-cpp-wasm-host 
-export CMAKE_PREFIX_PATH=$PREFIX
-export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
+export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot
 
 emcmake cmake \
         -DCMAKE_BUILD_TYPE=Release                        \
-        -DCMAKE_PREFIX_PATH=$PREFIX                       \
         -DCMAKE_INSTALL_PREFIX=$PREFIX                    \
         -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON               \
-        -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON            \
+        -DCMAKE_FIND_ROOT_PATH=$PREFIX                    \
+        -DSYSROOT_PATH=$SYSROOT_PATH                      \
         ..
 emmake make install
 ```
@@ -105,12 +104,20 @@ micromamba activate xeus-lite-host
 python -m pip install jupyterlite-xeus
 jupyter lite build --XeusAddon.prefix=$PREFIX
 ```
+
+We now need to shift necessary files like `xcpp.data` which contains the binary representation of the file(s)
+we want to include in our application. As of now this would contain all important files like Standard Headers,
+Libraries etc coming out of emscripten's sysroot. Assuming we are still inside build we should do the following
+```bash
+cp $PREFIX/bin/xcpp.data _output/extensions/@jupyterlite/xeus/static
+cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static
+```
+
 Once the Jupyter Lite site has built you can test the website locally by executing
 ```bash
 jupyter lite serve --XeusAddon.prefix=$PREFIX
 ```
 
-
 ## Trying it online
 
 To try out xeus-cpp interactively in your web browser, just click on the binder link:
@@ -135,7 +142,7 @@ http://xeus-cpp.readthedocs.io
 
 | `xeus-cpp` | `xeus-zmq`      | `CppInterOp` | `pugixml` | `cpp-argparse`| `nlohmann_json` |
 |------------|-----------------|--------------|-----------|---------------|-----------------|
-|  main      |  >=3.0.0,<4.0.0 | >=1.4.0      | ~1.8.1    | <3.1          | >=3.11.3,<4.0   |
+|  main      |  >=3.0.0,<4.0.0 | >=1.5.0      | ~1.8.1    | <3.1          | >=3.11.3,<4.0   |
 |  0.5.0     |  >=3.0.0,<4.0.0 | >=1.3.0      | ~1.8.1    | <3.1          | >=3.11.3,<4.0   |
 
 Versions prior to `0.5.0` have an additional dependency on [xtl](https://github.com/xtensor-stack/xtl), [clang](https://github.com/llvm/llvm-project/) & [cppzmq](https://github.com/zeromq/cppzmq)

docs/source/InstallationAndUsage.rst

@@ -1,7 +1,7 @@
 Installation And Usage
 --------------------
 
-Installation from source
+Installation from source (non wasm build instructions)
 ========================
 
 To ensure that the installation works, it is preferable to install `xeus-cpp` in a
@@ -42,6 +42,65 @@ with a custom installation prefix if need be)
     -D CMAKE_INSTALL_LIBDIR=lib
     make && make install
 
+Installation within a mamba environment (wasm build instructions)
+========================
+
+These instructions will assume you have cmake installed on your system. First clone the repository, and move into that directory
+.. code-block:: bash
+    git clone --depth=1 https://github.com/compiler-research/xeus-cpp.git
+    cd ./xeus-cpp
+
+
+You'll now want to make sure you're using emsdk version "3.1.45" and activate it. You can get this by executing the following
+
+.. code-block:: bash
+    cd $HOME
+    git clone https://github.com/emscripten-core/emsdk.git
+    cd emsdk
+    ./emsdk install 3.1.45
+    ./emsdk activate 3.1.45
+    source $HOME/emsdk/emsdk_env.sh
+
+
+You are now in a position to build the xeus-cpp kernel. You build it by executing the following
+
+.. code-block:: bash
+    micromamba create -f environment-wasm-host.yml --platform=emscripten-wasm32
+    mkdir build
+    pushd build
+    export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-cpp-wasm-host 
+    export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot
+    emcmake cmake \
+            -DCMAKE_BUILD_TYPE=Release                        \
+            -DCMAKE_INSTALL_PREFIX=$PREFIX                    \
+            -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON               \
+            -DCMAKE_FIND_ROOT_PATH=$PREFIX                    \
+            -DSYSROOT_PATH=$SYSROOT_PATH                      \
+            ..
+    emmake make install
+
+
+To build Jupyter Lite with this kernel without creating a website you can execute the following
+
+.. code-block:: bash
+    micromamba create -n xeus-lite-host jupyterlite-core
+    micromamba activate xeus-lite-host
+    python -m pip install jupyterlite-xeus
+    jupyter lite build --XeusAddon.prefix=$PREFIX
+
+We now need to shift necessary files like `xcpp.data` which contains the binary representation of the file(s)
+we want to include in our application. As of now this would contain all important files like Standard Headers,
+Libraries etc coming out of emscripten's sysroot. Assuming we are still inside build we should do the following
+
+.. code-block:: bash
+    cp $PREFIX/bin/xcpp.data _output/extensions/@jupyterlite/xeus/static
+    cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static
+
+Once the Jupyter Lite site has built you can test the website locally by executing
+
+.. code-block:: bash
+    jupyter lite serve --XeusAddon.prefix=$PREFIX
+
 Installing from conda-forge
 ===========================
 
@@ -51,4 +110,4 @@ If you have conda installed then you can install xeus-cpp using the follwing com
 
     conda install conda-forge::xeus-cpp
 
-Xeus-cpp is available for Linux, MacOS and Windows.
+Xeus-cpp is available for Linux, MacOS and Windows.

environment-dev.yml

@@ -10,7 +10,7 @@ dependencies:
   - xeus>=5.0.0
   - xeus-zmq>=3.0,<4.0
   - nlohmann_json=3.11.3
-  - CppInterOp>=1.4.0
+  - CppInterOp>=1.5.0
   - pugixml
   - cpp-argparse <3.1
   - zlib

environment-wasm-host.yml

@@ -6,6 +6,6 @@ dependencies:
   - nlohmann_json
   - xeus-lite
   - xeus
-  - CppInterOp>=1.4.0
+  - CppInterOp>=1.5.0
   - cpp-argparse
   - pugixml