update ci yml files (#12)

* update ci yml files

* adding the command for building pybind11 in the CI

* adding the command for building pybind11 in the CI

* minor corrections

* meson updated the yml codes

* making the numpy example optional (easier for the CI's)

Author: noabauma

.github/workflows/master-test-workflow.yml

@@ -40,8 +40,31 @@ jobs:
         with:
           submodules: 'true'
 
+      - name: Build Pybind11
+        run: |
+          cd extern/pybind11
+          mkdir -p build
+          cd build
+          cmake .. -DPython_EXECUTABLE=/usr/bin/python3
+          make check -j"$(nproc)"
+
+          mkdir -p mock_install/share/pkgconfig
+          cat <<EOF > mock_install/share/pkgconfig/pybind11.pc
+          prefix=$(pwd)/..
+          includedir=\${prefix}/include
+
+          Name: pybind11
+          Description: Seamless operability between C++11 and Python
+          Version: 2.13.6
+          Cflags: -I\${includedir}
+          EOF
+
+          echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV
+
+          export LD_PRELOAD=/usr/local/lib/libnosv.so
+        
       - name: Setup
-        run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true
+        run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true
 
       - name: Compile
         run: source /home/hicr/.bashrc && meson compile -C build

.github/workflows/pr-development-workflow.yml

@@ -231,7 +231,7 @@ jobs:
           docker run --name taskr --shm-size=1024M --privileged -v $PWD:/home/hicr/taskr -w /home/hicr/taskr -td ${{ env.DOCKERIMAGE }}:${{ inputs.arch }}-latest bash
 
       - name: Setup
-        run: docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true  -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true"
+        run: docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true  -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true"
 
       - name: Compile
         run: docker exec -u hicr taskr bash -c "meson compile -C build"
@@ -270,8 +270,31 @@ jobs:
         with:
           submodules: 'true'
 
+      - name: Build Pybind11
+        run: |
+          cd extern/pybind11
+          mkdir -p build
+          cd build
+          cmake .. -DPython_EXECUTABLE=/usr/bin/python3
+          make check -j"$(nproc)"
+
+          mkdir -p mock_install/share/pkgconfig
+          cat <<EOF > mock_install/share/pkgconfig/pybind11.pc
+          prefix=$(pwd)/..
+          includedir=\${prefix}/include
+
+          Name: pybind11
+          Description: Seamless operability between C++11 and Python
+          Version: 2.13.6
+          Cflags: -I\${includedir}
+          EOF
+
+          echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV
+
+          export LD_PRELOAD=/usr/local/lib/libnosv.so
+        
       - name: Setup
-        run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true
+        run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true
 
       - name: Compile
         run: source /home/hicr/.bashrc && meson compile -C build

examples/matmul/python/main.py

@@ -27,7 +27,7 @@ def main():
     # Running matmul example
     matmul_cpp_Driver(runtime)
 
-    matmul_numpy_Driver(runtime)
+    # matmul_numpy_Driver(runtime)
 
 
 if __name__ == "__main__":

examples/matmul/python/matmul.cpp

@@ -19,14 +19,22 @@
 
 #include <taskr/taskr.hpp>
 
+#ifdef ENABLE_INSTRUMENTATION
+  #include <tracr.hpp>
+#endif
+
 #define mytype float
 
 /**
  * Compute mmm
  */
 void matmul(taskr::Task *)
 {
-  const size_t N = 1000;
+#ifdef ENABLE_INSTRUMENTATION
+  INSTRUMENTATION_VMARKER_SET(MARK_COLOR_RED);
+#endif
+
+  const size_t N = 200;
 
   // Allocate memory
   volatile mytype *A = (mytype *)calloc(1, N * N * sizeof(mytype));
@@ -56,6 +64,10 @@ void matmul(taskr::Task *)
   free((mytype *)A);
   free((mytype *)B);
   free((mytype *)C);
+
+#ifdef ENABLE_INSTRUMENTATION
+  INSTRUMENTATION_VMARKER_RESET();
+#endif
 }
 
 PYBIND11_MODULE(cpp_matmul, m)

examples/matmul/python/matmul.py

@@ -15,7 +15,6 @@
 """
 
 import time
-import numpy as np
 
 import taskr
 import cpp_matmul
@@ -45,15 +44,14 @@ def matmul_cpp_Driver(runtime):
 
 
 
-
-
 def matmul_numpy_Driver(runtime):
+  import numpy as np
   # Initializing taskr
   runtime.initialize()
 
   def matmul_numpy(task):
     N = 1000
-    A = np.zeros((N,N))
+    A = np.empty((N,N))
     B = np.empty((N,N))
     C = np.empty((N,N))
 

examples/simple/python/main.py

@@ -13,13 +13,15 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-
 import taskr
+
 import simple
 
+NWORKERS = 4
+
 def main():
-    # Initialize taskr with the wanted compute manager backend and number of PUs
-    t = taskr.taskr(taskr.HiCRBackend.nosv, 2)
+    # Initialize taskr with the wanted HiCR backend and number of Workers
+    t = taskr.taskr(backend=taskr.HiCRBackend.nosv, num_workers=NWORKERS)
 
     # Get the runtime
     runtime = t.get_runtime()

examples/simple/python/simple.py

@@ -13,7 +13,6 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-
 import taskr
 
 NTASKS = 2
@@ -22,8 +21,8 @@ def simple(runtime):
   # Initializing taskr
   runtime.initialize()
 
+  # Create tasks
   fc = lambda task : print(f"Hello, I am task {task.getLabel()}")
-
   taskfc = taskr.Function(fc)
 
   # Adding to tasks to taskr

include/pytaskr/meson.build

@@ -2,7 +2,14 @@
 # Manually compile like this:
 # c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3-config --includes) -Iextern/pybind11/include example.cpp -o example$(python3-config --extension-suffix)
 
-py = import('python').find_installation(pure: false)
+python_mod = import('python')
+
+py = python_mod.find_installation(pure: false)
+
+if not py.found()
+  py = python_mod.find_installation('/usr/bin/python3', pure: false)
+endif
+
 pybind11_dep = dependency('pybind11', required: true)
 
 py.extension_module('taskr',

include/pytaskr/pytaskr.cpp

@@ -48,7 +48,7 @@ PYBIND11_MODULE(taskr, m)
     .def("setServiceWorkerCallbackHandler", &Runtime::setServiceWorkerCallbackHandler)
     .def("setTaskWorkerCallbackHandler", &Runtime::setTaskWorkerCallbackHandler)
     .def("initialize", &Runtime::initialize)
-    .def("addTask", &Runtime::addTask, py::keep_alive<1, 2>()) // keep_alive as the task should be alive until runtime's destructor
+    .def("addTask", &Runtime::addTask, py::keep_alive<1, 2>(), py::arg("task")) // keep_alive as the task should be alive until runtime's destructor
     .def("resumeTask", &Runtime::resumeTask)
     .def("run", &Runtime::run, py::call_guard<py::gil_scoped_release>())
     .def("await_", &Runtime::await, py::call_guard<py::gil_scoped_release>()) // Release GIL is important otherwise non-finished tasks are getting blocked
@@ -57,12 +57,12 @@ PYBIND11_MODULE(taskr, m)
     .def("addService", &Runtime::addService);
 
   // TaskR's Function class
-  py::class_<Function>(m, "Function").def(py::init<const function_t>());
+  py::class_<Function>(m, "Function").def(py::init<const function_t>(), py::arg("fc"));
 
   // TaskR's Task class
   py::class_<Task>(m, "Task")
     .def(py::init<Function *, const workerId_t>(), py::arg("fc"), py::arg("workerAffinity") = -1)
-    .def(py::init<const label_t, Function *, const workerId_t>(), py::arg("label"), py::arg("fc"), py::arg("workerAffinity") = -1)
+    .def(py::init<const label_t, Function *, const workerId_t>(), py::arg("label"), py::arg("taskfc"), py::arg("workerAffinity") = -1)
     .def("getLabel", &Task::getLabel)
     .def("setLabel", &Task::setLabel)
     .def("getWorkerAffinity", &Task::getWorkerAffinity)