[DAPHNE-#719] Add support for command-line arguments in DaphneLib

- Introduced two ways to pass DAPHNE arguments: array and string formats.
- Updated documentation
- Added testcases

Author: ldirry

doc/DaphneLib/Overview.md

@@ -556,12 +556,80 @@ tensor([[[100.5474, 100.9653],
          [100.3148, 100.3607]]], dtype=torch.float64)
 ```
 
+## Command-Line Arguments to Influence DAPHNE Behavior
+DAPHNE provides a range of command-line arguments that allow users to control its behavior and customize execution settings.
+These arguments can also be passed from DaphneLib.
+
+*Example Array Format:*
+
+```python
+from daphne.context.daphne_context import DaphneContext
+import numpy as np
+
+m1 = np.array([1, 2, 3])
+dc = DaphneContext()
+X = dc.from_numpy(m1)
+X.print().compute(daphne_args=["--explain", "parsing_simplified, parsing", "--timing"])
+```
+
+*Example String Format:*
+
+```python
+from daphne.context.daphne_context import DaphneContext
+import numpy as np
+
+m1 = np.array([1, 2, 3])
+dc = DaphneContext()
+X = dc.from_numpy(m1)
+X.print().compute(daphne_args="--explain=parsing_simplified,parsing --timing")
+```
+
+*Output (memory addresses may vary):*
+
+```
+IR after parsing:
+module {
+  func.func @main() {
+    %0 = "daphne.constant"() {value = 0 : si64} : () -> si64
+    %1 = "daphne.constant"() {value = 43781056 : si64} : () -> si64
+    %2 = "daphne.constant"() {value = 3 : si64} : () -> si64
+    %3 = "daphne.constant"() {value = 1 : si64} : () -> si64
+    %4 = "daphne.constant"() {value = 2 : si64} : () -> si64
+    %5 = "daphne.cast"(%0) : (si64) -> ui32
+    %6 = "daphne.cast"(%1) : (si64) -> ui32
+    %7 = "daphne.receiveFromNumpy"(%5, %6, %2, %3) : (ui32, ui32, si64, si64) -> !daphne.Matrix<?x?xsi64>
+    %8 = "daphne.constant"() {value = true} : () -> i1
+    %9 = "daphne.constant"() {value = false} : () -> i1
+    "daphne.print"(%7, %8, %9) : (!daphne.Matrix<?x?xsi64>, i1, i1) -> ()
+    "daphne.return"() : () -> ()
+  }
+}
+IR after parsing and some simplifications:
+module {
+  func.func @main() {
+    %0 = "daphne.constant"() {value = 43781056 : ui32} : () -> ui32
+    %1 = "daphne.constant"() {value = 0 : ui32} : () -> ui32
+    %2 = "daphne.constant"() {value = false} : () -> i1
+    %3 = "daphne.constant"() {value = true} : () -> i1
+    %4 = "daphne.constant"() {value = 3 : si64} : () -> si64
+    %5 = "daphne.constant"() {value = 1 : si64} : () -> si64
+    %6 = "daphne.receiveFromNumpy"(%1, %0, %4, %5) : (ui32, ui32, si64, si64) -> !daphne.Matrix<?x?xsi64>
+    "daphne.print"(%6, %3, %2) : (!daphne.Matrix<?x?xsi64>, i1, i1) -> ()
+    "daphne.return"() : () -> ()
+  }
+}
+DenseMatrix(3x1, int64_t)
+1
+2
+3
+{"startup_seconds": 0.0238036, "parsing_seconds": 0.00178526, "compilation_seconds": 0.0734249, "execution_seconds": 0.0232295, "total_seconds": 0.122243}
+```
+
 ## Known Limitations
 
 DaphneLib is still in an early development stage.
 Thus, there are a few limitations that users should be aware of.
 We plan to fix all of these limitations in the future.
 
-- Using DAPHNE's command-line arguments to influence its behavior is not supported yet.
 - Some DaphneDSL built-in functions are not represented by DaphneLib methods yet.
 - High-level primitives for integrated data analysis pipelines, which are implemented in DaphneDSL, cannot be called from DaphneLib yet.

src/api/daphnelib/daphnelib.cpp

@@ -31,9 +31,4 @@ extern "C" DaphneLibResult getResult() { return daphneLibRes; }
  * @brief Invokes DAPHNE with the specified DaphneDSL script and path to lib
  * dir.
  */
-extern "C" int daphne(const char *libDirPath, const char *scriptPath) {
-    const char *argv[] = {"daphne", "--libdir", libDirPath, scriptPath};
-    int argc = 4;
-
-    return mainInternal(argc, argv, &daphneLibRes);
-}
+extern "C" int daphne(int argc, const char **argv) { return mainInternal(argc, argv, &daphneLibRes); }

src/api/python/daphne/operator/nodes/matrix.py

@@ -101,8 +101,8 @@ def getDType(self, d_type):
     def _is_numpy(self) -> bool:
         return self._np_array is not None
 
-    def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None) -> Union[np.array]:
-        return super().compute(type=type, verbose=verbose, asTensorFlow=asTensorFlow, asPyTorch=asPyTorch, shape=shape)
+    def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None, daphne_args=None) -> Union[np.array]:
+        return super().compute(type=type, verbose=verbose, asTensorFlow=asTensorFlow, asPyTorch=asPyTorch, shape=shape, daphne_args=daphne_args)
 
     def __add__(self, other: VALID_ARITHMETIC_TYPES) -> 'Matrix':
         return Matrix(self.daphne_context, '+', [self, other])

src/api/python/daphne/operator/operation_node.py

@@ -91,7 +91,7 @@ def update_node_in_input_list(self, new_node, current_node):
         current_index = self._unnamed_input_nodes.index(current_node)
         self._unnamed_input_nodes[current_index] = new_node
 
-    def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None, useIndexColumn=False):
+    def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyTorch=False, shape=None, useIndexColumn=False, daphne_args=None):
         """
         Compute function for processing the Daphne Object or operation node and returning the results.
         The function builds a DaphneDSL script from the node and its context, executes it, and processes the results
@@ -103,6 +103,7 @@ def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyT
         :param asPyTorch: If True and the result is a matrix, the output will be converted to a PyTorch tensor.
         :param shape: If provided and the result is a matrix, it defines the shape to reshape the resulting tensor (either TensorFlow or PyTorch).
         :param useIndexColumn: If True and the result is a DataFrame, uses the column named "index" as the DataFrame's index.
+        :param daphne_args: Optional arguments specifically for Daphne DSL script execution, which can customize runtime behavior.
 
         :return: Depending on the parameters and the operation's output type, this function can return:
             - A pandas DataFrame for frame outputs.
@@ -123,7 +124,7 @@ def compute(self, type="shared memory", verbose=False, asTensorFlow=False, asPyT
             if verbose:
                 exec_start_time = time.time()
 
-            self._script.execute()
+            self._script.execute(daphne_args)
             self._script.clear(self)
 
             if verbose:

src/api/python/daphne/script_building/script.py

@@ -27,6 +27,7 @@
 
 import ctypes
 import os
+import shlex
 from typing import List, Dict, TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -88,23 +89,43 @@ def clear(self, dag_root:DAGNode):
         self._dfs_clear_dag_nodes(dag_root)
         self._variable_counter = 0
 
-    def execute(self):
+    def execute(self, daphne_args):
         temp_out_path = os.path.join(TMP_PATH, "tmpdaphne.daphne")
-        temp_out_file = open(temp_out_path, "w")
-        temp_out_file.writelines(self.daphnedsl_script)
-        temp_out_file.close()
-        
-        #os.environ['OPENBLAS_NUM_THREADS'] = '1'
-        res = DaphneLib.daphne(ctypes.c_char_p(str.encode(PROTOTYPE_PATH)), ctypes.c_char_p(str.encode(temp_out_path)))
+        with open(temp_out_path, "w") as temp_out_file:
+            temp_out_file.writelines(self.daphnedsl_script)
+
+        # Construct argv
+        argv = ["daphne", "--libdir", PROTOTYPE_PATH]
+
+        # Handle daphne_args
+        if isinstance(daphne_args, str):
+            argv.extend(shlex.split(daphne_args))
+        elif isinstance(daphne_args, list):
+            argv.extend(daphne_args)
+        elif daphne_args is None:
+            pass
+        else:
+            raise TypeError("daphne_args must be a string or a list")
+
+        # Add the script path
+        argv.append(temp_out_path)
+
+        # Convert argv to ctypes
+        argc = len(argv)
+        argv_ctypes = (ctypes.c_char_p * argc)(* [arg.encode('utf-8') for arg in argv])
+
+        # Set argument and return types for the C function
+        DaphneLib.daphne.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)]
+        DaphneLib.daphne.restype = ctypes.c_int
+
+        res = DaphneLib.daphne(ctypes.c_int(argc), argv_ctypes)
         if res != 0:
             # Error message with DSL code line.
             error_message = DaphneLib.getResult().error_message.decode("utf-8")
+            raise RuntimeError(f"Error in DaphneDSL script: {error_message}")
             # Remove DSL code line from error message.
             # index_code_line = error_message.find("Source file ->") - 29
             # error_message = error_message[:index_code_line]
-            
-            raise RuntimeError(f"Error in DaphneDSL script: {error_message}")
-        #os.environ['OPENBLAS_NUM_THREADS'] = '32'
 
     def _dfs_dag_nodes(self, dag_node: VALID_INPUT_TYPES)->str:
         """Uses Depth-First-Search to create code from DAG

test/api/python/DaphneLibTest.cpp

@@ -26,6 +26,9 @@
 #include <cstdlib>
 #include <cstring>
 
+#include <fstream>
+#include <sstream>
+
 const std::string dirPath = "test/api/python/";
 
 #define MAKE_TEST_CASE(name)                                                                                           \
@@ -65,6 +68,19 @@ const std::string dirPath = "test/api/python/";
         const std::string prefix = dirPath + name;                                                                     \
         compareDaphneLibToStr(str, prefix + ".py");                                                                    \
     }
+#define MAKE_TEST_CASE_FILE(name)                                                                                      \
+    TEST_CASE(name ".py", TAG_DAPHNELIB) {                                                                             \
+        const std::string prefix = dirPath + name;                                                                     \
+        std::ifstream fileStream(prefix + ".txt");                                                                     \
+        std::stringstream buffer;                                                                                      \
+        buffer << fileStream.rdbuf();                                                                                  \
+        std::stringstream out;                                                                                         \
+        std::stringstream err;                                                                                         \
+        int status = runDaphneLib(out, err, (prefix + ".py").c_str());                                                 \
+        CHECK(status == StatusCode::SUCCESS);                                                                          \
+        CHECK(err.str() != "");                                                                                        \
+        CHECK(out.str() == buffer.str());                                                                              \
+    }
 
 MAKE_TEST_CASE("data_transfer_numpy_1")
 MAKE_TEST_CASE("data_transfer_numpy_2")
@@ -158,3 +174,4 @@ MAKE_TEST_CASE("user_def_func_3_inputs")
 // MAKE_TEST_CASE_PARAMETRIZED("user_def_func_with_condition", "param=3.8")
 // MAKE_TEST_CASE("user_def_func_with_for_loop")
 // MAKE_TEST_CASE("user_def_func_with_while_loop")
+MAKE_TEST_CASE_FILE("daphne_args")

test/api/python/daphne_args.py

@@ -0,0 +1,10 @@
+import numpy as np
+from daphne.context.daphne_context import DaphneContext
+
+m1 = np.array([3, 9, 12])
+dc = DaphneContext()
+X = dc.from_numpy(m1)
+X.print().compute(daphne_args=["--vec", "--pin-workers"])
+X.print().compute(daphne_args=["--explain", "parsing_simplified"])
+X.print().compute(daphne_args=["--explain", "parsing_simplified,parsing", "--timing"])
+X.print().compute(daphne_args="--explain=parsing_simplified,parsing --timing")

test/api/python/daphne_args.txt

@@ -0,0 +1,16 @@
+DenseMatrix(3x1, int64_t)
+3
+9
+12
+DenseMatrix(3x1, int64_t)
+3
+9
+12
+DenseMatrix(3x1, int64_t)
+3
+9
+12
+DenseMatrix(3x1, int64_t)
+3
+9
+12