Optimize test_bounded_mean test execution (#333)

* added installation for clang-format in docker file

* moved pydp folver under src

* moved pydp folver under src

* moved pydp folver under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* moved pydp folder under src

* Explicitly tell pybind which python to use

* Fix CI tests

* Enable protobuf serde for testing

* Fix style errors

* removed bin files

Author: alejandrosame

.github/workflows/tests.yml

@@ -49,6 +49,9 @@ jobs:
         make build
     - name: Build PyDP with coverage (Windows)
       if: runner.os == 'Windows'
+      # There seems to be an issue with poetry and windows in this case. We need to
+      # install again to make pydp importable.
+      # Related ref: https://github.com/python-poetry/poetry/issues/3075
       run: |
         bazel clean --expunge
 

src/bindings/PyDP/proto/proto.cpp

@@ -1,3 +1,4 @@
+#include <fstream>
 #include <string>
 
 #include "../pydp_lib/casting.hpp"  // our caster helper library
@@ -15,7 +16,18 @@ namespace dp = differential_privacy;
 
 void init_proto(py::module &m) {
   py::class_<dp::Output> output(m, "Output");
-  py::class_<dp::Summary>(m, "Summary");
-
   output.def(py::init());
+
+  // Add save and load functions to use protobuf serde in tests
+  py::class_<dp::Summary>(m, "Summary")
+      .def(py::init())
+      .def("save",
+           [](dp::Summary &pythis, std::string &filename) {
+             fstream output(filename, ios::out | ios::trunc | ios::binary);
+             pythis.SerializeToOstream(&output);
+           })
+      .def("load", [](dp::Summary &pythis, std::string &filename) {
+        fstream input(filename, ios::in | ios::binary);
+        pythis.ParseFromIstream(&input);
+      });
 }

tests/algorithms/test_bounded_mean.py

@@ -1,5 +1,8 @@
+import os
 import pytest
 from pydp.algorithms.laplacian import BoundedMean
+from pydp._pydp import Summary
+
 
 expect_near = lambda expected, actual, tol: (
     expected + tol >= actual and expected - tol <= actual
@@ -25,12 +28,42 @@ def test_bounded_mean():
     # assert isinstance(bm2.quick_result([1.5, 2, 2.5]), float)
 
 
-def test_bounded_mean_int64():
-    example_list = [5] * 100000000
-    x = BoundedMean(1.0, 0, 10, dtype="int64")
-    for _ in range(5):
-        x.add_entries(example_list)
+@pytest.fixture(scope="function")
+def make_loaded_object(request):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+    def _make_loaded_object(value, size, iter):
+        dump_filepath = os.path.join(
+            dir_path,
+            request.module.__name__,
+            "{}_data.proto".format(request.function.__name__),
+        )
+
+        # Algorithm to initialize
+        x = BoundedMean(1.0, 0, 10, dtype="int64")
+
+        if os.path.exists(dump_filepath):
+            # Search for data dump to import
+            data = Summary()
+            data.load(dump_filepath)
+            x.merge(data)
+        else:
+            # No data dump found, we have to init the alforithm from scratch
+            # Add entries into algorithm
+            example_list = [value] * size
+            for _ in range(iter):
+                x.add_entries(example_list)
+
+            # Dump the initialized algorithm data to retrieve in future tests
+            x.serialize().save(dump_filepath)
+
+        return x
+
+    return _make_loaded_object
+
 
+def test_bounded_mean_int64(make_loaded_object):
+    x = make_loaded_object(5, 100000000, 5)
     assert expect_near(5.0, x.result(), 0.1)
 
 

tests/algorithms/test_bounded_mean/test_bounded_mean_int64_data.proto

@@ -0,0 +1,2 @@
+M
+;type.googleapis.com/differential_privacy.BoundedMeanSummary€Êµî
ۘܬ