chore: add wasm_par_mq crate for unsafe coop wasm parallelism

Author: nsarlin-zama

.github/workflows/aws_tfhe_wasm_tests.yml

@@ -117,6 +117,8 @@ jobs:
         run: |
           make install_chrome_browser
           make install_chrome_web_driver
+          make install_firefox_browser
+          make install_firefox_web_driver
 
       - name: Run fmt checks
         run: |
@@ -130,6 +132,11 @@ jobs:
         run: |
           make test_web_js_api_parallel_chrome_ci
 
+      - name: Run wasm_par_mq tests
+        run: |
+          make test_wasm_par_mq_chrome_ci
+          make test_wasm_par_mq_firefox_ci
+
       - name: Run x86_64/wasm zk compatibility tests
         run: |
           make test_zk_wasm_x86_compat_ci

.gitignore

@@ -32,6 +32,7 @@ venv/
 web-test-runner/
 node_modules/
 package-lock.json
+utils/wasm-par-mq/examples/*/pkg/
 
 # Python .env
 .env

Cargo.toml

@@ -16,6 +16,9 @@ members = [
     "utils/tfhe-backward-compat-data",
     "utils/tfhe-backward-compat-data/crates/add_new_version",
     "utils/param_dedup",
+    "utils/wasm-par-mq",
+    "utils/wasm-par-mq/web_tests",
+    "utils/wasm-par-mq/examples/msm",
     "tests",
     "mockups/tfhe-hpu-mockup",
     "apps/test-vectors",
@@ -46,6 +49,9 @@ bincode = "=1.3.3"
 cmake = "0.1"
 pkg-config = "0.3"
 clap = { version = "4.5", features = ["derive"] }
+js-sys = "0.3"
+serde-wasm-bindgen = "0.6.5"
+paste = "1.0.15"
 
 [profile.bench]
 lto = "fat"

Makefile

@@ -534,6 +534,11 @@ clippy_param_dedup: install_rs_check_toolchain
 	RUSTFLAGS="$(RUSTFLAGS)" cargo "$(CARGO_RS_CHECK_TOOLCHAIN)" clippy --all-targets \
 		-p param_dedup -- --no-deps -D warnings
 
+.PHONY: clippy_wasm_par_mq # Run clippy lints on wasm-par-mq and its examples
+clippy_wasm_par_mq: install_rs_check_toolchain
+	RUSTFLAGS="$(RUSTFLAGS)" cargo "$(CARGO_RS_CHECK_TOOLCHAIN)" clippy --all-targets --all-features \
+		-p wasm-par-mq -p wasm-par-mq-web-tests -p wasm-par-mq-example-msm -- --no-deps -D warnings
+
 .PHONY: clippy_backward_compat_data # Run clippy lints on tfhe-backward-compat-data
 clippy_backward_compat_data: install_rs_check_toolchain # the toolchain is selected with toolchain.toml
 	@# Some old crates are x86 specific, only run in that case
@@ -559,7 +564,7 @@ clippy_test_vectors: install_rs_check_toolchain
 clippy_all: clippy_rustdoc clippy clippy_boolean clippy_shortint clippy_integer clippy_all_targets \
 clippy_c_api clippy_js_wasm_api clippy_tasks clippy_core clippy_tfhe_csprng clippy_zk_pok clippy_trivium \
 clippy_versionable clippy_tfhe_lints clippy_ws_tests clippy_bench clippy_param_dedup \
-clippy_test_vectors clippy_backward_compat_data
+clippy_test_vectors clippy_backward_compat_data clippy_wasm_par_mq
 
 .PHONY: clippy_fast # Run main clippy targets
 clippy_fast: clippy_rustdoc clippy clippy_all_targets clippy_c_api clippy_js_wasm_api clippy_tasks \
@@ -1099,7 +1104,7 @@ test_high_level_api_gpu_fast: install_cargo_nextest # Run all the GPU tests for
 test_high_level_api_gpu: install_cargo_nextest # Run all the GPU tests for high_level_api
 	RUSTFLAGS="$(RUSTFLAGS)" cargo nextest run --cargo-profile $(CARGO_PROFILE) \
 		--test-threads=4 --features=integer,internal-keycache,gpu,zk-pok -p tfhe \
-  	-E "test(/high_level_api::.*gpu.*/)"
+		-E "test(/high_level_api::.*gpu.*/)"
 
 test_list_gpu: install_cargo_nextest
 	RUSTFLAGS="$(RUSTFLAGS)" cargo nextest list --cargo-profile $(CARGO_PROFILE) \
@@ -1397,6 +1402,54 @@ test_web_js_api_parallel_firefox_ci: setup_venv
 	nvm use $(NODE_VERSION) && \
 	$(MAKE) test_web_js_api_parallel_firefox
 
+WASM_PAR_MQ_TEST_DIR=utils/wasm-par-mq/web_tests
+
+.PHONY: build_wasm_par_mq_tests # Build the wasm-par-mq test WASM package
+build_wasm_par_mq_tests: install_wasm_pack
+	cd $(WASM_PAR_MQ_TEST_DIR) && \
+	RUSTFLAGS="$(WASM_RUSTFLAGS)" wasm-pack build --target=web --out-dir pkg
+
+# This is an internal target, not meant to be called on its own.
+run_wasm_par_mq_tests: build_wasm_par_mq_tests setup_venv
+	cd $(WASM_PAR_MQ_TEST_DIR) && npm install && npm run build
+	source venv/bin/activate && \
+	python ci/webdriver.py \
+	--browser-path $(browser_path) \
+	--driver-path $(driver_path) \
+	--browser-kind $(browser_kind) \
+	--server-cmd "npm run server" \
+	--server-workdir "$(WASM_PAR_MQ_TEST_DIR)" \
+	--index-path "$(WASM_PAR_MQ_TEST_DIR)/index.html" \
+	--id-pattern Test
+
+test_wasm_par_mq_chrome: browser_path = "$(WEB_RUNNER_DIR)/chrome/chrome-linux64/chrome"
+test_wasm_par_mq_chrome: driver_path = "$(WEB_RUNNER_DIR)/chrome/chromedriver-linux64/chromedriver"
+test_wasm_par_mq_chrome: browser_kind = chrome
+
+.PHONY: test_wasm_par_mq_chrome # Run wasm-par-mq tests on Chrome
+test_wasm_par_mq_chrome: run_wasm_par_mq_tests
+
+.PHONY: test_wasm_par_mq_chrome_ci # Run wasm-par-mq tests on Chrome in CI
+test_wasm_par_mq_chrome_ci: setup_venv
+	source ~/.nvm/nvm.sh && \
+	nvm install $(NODE_VERSION) && \
+	nvm use $(NODE_VERSION) && \
+	$(MAKE) test_wasm_par_mq_chrome
+
+test_wasm_par_mq_firefox: browser_path = "$(WEB_RUNNER_DIR)/firefox/firefox/firefox"
+test_wasm_par_mq_firefox: driver_path = "$(WEB_RUNNER_DIR)/firefox/geckodriver"
+test_wasm_par_mq_firefox: browser_kind = firefox
+
+.PHONY: test_wasm_par_mq_firefox # Run wasm-par-mq tests on Firefox
+test_wasm_par_mq_firefox: run_wasm_par_mq_tests
+
+.PHONY: test_wasm_par_mq_firefox_ci # Run wasm-par-mq tests on Firefox in CI
+test_wasm_par_mq_firefox_ci: setup_venv
+	source ~/.nvm/nvm.sh && \
+	nvm install $(NODE_VERSION) && \
+	nvm use $(NODE_VERSION) && \
+	$(MAKE) test_wasm_par_mq_firefox
+
 .PHONY: no_tfhe_typo # Check we did not invert the h and f in tfhe
 no_tfhe_typo:
 	@./scripts/no_tfhe_typo.sh
@@ -2045,6 +2098,7 @@ pcc_batch_2:
 	$(call run_recipe_with_details,check_fmt_js)  # This needs to stay there, CI pipeline rely on this recipe to conditionally install Node
 	$(call run_recipe_with_details,clippy_test_vectors)
 	$(call run_recipe_with_details,check_test_vectors)
+	$(call run_recipe_with_details,clippy_wasm_par_mq)
 
 .PHONY: pcc_batch_3 # duration: 6'50''
 pcc_batch_3:

backends/tfhe-hpu-backend/Cargo.toml

@@ -31,7 +31,7 @@ tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 serde = { version = "1", features = ["derive"] }
 toml = { version = "0.8", features = [] }
-paste = "1.0.15"
+paste = { workspace = true }
 thiserror = "1.0.61"
 bytemuck = { workspace = true }
 anyhow = "1.0.82"

tasks/src/check_tfhe_docs_are_tested.rs

@@ -5,13 +5,14 @@ use std::io::{Error, ErrorKind};
 // TODO use .gitignore or git to resolve ignored files
 const DIR_TO_IGNORE: [&str; 3] = [".git", "target", "apps/test-vectors"];
 
-const FILES_TO_IGNORE: [&str; 10] = [
+const FILES_TO_IGNORE: [&str; 11] = [
     // This contains fragments of code that are unrelated to TFHE-rs
     "tfhe/docs/tutorials/sha256-bool.md",
     // TODO: This contains code that could be executed as a trivium docstring
     "apps/trivium/README.md",
     // TODO: should we test this ?
     "utils/tfhe-versionable/README.md",
+    "utils/wasm-par-mq/README.md",
     // TODO: find a way to test the tfhe-fft readme
     "tfhe-fft/README.md",
     // TODO: find a way to test the tfhe-ntt readme

tfhe-benchmark/Cargo.toml

@@ -24,7 +24,7 @@ dyn-stack = { workspace = true, features = ["default"] }
 itertools = "0.14"
 serde = { version = "1.0", default-features = false }
 serde_json = "1.0.94"
-paste = "1.0.7"
+paste = { workspace = true }
 rand = { workspace = true }
 rayon = { workspace = true }
 tfhe = { path = "../tfhe", default-features = false }

tfhe-fft/Cargo.toml

@@ -18,7 +18,7 @@ pulp = { workspace = true }
 serde = { workspace = true, optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-js-sys = "0.3"
+js-sys = { workspace = true }
 
 [features]
 default = ["std", "avx512"]

tfhe-zk-pok/Cargo.toml

@@ -14,8 +14,8 @@ rust-version.workspace = true
 
 [dependencies]
 ark-bls12-381 = "0.5.0"
-ark-ec = { version = "0.5.0", features = ["parallel"] }
-ark-ff = { version = "0.5.0", features = ["parallel"] }
+ark-ec = { workspace = true, features = ["parallel"] }
+ark-ff = { workspace = true, features = ["parallel"] }
 ark-poly = { version = "0.5.0", features = ["parallel"] }
 rand = { workspace = true }
 rayon = { workspace = true }

tfhe/Cargo.toml

@@ -69,7 +69,7 @@ pulp = { workspace = true, features = ["default"] }
 tfhe-cuda-backend = { version = "0.13.0", path = "../backends/tfhe-cuda-backend", optional = true }
 aligned-vec = { workspace = true, features = ["default", "serde"] }
 dyn-stack = { workspace = true, features = ["default"] }
-paste = "1.0.7"
+paste = { workspace = true }
 fs2 = { version = "0.4.3", optional = true }
 # Used for OPRF in shortint and rerand
 sha3 = { version = "0.10", optional = true }
@@ -86,9 +86,9 @@ wasm-bindgen = { workspace = true, features = [
     "serde-serialize",
 ], optional = true }
 wasm-bindgen-rayon = { version = "1.3.0", optional = true }
-js-sys = { version = "0.3", optional = true }
+js-sys = { workspace = true, optional = true }
 console_error_panic_hook = { version = "0.1.7", optional = true }
-serde-wasm-bindgen = { version = "0.6.0", optional = true }
+serde-wasm-bindgen = { workspace = true, optional = true }
 getrandom = { workspace = true, optional = true }
 bytemuck = { workspace = true }