Replace SPHINCS+ with SLH-DSA for OQS_ALGS_ENABLED=STD (#2290)

* can pass slh_dsa schemes to copy_from_upstream

Signed-off-by: Ganyu (Bruce) Xu <[email protected]>

* hacked copy_from_slh_dsa_c to extract SLH-DSA schemes, implemented replace_one_fragment so list_standardized_algs.fragment can be rendered with a separate set of instructions

Signed-off-by: Ganyu (Bruce) Xu <[email protected]>

* Exclude pre-hashed SLH-DSA variants from OQS_ALGS_ENABLED=STD

Signed-off-by: Ganyu (Bruce) Xu <[email protected]>

---------

Signed-off-by: Ganyu (Bruce) Xu <[email protected]>

Author: xuganyu96

.CMake/alg_support.cmake

@@ -433,7 +433,7 @@ if(NOT ((OQS_MINIMAL_BUILD STREQUAL "") OR (OQS_MINIMAL_BUILD STREQUAL "OFF")))
 	filter_algs("${OQS_MINIMAL_BUILD}")
 elseif (${OQS_ALGS_ENABLED} STREQUAL "STD")
 ##### OQS_COPY_FROM_UPSTREAM_FRAGMENT_LIST_STANDARDIZED_ALGS_START
-	filter_algs("KEM_ml_kem_512;KEM_ml_kem_768;KEM_ml_kem_1024;SIG_ml_dsa_44;SIG_ml_dsa_65;SIG_ml_dsa_87;SIG_falcon_512;SIG_falcon_1024;SIG_falcon_padded_512;SIG_falcon_padded_1024;SIG_sphincs_sha2_128f_simple;SIG_sphincs_sha2_128s_simple;SIG_sphincs_sha2_192f_simple;SIG_sphincs_sha2_192s_simple;SIG_sphincs_sha2_256f_simple;SIG_sphincs_sha2_256s_simple;SIG_sphincs_shake_128f_simple;SIG_sphincs_shake_128s_simple;SIG_sphincs_shake_192f_simple;SIG_sphincs_shake_192s_simple;SIG_sphincs_shake_256f_simple;SIG_sphincs_shake_256s_simple")
+	filter_algs("KEM_ml_kem_512;KEM_ml_kem_768;KEM_ml_kem_1024;SIG_ml_dsa_44;SIG_ml_dsa_65;SIG_ml_dsa_87;SIG_falcon_512;SIG_falcon_1024;SIG_falcon_padded_512;SIG_falcon_padded_1024;SIG_slh_dsa_pure_sha2_128s;SIG_slh_dsa_pure_sha2_128f;SIG_slh_dsa_pure_sha2_192s;SIG_slh_dsa_pure_sha2_192f;SIG_slh_dsa_pure_sha2_256s;SIG_slh_dsa_pure_sha2_256f;SIG_slh_dsa_pure_shake_128s;SIG_slh_dsa_pure_shake_128f;SIG_slh_dsa_pure_shake_192s;SIG_slh_dsa_pure_shake_192f;SIG_slh_dsa_pure_shake_256s;SIG_slh_dsa_pure_shake_256f")
 ##### OQS_COPY_FROM_UPSTREAM_FRAGMENT_LIST_STANDARDIZED_ALGS_END
 elseif(${OQS_ALGS_ENABLED} STREQUAL "NIST_R4")
 	filter_algs("KEM_classic_mceliece_348864;KEM_classic_mceliece_348864f;KEM_classic_mceliece_460896;KEM_classic_mceliece_460896f;KEM_classic_mceliece_6688128;KEM_classic_mceliece_6688128f;KEM_classic_mceliece_6960119;KEM_classic_mceliece_6960119f;KEM_classic_mceliece_8192128;KEM_classic_mceliece_8192128f;KEM_hqc_128;KEM_hqc_192;KEM_hqc_256;KEM_bike_l1;KEM_bike_l3;KEM_bike_l5")

scripts/copy_from_upstream/.CMake/alg_support.cmake/list_standardized_algs.fragment

@@ -5,7 +5,7 @@
         {%- if 'alias_scheme' in scheme -%}KEM_{{ family['name'] }}_{{ scheme['alias_scheme'] }}{%- else -%}KEM_{{ family['name'] }}_{{ scheme['scheme'] }}{%- endif -%};
     {%- endfor -%}
 {%- endfor -%}
-{%- for family in instructions['sigs'] if family['name'] in ['ml_dsa', 'falcon', 'sphincs'] -%}
+{%- for family in instructions['sigs'] if family['name'] in ['ml_dsa', 'falcon', 'slh_dsa'] -%}
     {%- set outer_loop = loop -%}
     {%- for scheme in family['schemes'] -%}
         {%- if 'alias_scheme' in scheme -%}SIG_{{ family['name'] }}_{{ scheme['alias_scheme'] }}{%- else -%}SIG_{{ family['name'] }}_{{ scheme['scheme'] }}{%- endif -%}{%- if not (outer_loop.last and loop.last) -%};{%- endif -%}

scripts/copy_from_upstream/copy_from_slh_dsa_c.py

@@ -337,5 +337,18 @@ def main():
     # apply patches
     apply_patches(slh_patch_dir)
 
+    # NOTE: from [issue 2203](https://github.com/open-quantum-safe/liboqs/issues/2203)
+    # SLH-DSA is not described in copy_from_upstream.yml. It is instead described
+    # here in this separate module. This makes replacing SPHINCS+ with SLH-DSA
+    # in list_standardized_algs.fragment non-trivial because this Jinja template
+    # is rendered from copy_from_upstream.yml.
+    # As a necessary hack, the list of variants (e.g. "pure_sha2_128s") is returned
+    # so that copy_from_upstream.py can use this list to construct a dictionary
+    # that resembles the structure of copy_from_upstream.yml.
+    # In the near future I want to consider refactoring build configuration
+    # management and upstream integration scripts. The status quo is a mess and
+    # will make future integrations all the more difficult.
+    return variants
+
 if __name__ == "__main__":
     main()

scripts/copy_from_upstream/copy_from_upstream.py

@@ -16,6 +16,7 @@
 import platform
 import update_upstream_alg_docs
 import copy_from_slh_dsa_c
+from copy import deepcopy
 
 # kats of all algs
 kats = {}
@@ -97,6 +98,46 @@ def generator_all(filename, instructions):
     contents = jinja2.Template(template).render({'instructions': instructions})
     file_put_contents(filename, contents)
 
+# TODO: consider refactoring replacer by calling replace_one_fragment
+def replace_one_fragment(
+    dst_path: str,
+    template_path: str,
+    instructions: dict,
+    delimiter: str,
+    libjade: bool = False,
+):
+    """Replace a single fragment with a rendered Jinja template
+
+    :param dst_path: path to the rendered file, relative to LIBOQS_DIR
+    :param template_path: path to the Jinja template file, relative to LIBOQS_DIR
+    :param instructions: copy_from_upstream.yml or some patched version
+    :param delimiter: how the identifer for the fragment in the destination file
+    is prefixed
+    """
+    liboqs_dir = os.environ.get("LIBOQS_DIR", None)
+    if not liboqs_dir:
+        raise KeyError("Environment variable LIBOQS_DIR is missing")
+    dst_path = os.path.join(liboqs_dir, dst_path)
+    template_path = os.path.join(liboqs_dir, template_path)
+    with open(template_path, "r") as template_f, open(dst_path, "r") as dst_f:
+        template = template_f.read()
+        dst_content = dst_f.read()
+    identifier, _ = os.path.splitext(os.path.basename(template_path))
+    jade_or_upstream = "LIBJADE" if libjade else "UPSTREAM"
+    identifier_start = f"{delimiter} OQS_COPY_FROM_{jade_or_upstream}_FRAGMENT_{identifier.upper()}_START"
+    identifier_end = f"{delimiter} OQS_COPY_FROM_{jade_or_upstream}_FRAGMENT_{identifier.upper()}_END"
+    preamble = dst_content[: dst_content.find(identifier_start)]
+    postamble = dst_content[dst_content.find(identifier_end) :]
+    dst_content = (
+        preamble
+        + identifier_start
+        + jinja2.Template(template).render(
+            {"instructions": instructions, "non_upstream_kems": non_upstream_kems}
+        )
+        + postamble
+    )
+    with open(dst_path, "w") as f:
+        f.write(dst_content)
 
 def replacer(filename, instructions, delimiter, libjade=False):
     fragments = glob.glob(
@@ -701,14 +742,29 @@ def process_families(instructions, basedir, with_kat, with_generator, with_libja
             )
 
 
-def copy_from_upstream():
+def copy_from_upstream(slh_dsa_inst: dict):
+    """Integrate upstreams implementations and algorithms described in 
+    copy_from_upstream.yml.
+
+    :param slh_dsa_inst: instruction for integrating SLH-DSA, only used for 
+    rendering alg_support.cmake
+    """
     for t in ["kem", "sig"]:
         with open(os.path.join(os.environ['LIBOQS_DIR'], 'tests', 'KATs', t, 'kats.json'), 'r') as fp:
             kats[t] = json.load(fp)
 
     instructions = load_instructions('copy_from_upstream.yml')
+    patched_inst: dict = deepcopy(instructions)
+    patched_inst["sigs"].append(slh_dsa_inst["sigs"][0])
     process_families(instructions, os.environ['LIBOQS_DIR'], True, True)
     replacer('.CMake/alg_support.cmake', instructions, '#####')
+    # NOTE: issue 2203, only for replacing list of standardized algs
+    replace_one_fragment(
+        ".CMake/alg_support.cmake",
+        "scripts/copy_from_upstream/.CMake/alg_support.cmake/list_standardized_algs.fragment",
+        patched_inst,
+        "#####"
+    )
     replacer('CMakeLists.txt', instructions, '#####')
     replacer('src/oqsconfig.h.cmake', instructions, '/////')
     replacer('src/CMakeLists.txt', instructions, '#####')
@@ -839,9 +895,20 @@ def verify_from_upstream():
 
 if args.operation == "copy":
     # copy_from_slh_dsa_c will modify slh_dsa.yml before copy_from_upstream modifies md files
-    copy_from_slh_dsa_c.main()
+    slh_dsa_schemes: list[str] = copy_from_slh_dsa_c.main()
+    slh_dsa_instruction = {
+        "sigs": [
+            {
+                "name": "slh_dsa",
+                "schemes": [
+                    {"scheme": scheme} for scheme in slh_dsa_schemes
+                    if "pure" in scheme
+                ]
+            }
+        ]
+    }
     os.chdir(os.path.join(os.environ['LIBOQS_DIR'],"scripts","copy_from_upstream"))
-    copy_from_upstream()
+    copy_from_upstream(slh_dsa_instruction)
 elif args.operation == "libjade":
     copy_from_libjade()
 elif args.operation == "verify":