[kernel][arm64] Use FEAT_MOPS memcpy, memset when available

Use code-patching to switch between the current (cortex-strings)
assembly versions of memcpy and memset, and new versions taking
advantage of the FEAT_MOPS instructions.

Change-Id: I09bbdd877316976769183fafdcc7f651f2710ccd
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/1069632
Reviewed-by: Travis Geiselbrecht <[email protected]>
Fuchsia-Auto-Submit: Roland McGrath <[email protected]>
Commit-Queue: Auto-Submit <[email protected]>

Author: frobtech

zircon/kernel/arch/arm64/code-patches/BUILD.gn

@@ -9,14 +9,15 @@ import("//build/toolchain/toolchain_environment.gni")
 
 if (toolchain_environment == "kernel.phys") {
   source_set("code-patches") {
+    public_deps = [ ":headers" ]
     sources = [ "code-patches.cc" ]
     deps = [
       "//zircon/kernel/arch/arm64/phys:arch-phys-info",
+      "//zircon/kernel/lib/arch",
       "//zircon/kernel/lib/code-patching",
       "//zircon/kernel/phys:handoff",
       "//zircon/kernel/phys:symbolize",
     ]
-    public_deps = [ ":headers" ]
   }
 }
 

zircon/kernel/arch/arm64/code-patches/code-patches.cc

@@ -4,6 +4,7 @@
 // license that can be found in the LICENSE file or at
 // https://opensource.org/licenses/MIT
 
+#include <lib/arch/arm64/feature.h>
 #include <lib/code-patching/code-patches.h>
 #include <zircon/assert.h>
 
@@ -20,6 +21,20 @@
 bool ArchPatchCode(code_patching::Patcher& patcher, const ArchPatchInfo& info,
                    ktl::span<ktl::byte> insns, CodePatchId case_id,
                    fit::inline_function<void(ktl::initializer_list<ktl::string_view>)> print) {
+  constexpr auto if_mops = [](ktl::string_view mops,
+                              ktl::string_view not_mops) -> ktl::string_view {
+    auto isar2 = arch::ArmIdAa64IsaR2El1::Read();
+    bool feat_mops = isar2.mops() != arch::ArmIdAa64IsaR2El1::Mops::kNone;
+    return feat_mops ? mops : not_mops;
+  };
+
+  auto do_alternative = [&patcher, insns, &print](ktl::string_view name,
+                                                  ktl::string_view alternative) -> bool {
+    patcher.MandatoryPatchWithAlternative(insns, alternative);
+    print({"using ", name, " alternative \"", alternative, "\""});
+    return true;
+  };
+
   switch (case_id) {
     case CodePatchId::kSelfTest:
       patcher.NopFill(insns);
@@ -79,6 +94,13 @@ bool ArchPatchCode(code_patching::Patcher& patcher, const ArchPatchInfo& info,
       print({"CPU workaround SMCCC function is "sv, choice});
       return true;
     }
+
+    case CodePatchId::k__UnsanitizedMemcpy:
+      return do_alternative("memcpy", if_mops("memcpy-mops", "memcpy-cortex"));
+
+    case CodePatchId::k__UnsanitizedMemset:
+      return do_alternative("memset", if_mops("memset-mops", "memset-cortex"));
   }
+
   return false;
 }

zircon/kernel/arch/arm64/code-patches/include/arch/code-patches/case-id.h

@@ -24,6 +24,13 @@ enum class CodePatchId : uint32_t {
   // The patched area is a single `mov w0, #...` instruction.  It gets patched
   // with the SMCCC function number used for SMCCC_ARCH_WORKAROUND_3.
   kSmcccWorkaroundFunction,
+
+  // Relates to the optimizations available for C string utilities.
+  //
+  // Note: the "__" is intentional as the function name has two leading
+  // underscores.
+  k__UnsanitizedMemcpy,
+  k__UnsanitizedMemset,
 };
 
 // The callback accepts an initializer-list of something constructible with
@@ -35,6 +42,8 @@ inline constexpr auto WithCodePatchNames = [](auto&& callback) {
       {CodePatchId::kSelfTest, "SELF_TEST"},
       {CodePatchId::kSmcccConduit, "SMCCC_CONDUIT"},
       {CodePatchId::kSmcccWorkaroundFunction, "SMCCC_WORKAROUND_FUNCTION"},
+      {CodePatchId::k__UnsanitizedMemcpy, "__UNSANITIZED_MEMCPY"},
+      {CodePatchId::k__UnsanitizedMemset, "__UNSANITIZED_MEMSET"},
   });
 };
 

zircon/kernel/lib/code-patching/code-patching.gni

@@ -158,8 +158,7 @@ template("code_patching_hermetic_alternative") {
     blob_entrypoint = target_name
   }
 
-  blob_path =
-      rebase_path("$target_out_dir/$blob_entrypoint.bin", root_build_dir)
+  blob_path = rebase_path("$target_out_dir/$target_name.bin", root_build_dir)
 
   hermetic_code_blob(target_name) {
     forward_variables_from(invoker,
@@ -182,7 +181,7 @@ template("code_patching_hermetic_alternative") {
       code_patching_hermetic_alternatives += [
         {
           label = target_label
-          name = blob_entrypoint
+          name = target_name
           path = blob_path
         },
       ]
@@ -192,7 +191,7 @@ template("code_patching_hermetic_alternative") {
       distribution_entries += [
         {
           source = blob_path
-          destination = "code-patches/$entrypoint"
+          destination = "code-patches/$target_name"
           label = target_label
         },
       ]

zircon/kernel/lib/libc/string/arch/arm64/BUILD.gn

@@ -4,6 +4,12 @@
 # license that can be found in the LICENSE file or at
 # https://opensource.org/licenses/MIT
 
+import("//build/toolchain/toolchain_environment.gni")
+import("//zircon/kernel/lib/code-patching/code-patching.gni")
+
+cortex_strings_dir = "//zircon/third_party/lib/cortex-strings"
+not_needed([ "cortex_strings_dir" ])
+
 source_set("arm64") {
   if (current_os == "win" || toolchain_variant.tags + [ "strict-align" ] -
                              [ "strict-align" ] != toolchain_variant.tags) {
@@ -14,13 +20,48 @@ source_set("arm64") {
       "//zircon/kernel/lib/libc/string/memset.c",
     ]
     deps = [ "//zircon/kernel/lib/libc:headers" ]
+  } else if (toolchain_environment == "kernel") {
+    # In the kernel proper, code-patching chooses the implementation.
+    deps = [
+      ":__unsanitized_memcpy",
+      ":__unsanitized_memset",
+    ]
   } else {
-    # These use '#include "third_party/lib/cortex-strings/src/aarch64/..."'.
-    # and '#include "third_party/lib/cortex-strings/no-neon/src/aarch64/..."'.
-    include_dirs = [ "//zircon/" ]
     sources = [
-      "memcpy.S",
-      "memset.S",
+      "$cortex_strings_dir/no-neon/src/aarch64/memset.S",
+      "$cortex_strings_dir/src/aarch64/memcpy.S",
     ]
   }
 }
+
+if (toolchain_environment == "kernel") {
+  foreach(function,
+          [
+            "memcpy",
+            "memset",
+          ]) {
+    code_patching_hermetic_stub("__unsanitized_${function}") {
+      aliases = [ function ]
+      deps = [
+        ":${function}-cortex",
+        ":${function}-mops",
+      ]
+    }
+
+    code_patching_hermetic_alternative("${function}-mops") {
+      entrypoint = function
+      sources = [ "${function}-mops.S" ]
+      deps = [ "//zircon/kernel/lib/arch:headers" ]
+    }
+  }
+
+  code_patching_hermetic_alternative("memcpy-cortex") {
+    entrypoint = "memcpy"
+    sources = [ "$cortex_strings_dir/src/aarch64/memcpy.S" ]
+  }
+
+  code_patching_hermetic_alternative("memset-cortex") {
+    entrypoint = "memset"
+    sources = [ "$cortex_strings_dir/no-neon/src/aarch64/memset.S" ]
+  }
+}

zircon/kernel/lib/libc/string/arch/arm64/memcpy-mops.S

@@ -0,0 +1,28 @@
+// Copyright 2025 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+
+#include <lib/arch/asm.h>
+
+.arch_extension mops
+
+.function memcpy, global
+
+  // The return value is always just the first argument and they both go in x0.
+  // So use a different register for the destination pointer that gets updated.
+  retval .req x0
+  dst .req x3
+  src .req x1
+  count .req x2
+
+  mov dst, retval
+
+  cpyfp [dst]!, [src]!, count!
+  cpyfm [dst]!, [src]!, count!
+  cpyfe [dst]!, [src]!, count!
+
+  ret
+
+.end_function

zircon/kernel/lib/libc/string/arch/arm64/memset-mops.S

@@ -0,0 +1,28 @@
+// Copyright 2025 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+
+#include <lib/arch/asm.h>
+
+.arch_extension mops
+
+.function memset, global
+
+  // The return value is always just the first argument and they both go in x0.
+  // So use a different register for the destination pointer that gets updated.
+  retval .req x0
+  dst .req x3
+  byte .req x1
+  count .req x2
+
+  mov dst, retval
+
+  setp [dst]!, count!, byte
+  setm [dst]!, count!, byte
+  sete [dst]!, count!, byte
+
+  ret
+
+.end_function