Merge pull request pmodels#7351 from hzhou/2503_xpmem_thresh

ch4/ipc: adjust xpmem threshold

Approved-by: Ken Raffenetti

Author: hzhou

src/mpid/ch4/shm/ipc/xpmem/xpmem_post.h

@@ -26,7 +26,7 @@
     - name        : MPIR_CVAR_CH4_IPC_XPMEM_P2P_THRESHOLD
       category    : CH4
       type        : int
-      default     : 1024
+      default     : 65536
       class       : none
       verbosity   : MPI_T_VERBOSITY_USER_BASIC
       scope       : MPI_T_SCOPE_ALL_EQ
@@ -35,6 +35,20 @@
         bytes), then enable XPMEM-based single copy protocol for intranode communication. The
         environment variable is valid only when the XPMEM submodule is enabled.
 
+    - name        : MPIR_CVAR_CH4_IPC_XPMEM_P2P_UPPER_THRESHOLD
+      category    : CH4
+      type        : int
+      default     : -1
+      class       : none
+      verbosity   : MPI_T_VERBOSITY_USER_BASIC
+      scope       : MPI_T_SCOPE_ALL_EQ
+      description : >-
+        If a send message size is greater than or equal to MPIR_CVAR_CH4_IPC_XPMEM_P2P_UPPER_THRESHOLD (in
+        bytes), then skip XPMEM-based single copy protocol for intranode communication. The
+        environment variable is valid only when the XPMEM submodule is enabled. The default is -1, which
+        does not limit the upper threshold.
+
+
 === END_MPI_T_CVAR_INFO_BLOCK ===
 */
 
@@ -53,7 +67,9 @@ MPL_STATIC_INLINE_PREFIX int MPIDI_XPMEM_get_ipc_attr(const void *buf, MPI_Aint
     MPIDI_Datatype_get_info(count, datatype, dt_contig, data_sz, dt_ptr, true_lb);
 
     if (!MPIR_CVAR_CH4_XPMEM_ENABLE || buf == MPI_BOTTOM ||
-        data_sz < MPIR_CVAR_CH4_IPC_XPMEM_P2P_THRESHOLD) {
+        data_sz < MPIR_CVAR_CH4_IPC_XPMEM_P2P_THRESHOLD ||
+        (MPIR_CVAR_CH4_IPC_XPMEM_P2P_UPPER_THRESHOLD > 0 &&
+         data_sz > MPIR_CVAR_CH4_IPC_XPMEM_P2P_UPPER_THRESHOLD)) {
         goto fn_exit;
     } else {
         ipc_attr->ipc_type = MPIDI_IPCI_TYPE__XPMEM;