Merge branch 'ikawrakow:main' into main

Author: Thireus

ggml/CMakeLists.txt

@@ -104,6 +104,7 @@ endif()
 
 # ggml core
 set(GGML_SCHED_MAX_COPIES  "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
+set(GGML_MAX_CONTEXTS       "" CACHE STRING "ggml: max model contexts (override only; defaults to 64 in the code)")
 
 # 3rd party libs / backends
 option(GGML_ACCELERATE                      "ggml: enable Accelerate framework"               ON)

ggml/src/CMakeLists.txt

@@ -3,6 +3,9 @@ include(CheckCXXCompilerFlag)
 unset(GGML_CDEF_PUBLIC)
 
 add_compile_definitions(GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES})
+if (GGML_MAX_CONTEXTS)
+    add_compile_definitions(GGML_MAX_CONTEXTS=${GGML_MAX_CONTEXTS})
+endif()
 
 # enable libstdc++ assertions for debug builds
 if (CMAKE_SYSTEM_NAME MATCHES "Linux")

src/llama.cpp

@@ -4265,6 +4265,24 @@ struct llama_model_loader {
             trace = atoi(getenv("LLAMA_TRACE"));
         }
 
+        #ifdef _WIN32
+            // Only bump maxstdio if the user really wants large contexts:
+            #if defined(GGML_MAX_CONTEXTS) && (GGML_MAX_CONTEXTS > 512)
+                // Cap at MSVC's hard limit of 8192 - https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160
+                #if (GGML_MAX_CONTEXTS > 8192)
+                    #define _GGML_STDIO_TARGET 8192
+                #else
+                    #define _GGML_STDIO_TARGET GGML_MAX_CONTEXTS
+                #endif
+                int _setmaxstdio_ret = _setmaxstdio(_GGML_STDIO_TARGET);
+                if (_setmaxstdio_ret == -1) {
+                    LLAMA_LOG_INFO("%s: failed to set max stdio to %d. (setmaxstdio returned -1)\n", __func__, _GGML_STDIO_TARGET);
+                } else {
+                    LLAMA_LOG_INFO("%s: max stdio successfully set to %d\n", __func__, _setmaxstdio_ret);
+                }
+            #endif // GGML_MAX_CONTEXTS > 512
+        #endif // _WIN32
+
         if (param_overrides_p != nullptr) {
             for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {
                 kv_overrides.insert({std::string(p->key), *p});