better way of checking for avx2 support

Author: LostRuins

.github/workflows/kcpp-build-release-win-oldpc.yaml

@@ -33,7 +33,7 @@ jobs:
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5 py-cpuinfo==9.0.0
+          pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5
 
       - name: Display full Visual Studio info Before
         run: |

.github/workflows/kcpp-build-release-win.yaml

@@ -33,7 +33,7 @@ jobs:
       - name: Install python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5 py-cpuinfo==9.0.0
+          pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5
 
       - name: Display full Visual Studio info Before
         run: |

Makefile

@@ -904,6 +904,8 @@ quantize_clip: tools/mtmd/clip.cpp tools/mtmd/clip.h tools/quantclip.cpp ggml_v3
 #window simple clinfo
 simpleclinfo: simpleclinfo.cpp
 	$(CXX) $(CXXFLAGS) $^ lib/OpenCL.lib lib/clblast.lib -o $@ $(LDFLAGS)
+simplecpuinfo: simplecpuinfo.cpp
+	$(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS)
 
 build-info.h:
 	$(DONOTHING)

environment-nocuda.yaml

@@ -19,4 +19,3 @@ dependencies:
   - tk=*=xft_*
   - pip:
     - customtkinter
-    - py-cpuinfo

environment.yaml

@@ -22,4 +22,3 @@ dependencies:
   - tk=*=xft_*
   - pip:
     - customtkinter
-    - py-cpuinfo

klite.embd

@@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
 -->
 
 <script id="init-config">
-	const LITEVER = 256;
+	const LITEVER = 257;
 	const urlParams = new URLSearchParams(window.location.search);
 	var localflag = urlParams.get('local'); //this will be replaced automatically in embedded kcpp
 	const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@@ -3213,7 +3213,7 @@ Current version indicated by LITEVER below.
 	var passed_ai_warning_local = false;
 	var welcome = "";
 	var personal_notes = "";
-	var voice_typing_mode = 0; //0=off, 1=on, 2=ptt
+	var voice_typing_mode = 0; //0=off, 1=on, 2=ptt, 3=ttt
 	var koboldcpp_has_whisper = false; //does backend support voice typing
 	var voice_is_recording = false; //currently recording voice?
 	var voice_is_processing = false; //currently processing voice?
@@ -5636,7 +5636,9 @@ Current version indicated by LITEVER below.
 					let evs = [];
 					let m;
 					while ((m = /^data: ?(.*)(\r?\n){2}/m.exec(ctrl.buf)) !== null) {
-						try{evs.push({data: JSON.parse(m[1])});} catch (e) {}
+						try{evs.push({data: JSON.parse(m[1])});} catch (e) {
+							console.log("Cannot parse a chunk: " + chunk);
+						}
 						ctrl.buf = ctrl.buf.substring(m.index + m[0].length);
 					}
 					if (evs.length) {
@@ -15310,6 +15312,12 @@ Current version indicated by LITEVER below.
 	var recent_voice_duration = 0;
 	function ptt_start()
 	{
+		if(voice_typing_mode==3 && voice_is_speaking)
+		{
+			//stop toggle to talk
+			ptt_end(true);
+			return;
+		}
 		if(voice_typing_mode>0)
 		{
 			voice_is_speaking = true;
@@ -15328,8 +15336,12 @@ Current version indicated by LITEVER below.
 			}
 		}
 	}
-	function ptt_end()
+	function ptt_end(force=false)
 	{
+		if(voice_typing_mode==3 && !force)
+		{
+			return; //do nothing for toggle to talk unless forced
+		}
 		var voice_end_delay = localsettings.voice_end_delay;
 		if(voice_typing_mode>0)
 		{
@@ -19471,7 +19483,7 @@ Current version indicated by LITEVER below.
 					document.getElementById("btnsend").innerHTML = "<div class='showmiclivebig'></div><span style='font-size:12px'>Record</span>";
 				} else if (ready_to_record()) {
 					document.getElementById("chat_msg_send_btn").classList.add("showmic");
-					document.getElementById("btnsend").innerHTML = "<div class='showmicbig'></div><span style='font-size:12px'>"+(voice_typing_mode==1?"Standby":"PTT")+"</span>";
+					document.getElementById("btnsend").innerHTML = "<div class='showmicbig'></div><span style='font-size:12px'>"+(voice_typing_mode==1?"Standby":(voice_typing_mode==2?"PTT":"TTT"))+"</span>";
 				} else {
 					document.getElementById("chat_msg_send_btn").classList.add("showmicoff");
 					document.getElementById("btnsend").innerHTML = "<div class='showmicoffbig'></div><span style='font-size:12px'>Busy</span>";
@@ -23373,7 +23385,7 @@ Current version indicated by LITEVER below.
 				</div>
 				<div style="position: relative; padding-right: 2px;">
 					<button type="button" class="btn wait mainnav" id="btnsend" disabled
-						onclick="submit_generation_button(false)" onmousedown="ptt_start()" onmouseup="ptt_end()">Loading</button>
+						onclick="submit_generation_button(false)" onpointerdown="ptt_start()" onpointerup="ptt_end()">Loading</button>
 						<a href="#" id="abortgen" class="hidden bg_black mainnav" style="text-align: center;color: #ffaaaa;" onclick="abort_generation()"><b style="display: block;">[ABORT]</b></a>
 						<span id="searchingtxt" class="hidden bg_black mainnav" style="text-align: center;color: #49a8e4; font-size: 9px;"><b style="display: block;">WebSearch...</b></span>
 				</div>
@@ -24373,6 +24385,7 @@ Current version indicated by LITEVER below.
 								<option value="0">Off</option>
 								<option value="1">Detect Voice</option>
 								<option value="2">Push-To-Talk</option>
+								<option value="3">Toggle-To-Talk</option>
 							</select>
 						</div>
 						<div class="inlinelabel" style="font-size: 11px;">
@@ -25166,9 +25179,9 @@ Current version indicated by LITEVER below.
 			</div>
 
 			<div id="koboldcustom" class="menutext">
-				You can use this to connect to a KoboldAI instance running via a remote tunnel such as <span class="color_orange" style="font-weight: bold;">trycloudflare, localtunnel, ngrok</span>.<br><br>
-				Localhost IPs require host mode enabled. You can use the remote address displayed in the <span class="color_orange" style="font-weight: bold;">terminal console</span> or <span class="color_orange" style="font-weight: bold;">colab window</span>, note that the model must be loaded first.<br><br>
-				<span class="color_green" style="font-weight: bold;">Please input URL of the KoboldAI instance.</span><br><br>
+				You can use this to connect to a KoboldCpp or KoboldAI instance running via a remote tunnel such as <span class="color_orange" style="font-weight: bold;">trycloudflare, localtunnel, ngrok</span>.<br><br>
+				You can use the remote address displayed in the <span class="color_orange" style="font-weight: bold;">terminal console</span> or <span class="color_orange" style="font-weight: bold;">colab window</span>, note that the model must be loaded first.<br><br>
+				<span class="color_green" style="font-weight: bold;">Please input URL of the KoboldCpp or KoboldAI instance.</span><br><br>
 				<input class="form-control" title="Enter KoboldCpp Custom Endpoint" id="customkoboldendpoint" placeholder="https://sample-remote-address.trycloudflare.com" value="">
 				<input class="form-control" title="Enter KoboldCpp API Key" type="password" id="customkoboldkey" placeholder="KoboldAI API Key (Optional)" value="" onfocus="focus_api_keys()" onblur="blur_api_keys()"><br>
 				<div class="borderbox flex flex-push-right">

koboldcpp.py

@@ -678,23 +678,34 @@ def strip_base64_prefix(encoded_data):
         encoded_data = encoded_data.split(',', 1)[-1]
     return encoded_data
 
-def old_cpu_check(): #return 0 if has avx2, 1 if has avx, 2 if has nothing
+def old_cpu_check(): #return -1 for pass, 0 if has avx2, 1 if has avx, 2 if has nothing
     shouldcheck = ((sys.platform == "linux" and platform.machine().lower() in ("x86_64", "amd64")) or
                   (os.name == 'nt' and platform.machine().lower() in ("amd64", "x86_64")))
     if not shouldcheck:
-        return 0 #doesnt deal with avx at all.
+        return -1 #doesnt deal with avx at all.
     try:
-        import cpuinfo
-        info = cpuinfo.get_cpu_info()
-        flags = info.get('flags', [])
-        if 'avx2' in flags:
-            return 0
-        elif 'avx' in flags:
-            return 1
-        else:
-            return 2
+        retflags = 0
+        if sys.platform == "linux":
+            with open('/proc/cpuinfo', 'r') as f:
+                cpuinfo = f.read()
+                cpuinfo = cpuinfo.lower()
+                if 'avx' not in cpuinfo and 'avx2' not in cpuinfo:
+                    retflags = 2
+                elif 'avx2' not in cpuinfo:
+                    retflags = 1
+        elif os.name == 'nt':
+            basepath = os.path.abspath(os.path.dirname(__file__))
+            output = ""
+            data = None
+            output = subprocess.run([os.path.join(basepath, "simplecpuinfo.exe")], capture_output=True, text=True, check=True, creationflags=subprocess.CREATE_NO_WINDOW | subprocess.DETACHED_PROCESS, encoding='utf-8', timeout=6).stdout
+            data = json.loads(output)
+            if data["avx2"]==0 and data["avx"]==0:
+                retflags = 2
+            elif data["avx2"]==0:
+                retflags = 1
+        return retflags
     except Exception:
-        return 0 #cannot determine
+        return -1 #cannot determine
 
 
 def unpack_to_dir(destpath = ""):
@@ -1286,7 +1297,7 @@ def auto_set_backend_cli():
     # check for avx2 and avx support
     is_oldpc_ver = "Use CPU" not in runopts #on oldcpu ver, default lib does not exist
     cpusupport = old_cpu_check() # 0 if has avx2, 1 if has avx, 2 if has nothing
-    eligible_cuda = (cpusupport==0 and not is_oldpc_ver) or (cpusupport==1 and is_oldpc_ver)
+    eligible_cuda = (cpusupport<1 and not is_oldpc_ver) or (cpusupport<2 and is_oldpc_ver)
     if not eligible_cuda:
         if cpusupport==1:
             args.noavx2 = True
@@ -1297,17 +1308,17 @@ def auto_set_backend_cli():
     if eligible_cuda and exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and any(CUDevicesNames):
         if "Use CuBLAS" in runopts or "Use hipBLAS (ROCm)" in runopts:
             args.usecublas = ["normal","mmq"]
-            print("Auto Selected CUDA Backend...\n")
+            print(f"Auto Selected CUDA Backend (flag={cpusupport})\n")
             found_new_backend = True
     elif exitcounter < 100 and (1 in VKIsDGPU) and ("Use Vulkan" in runopts or "Use Vulkan (Old CPU)" in runopts):
         for i in range(0,len(VKIsDGPU)):
             if VKIsDGPU[i]==1:
                 args.usevulkan = []
-                print("Auto Selected Vulkan Backend...\n")
+                print(f"Auto Selected Vulkan Backend (flag={cpusupport})\n")
                 found_new_backend = True
                 break
     if not found_new_backend:
-        print("No GPU Backend found...\n")
+        print(f"Auto Selected Default Backend (flag={cpusupport})\n")
 
 def load_model(model_filename):
     global args
@@ -4668,30 +4679,30 @@ def auto_set_backend_gui(manual_select=False):
         # check for avx2 and avx support
         is_oldpc_ver = "Use CPU" not in runopts #on oldcpu ver, default lib does not exist
         cpusupport = old_cpu_check() # 0 if has avx2, 1 if has avx, 2 if has nothing
-        eligible_cuda = (cpusupport==0 and not is_oldpc_ver) or (cpusupport==1 and is_oldpc_ver)
+        eligible_cuda = (cpusupport<1 and not is_oldpc_ver) or (cpusupport<2 and is_oldpc_ver)
 
         #autopick cublas if suitable, requires at least 3.5GB VRAM to auto pick
         #we do not want to autoselect hip/cublas if the user has already changed their desired backend!
         if eligible_cuda and exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and (any(CUDevicesNames) or any(CLDevicesNames)) and runmode_untouched:
             if "Use CuBLAS" in runopts:
                 runopts_var.set("Use CuBLAS")
                 gpu_choice_var.set("1")
-                print("Auto Selected CUDA Backend...\n")
+                print(f"Auto Selected CUDA Backend (flag={cpusupport})\n")
                 found_new_backend = True
             elif "Use hipBLAS (ROCm)" in runopts:
                 runopts_var.set("Use hipBLAS (ROCm)")
                 gpu_choice_var.set("1")
-                print("Auto Selected HIP Backend...\n")
+                print(f"Auto Selected HIP Backend (flag={cpusupport})\n")
                 found_new_backend = True
         elif exitcounter < 100 and (1 in VKIsDGPU) and runmode_untouched and ("Use Vulkan" in runopts or "Use Vulkan (Old CPU)" in runopts):
             for i in range(0,len(VKIsDGPU)):
                 if VKIsDGPU[i]==1:
-                    if cpusupport==0 and "Use Vulkan" in runopts:
+                    if cpusupport<1 and "Use Vulkan" in runopts:
                         runopts_var.set("Use Vulkan")
                     else:
                         runopts_var.set("Use Vulkan (Old CPU)")
                     gpu_choice_var.set(str(i+1))
-                    print("Auto Selected Vulkan Backend...\n")
+                    print(f"Auto Selected Vulkan Backend (flag={cpusupport})\n")
                     found_new_backend = True
                     break
         else:
@@ -4700,7 +4711,7 @@ def auto_set_backend_gui(manual_select=False):
             elif runopts_var.get()=="Use CPU" and cpusupport==2 and "Failsafe Mode (Older CPU)" in runopts:
                 runopts_var.set("Failsafe Mode (Older CPU)")
         if not found_new_backend:
-            print("Auto Selected Default Backend...\n")
+            print(f"Auto Selected Default Backend (flag={cpusupport})\n")
         changed_gpu_choice_var()
 
     def on_picked_model_file(filepath):

make_pyinstaller.bat

@@ -1,3 +1,3 @@
 call create_ver_file.bat
 PyInstaller  --onedir --noconfirm --clean --console --runtime-hook "./tools/kcpplauncherhook.py" --icon "./niko.ico" --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-launcher"
-PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --collect-all psutil --icon "./niko.ico" --add-data "./dist/koboldcpp-launcher/koboldcpp-launcher.exe;." --add-data "./simpleclinfo.exe;." --add-data "./aria2c-win.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_adapters;./kcpp_adapters" --add-data "./koboldcpp.py;." --add-data "./json_to_gbnf.py;." --add-data "./LICENSE.md;." --add-data "./MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md;." --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./kcpp_sdui.embd;." --add-data "./taesd.embd;." --add-data "./taesd_xl.embd;." --add-data "./taesd_f.embd;." --add-data "./taesd_3.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./koboldcpp_clblast_failsafe.dll;." --add-data "./koboldcpp_vulkan_noavx2.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_vulkan.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-nocuda.exe"
+PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --collect-all psutil --icon "./niko.ico" --add-data "./dist/koboldcpp-launcher/koboldcpp-launcher.exe;." --add-data "./simpleclinfo.exe;." --add-data "./simplecpuinfo.exe;." --add-data "./aria2c-win.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_adapters;./kcpp_adapters" --add-data "./koboldcpp.py;." --add-data "./json_to_gbnf.py;." --add-data "./LICENSE.md;." --add-data "./MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md;." --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./kcpp_sdui.embd;." --add-data "./taesd.embd;." --add-data "./taesd_xl.embd;." --add-data "./taesd_f.embd;." --add-data "./taesd_3.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./koboldcpp_clblast_failsafe.dll;." --add-data "./koboldcpp_vulkan_noavx2.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_vulkan.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-nocuda.exe"

make_pyinstaller_cuda.bat

@@ -1,3 +1,3 @@
 call create_ver_file.bat
 PyInstaller  --onedir --noconfirm --clean --console --runtime-hook "./tools/kcpplauncherhook.py" --icon "./niko.ico" --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-launcher"
-PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --collect-all psutil --icon "./nikogreen.ico" --add-data "./dist/koboldcpp-launcher/koboldcpp-launcher.exe;." --add-data "./simpleclinfo.exe;." --add-data "./aria2c-win.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_adapters;./kcpp_adapters" --add-data "./koboldcpp.py;." --add-data "./json_to_gbnf.py;." --add-data "./LICENSE.md;." --add-data "./MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md;." --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./kcpp_sdui.embd;." --add-data "./taesd.embd;." --add-data "./taesd_xl.embd;." --add-data "./taesd_f.embd;." --add-data "./taesd_3.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./koboldcpp_clblast_failsafe.dll;." --add-data "./koboldcpp_vulkan_noavx2.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_vulkan.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cublas64_12.dll;." --add-data "./cublasLt64_12.dll;." --add-data "./cudart64_12.dll;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/msvcp140_codecvt_ids.dll;." --add-data "C:/Windows/System32/vcruntime140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp.exe"
+PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --collect-all psutil --icon "./nikogreen.ico" --add-data "./dist/koboldcpp-launcher/koboldcpp-launcher.exe;." --add-data "./simpleclinfo.exe;." --add-data "./simplecpuinfo.exe;." --add-data "./aria2c-win.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_adapters;./kcpp_adapters" --add-data "./koboldcpp.py;." --add-data "./json_to_gbnf.py;." --add-data "./LICENSE.md;." --add-data "./MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md;." --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./kcpp_sdui.embd;." --add-data "./taesd.embd;." --add-data "./taesd_xl.embd;." --add-data "./taesd_f.embd;." --add-data "./taesd_3.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./koboldcpp_clblast_failsafe.dll;." --add-data "./koboldcpp_vulkan_noavx2.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_vulkan.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cublas64_12.dll;." --add-data "./cublasLt64_12.dll;." --add-data "./cudart64_12.dll;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/msvcp140_codecvt_ids.dll;." --add-data "C:/Windows/System32/vcruntime140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp.exe"