Enable marlin as default GPTQ kernel

cyang49 · cyang49 · commit e1b2110e61be · 2024-03-22T12:50:11.000Z
Signed-off-by: Chih-Chieh Yang &lt;7364402+cyang49@users.noreply.github.com&gt;
Signed-off-by: Chih-Chieh-Yang &lt;7364402+cyang49@users.noreply.github.com&gt;
diff --git a/server/text_generation_server/utils/layers.py b/server/text_generation_server/utils/layers.py
@@ -15,7 +15,7 @@
 HAS_BITS_AND_BYTES = False
 EXLLAMA_VERSION = None
 HAS_GPTQ_CUDA = False
-GPTQ_CUDA_TYPE = os.getenv("GPTQ_CUDA_TYPE", "exllama").lower()
+GPTQ_CUDA_TYPE = os.getenv("GPTQ_CUDA_TYPE", "marlin").lower()
 GPTQ_CUDA_LINEAR = None
 
 if torch.cuda.is_available():
