Refactoring

Author: countzero

README.md

@@ -64,9 +64,9 @@ CLEAN_CACHE=True
 #
 #     IQ2_XXS :  2.06 bpw quantization
 #     IQ2_XS  :  2.31 bpw quantization
-#     IQ3_XXS :  3.06 bpw quantization
 #     Q2_K    :  2.63G, +0.6717 ppl @ LLaMA-v1-7B
 #     Q2_K_S  :  2.16G, +9.0634 ppl @ LLaMA-v1-7B
+#     IQ3_XXS :  3.06 bpw quantization
 #     Q3_K_XS :  3-bit extra small quantization
 #     Q3_K_S  :  2.75G, +0.5551 ppl @ LLaMA-v1-7B
 #     Q3_K_M  :  3.07G, +0.2496 ppl @ LLaMA-v1-7B

quantize_weights_for_llama.cpp.ps1

@@ -56,13 +56,17 @@ ForEach ($repositoryName in $repositoryDirectories) {
             Invoke-Expression "$convertCommand --outfile `"${unquantizedModelPath}`" `"${sourceDirectoryPath}`""
         }
 
-        if (("IQ2_XXS IQ2_XS IQ3_XXS".Contains($type)) -and !(Test-Path -Path $importanceMatrixPath)) {
+        # We do need to compute an importance matrix for 2-bit quantized models:
+        # https://github.com/ggerganov/llama.cpp/tree/master/examples/imatrix
+        $requiresImportanceMatrix = "IQ2_XXS IQ2_XS Q2_K_S".Contains($type)
+
+        if ($requiresImportanceMatrix -and !(Test-Path -Path $importanceMatrixPath)) {
 
             Write-Host "Computing importance matrix for ${unquantizedModelPath} at ${importanceMatrixPath}..." -ForegroundColor "DarkYellow"
 
-            $imatrixCommand = "${llamaCppDirectory}\build\bin\Release\imatrix.exe"
+            $matrixCommand = "${llamaCppDirectory}\build\bin\Release\imatrix.exe"
 
-            Invoke-Expression "$imatrixCommand -m `"${unquantizedModelPath}`" -f `"${trainingDataPath}`" -o `"${importanceMatrixPath}`""
+            Invoke-Expression "$matrixCommand -m `"${unquantizedModelPath}`" -f `"${trainingDataPath}`" -o `"${importanceMatrixPath}`" -ngl 99"
         }
 
         if (!(Test-Path -Path $quantizedModelPath)) {
@@ -71,6 +75,10 @@ ForEach ($repositoryName in $repositoryDirectories) {
 
             $quantizeCommand = "${llamaCppDirectory}\build\bin\Release\quantize.exe"
 
+            if ($requiresImportanceMatrix) {
+                $quantizeCommand = "${quantizeCommand} --imatrix=`"${importanceMatrixPath}`""
+            }
+
             Invoke-Expression "$quantizeCommand `"${unquantizedModelPath}`" `"${quantizedModelPath}`" `"${type}`""
         }
     }