8-bits precision quantized distilbert

Author: Pierrci

README.md

@@ -15,8 +15,9 @@ It provides 48 passages from the dataset for users to choose from.
 ![demo gif](media/distilbert_qa.gif "Demo running offline on a Samsung Galaxy S8")
 
 > Available models:
-> * "original" converted DistilBERT (266MB)
-> * FP16 post-training-quantized DistilBERT (67MB)
+> * "original" converted DistilBERT (254MB)
+> * FP16 post-training-quantized DistilBERT (131MB)
+> * "hybrid" (8-bits precision weights) post-training-quantized DistilBERT (64MB)
 
 ### Coming soon: GPT-2, quantization... and much more!
 
@@ -81,6 +82,7 @@ To choose which model to use in the app:
 ```java
 "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384.tflite": "model.tflite", // <- "original" converted DistilBERT (default)
 // "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-fp16.tflite": "model.tflite", // <- fp16 quantized version of DistilBERT
+// "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-8bits.tflite": "model.tflite", // <- hybrid quantized version of DistilBERT
 ```
 
 ## Models generation

app/download.gradle

@@ -3,7 +3,8 @@ apply plugin: 'de.undercouch.download'
 task downloadLiteModel {
     def downloadFiles = [
         'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384.tflite': 'model.tflite',
-//        'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-fp16.tflite': 'model.tflite', // FP16 version
+//        'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-fp16.tflite': 'model.tflite', // FP16 quantization version
+//        'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-8bits.tflite': 'model.tflite', // hybrid quantization version
     ]
     downloadFiles.each { key, value ->
       download {

models_generation/distilbert.py

@@ -14,7 +14,13 @@
 # For normal conversion:
 converter.target_spec.supported_ops = [tf.lite.OpsSet.SELECT_TF_OPS]
 
-# For FP16 conversion:
+# For conversion with FP16 quantization:
+# converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS]
+# converter.target_spec.supported_types = [tf.float16]
+# converter.optimizations = [tf.lite.Optimize.DEFAULT]
+# converter.experimental_new_converter = True
+
+# For conversion with hybrid quantization:
 # converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS]
 # converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
 # converter.experimental_new_converter = True