fp16 distilbert

Author: Pierrci

README.md

@@ -14,6 +14,10 @@ It provides 48 passages from the dataset for users to choose from.
 
 ![demo gif](media/distilbert_qa.gif "Demo running offline on a Samsung Galaxy S8")
 
+> Available models:
+> * "original" converted DistilBERT (266MB)
+> * FP16 post-training-quantized DistilBERT (67MB)
+
 ### Coming soon: GPT-2, quantization... and much more!
 
 ---
@@ -69,6 +73,16 @@ adb install app/build/outputs/apk/debug/app-debug.apk
 
 ---
 
+## Change the model
+
+To choose which model to use in the app:
+*   Remove/rename the current `model.tflite` file in `app/src/main/assets`
+*   Comment/uncomment the model to download in the `download.gradle` config file:
+```java
+"https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384.tflite": "model.tflite", // <- "original" converted DistilBERT (default)
+// "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-fp16.tflite": "model.tflite", // <- fp16 quantized version of DistilBERT
+```
+
 ## Models generation
 
 Example scripts used to convert models are available in the `models_generation` directory.

app/download.gradle

@@ -3,6 +3,7 @@ apply plugin: 'de.undercouch.download'
 task downloadLiteModel {
     def downloadFiles = [
         'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384.tflite': 'model.tflite',
+//        'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-384-fp16.tflite': 'model.tflite', // FP16 version
     ]
     downloadFiles.each { key, value ->
       download {

app/libs/tensorflow-lite-with-select-tf-ops-0.0.0-nightly.aar

@@ -6,7 +6,7 @@ buildscript {
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.5.0'
+        classpath 'com.android.tools.build:gradle:3.5.3'
         classpath 'de.undercouch:gradle-download-task:4.0.0'
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files

app/libs/tensorflow-lite-with-select-tf-ops-0.0.0-nightly.pom

@@ -10,8 +10,15 @@
 print(model.outputs)
 
 converter = tf.lite.TFLiteConverter.from_keras_model(model)
+
+# For normal conversion:
 converter.target_spec.supported_ops = [tf.lite.OpsSet.SELECT_TF_OPS]
 
+# For FP16 conversion:
+# converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS]
+# converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
+# converter.experimental_new_converter = True
+
 tflite_model = converter.convert()
 
 open("distilbert-squad-384.tflite", "wb").write(tflite_model)