Merge pull request #31 from ARM-software/add_accuracy_scores

tom-arm · web-flow · commit 5f2fea4b7628 · 2021-08-26T14:23:23.000+01:00
Added accuracy scores to table
diff --git a/README.md b/README.md
diff --git a/models/anomaly_detection/micronet_medium/tflite_int8/README.md b/models/anomaly_detection/micronet_medium/tflite_int8/README.md
@@ -36,7 +36,7 @@ Dataset: Dcase 2020 Task 2 Slide Rail
 
 | Metric | Value |
 |--------|-------|
-| AUC | 0.9632 |
+| AUC | 0.963 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/anomaly_detection/micronet_medium/tflite_int8/definition.yaml b/models/anomaly_detection/micronet_medium/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   DCASE 2020 Task 2 Slide rail:
-    AUC: 0.9632
+    AUC: 0.963
 description: This is a fully quantized version (asymmetrical int8) of the MicroNet
   Medium model developed by Arm, from the MicroNets paper. It is trained on the 'slide
   rail' task from http://dcase.community/challenge2020/task-unsupervised-detection-of-anomalous-sounds.
diff --git a/models/anomaly_detection/micronet_small/tflite_int8/README.md b/models/anomaly_detection/micronet_small/tflite_int8/README.md
@@ -36,7 +36,7 @@ Dataset: Dcase 2020 Task 2 Slide Rail
 
 | Metric | Value |
 |--------|-------|
-| AUC | 0.9548 |
+| AUC | 0.955 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/anomaly_detection/micronet_small/tflite_int8/definition.yaml b/models/anomaly_detection/micronet_small/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   DCASE 2020 Task 2 Slide rail:
-    AUC: 0.9548
+    AUC: 0.955
 description: This is a fully quantized version (asymmetrical int8) of the MicroNet
   Small model developed by Arm, from the MicroNets paper. It is trained on the 'slide
   rail' task from http://dcase.community/challenge2020/task-unsupervised-detection-of-anomalous-sounds.
diff --git a/models/image_classification/mobilenet_v2_1.0_224/tflite_int8/README.md b/models/image_classification/mobilenet_v2_1.0_224/tflite_int8/README.md
@@ -39,7 +39,7 @@ Dataset: ILSVRC 2012
 
 | Metric | Value |
 |--------|-------|
-| Top 1 Accuracy | 69.68 |
+| Top 1 Accuracy | 0.697 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/image_classification/mobilenet_v2_1.0_224/tflite_int8/definition.yaml b/models/image_classification/mobilenet_v2_1.0_224/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   ILSVRC 2012:
-    top-1-accuracy: '69.68'
+    top-1-accuracy: 0.697
 description: "INT8 quantised version of MobileNet v2 model. Trained on ImageNet."
 license:
 - Apache-2.0
diff --git a/models/image_classification/mobilenet_v2_1.0_224/tflite_uint8/README.md b/models/image_classification/mobilenet_v2_1.0_224/tflite_uint8/README.md
@@ -25,7 +25,7 @@ A guide on how to deploy this model using the Arm NN SDK can be found [here](htt
 ## Performance
 | Platform | Optimized |
 |----------|:---------:|
-| Cortex-A |:heavy_check_mark:         |
+| Cortex-A |:heavy_multiplication_x:         |
 | Cortex-M |:heavy_multiplication_x:         |
 | Mali GPU |:heavy_check_mark:         |
 | Ethos U  |:heavy_check_mark:         |
diff --git a/models/keyword_spotting/cnn_large/tflite_int8/README.md b/models/keyword_spotting/cnn_large/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 92.92% |
+| Accuracy | 0.929 |
 
 ## Performance
 | Platform | Optimized |
@@ -42,7 +42,6 @@ Dataset: Google Speech Commands Test Set
 * :heavy_multiplication_x: - Will not run on this platform.
 
 
-
 ## Optimizations
 | Optimization |  Value  |
 |-----------------|---------|
diff --git a/models/keyword_spotting/cnn_large/tflite_int8/definition.yaml b/models/keyword_spotting/cnn_large/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 92.92%
+    Accuracy: 0.929
 description: 'This is a fully quantized version (asymmetrical int8) of the CNN Large
   model developed by Arm, with training checkpoints, from the Hello Edge paper. Code
   to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/cnn_medium/tflite_int8/README.md b/models/keyword_spotting/cnn_medium/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 91.33% |
+| Accuracy | 0.913 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/keyword_spotting/cnn_medium/tflite_int8/definition.yaml b/models/keyword_spotting/cnn_medium/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 91.33%
+    Accuracy: 0.913
 description: 'This is a fully quantized version (asymmetrical int8) of the CNN Medium
   model developed by Arm, with training checkpoints, from the Hello Edge paper. Code
   to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/cnn_small/tflite_int8/README.md b/models/keyword_spotting/cnn_small/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 91.41% |
+| Accuracy | 0.914 |
 
 ## Performance
 | Platform | Optimized |
@@ -41,8 +41,6 @@ Dataset: Google Speech Commands Test Set
 * :heavy_check_mark: - Will run on this platform.
 * :heavy_multiplication_x: - Will not run on this platform.
 
-
-
 ## Optimizations
 | Optimization |  Value  |
 |-----------------|---------|
diff --git a/models/keyword_spotting/cnn_small/tflite_int8/definition.yaml b/models/keyword_spotting/cnn_small/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 91.41%
+    Accuracy: 0.914
 description: 'This is a fully quantized version (asymmetrical int8) of the CNN Small
   model developed by Arm, with training checkpoints, from the Hello Edge paper. Code
   to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/dnn_large/tflite_int8/README.md b/models/keyword_spotting/dnn_large/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 86.28% |
+| Accuracy | 0.863 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/keyword_spotting/dnn_large/tflite_int8/definition.yaml b/models/keyword_spotting/dnn_large/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 86.28%
+    Accuracy: 0.863
 description: 'This is a fully quantized version (asymmetrical int8) of the DNN Large
   model developed by Arm, with training checkpoints, from the Hello Edge paper. Code
   to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/dnn_medium/tflite_int8/README.md b/models/keyword_spotting/dnn_medium/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 84.64% |
+| Accuracy | 0.846 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/keyword_spotting/dnn_medium/tflite_int8/definition.yaml b/models/keyword_spotting/dnn_medium/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 84.64%
+    Accuracy: 0.846
 description: 'This is a fully quantized version (asymmetrical int8) of the DNN Medium
   model developed by Arm, with training checkpoints, from the Hello Edge paper. Code
   to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/dnn_small/tflite_int8/README.md b/models/keyword_spotting/dnn_small/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 82.70% |
+| Accuracy | 0.827 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/keyword_spotting/dnn_small/tflite_int8/definition.yaml b/models/keyword_spotting/dnn_small/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 82.70%
+    Accuracy: 0.827
 description: 'This is a fully quantized version (asymmetrical int8) of the DNN Small
   model developed by Arm, with training checkpoints, from the Hello Edge paper. Code
   to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/ds_cnn_large/tflite_clustered_fp32/README.md b/models/keyword_spotting/ds_cnn_large/tflite_clustered_fp32/README.md
@@ -39,7 +39,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Top 1 Accuracy | 0.9495 |
+| Top 1 Accuracy | 0.950 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/keyword_spotting/ds_cnn_large/tflite_clustered_fp32/definition.yaml b/models/keyword_spotting/ds_cnn_large/tflite_clustered_fp32/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   SpeechCommands:
-    top_1_accuracy: 0.9495
+    top_1_accuracy: 0.950
 description: 'This is a clustered (32 clusters, kmeans++ centroid initialization)
   and retrained (fine-tuned) FP32 version of the DS-CNN Large model developed by Arm
   from the Hello Edge paper. Code for the original DS-CNN implementation can be found
diff --git a/models/keyword_spotting/ds_cnn_large/tflite_clustered_int8/README.md b/models/keyword_spotting/ds_cnn_large/tflite_clustered_int8/README.md
@@ -25,7 +25,7 @@ Code to recreate this model can be found here: https://github.com/ARM-software/M
 ## Performance
 | Platform | Optimized |
 |----------|:---------:|
-| Cortex-A |:heavy_check_mark:         |
+| Cortex-A |:heavy_multiplication_x:         |
 | Cortex-M |:heavy_check_mark:         |
 | Mali GPU |:heavy_check_mark:         |
 | Ethos U  |:heavy_check_mark:         |
@@ -39,7 +39,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Top 1 Accuracy | 0.9401 |
+| Top 1 Accuracy | 0.940 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/keyword_spotting/ds_cnn_large/tflite_clustered_int8/definition.yaml b/models/keyword_spotting/ds_cnn_large/tflite_clustered_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   SpeechCommands:
-    top_1_accuracy: 0.9401
+    top_1_accuracy: 0.940
 description: 'This is a clustered (32 clusters, kmeans++ centroid initialization),
   retrained (fine-tuned) and fully quantized version (INT8) of the DS-CNN Large model
   developed by Arm from the Hello Edge paper. Code for the original DS-CNN implementation
diff --git a/models/keyword_spotting/ds_cnn_large/tflite_int8/README.md b/models/keyword_spotting/ds_cnn_large/tflite_int8/README.md
@@ -27,12 +27,12 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 94.58% |
+| Accuracy | 0.946 |
 
 ## Performance
 | Platform | Optimized |
 |----------|:---------:|
-| Cortex-A |:heavy_check_mark:         |
+| Cortex-A |:heavy_multiplication_x:         |
 | Cortex-M |:heavy_check_mark:         |
 | Mali GPU |:heavy_check_mark:         |
 | Ethos U  |:heavy_check_mark:         |
diff --git a/models/keyword_spotting/ds_cnn_large/tflite_int8/definition.yaml b/models/keyword_spotting/ds_cnn_large/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 94.58%
+    Accuracy: 0.946
 description: 'This is a fully quantized version (asymmetrical int8) of the DS-CNN
   Large model developed by Arm, with training checkpoints, from the Hello Edge paper.
   Code to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/ds_cnn_medium/tflite_int8/README.md b/models/keyword_spotting/ds_cnn_medium/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 93.35% |
+| Accuracy | 0.934 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/keyword_spotting/ds_cnn_medium/tflite_int8/definition.yaml b/models/keyword_spotting/ds_cnn_medium/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 93.35%
+    Accuracy: 0.934
 description: 'This is a fully quantized version (asymmetrical int8) of the DS-CNN
   Medium model developed by Arm, with training checkpoints, from the Hello Edge paper.
   Code to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/ds_cnn_small/tflite_int8/README.md b/models/keyword_spotting/ds_cnn_small/tflite_int8/README.md
@@ -27,7 +27,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 93.35% |
+| Accuracy | 0.934 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/keyword_spotting/ds_cnn_small/tflite_int8/definition.yaml b/models/keyword_spotting/ds_cnn_small/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 93.35%
+    Accuracy: 0.934
 description: 'This is a fully quantized version (asymmetrical int8) of the DS-CNN
   Small model developed by Arm, with training checkpoints, from the Hello Edge paper.
   Code to recreate this model can be found here: https://github.com/ARM-software/ML-examples/tree/master/tflu-kws-cortex-m'
diff --git a/models/keyword_spotting/micronet_large/tflite_int8/README.md b/models/keyword_spotting/micronet_large/tflite_int8/README.md
@@ -36,7 +36,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 96.48% |
+| Accuracy | 0.965 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/keyword_spotting/micronet_large/tflite_int8/definition.yaml b/models/keyword_spotting/micronet_large/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 96.48%
+    Accuracy: 0.965
 description: This is a fully quantized version (asymmetrical int8) of the MicroNet
   Large model developed by Arm, from the MicroNets paper. This model is trained on
   the 'Google Speech Commands' dataset.
diff --git a/models/keyword_spotting/micronet_medium/tflite_int8/README.md b/models/keyword_spotting/micronet_medium/tflite_int8/README.md
@@ -36,7 +36,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 95.77% |
+| Accuracy | 0.958 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/keyword_spotting/micronet_medium/tflite_int8/definition.yaml b/models/keyword_spotting/micronet_medium/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 95.77%
+    Accuracy: 0.958
 description: This is a fully quantized version (asymmetrical int8) of the MicroNet
   Medium model developed by Arm, from the MicroNets paper. This model is trained on
   the 'Google Speech Commands' dataset.
diff --git a/models/keyword_spotting/micronet_small/tflite_int8/README.md b/models/keyword_spotting/micronet_small/tflite_int8/README.md
@@ -36,7 +36,7 @@ Dataset: Google Speech Commands Test Set
 
 | Metric | Value |
 |--------|-------|
-| Accuracy | 95.32% |
+| Accuracy | 0.953 |
 
 ## Optimizations
 | Optimization |  Value  |
diff --git a/models/keyword_spotting/micronet_small/tflite_int8/definition.yaml b/models/keyword_spotting/micronet_small/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   Google Speech Commands test set:
-    Accuracy: 95.32%
+    Accuracy: 0.953
 description: This is a fully quantized version (asymmetrical int8) of the MicroNet
   Small model developed by Arm, from the MicroNets paper. This model is trained on
   the 'Google Speech Commands' dataset.
diff --git a/models/object_detection/ssd_mobilenet_v1/tflite_fp32/README.md b/models/object_detection/ssd_mobilenet_v1/tflite_fp32/README.md
@@ -30,7 +30,7 @@ Dataset: Coco Validation 2017
 
 | Metric | Value |
 |--------|-------|
-| mAP | 0.21 |
+| mAP | 0.210 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/object_detection/ssd_mobilenet_v1/tflite_fp32/definition.yaml b/models/object_detection/ssd_mobilenet_v1/tflite_fp32/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   coco_validation_2017:
-    mAP: 0.21
+    mAP: 0.210
 description: SSD MobileNet v1 is a object detection network, that localizes and identifies
   objects in an input image. This is a TF Lite floating point version that takes a
   300x300 input image and outputs detections for this image. This model is trained
diff --git a/models/object_detection/ssd_mobilenet_v1/tflite_int8/definition.yaml b/models/object_detection/ssd_mobilenet_v1/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   COCO 2017 Validation:
-    mAP: '0.234'
+    mAP: 0.234
 description: SSD MobileNet v1 is a object detection network, that localizes and identifies
   objects in an input image. This is a TF Lite quantized version that takes a 300x300
   input image and outputs detections for this image. This model is converted from
diff --git a/models/object_detection/ssd_mobilenet_v1/tflite_uint8/README.md b/models/object_detection/ssd_mobilenet_v1/tflite_uint8/README.md
@@ -27,12 +27,12 @@ Dataset: Coco Validation 2017
 
 | Metric | Value |
 |--------|-------|
-| mAP | 0.18 |
+| mAP | 0.180 |
 
 ## Performance
 | Platform | Optimized |
 |----------|:---------:|
-| Cortex-A |:heavy_check_mark:         |
+| Cortex-A |:heavy_multiplication_x:         |
 | Cortex-M |:heavy_multiplication_x:         |
 | Mali GPU |:heavy_check_mark:         |
 | Ethos U  |:heavy_multiplication_x:         |
diff --git a/models/object_detection/ssd_mobilenet_v1/tflite_uint8/definition.yaml b/models/object_detection/ssd_mobilenet_v1/tflite_uint8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   coco_validation_2017:
-    mAP: 0.18
+    mAP: 0.180
 description: SSD MobileNet v1 is a object detection network, that localizes and identifies
   objects in an input image. This is a TF Lite quantized version that takes a 300x300
   input image and outputs detections for this image. This model is trained and quantized
diff --git a/models/speech_recognition/wav2letter/tflite_int8/README.md b/models/speech_recognition/wav2letter/tflite_int8/README.md
@@ -24,7 +24,7 @@ Dataset: Librispeech
 
 | Metric | Value |
 |--------|-------|
-| Ler | 0.08771 |
+| Ler | 0.0877 |
 
 ## Performance
 | Platform | Optimized |
diff --git a/models/speech_recognition/wav2letter/tflite_int8/definition.yaml b/models/speech_recognition/wav2letter/tflite_int8/definition.yaml
@@ -1,6 +1,6 @@
 benchmark:
   LibriSpeech:
-    LER: 0.08771
+    LER: 0.0877
 description: Wav2letter is a convolutional speech recognition neural network. This
   implementation was created by Arm and quantized to the INT8 datatype.
 license:
diff --git a/models/speech_recognition/wav2letter/tflite_pruned_int8/README.md b/models/speech_recognition/wav2letter/tflite_pruned_int8/README.md
diff --git a/models/speech_recognition/wav2letter/tflite_pruned_int8/definition.yaml b/models/speech_recognition/wav2letter/tflite_pruned_int8/definition.yaml
diff --git a/models/visual_wake_words/micronet_vww2/tflite_int8/README.md b/models/visual_wake_words/micronet_vww2/tflite_int8/README.md
diff --git a/models/visual_wake_words/micronet_vww2/tflite_int8/definition.yaml b/models/visual_wake_words/micronet_vww2/tflite_int8/definition.yaml
diff --git a/models/visual_wake_words/micronet_vww3/tflite_int8/README.md b/models/visual_wake_words/micronet_vww3/tflite_int8/README.md
diff --git a/models/visual_wake_words/micronet_vww3/tflite_int8/definition.yaml b/models/visual_wake_words/micronet_vww3/tflite_int8/definition.yaml
diff --git a/models/visual_wake_words/micronet_vww4/tflite_int8/README.md b/models/visual_wake_words/micronet_vww4/tflite_int8/README.md
diff --git a/models/visual_wake_words/micronet_vww4/tflite_int8/definition.yaml b/models/visual_wake_words/micronet_vww4/tflite_int8/definition.yaml
