Add Sanity check to score calibration file

(1) Check if each row of the score calibration file has 3 or 4 elements
(2) Check if scale (the first element) is non-negative.

PiperOrigin-RevId: 399709641

Author: lu-wang-g

tensorflow_lite_support/metadata/metadata_schema.fbs

@@ -119,7 +119,8 @@ enum AssociatedFileType : byte {
   // Contains sigmoid-based score calibration parameters, formatted as CSV.
   // Lines contain for each index of an output tensor the scale, slope, offset
   // and (optional) min_score parameters to be used for sigmoid fitting (in this
-  // order and in `strtof`-compatible [1] format).
+  // order and in `strtof`-compatible [1] format). Scale should be a
+  // non-negative value.
   // A line may be left empty to default calibrated scores for this index to
   // default_score.
   // In summary, each line should thus contain 0, 3 or 4 comma-separated values.

tensorflow_lite_support/metadata/python/metadata_writers/metadata_info.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 """Helper classes for common model metadata information."""
 
+import csv
 import os
 from typing import Optional, List, Type
 
@@ -273,11 +274,28 @@ def __init__(self,
       file_path: file_path of the score calibration file [1].
       [1]:
         https://github.com/tensorflow/tflite-support/blob/5e0cdf5460788c481f5cd18aab8728ec36cf9733/tensorflow_lite_support/metadata/metadata_schema.fbs#L122
+
+    Raises:
+      ValueError: if the score_calibration file is malformed.
     """
     self._score_transformation_type = score_transformation_type
     self._default_score = default_score
     self._file_path = file_path
 
+    # Sanity check the score calibration file.
+    with open(self._file_path) as calibration_file:
+      csv_reader = csv.reader(calibration_file, delimiter=",")
+      for row in csv_reader:
+        if row and len(row) != 3 and len(row) != 4:
+          raise ValueError(
+              "Expected empty lines or 3 or 4 parameters per line in score" +
+              " calibration file, but got {}.".format(len(row)))
+
+        if row and float(row[0]) < 0:
+          raise ValueError(
+              "Expected scale to be a non-negative value, but got {}.".format(
+                  float(row[0])))
+
   def create_metadata(self) -> _metadata_fb.ProcessUnitT:
     """Creates the score calibration metadata based on the information.
 

tensorflow_lite_support/metadata/python/tests/metadata_writers/BUILD

@@ -41,7 +41,10 @@ py_test(
 py_test(
     name = "metadata_info_test",
     srcs = ["metadata_info_test.py"],
-    data = ["//tensorflow_lite_support/metadata/python/tests/testdata:test_files"],
+    data = [
+        "//tensorflow_lite_support/metadata/python/tests/testdata:test_files",
+        "//tensorflow_lite_support/metadata/python/tests/testdata/image_classifier:test_files",
+    ],
     python_version = "PY3",
     srcs_version = "PY3",
     deps = [

tensorflow_lite_support/metadata/python/tests/metadata_writers/audio_classifier_test.py

@@ -27,7 +27,6 @@
 _MULTIHEAD_MODEL = "../testdata/audio_classifier/two_heads.tflite"
 _YAMNET_LABEL_FILE = "../testdata/audio_classifier/yamnet_521_labels.txt"
 _LABEL_FILE = "../testdata/audio_classifier/labelmap.txt"
-_SCORE_CALIBRATION_FILE = "../testdata/audio_classifier/score_calibration.txt"
 _DEFAULT_SCORE_CALIBRATION_VALUE = 0.2
 _JSON_FOR_INFERENCE_DYNAMIC = "../testdata/audio_classifier/yamnet_tfhub.json"
 _JSON_FOR_INFERENCE_FIXED = "../testdata/audio_classifier/yamnet_wavin_quantized_mel_relu6.json"
@@ -46,7 +45,8 @@ def test_create_for_inference_should_succeed_dynamic_input_shape_model(self):
         _CHANNELS, [_LABEL_FILE],
         metadata_info.ScoreCalibrationMd(
             _metadata_fb.ScoreTransformationType.LOG,
-            _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
+            _DEFAULT_SCORE_CALIBRATION_VALUE,
+            test_utils.create_calibration_file(self.get_temp_dir())))
 
     metadata_json = writer.get_metadata_json()
     expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_DYNAMIC, "r")
@@ -59,7 +59,8 @@ def test_create_for_inference_should_succeed_with_fixed_input_shape_model(
         [_YAMNET_LABEL_FILE],
         metadata_info.ScoreCalibrationMd(
             _metadata_fb.ScoreTransformationType.LOG,
-            _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
+            _DEFAULT_SCORE_CALIBRATION_VALUE,
+            test_utils.create_calibration_file(self.get_temp_dir())))
 
     metadata_json = writer.get_metadata_json()
     expected_json = test_utils.load_file(_JSON_FOR_INFERENCE_FIXED, "r")
@@ -83,6 +84,11 @@ def test_create_from_metadata_info_by_default_succeeds_for_multihead(self):
     self.assertEqual(metadata_json, expected_json)
 
   def test_create_from_metadata_info_succeeds_for_multihead(self):
+    calibration_file1 = test_utils.create_calibration_file(
+        self.get_temp_dir(), "score_cali_1.txt")
+    calibration_file2 = test_utils.create_calibration_file(
+        self.get_temp_dir(), "score_cali_2.txt")
+
     general_md = metadata_info.GeneralMd(name="AudioClassifier")
     input_md = metadata_info.InputAudioTensorMd(
         name="audio_clip", sample_rate=_SAMPLE_RATE, channels=_CHANNELS)
@@ -97,7 +103,7 @@ def test_create_from_metadata_info_succeeds_for_multihead(self):
         ],
         score_calibration_md=metadata_info.ScoreCalibrationMd(
             _metadata_fb.ScoreTransformationType.LOG,
-            _DEFAULT_SCORE_CALIBRATION_VALUE, "score_cali_1.txt"),
+            _DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file1),
         tensor_name="Identity_1")
     output_head_md_2 = metadata_info.ClassificationTensorMd(
         name="head2",
@@ -107,7 +113,7 @@ def test_create_from_metadata_info_succeeds_for_multihead(self):
         ],
         score_calibration_md=metadata_info.ScoreCalibrationMd(
             _metadata_fb.ScoreTransformationType.LOG,
-            _DEFAULT_SCORE_CALIBRATION_VALUE, "score_cali_2.txt"),
+            _DEFAULT_SCORE_CALIBRATION_VALUE, calibration_file2),
         tensor_name="Identity")
 
     writer = (
@@ -139,7 +145,8 @@ def test_create_for_inference_fails_with_wrong_sample_rate(
           _CHANNELS, [_LABEL_FILE],
           metadata_info.ScoreCalibrationMd(
               _metadata_fb.ScoreTransformationType.LOG,
-              _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
+              _DEFAULT_SCORE_CALIBRATION_VALUE,
+              test_utils.create_calibration_file(self.get_temp_dir())))
 
     self.assertEqual(
         "sample_rate should be positive, but got {}.".format(wrong_sample_rate),
@@ -164,7 +171,8 @@ def test_create_for_inference_fails_with_wrong_channels(self, wrong_channels):
           wrong_channels, [_LABEL_FILE],
           metadata_info.ScoreCalibrationMd(
               _metadata_fb.ScoreTransformationType.LOG,
-              _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
+              _DEFAULT_SCORE_CALIBRATION_VALUE,
+              test_utils.create_calibration_file(self.get_temp_dir())))
 
     self.assertEqual(
         "channels should be positive, but got {}.".format(wrong_channels),

tensorflow_lite_support/metadata/python/tests/metadata_writers/image_classifier_test.py

@@ -54,7 +54,8 @@ def test_create_for_inference_should_succeed(self, model_file, golden_json):
         [_LABEL_FILE],
         metadata_info.ScoreCalibrationMd(
             _metadata_fb.ScoreTransformationType.LOG,
-            _DEFAULT_SCORE_CALIBRATION_VALUE, _SCORE_CALIBRATION_FILE))
+            _DEFAULT_SCORE_CALIBRATION_VALUE,
+            test_utils.get_resource_path(_SCORE_CALIBRATION_FILE)))
 
     metadata_json = writer.get_metadata_json()
     expected_json = test_utils.load_file(golden_json, "r")

tensorflow_lite_support/metadata/python/tests/metadata_writers/metadata_info_test.py

@@ -15,7 +15,6 @@
 """Tests for metadata info classes."""
 
 from absl.testing import parameterized
-
 import tensorflow as tf
 
 import flatbuffers
@@ -25,6 +24,9 @@
 from tensorflow_lite_support.metadata.python.metadata_writers import metadata_info
 from tensorflow_lite_support.metadata.python.tests.metadata_writers import test_utils
 
+_SCORE_CALIBRATION_FILE = test_utils.get_resource_path(
+    "../testdata/image_classifier/score_calibration.txt")
+
 
 class GeneralMdTest(tf.test.TestCase):
 
@@ -280,7 +282,6 @@ class ClassificationTensorMdTest(tf.test.TestCase, parameterized.TestCase):
   _DESCRIPTION = "The classification result tensor."
   _LABEL_FILE_EN = "labels.txt"
   _LABEL_FILE_CN = "labels_cn.txt"  # Locale label file in Chinese.
-  _SCORE_CALIBRATION_FILE = "score_calibration.txt"
   _CALIBRATION_DEFAULT_SCORE = 0.2
   _EXPECTED_FLOAT_TENSOR_JSON = "../testdata/classification_tensor_float_meta.json"
   _EXPECTED_UINT8_TENSOR_JSON = "../testdata/classification_tensor_uint8_meta.json"
@@ -307,7 +308,7 @@ def test_create_metadata_should_succeed(self, tensor_type, golden_json):
         file_path=self._LABEL_FILE_CN, locale="cn")
     score_calibration_md = metadata_info.ScoreCalibrationMd(
         _metadata_fb.ScoreTransformationType.IDENTITY,
-        self._CALIBRATION_DEFAULT_SCORE, self._SCORE_CALIBRATION_FILE)
+        self._CALIBRATION_DEFAULT_SCORE, _SCORE_CALIBRATION_FILE)
 
     tesnor_md = metadata_info.ClassificationTensorMd(
         name=self._NAME,
@@ -397,16 +398,15 @@ def test_create_metadata_should_succeed(self):
     self.assertEqual(metadata_json, expected_json)
 
 
-class ScoreClalibrationMdTest(tf.test.TestCase):
+class ScoreCalibrationMdTest(tf.test.TestCase):
   _DEFAULT_VALUE = 0.2
-  _SCORE_CALIBRATION_FILE = "score_calibration.txt"
   _EXPECTED_TENSOR_JSON = "../testdata/score_calibration_tensor_meta.json"
   _EXPECTED_MODEL_META_JSON = "../testdata/score_calibration_file_meta.json"
 
   def test_create_metadata_should_succeed(self):
     score_calibration_md = metadata_info.ScoreCalibrationMd(
         _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE,
-        self._SCORE_CALIBRATION_FILE)
+        _SCORE_CALIBRATION_FILE)
     score_calibration_metadata = score_calibration_md.create_metadata()
 
     metadata_json = _metadata.convert_to_json(
@@ -418,7 +418,7 @@ def test_create_metadata_should_succeed(self):
   def test_create_score_calibration_file_md_should_succeed(self):
     score_calibration_md = metadata_info.ScoreCalibrationMd(
         _metadata_fb.ScoreTransformationType.LOG, self._DEFAULT_VALUE,
-        self._SCORE_CALIBRATION_FILE)
+        _SCORE_CALIBRATION_FILE)
     score_calibration_file_md = (
         score_calibration_md.create_score_calibration_file_md())
     file_metadata = score_calibration_file_md.create_metadata()
@@ -435,6 +435,28 @@ def test_create_score_calibration_file_md_should_succeed(self):
     expected_json = test_utils.load_file(self._EXPECTED_MODEL_META_JSON, "r")
     self.assertEqual(metadata_json, expected_json)
 
+  def test_create_score_calibration_file_fails_with_less_colunms(self):
+    malformed_calibration_file = test_utils.create_calibration_file(
+        self.get_temp_dir(), content="1.0,0.2")
+
+    with self.assertRaisesRegex(
+        ValueError,
+        "Expected empty lines or 3 or 4 parameters per line in score" +
+        " calibration file, but got 2."):
+      metadata_info.ScoreCalibrationMd(_metadata_fb.ScoreTransformationType.LOG,
+                                       self._DEFAULT_VALUE,
+                                       malformed_calibration_file)
+
+  def test_create_score_calibration_file_fails_with_negative_scale(self):
+    malformed_calibration_file = test_utils.create_calibration_file(
+        self.get_temp_dir(), content="-1.0,0.2,0.1")
+
+    with self.assertRaisesRegex(
+        ValueError, "Expected scale to be a non-negative value, but got -1.0."):
+      metadata_info.ScoreCalibrationMd(_metadata_fb.ScoreTransformationType.LOG,
+                                       self._DEFAULT_VALUE,
+                                       malformed_calibration_file)
+
 
 def _create_dummy_model_metadata_with_tensor(
     tensor_metadata: _metadata_fb.TensorMetadataT) -> bytes:

tensorflow_lite_support/metadata/python/tests/metadata_writers/test_utils.py

@@ -14,10 +14,21 @@
 # ==============================================================================
 """Test utils for MetadataWriter."""
 
+import os
 from typing import Union
 from tensorflow.python.platform import resource_loader
 
 
+def create_calibration_file(file_dir: str,
+                            file_name: str = "score_calibration.txt",
+                            content: str = "1.0,2.0,3.0,4.0") -> str:
+  """Creates the calibration file."""
+  calibration_file = os.path.join(file_dir, file_name)
+  with open(calibration_file, mode="w") as file:
+    file.write(content)
+  return calibration_file
+
+
 def load_file(file_name: str, mode: str = "rb") -> Union[str, bytes]:
   """Loads files from resources."""
   file_path = get_resource_path(file_name)