feat: add conformer small no decay result

Author: nglehuy

examples/configs/librispeech/data.yml.j2

@@ -30,15 +30,28 @@ data_config:
     metadata: {{metadata}}
     indefinite: True
 
-  test_dataset_config:
-    enabled: True
-    sample_rate: 16000
-    data_paths:
-      - {{datadir}}/test-clean/transcripts.tsv
-    tfrecords_dir: {{datadir}}/tfrecords
-    shuffle: False
-    cache: False
-    buffer_size: null
-    drop_remainder: False
-    stage: test
-    indefinite: False
+  test_dataset_configs:
+    - name: test-clean
+      enabled: True
+      sample_rate: 16000
+      data_paths:
+        - {{datadir}}/test-clean/transcripts.tsv
+      tfrecords_dir: {{datadir}}/tfrecords
+      shuffle: False
+      cache: False
+      buffer_size: null
+      drop_remainder: False
+      stage: test
+      indefinite: False
+    - name: test-other
+      enabled: True
+      sample_rate: 16000
+      data_paths:
+        - {{datadir}}/test-other/transcripts.tsv
+      tfrecords_dir: {{datadir}}/tfrecords
+      shuffle: False
+      cache: False
+      buffer_size: null
+      drop_remainder: False
+      stage: test
+      indefinite: False

examples/models/transducer/conformer/results/sentencepiece/README.md

@@ -5,6 +5,12 @@
       - [2. Batch Loss](#2-batch-loss)
     - [Training Learning Rate](#training-learning-rate)
     - [Results](#results)
+- [SentencePiece 1k + Small + LibriSpeech + Without Weight Decay](#sentencepiece-1k--small--librispeech--without-weight-decay)
+    - [Training Loss](#training-loss-1)
+      - [1. Epoch Loss](#1-epoch-loss-1)
+      - [2. Batch Loss](#2-batch-loss-1)
+    - [Training Learning Rate](#training-learning-rate-1)
+    - [Results](#results-1)
 
 
 # SentencePiece 1k + Small + LibriSpeech
@@ -17,7 +23,6 @@
 | Device            | Google Colab TPUs                  |
 | Global Batch Size | 2 * 16 * 8 = 256 (as 8 TPUs)       |
 | Max Epochs        | 300                                |
-| Training time     |                                    |
 
 
 ### Training Loss
@@ -64,3 +69,60 @@ Pretrain Model here: [link]()
   },
 ]
 ```
+
+# SentencePiece 1k + Small + LibriSpeech + Without Weight Decay
+
+
+| Category          | Description                                          |
+| :---------------- | :--------------------------------------------------- |
+| Config            | [small-no-decay.yml.j2](../../small-no-decay.yml.j2) |
+| Tensorflow        | **2.13.x**                                           |
+| Device            | Google Colab TPUs                                    |
+| Global Batch Size | 2 * 16 * 8 = 256 (as 8 TPUs)                         |
+| Max Epochs        | 300                                                  |
+
+
+### Training Loss
+
+#### 1. Epoch Loss
+
+![Epoch Loss](./figs/conformer-small-no-decay-sp1k-epoch-loss.svg)
+
+#### 2. Batch Loss
+
+![Batch Loss](./figs/conformer-small-no-decay-sp1k-batch-loss.svg)
+
+### Training Learning Rate
+
+![Learning Rate](./figs/conformer-small-no-decay-sp1k-lr.svg)
+
+
+### Results
+
+Pretrain Model here: [link]()
+
+```json
+[
+  {
+    "epoch": 115,
+    "test-clean": {
+      "greedy": {
+        "wer": 0.06327982349360925,
+        "cer": 0.02412176322239193,
+        "mer": 0.06283642132698737,
+        "wil": 0.110402410864341,
+        "wip": 0.889597589135659
+      }
+    },
+    "test-other": {
+      "greedy": {
+        "wer": 0.15083201192136483,
+        "cer": 0.07265414763270005,
+        "mer": 0.14853347882527798,
+        "wil": 0.25123406103539114,
+        "wip": 0.7487659389646089
+      }
+    }
+  },
+]
+```

examples/models/transducer/conformer/results/sentencepiece/figs/conformer-small-no-decay-sp1k-batch-loss.svg

@@ -29,17 +29,17 @@ def main(
     config_path: str,
     dataset_type: str,
     datadir: str,
+    outputdir: str,
     h5: str = None,
     mxp: str = "none",
     bs: int = 1,
     device: int = 0,
     cpu: bool = False,
     jit_compile: bool = False,
-    output: str = "test.tsv",
     repodir: str = os.path.realpath(os.path.join(os.path.dirname(__file__), "..")),
 ):
-    assert h5 and output
-    output = file_util.preprocess_paths(output)
+    outputdir = file_util.preprocess_paths(outputdir, isdir=True)
+    checkpoint_name = os.path.splitext(os.path.basename(h5))[0]
 
     env_util.setup_seed()
     env_util.setup_devices([device], cpu=cpu)
@@ -50,34 +50,41 @@ def main(
 
     tokenizer = tokenizers.get(config)
 
-    test_dataset = datasets.get(tokenizer=tokenizer, dataset_config=config.data_config.test_dataset_config, dataset_type=dataset_type)
-    test_data_loader = test_dataset.create(batch_size)
-
     model: BaseModel = tf.keras.models.model_from_config(config.model_config)
     model.tokenizer = tokenizer
     model.make(batch_size=batch_size)
     model.load_weights(h5, by_name=file_util.is_hdf5_filepath(h5), skip_mismatch=False)
     model.jit_compile = jit_compile
     model.summary()
 
-    overwrite = True
-    if tf.io.gfile.exists(output):
-        while overwrite not in ["yes", "no"]:
-            overwrite = input(f"File {output} exists, overwrite? (yes/no): ").lower()
-        overwrite = overwrite == "yes"
-
-    if overwrite:
-        with file_util.save_file(output) as output_file_path:
-            model.predict(
-                test_data_loader,
-                verbose=1,
-                callbacks=[
-                    PredictLogger(test_dataset=test_dataset, output_file_path=output_file_path),
-                ],
-            )
-
-    evaluation_outputs = app_util.evaluate_hypotheses(output)
-    logger.info(json.dumps(evaluation_outputs, indent=2))
+    for test_data_config in config.data_config.test_dataset_configs:
+        if not test_data_config.name:
+            raise ValueError("Test dataset name must be provided")
+        logger.info(f"Testing dataset: {test_data_config.name}")
+
+        output = os.path.join(outputdir, f"{test_data_config.name}-{checkpoint_name}.tsv")
+
+        test_dataset = datasets.get(tokenizer=tokenizer, dataset_config=test_data_config, dataset_type=dataset_type)
+        test_data_loader = test_dataset.create(batch_size)
+
+        overwrite = True
+        if tf.io.gfile.exists(output):
+            while overwrite not in ["yes", "no"]:
+                overwrite = input(f"File {output} exists, overwrite? (yes/no): ").lower()
+            overwrite = overwrite == "yes"
+
+        if overwrite:
+            with file_util.save_file(output) as output_file_path:
+                model.predict(
+                    test_data_loader,
+                    verbose=1,
+                    callbacks=[
+                        PredictLogger(test_dataset=test_dataset, output_file_path=output_file_path),
+                    ],
+                )
+
+        evaluation_outputs = app_util.evaluate_hypotheses(output)
+        logger.info(json.dumps(evaluation_outputs, indent=2))
 
 
 if __name__ == "__main__":

examples/models/transducer/conformer/results/sentencepiece/figs/conformer-small-no-decay-sp1k-epoch-loss.svg

@@ -66,6 +66,7 @@ class DatasetConfig:
     def __init__(self, config: dict = None):
         if not config:
             config = {}
+        self.name: str = config.pop("name", "")
         self.enabled: bool = config.pop("enabled", True)
         self.stage: str = config.pop("stage", None)
         self.data_paths = config.pop("data_paths", None)
@@ -87,7 +88,10 @@ def __init__(self, config: dict = None):
             config = {}
         self.train_dataset_config = DatasetConfig(config.pop("train_dataset_config", {}))
         self.eval_dataset_config = DatasetConfig(config.pop("eval_dataset_config", {}))
-        self.test_dataset_config = DatasetConfig(config.pop("test_dataset_config", {}))
+        self.test_dataset_configs = [DatasetConfig(conf) for conf in config.pop("test_dataset_configs", [])]
+        _test_dataset_config = config.pop("test_dataset_config", None)
+        if _test_dataset_config:
+            self.test_dataset_configs.append(_test_dataset_config)
 
 
 class LearningConfig:
@@ -114,8 +118,7 @@ def __init__(self, data: Union[str, dict], training=True, **kwargs):
         self.decoder_config = DecoderConfig(config.pop("decoder_config", {}))
         self.model_config: dict = config.pop("model_config", {})
         self.data_config = DataConfig(config.pop("data_config", {}))
-        _learning_config_dict = config.pop("learning_config", {})
-        self.learning_config = LearningConfig(_learning_config_dict) if training else None
+        self.learning_config = LearningConfig(config.pop("learning_config", {})) if training else None
         for k, v in config.items():
             setattr(self, k, v)
         logger.info(str(self))

examples/models/transducer/conformer/results/sentencepiece/figs/conformer-small-no-decay-sp1k-lr.svg

@@ -160,6 +160,7 @@ def __init__(
         metadata: str = None,
         sample_rate: int = 16000,
         stage: str = "train",
+        name: str = "",
         **kwargs,
     ):
         self.data_paths = data_paths or []
@@ -175,6 +176,7 @@ def __init__(
         self.total_steps = None  # for better training visualization
         self.metadata = metadata
         self.sample_rate = sample_rate
+        self.name = name
 
     def parse(self, *args, **kwargs):
         raise NotImplementedError()
@@ -199,6 +201,7 @@ def __init__(
         metadata: str = None,
         buffer_size: int = BUFFER_SIZE,
         sample_rate: int = 16000,
+        name: str = "",
         **kwargs,
     ):
         super().__init__(
@@ -212,6 +215,8 @@ def __init__(
             metadata=metadata,
             indefinite=indefinite,
             sample_rate=sample_rate,
+            name=name,
+            **kwargs,
         )
         self.entries = []
         self.tokenizer = tokenizer
@@ -413,6 +418,7 @@ def __init__(
         buffer_size: int = BUFFER_SIZE,
         compression_type: str = "GZIP",
         sample_rate: int = 16000,
+        name: str = "",
         **kwargs,
     ):
         super().__init__(
@@ -427,6 +433,7 @@ def __init__(
             metadata=metadata,
             indefinite=indefinite,
             sample_rate=sample_rate,
+            name=name,
             **kwargs,
         )
         if not self.stage: