ConvertToTarredAudioDataset processor implemetation

.github/workflows/tests.yml

@@ -72,7 +72,7 @@ jobs:
         sudo apt-get install -y libsndfile1 ffmpeg sox libsox-fmt-mp3
         pip install pytorch_lightning
         pip install Cython wheel  # need to pre-install to avoid error in nemo installation
-        pip install nemo-toolkit[asr,nlp]==1.23.0
+        pip install nemo-toolkit[asr,nlp]==2.2.1
         pip install nemo_text_processing
         pip install -r requirements/huggingface.txt
         pip install certifi #this needed to avoid problems with certificates [COORAL]
@@ -85,6 +85,7 @@ jobs:
         AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
         AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
         CLEAN_UP_TMP_PATH: 1
+        USE_THREADING_BACKEND: 1
       run: |
 
         wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem #downloading cert manually [for CORAL]

docs/src/sdp/api.rst

@@ -311,6 +311,9 @@ Files management
 .. autodata:: sdp.processors.RemoveFiles
    :annotation:
 
+.. autodata:: sdp.processors.ConvertToTarredAudioDataset
+   :annotation:
+
 Data filtering
 ''''''''''''''
 

requirements/docs.txt

@@ -4,3 +4,4 @@ Sphinx
 sphinx-book-theme
 sphinx-copybutton
 sphinxext-opengraph
+tabulate

requirements/main.txt

@@ -22,10 +22,12 @@ jiwer>=3.1.0,<4.0.0
 pyarrow>=8.0.0,<14.0.0
 datasets>=2.14.0,<3.0.0
 # toloka-kit  # Temporarily disabled due to Toloka's technical pause; keep as reference for past and future API support
-# for some processers, additionally https://github.com/NVIDIA/NeMo is required
+# for some processers, additionally https://github.com/NVIDIA/NeMo 2.2.1 is required
 # for some processers, additionally nemo_text_processing is required
 # for mcv: apt-get update && apt-get upgrade -y && apt-get install -y sox libsox-fmt-all
+
 # for FasterWhisperInference processor is required: 
     # pip install pytorch-lightning nvidia-cublas-cu12 nvidia-cudnn-cu12==9.* faster_whisper
     # export LD_LIBRARY_PATH=`python3 -c 'import os; import nvidia.cublas.lib; import nvidia.cudnn.lib; print(os.path.dirname(nvidia.cublas.lib.__file__) + ":" + os.path.dirname(nvidia.cudnn.lib.__file__))'`
-# for vLLMInference processor is required: pip install "optree>=0.13.0" vllm
+# for vLLMInference processor is required: pip install "optree>=0.13.0" vllm
+# for ConvertToTarredAudioDatasetConfig processor can be additionally required: pip install lhotse "nemo-toolkit[common]==2.2.1"

sdp/processors/__init__.py

@@ -161,7 +161,9 @@
 from sdp.processors.manage_files.remove import (
     RemoveFiles,
 )
-
+from sdp.processors.manage_files.convert_to_tarred_audio_dataset import (
+    ConvertToTarredAudioDataset,
+)
 from sdp.processors.toloka.accept_if import AcceptIfWERLess
 from sdp.processors.toloka.create_pool import CreateTolokaPool
 from sdp.processors.toloka.create_project import CreateTolokaProject

sdp/processors/inference/asr/nemo/utils/frame_vad_infer_postprocess.yaml

@@ -36,4 +36,4 @@ out_manifest_filepath: null # if not specify it will automatically be "manifest_
 
 
 # json manifest line example
-# {"audio_filepath": "/path/to/audio_file.wav", "offset": 0, "duration": 1.23, "label": "infer", "text": "-"}
+# {"audio_filepath": "/path/to/audio_file.wav", "offset": 0, "duration": 1.23, "label": "infer", "text": "-"}

sdp/processors/inference/asr/nemo/utils/speech_to_text_with_vad.py

@@ -57,9 +57,8 @@
 import contextlib
 import json
 import os
-
 import time
-from dataclasses import dataclass, is_dataclass, field
+from dataclasses import dataclass, field, is_dataclass
 from pathlib import Path
 from typing import Callable, Optional
 
@@ -646,4 +645,4 @@ def run_asr_inference(manifest_filepath, cfg, record_fn) -> str:
 
 
 if __name__ == "__main__":
-    main()
+    main()