merge

Signed-off-by: Nikolay Karpov <nkarpov@nvidia.com>

Author: karpnv

.github/workflows/tests.yml

@@ -75,14 +75,20 @@ jobs:
         pip install nemo-toolkit[asr,nlp]==1.23.0
         pip install nemo_text_processing
         pip install -r requirements/huggingface.txt
+        pip install certifi #this needed to avoid problems with certificates [COORAL]
+        export SSL_CERT_FILE=$(python -m certifi)
         python -m pip cache purge
+        
 
     - name: Run all tests
       env:
         AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
         AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
         CLEAN_UP_TMP_PATH: 1
       run: |
+        wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem #downloading cert manually [for CORAL]
+        sudo cp incommon-rsa-ca2.pem     /usr/local/share/ca-certificates/incommon-rsa-server-ca-2.crt # [cert for CORAL]
+        sudo update-ca-certificates # [cert for CORAL]
         set -o pipefail # this will make sure next line returns non-0 exit code if tests fail
         python -m pytest tests/ --junitxml=pytest.xml --ignore=tests/test_tts_sdp_end_to_end.py --cov-report=term-missing:skip-covered --cov=sdp --durations=30 -rs | tee pytest-coverage.txt
 

dataset_configs/english/coraal/config.yaml

@@ -18,7 +18,7 @@ documentation: |
   This config performs the following data processing.
 
   1. Downloads CORAAL data based on the
-     `official file list <http://lingtools.uoregon.edu/coraal/coraal_download_list.txt>`_.
+     `official file list <https://lingtools.uoregon.edu/coraal/coraal_download_list.txt>`_. #Official mirror link
      There are a couple of errors in the links there, which are fixed in our code.
   2. Drops all utterances which contain only pauses. Set ``drop_pauses=False`` to undo.
   3. Groups all consecutive segments from the same speaker until 20 seconds duration

docs/src/conf.py

@@ -45,7 +45,6 @@
     "numpy",
     "tqdm",
     "soundfile",
-    "ndjson",
     "boto3",
     "webvtt_py",
     "python_docx",
@@ -189,3 +188,8 @@ def setup(app):
 ]
 # nitpick_ignore_regex = [('py:class', '*')]
 
+#adding this especially for coraal, temporary
+linkcheck_ignore = [
+    r'https://lingtools\.uoregon\.edu/coraal/coraal_download_list\.txt',
+]
+# https://lingtools.uoregon.edu/coraal/coraal_download_list.txt

requirements/main.txt

@@ -4,7 +4,7 @@ ffmpeg
 hydra-core
 joblib
 librosa>=0.10.0 # specify >=0.10.0 so that librosa.get_duration(path=...) will work
-numpy==1.26
+numpy>=1.26, <2.0 # module was used numpy 1.x and may crash in 2.x
 omegaconf
 pandas
 rarfile
@@ -18,7 +18,7 @@ python-docx
 pydub
 dask
 distributed
-
+jiwer>=3.1.0,<4.0.0
 # toloka-kit  # Temporarily disabled due to Toloka's technical pause; keep as reference for past and future API support
 # for some processers, additionally https://github.com/NVIDIA/NeMo is required
 # for some processers, additionally nemo_text_processing is required

requirements/tts.txt

@@ -1,4 +1,3 @@
-ndjson
 transformers
 accelerate
 torchaudio

sdp/processors/datasets/coraal/create_initial_manifest.py

@@ -31,15 +31,15 @@ def get_coraal_url_list():
     There are a few mistakes in the official url list that are fixed here.
     Can be overridden by tests to select a subset of urls.
     """
-    dataset_url = "http://lingtools.uoregon.edu/coraal/coraal_download_list.txt"
+    dataset_url = "https://lingtools.uoregon.edu/coraal/coraal_download_list.txt"
     urls = []
     for file_url in urllib.request.urlopen(dataset_url):
         file_url = file_url.decode('utf-8').strip()
         # fixing known errors in the urls
-        if file_url == 'http://lingtools.uoregon.edu/coraal/les/2021.07/LES_metadata_2018.10.06.txt':
-            file_url = 'http://lingtools.uoregon.edu/coraal/les/2021.07/LES_metadata_2021.07.txt'
-        if file_url == 'http://lingtools.uoregon.edu/coraal/vld/2021.07/VLD_metadata_2018.10.06.txt':
-            file_url = 'http://lingtools.uoregon.edu/coraal/vld/2021.07/VLD_metadata_2021.07.txt'
+        if file_url == 'https://lingtools.uoregon.edu/coraal/les/2021.07/LES_metadata_2018.10.06.txt':
+            file_url = 'https://lingtools.uoregon.edu/coraal/les/2021.07/LES_metadata_2021.07.txt'
+        if file_url == 'https://lingtools.uoregon.edu/coraal/vld/2021.07/VLD_metadata_2018.10.06.txt':
+            file_url = 'https://lingtools.uoregon.edu/coraal/vld/2021.07/VLD_metadata_2021.07.txt'
         urls.append(file_url)
     return urls
 

tests/test_tts_sdp_end_to_end.py

@@ -8,6 +8,7 @@
 from omegaconf import OmegaConf
 
 from sdp.run_processors import run_processors
+from sdp.utils.common import load_manifest
 
 DATASET_CONFIGS_ROOT = Path(__file__).parents[1] / "dataset_configs"
 
@@ -69,16 +70,14 @@ def test_tts_sdp_end_to_end(get_tts_ytc_data):
 
     assert os.path.exists(cfg.final_manifest)
     output_file_data = {}
-    with open(cfg.final_manifest, "r") as f:
-        output_data = ndjson.load(f)
-        for item in output_data:
-            output_file_data[item["audio_item_id"]] = item
+    output_data = load_manifest(cfg.final_manifest, encoding="utf8")
+    for item in output_data:
+        output_file_data[item["audio_item_id"]] = item
 
     reference_file_data = {}
-    with open(reference_manifest_file, "r") as f:
-        reference_data = ndjson.load(f)
-        for item in reference_data:
-            reference_file_data[item["audio_item_id"]] = item
+    reference_data = load_manifest(reference_manifest_file, encoding="utf8")
+    for item in reference_data:
+        reference_file_data[item["audio_item_id"]] = item
 
     assert len(output_file_data) == len(reference_file_data)
     assert len(output_file_data) == 2