add docs

Signed-off-by: Nikolay Karpov <nkarpov@nvidia.com>

Author: karpnv

dataset_configs/portuguese/unlabeled/config.yaml

@@ -1,22 +1,52 @@
+documentation: |
+  Unlabeled data processing pipeline
+  ############################
+
+  This pipeline processes unlabeled data for iterative-pseudo labelling training.
+  
+  The pipeline performs the following steps:
+  1. Creates initial manifest by searching all WAV files in raw_data_dir folder
+  2. Counts duration of each WAV file
+  3. Identify language using langid_ambernet NeMo model
+  4. Filers out audios that are marked by different language tag
+  5. Filers out audios that are longer than it can be processed
+  6. Applies VAD algorithm from NeMo repository 
+  7. Forms segments by joining adjacent segments up to duration threshold
+  8. Splits long audios into short segments 
+  9. Remove empty files and extra fields from manifest
+
+  Required inputs:
+  - workspace_dir: Directory for intermediate files that contanes subfolders:
+    ${workspace_dir}/wavs/ - folder with sourse long files 
+    ${workspace_dir}/sdp/ - folder to keep manifests
+    ${workspace_dir}/sdp/vad/ - folder to keep temprorary files from VAD algorithm
+    ${workspace_dir}/splited_wavs/ - folder to keep splited short files
+
+  - language_short: 2-letter language code
+  - nemo_path: Path to NeMo installation
+  - final_manifest: Path for final output manifest
+
 processors_to_run: "0:"
-workspace_dir: /mnt/ssd8/multilang/portuguese/yt/sdp
-final_manifest: ${workspace_dir}/final_manifest.json
-nemo_path: /home/nkarpov/workspace/NeMo_old
+workspace_dir: ??? # /mnt/ssd8/multilang/portuguese/yt
+manifest_dir: ${workspace_dir}/sdp
+language_short: pt
+nemo_path: ??? # /home/nkarpov/workspace/NeMo_old
+final_manifest: ${manifest_dir}/final_manifest.json
 
 processors:
   - _target_: sdp.processors.CreateInitialManifestByExt
-    raw_data_dir: /mnt/ssd8/multilang/portuguese/yt/wavs
+    raw_data_dir: ${workspace_dir}/wavs
     extension: wav
     output_file_key: audio_filepath
-    output_manifest_file: ${workspace_dir}/manifest0.json
+    output_manifest_file: ${manifest_dir}/manifest0.json
 
   - _target_: sdp.processors.GetAudioDuration
     audio_filepath_key: audio_filepath
     duration_key: duration
-    output_manifest_file: ${workspace_dir}/manifest1.json
+    output_manifest_file: ${manifest_dir}/manifest1.json
 
   - _target_: sdp.processors.AudioLid
-    output_manifest_file: ${workspace_dir}/manifest2.json
+    output_manifest_file: ${manifest_dir}/manifest2.json
     input_audio_key: audio_filepath
     output_lang_key: audio_lang
     device: cuda
@@ -25,53 +55,52 @@ processors:
     num_segments: 3
 
   - _target_: sdp.processors.PreserveByValue
-    output_manifest_file: ${workspace_dir}/manifest3.json
+    output_manifest_file: ${manifest_dir}/manifest3.json
     input_value_key: audio_lang
-    target_value: pt
+    target_value: ${language_short}
 
   - _target_: sdp.processors.PreserveByValue
-    output_manifest_file: ${workspace_dir}/manifest4.json
+    output_manifest_file: ${manifest_dir}/manifest4.json
     input_value_key: duration
     operator: le
     target_value: 20000.0
 
   - _target_: sdp.processors.Subprocess
-    cmd: "rm -rf ${workspace_dir}/vad/*"
+    cmd: "rm -rf ${manifest_dir}/vad/*"
 
   - _target_: sdp.processors.Subprocess
-    input_manifest_file: ${workspace_dir}/manifest4.json
-    output_manifest_file: ${workspace_dir}/vad
+    input_manifest_file: ${manifest_dir}/manifest4.json
+    output_manifest_file: ${manifest_dir}/vad
     input_manifest_arg: "manifest_filepath"
     output_manifest_arg: "output_dir"
-    cmd: "python ${nemo_path}/examples/asr/asr_vad/speech_to_text_with_vad.py audio_type=wav \
-    vad_model=vad_multilingual_frame_marblenet  vad_config=${nemo_path}/examples/asr/conf/vad/frame_vad_infer_postprocess.yaml"
+    cmd: "python ${nemo_path}/examples/asr/asr_vad/speech_to_text_with_vad.py audio_type=wav vad_model=vad_multilingual_frame_marblenet  vad_config=${nemo_path}/examples/asr/conf/vad/frame_vad_infer_postprocess.yaml"
 
   - _target_: sdp.processors.RenameFields
-    input_manifest_file: ${workspace_dir}/vad/temp_manifest_vad_rttm-onset0.3-offset0.3-pad_onset0.2-pad_offset0.2-min_duration_on0.2-min_duration_off0.2-filter_speech_firstTrue.json
-    output_manifest_file: ${workspace_dir}/manifest7.json
+    input_manifest_file: ${manifest_dir}/vad/temp_manifest_vad_rttm-onset0.3-offset0.3-pad_onset0.2-pad_offset0.2-min_duration_on0.2-min_duration_off0.2-filter_speech_firstTrue.json
+    output_manifest_file: ${manifest_dir}/manifest7.json
     rename_fields: {"audio_filepath":"source_filepath"}
 
   - _target_: sdp.processors.nemo.rttm.GetRttmSegments
-    output_manifest_file: ${workspace_dir}/manifest8.json
+    output_manifest_file: ${manifest_dir}/manifest8.json
     rttm_key: rttm_file
     output_file_key: audio_segments
     duration_key: duration
     duration_threshold: 20.0
 
   - _target_: sdp.processors.nemo.rttm.SplitAudioFile
-    output_manifest_file: ${workspace_dir}/manifest9.json
-    splited_audio_dir: /mnt/ssd8/multilang/portuguese/yt/splited_wavs/
+    output_manifest_file: ${manifest_dir}/manifest9.json
+    splited_audio_dir: ${workspace_dir}/splited_wavs/
     segments_key: audio_segments
     duration_key: duration
     input_file_key: source_filepath
     output_file_key: audio_filepath
 
   - _target_: sdp.processors.PreserveByValue
-    output_manifest_file: ${workspace_dir}/manifest10.json
+    output_manifest_file: ${manifest_dir}/manifest10.json
     input_value_key: duration
     operator: gt
     target_value: 0.0
 
   - _target_: sdp.processors.KeepOnlySpecifiedFields
-    output_manifest_file: ${workspace_dir}/manifest11.json
+    output_manifest_file: ${final_manifest}
     fields_to_keep: ["audio_filepath", "duration"]