Highlight sentence during voiceover playback (oppia#22689)

* Creats helper modules.

* Adds platform file for Azure TTS.

* Removes unwanted files

* Adds backend tests

* Add functionality to regenerate voiceovers from exploration editor

* Updates the branch with latest changes

* Fixes backend tests

* Updates docstring

* Fixes backend tests

* Fixes mypy checks and lint checks

* Fixes lint issues

* Adds backend test files in a shard.

* Updates dummy voiceovers!

* Adds backend coverage tests

* updates typo + shards

* temp

* remove comma from backend shard

* Adds saving functionality for generated voiceovers.

* Updates method names and removes consoles

* updates graph coloring after voiceover regeneration

* Adds backend unit tests

* Adds frontend tests

* Adds frontend test coverage

* Fixes mypy checks

* Fixes ts checks

* Hides feature behind feature flag.

* Fixes mypy checks

* Adds logic to highlight text during voiceover play

* Adds backend coverage tests

* Highlights in preview and player page

* Fixes acceptance test

* Adds confirmation modal for updating autogeneration suuport for voiceovers.

* Hides autogenerate voiceover button for non curated explorations

* Addresses review comments

* Adds services to highlight sentences

* Adds frontend tests

* adds input variable

* Updates modal information.

* Fixes lint issues

* Removes constants

* Updates highlighting logic

* Fixes lint issues

* Adds voiceover accuracy rules

* Adds logic to block voiceover addition/regeneration when there is no content.

* Updates code to handle superscripts

* Adds frontend tests coverage

* Fixes frontend tests

* Removes azure call dependencies

* Fixes lint issues.

* updates variable names

* voiceover addition option for customization args

* Adds backend test coverage

* Fixes lint and TS issues

* Commits changes for highlighting

* restructures code

* temp

* Fixes acceptance tests

* Fixes sentence highlight during language change

* Removes redundant method

* Adds frontend test coverage

* Fixes lint checks

* Fixes TS checks

* Fixes backend test

* Updates copyright year & fileoverview.

* Fixes minute bugs

* Fixes voiceover card loading issue

* Blocks language accent support if language code is not present in constants.

* Rephrases the names of some frontend tests

* Fixes spelling

Author: Nik-09

.github/CODEOWNERS

@@ -98,6 +98,7 @@
 /core/templates/services/entity-translations.services.ts @oppia/lace-frontend-reviewers
 /core/templates/services/entity-voiceovers.services.ts @oppia/lace-frontend-reviewers
 /core/templates/services/voiceover-language-management-service.ts @oppia/lace-frontend-reviewers
+/core/templates/services/automatic-voiceover-highlight-service.ts @oppia/lace-frontend-reviewers
 
 
 # Audio Voiceover project

assets/constants.ts

@@ -7752,7 +7752,6 @@ export default {
   "CONTRIBUTOR_CERTIFICATE_HEIGHT": 1313,
   "BRANCH_NAME": "",
   "SHORT_COMMIT_HASH": "",
-
   // Please consult the translation team before adding any entries here.
   // These words improve the quality of automatic voiceovers.
   "LANGUAGE_CODE_TO_MATH_SYMBOL_PRONUNCIATIONS": {
@@ -7804,5 +7803,14 @@ export default {
       "^2": "تربيع",
       "^3": "تكعيب"
     }
+  },
+  // Please consult the translation team before adding any entries here.
+  // These punctuation marks are used to identify sentence boundaries during
+  // voiceover playback.
+  "LANGUAGE_CODE_TO_SENTENCE_ENDING_PUNCTUATION_MARKS": {
+    "ar": "؟!",
+    "en": ".!?",
+    "pt": ".!?",
+    "hi": "।!?"
   }
 } as const;

core/domain/voiceover_regeneration_services.py

@@ -106,7 +106,6 @@ def parse_html(html_content: str) -> str:
     Returns:
         str. The plain text retrieved from the HTML content.
     """
-
     soup = bs4.BeautifulSoup(html_content, 'html.parser')
     for custom_tag_element in ALLOWED_CUSTOM_OPPIA_RTE_TAGS:
         for element in soup.find_all(custom_tag_element):
@@ -167,7 +166,6 @@ def synthesize_voiceover_for_html_string(
             feconf.OPPIA_AUTOMATIC_VOICEOVER_PROVIDER
         )
     )
-
     audio_offset_list: List[Dict[str, Union[str, float]]] = []
 
     is_cached_model_used_for_voiceovers = False

core/platform/speech_synthesis/azure_speech_synthesis_services.py

@@ -61,11 +61,6 @@
     </p>
 """
 
-# Standard arithmetic operators used to separate text with math expressions in
-# an SSML string.
-COMMONLY_USED_ARITHMETIC_EXPRESSIONS = [
-    '+', ' - ', '*', ' / ', '×', '÷', '=']
-
 
 class WordBoundaryCollection:
     """This class handles word boundary events to collect the time offsets
@@ -94,24 +89,6 @@ def word_boundary_event(self, event: speechsdk.SessionEventArgs) -> None:
         self.audio_offset_list.append(audio_offset_record)
 
 
-def is_mathematical_text(text: str) -> bool:
-    """The method determines whether the given text is mathematical by checking
-    for the presence of common arithmetic operators.
-
-    Args:
-        text: str. The text to be analyzed for the presence of arithmetic
-            operators.
-
-    Returns:
-        bool. A boolean value that signifies if the given text contains
-        mathematical content.
-    """
-    for expression in COMMONLY_USED_ARITHMETIC_EXPRESSIONS:
-        if expression in text:
-            return True
-    return False
-
-
 def get_azure_voicecode_from_language_accent_code(
         language_accent_code: str
 ) -> str:

core/platform/speech_synthesis/azure_speech_synthesis_services_test.py

@@ -291,15 +291,6 @@ def test_should_return_word_boundary_collection_correctly(self) -> None:
             word_boundary_collection.audio_offset_list,
             expected_word_boundary_collection)
 
-    def test_is_mathematical_text(self) -> None:
-        math_text = 'Convert 5 + 3 to words.'
-        self.assertTrue(
-            azure_speech_synthesis_services.is_mathematical_text(math_text))
-
-        non_math_text = 'This is a test text.'
-        self.assertFalse(
-            azure_speech_synthesis_services.is_mathematical_text(non_math_text))
-
     def _get_ssml_content(
         self, main_content: str, language_accent_code: str
     ) -> str:

core/templates/pages/exploration-editor-page/translation-tab/voiceover-card/voiceover-card.component.html

@@ -65,7 +65,7 @@ <h4 class="voiceover-type-heading">Manual Voiceovers</h4>
     </div>
 
     <div class="automatic-voiceovers"
-         *ngIf="isVoiceoverAutogenerationSupportedForSelectedAccent && isAutomaticVoiceoverRegenerationFromExpFeatureEnabled()">
+         *ngIf="shouldShowAutoVoiceoverRegenerationSection()">
       <h4 class="voiceover-type-heading">Automatic Voiceovers (generated using cloud service)</h4>
       <div class="voiceovers-component" *ngIf="automaticVoiceover">
         <button class="btn btn-secondary audio-button" (click)="playAndPauseVoiceover(automaticVoiceover.filename, 'auto')">

core/templates/pages/exploration-editor-page/translation-tab/voiceover-card/voiceover-card.component.spec.ts

@@ -917,4 +917,22 @@ describe('Voiceover card component', () => {
       component.isAutomaticVoiceoverRegenerationFromExpFeatureEnabled()
     ).toBeTrue();
   });
+
+  it('should show voiceover regeneration section', () => {
+    let explorationLinkedToStorySpy = spyOn(
+      contextService,
+      'isExplorationLinkedToStory'
+    );
+    component.isVoiceoverAutogenerationSupportedForSelectedAccent = true;
+    spyOn(
+      component,
+      'isAutomaticVoiceoverRegenerationFromExpFeatureEnabled'
+    ).and.returnValue(true);
+
+    explorationLinkedToStorySpy.and.returnValue(true);
+    expect(component.shouldShowAutoVoiceoverRegenerationSection()).toBeTrue();
+
+    explorationLinkedToStorySpy.and.returnValue(false);
+    expect(component.shouldShowAutoVoiceoverRegenerationSection()).toBeFalse();
+  });
 });

core/templates/pages/exploration-editor-page/translation-tab/voiceover-card/voiceover-card.component.ts

@@ -18,6 +18,7 @@
 
 import {
   Component,
+  ChangeDetectorRef,
   ElementRef,
   OnInit,
   ViewChild,
@@ -98,6 +99,7 @@ export class VoiceoverCardComponent implements OnInit, AfterViewChecked {
   constructor(
     private audioPlayerService: AudioPlayerService,
     private contextService: ContextService,
+    private changeDetectorRef: ChangeDetectorRef,
     private translationLanguageService: TranslationLanguageService,
     private translationTabActiveContentIdService: TranslationTabActiveContentIdService,
     private ngbModal: NgbModal,
@@ -149,10 +151,6 @@ export class VoiceoverCardComponent implements OnInit, AfterViewChecked {
         }
       )
     );
-    this.voiceoversAreLoaded =
-      Object.keys(
-        this.entityVoiceoversService.languageAccentCodeToEntityVoiceovers
-      ).length !== 0;
 
     if (!this.entityVoiceoversService.getActiveLanguageAccentCode()) {
       this.voiceoversAreLoaded = true;
@@ -212,6 +210,11 @@ export class VoiceoverCardComponent implements OnInit, AfterViewChecked {
     } else {
       this.isGenerateAutomaticVoiceoverOptionEnabled = false;
     }
+    this.isVoiceoverAutogenerationSupportedForSelectedAccent =
+      this.voiceoverLanguageManagementService.isAutogenerationSupportedGivenLanguageAccent(
+        this.languageAccentCode
+      );
+    this.changeDetectorRef.detectChanges();
   }
 
   isAutomaticVoiceoverRegenerationFromExpFeatureEnabled(): boolean {
@@ -487,6 +490,18 @@ export class VoiceoverCardComponent implements OnInit, AfterViewChecked {
     }
   }
 
+  isExplorationLinkedToStory(): boolean {
+    return this.contextService.isExplorationLinkedToStory();
+  }
+
+  shouldShowAutoVoiceoverRegenerationSection(): boolean {
+    return (
+      this.isVoiceoverAutogenerationSupportedForSelectedAccent &&
+      this.isAutomaticVoiceoverRegenerationFromExpFeatureEnabled() &&
+      this.isExplorationLinkedToStory()
+    );
+  }
+
   deleteManualVoiceover(): void {
     const modalRef = this.ngbModal.open(VoiceoverRemovalConfirmModalComponent, {
       backdrop: 'static',

core/templates/pages/exploration-player-page/layout-directives/audio-bar.component.spec.ts

@@ -326,7 +326,7 @@ describe('Audio Bar Component', () => {
       expect(loadAndPlaySpy).toHaveBeenCalled();
     });
 
-    it('should be able update diplayable language accent code', () => {
+    it('should be able to update displayable language accent code for manual voiceover', () => {
       let manualVoiceoverBackendDict: VoiceoverBackendDict = {
         filename: 'a.mp3',
         file_size_bytes: 200000,
@@ -336,6 +336,7 @@ describe('Audio Bar Component', () => {
       let contentIdToVoiceoversMappingBackendDict = {
         content0: {
           manual: manualVoiceoverBackendDict,
+          auto: undefined,
         },
       };
       let entityId = 'exploration_1';
@@ -372,6 +373,53 @@ describe('Audio Bar Component', () => {
       expect(component.voiceoverToBePlayed.filename).toEqual('a.mp3');
     });
 
+    it('should be able to update displayable language accent code for auto voiceover', () => {
+      let autoVoiceoverBackendDict: VoiceoverBackendDict = {
+        filename: 'b.mp3',
+        file_size_bytes: 200000,
+        needs_update: false,
+        duration_secs: 10.0,
+      };
+      let contentIdToVoiceoversMappingBackendDict = {
+        content0: {
+          manual: undefined,
+          auto: autoVoiceoverBackendDict,
+        },
+      };
+      let entityId = 'exploration_1';
+      let entityType = 'exploration';
+      let entityVersion = 1;
+      let languageAccentCode = 'en-US';
+      let entityVoiceoversBackendDict = {
+        entity_id: entityId,
+        entity_type: entityType,
+        entity_version: entityVersion,
+        language_accent_code: languageAccentCode,
+        voiceovers_mapping: contentIdToVoiceoversMappingBackendDict,
+      };
+      let entityVoiceovers = EntityVoiceovers.createFromBackendDict(
+        entityVoiceoversBackendDict
+      );
+      let languageAccentDecriptions = ['en-US', 'en-IN'];
+
+      spyOn(
+        voiceoverPlayerService,
+        'getLanguageAccentDescriptions'
+      ).and.returnValue(languageAccentDecriptions);
+      spyOn(
+        entityVoiceoversService,
+        'getActiveEntityVoiceovers'
+      ).and.returnValue(entityVoiceovers);
+      spyOn(playerPositionService, 'getCurrentStateName');
+      spyOn(audioPreloaderService, 'restartAudioPreloader');
+      voiceoverPlayerService.activeContentId = 'content0';
+
+      component.voiceoverToBePlayed = undefined;
+      component.updateDisplayableLanguageAccentDescription();
+
+      expect(component.voiceoverToBePlayed.filename).toEqual('b.mp3');
+    });
+
     it('should pause uploaded voiceover when pause button is clicked', () => {
       component.voiceoverToBePlayed = Voiceover.createFromBackendDict({
         filename: 'audio-en.mp3',

core/templates/pages/exploration-player-page/layout-directives/audio-bar.component.ts

@@ -217,10 +217,21 @@ export class AudioBarComponent {
 
     let contentId = this.voiceoverPlayerService.activeContentId;
 
-    this.voiceoverToBePlayed = entityVoiceover.getManualVoiceover(
+    let manualVoiceover = entityVoiceover.getManualVoiceover(
       contentId
     ) as Voiceover;
+    let automaticVoiceover = entityVoiceover.getAutomaticVoiceover(
+      contentId
+    ) as Voiceover;
+
+    if (manualVoiceover && manualVoiceover.needsUpdate === false) {
+      this.voiceoverToBePlayed = manualVoiceover;
+    } else if (automaticVoiceover && automaticVoiceover.needsUpdate === false) {
+      this.voiceoverToBePlayed = automaticVoiceover;
+    }
 
+    this.audioPreloaderService.contentIdsToVoiceovers =
+      this.entityVoiceoversService.getAllContentIdsToVoiceovers();
     this.audioPreloaderService.restartAudioPreloader(
       this.playerPositionService.getCurrentStateName()
     );