Skip to content

Commit 6994458

Browse files
rahul-lohrarahul-lohra
committed
1. Expose CC start and stop Rest Apis
2. Create its response model 3. Create WS events and map it 4. Add CaptionManager to handle cc relates properties 5. Update closedCaptionMode data type in TranscriptionSettingsResponse 6. Add closedCaptionUi argument
1 parent f9f8de7 commit 6994458

File tree

19 files changed

+659
-19
lines changed

19 files changed

+659
-19
lines changed

‎stream-video-android-core/api/stream-video-android-core.api‎

Lines changed: 151 additions & 15 deletions
Large diffs are not rendered by default.

‎stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,9 @@ import org.openapitools.client.models.PinResponse
7878
import org.openapitools.client.models.RejectCallResponse
7979
import org.openapitools.client.models.SendCallEventResponse
8080
import org.openapitools.client.models.SendReactionResponse
81+
import org.openapitools.client.models.StartClosedCaptionResponse
8182
import org.openapitools.client.models.StartTranscriptionResponse
83+
import org.openapitools.client.models.StopClosedCaptionResponse
8284
import org.openapitools.client.models.StopLiveResponse
8385
import org.openapitools.client.models.StopTranscriptionResponse
8486
import org.openapitools.client.models.UnpinResponse
@@ -1290,6 +1292,14 @@ public class Call(
12901292
return clientImpl.listTranscription(type, id)
12911293
}
12921294

1295+
suspend fun startClosedCaptions(): Result<StartClosedCaptionResponse> {
1296+
return clientImpl.startClosedCaptions(type, id)
1297+
}
1298+
1299+
suspend fun stopClosedCaptions(): Result<StopClosedCaptionResponse> {
1300+
return clientImpl.stopClosedCaptions(type, id)
1301+
}
1302+
12931303
/**
12941304
* Sets the preferred incoming video resolution.
12951305
*

‎stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import android.util.Log
2020
import androidx.compose.runtime.Stable
2121
import io.getstream.log.taggedLogger
2222
import io.getstream.video.android.core.call.RtcSession
23+
import io.getstream.video.android.core.closedcaptions.ClosedCaptionManager
2324
import io.getstream.video.android.core.events.AudioLevelChangedEvent
2425
import io.getstream.video.android.core.events.ChangePublishQualityEvent
2526
import io.getstream.video.android.core.events.ConnectionQualityChangeEvent
@@ -99,6 +100,9 @@ import org.openapitools.client.models.CallTranscriptionFailedEvent
99100
import org.openapitools.client.models.CallTranscriptionStartedEvent
100101
import org.openapitools.client.models.CallTranscriptionStoppedEvent
101102
import org.openapitools.client.models.CallUpdatedEvent
103+
import org.openapitools.client.models.ClosedCaptionEndedEvent
104+
import org.openapitools.client.models.ClosedCaptionEvent
105+
import org.openapitools.client.models.ClosedCaptionStartedEvent
102106
import org.openapitools.client.models.ConnectedEvent
103107
import org.openapitools.client.models.CustomVideoEvent
104108
import org.openapitools.client.models.EgressHLSResponse
@@ -573,6 +577,12 @@ public class CallState(
573577

574578
internal var acceptedOnThisDevice: Boolean = false
575579

580+
/**
581+
* This [ClosedCaptionManager] is responsible for handling closed captions during the call.
582+
* This includes processing events related to closed captions and maintaining their state.
583+
*/
584+
public val closedCaptionManager = ClosedCaptionManager()
585+
576586
fun handleEvent(event: VideoEvent) {
577587
logger.d { "Updating call state with event ${event::class.java}" }
578588
when (event) {
@@ -949,6 +959,12 @@ public class CallState(
949959
is CallTranscriptionFailedEvent -> {
950960
_transcribing.value = false
951961
}
962+
963+
is ClosedCaptionStartedEvent,
964+
is ClosedCaptionEvent,
965+
is ClosedCaptionEndedEvent,
966+
->
967+
closedCaptionManager.handleEvent(event)
952968
}
953969
}
954970

@@ -1244,6 +1260,7 @@ public class CallState(
12441260
_team.value = response.team
12451261

12461262
updateRingingState()
1263+
closedCaptionManager.handleCallUpdate(response)
12471264
}
12481265

12491266
fun updateFromResponse(response: GetOrCreateCallResponse) {

‎stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/StreamVideoClient.kt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,12 @@ import org.openapitools.client.models.SendCallEventRequest
114114
import org.openapitools.client.models.SendCallEventResponse
115115
import org.openapitools.client.models.SendReactionRequest
116116
import org.openapitools.client.models.SendReactionResponse
117+
import org.openapitools.client.models.StartClosedCaptionResponse
117118
import org.openapitools.client.models.StartHLSBroadcastingResponse
118119
import org.openapitools.client.models.StartRecordingRequest
119120
import org.openapitools.client.models.StartTranscriptionRequest
120121
import org.openapitools.client.models.StartTranscriptionResponse
122+
import org.openapitools.client.models.StopClosedCaptionResponse
121123
import org.openapitools.client.models.StopLiveResponse
122124
import org.openapitools.client.models.StopTranscriptionResponse
123125
import org.openapitools.client.models.UnblockUserRequest
@@ -1115,6 +1117,18 @@ internal class StreamVideoClient internal constructor(
11151117
coordinatorConnectionModule.api.listTranscriptions(type, id)
11161118
}
11171119
}
1120+
1121+
suspend fun startClosedCaptions(type: String, id: String): Result<StartClosedCaptionResponse> {
1122+
return apiCall {
1123+
coordinatorConnectionModule.api.startClosedCaptions(type, id)
1124+
}
1125+
}
1126+
1127+
suspend fun stopClosedCaptions(type: String, id: String): Result<StopClosedCaptionResponse> {
1128+
return apiCall {
1129+
coordinatorConnectionModule.api.stopClosedCaptions(type, id)
1130+
}
1131+
}
11181132
}
11191133

11201134
/** Extension function that makes it easy to use on kotlin, but keeps Java usable as well */

‎stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/*
2+
* Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
3+
*
4+
* Licensed under the Stream License;
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://github.com/GetStream/stream-video-android/blob/main/LICENSE
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.getstream.video.android.core.closedcaptions
18+
19+
import kotlinx.coroutines.CoroutineScope
20+
import kotlinx.coroutines.Dispatchers
21+
import kotlinx.coroutines.Job
22+
import kotlinx.coroutines.SupervisorJob
23+
import kotlinx.coroutines.delay
24+
import kotlinx.coroutines.flow.MutableStateFlow
25+
import kotlinx.coroutines.flow.StateFlow
26+
import kotlinx.coroutines.flow.asStateFlow
27+
import kotlinx.coroutines.launch
28+
import kotlinx.coroutines.sync.Mutex
29+
import kotlinx.coroutines.sync.withLock
30+
import org.openapitools.client.models.CallClosedCaption
31+
import org.openapitools.client.models.CallResponse
32+
import org.openapitools.client.models.ClosedCaptionEndedEvent
33+
import org.openapitools.client.models.ClosedCaptionEvent
34+
import org.openapitools.client.models.ClosedCaptionStartedEvent
35+
import org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode
36+
import org.openapitools.client.models.VideoEvent
37+
38+
/**
39+
* Manages the lifecycle, state, and configuration of closed captions for a video call.
40+
*
41+
* The [ClosedCaptionManager] is responsible for handling caption updates, maintaining caption states,
42+
* and auto-removing captions based on the provided [ClosedCaptionsConfig]. It ensures thread-safe
43+
* operations using a [Mutex] and manages jobs for scheduled caption removal using [CoroutineScope].
44+
*
45+
* @property config The configuration that defines how closed captions are managed,
46+
* including auto-dismiss behavior, maximum number of captions to retain, and dismiss time.
47+
*/
48+
49+
class ClosedCaptionManager(private var config: ClosedCaptionsConfig = ClosedCaptionsConfig()) {
50+
51+
/**
52+
* Holds the current list of closed captions. This list is updated dynamically
53+
* and contains at most [ClosedCaptionsConfig.maxCaptions] captions.
54+
*/
55+
56+
private val _closedCaptions: MutableStateFlow<List<CallClosedCaption>> =
57+
MutableStateFlow(emptyList())
58+
val closedCaptions: StateFlow<List<CallClosedCaption>> = _closedCaptions.asStateFlow()
59+
60+
private val _ccMode =
61+
MutableStateFlow<ClosedCaptionMode>(ClosedCaptionMode.Disabled)
62+
val ccMode: StateFlow<ClosedCaptionMode> = _ccMode.asStateFlow()
63+
64+
/**
65+
* Tracks whether closed captioning is currently active for the call.
66+
* True if captioning is ongoing, false otherwise.
67+
*/
68+
private val _closedCaptioning: MutableStateFlow<Boolean> = MutableStateFlow(false)
69+
val closedCaptioning: StateFlow<Boolean> = _closedCaptioning
70+
71+
/**
72+
* Manages the job responsible for automatically removing closed captions after a delay.
73+
*/
74+
private var removalJob: Job? = null
75+
private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob())
76+
77+
/**
78+
* Ensures thread-safe updates to the list of closed captions.
79+
*/
80+
private val mutex = Mutex()
81+
82+
/**
83+
* Updates the current configuration for the closed captions manager.
84+
*
85+
* @param newConfig The new configuration to apply. This affects behavior such as auto-dismiss
86+
* and the number of captions retained.
87+
*/
88+
fun setConfig(newConfig: ClosedCaptionsConfig) {
89+
config = newConfig
90+
}
91+
92+
/**
93+
* Handles updates from the call response to determine the availability and state
94+
* of closed captions.
95+
*
96+
* @param callResponse The response containing transcription and caption settings for the call.
97+
*/
98+
fun handleCallUpdate(callResponse: CallResponse) {
99+
_closedCaptioning.value = callResponse.captioning
100+
_ccMode.value = callResponse.settings.transcription.closedCaptionMode
101+
}
102+
103+
/**
104+
* Processes incoming events related to closed captions, such as new captions being added,
105+
* captioning starting, or captioning ending.
106+
*
107+
* @param videoEvent The event containing closed captioning information.
108+
*/
109+
fun handleEvent(videoEvent: VideoEvent) {
110+
when (videoEvent) {
111+
is ClosedCaptionEvent -> {
112+
addCaption(videoEvent)
113+
_closedCaptioning.value = true
114+
}
115+
116+
is ClosedCaptionStartedEvent -> {
117+
_closedCaptioning.value = true
118+
}
119+
120+
is ClosedCaptionEndedEvent -> {
121+
_closedCaptioning.value = false
122+
}
123+
}
124+
}
125+
126+
/**
127+
* Adds a new caption to the list and manages the auto-dismiss logic.
128+
*
129+
* @param event The event containing the closed caption data to add.
130+
*/
131+
private fun addCaption(event: ClosedCaptionEvent) {
132+
scope.launch {
133+
mutex.withLock {
134+
// Add the caption and keep the latest 3
135+
_closedCaptions.value =
136+
(_closedCaptions.value + event.closedCaption).takeLast(config.maxCaptions)
137+
}
138+
139+
if (config.autoDismissCaptions) {
140+
removalJob?.cancel()
141+
scheduleRemoval()
142+
}
143+
}
144+
}
145+
146+
/**
147+
* Schedules the removal of the oldest caption after the specified [ClosedCaptionsConfig.captionsAutoDismissTime].
148+
*
149+
*/
150+
private fun scheduleRemoval() {
151+
removalJob = scope.launch {
152+
delay(config.captionsAutoDismissTime)
153+
mutex.withLock {
154+
if (_closedCaptions.value.isNotEmpty()) {
155+
_closedCaptions.value =
156+
_closedCaptions.value.drop(1) // Remove the oldest caption
157+
}
158+
}
159+
if (_closedCaptions.value.isNotEmpty()) {
160+
scheduleRemoval() // Continue scheduling removal for remaining captions
161+
}
162+
}
163+
}
164+
}

‎stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsConfig.kt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
3+
*
4+
* Licensed under the Stream License;
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://github.com/GetStream/stream-video-android/blob/main/LICENSE
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.getstream.video.android.core.closedcaptions
18+
19+
private const val DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS = 6000L
20+
21+
/**
22+
* Configuration for managing closed captions in the [ClosedCaptionManager].
23+
*
24+
* @param captionsAutoDismissTime The duration (in milliseconds) after which captions will be automatically removed.
25+
* Set to [DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS] by default.
26+
*
27+
* @param autoDismissCaptions Determines whether closed captions should be automatically dismissed after a delay.
28+
* If set to `false`, captions will remain visible indefinitely.
29+
*
30+
* @param maxCaptions The maximum number of closed captions to retain in the [ClosedCaptionManager.closedCaptions] flow.
31+
* Must be greater than or equal to [io.getstream.video.android.compose.ui.components.closedcaptions.ClosedCaptionsThemeConfig.maxVisibleCaptions]
32+
* to ensure the UI has sufficient data to render.
33+
*
34+
*/
35+
36+
data class ClosedCaptionsConfig(
37+
val captionsAutoDismissTime: Long = DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS,
38+
val autoDismissCaptions: Boolean = true,
39+
val maxCaptions: Int = 3, // Default to keep the latest 3 captions
40+
)

‎stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/socket/common/parser2/MoshiVideoParser.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ internal class MoshiVideoParser : VideoParser {
108108
org.openapitools.client.models.TranscriptionSettingsResponse.Mode.ModeAdapter(),
109109
),
110110
)
111+
.add(
112+
lenientAdapter(
113+
org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode.ClosedCaptionModeAdapter(),
114+
),
115+
)
111116
.add(
112117
lenientAdapter(
113118
org.openapitools.client.models.VideoSettingsRequest.CameraFacing.CameraFacingAdapter(),

‎stream-video-android-core/src/main/kotlin/org/openapitools/client/apis/ProductvideoApi.kt‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,13 @@ import org.openapitools.client.models.SendCallEventRequest
7272
import org.openapitools.client.models.SendCallEventResponse
7373
import org.openapitools.client.models.SendReactionRequest
7474
import org.openapitools.client.models.SendReactionResponse
75+
import org.openapitools.client.models.StartClosedCaptionResponse
7576
import org.openapitools.client.models.StartHLSBroadcastingResponse
7677
import org.openapitools.client.models.StartRecordingRequest
7778
import org.openapitools.client.models.StartRecordingResponse
7879
import org.openapitools.client.models.StartTranscriptionRequest
7980
import org.openapitools.client.models.StartTranscriptionResponse
81+
import org.openapitools.client.models.StopClosedCaptionResponse
8082
import org.openapitools.client.models.StopHLSBroadcastingResponse
8183
import org.openapitools.client.models.StopLiveResponse
8284
import org.openapitools.client.models.StopRecordingResponse
@@ -854,4 +856,39 @@ interface ProductvideoApi {
854856
@Body unpinRequest: UnpinRequest
855857
): UnpinResponse
856858

859+
860+
/**
861+
* Start CC for a call
862+
* Responses:
863+
* - 201: Successful response
864+
* - 400: Bad request
865+
* - 429: Too many requests
866+
*
867+
* @param type
868+
* @param id
869+
* @return [StartClosedCaptionResponse]
870+
*/
871+
@POST("/video/call/{type}/{id}/start_closed_captions")
872+
suspend fun startClosedCaptions(
873+
@Path("type") type: String,
874+
@Path("id") id: String,
875+
): StartClosedCaptionResponse
876+
877+
878+
/**
879+
* Stops CC for a call
880+
* Responses:
881+
* - 201: Successful response
882+
* - 400: Bad request
883+
* - 429: Too many requests
884+
*
885+
* @param type
886+
* @param id
887+
* @return [StopClosedCaptionResponse]
888+
*/
889+
@POST("/video/call/{type}/{id}/stop_closed_captions")
890+
suspend fun stopClosedCaptions(
891+
@Path("type") type: String,
892+
@Path("id") id: String,
893+
): StopClosedCaptionResponse
857894
}

‎stream-video-android-core/src/main/kotlin/org/openapitools/client/infrastructure/Serializer.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ object Serializer {
4444
.add(org.openapitools.client.models.RecordSettingsRequest.Quality.QualityAdapter())
4545
.add(org.openapitools.client.models.TranscriptionSettingsRequest.Mode.ModeAdapter())
4646
.add(org.openapitools.client.models.TranscriptionSettingsResponse.Mode.ModeAdapter())
47+
.add(org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode.ClosedCaptionModeAdapter())
4748
.add(org.openapitools.client.models.VideoSettingsRequest.CameraFacing.CameraFacingAdapter())
4849
.add(org.openapitools.client.models.VideoSettingsResponse.CameraFacing.CameraFacingAdapter())
4950
.add(BigDecimalAdapter())

‎stream-video-android-core/src/main/kotlin/org/openapitools/client/models/CallResponse.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ data class CallResponse (
109109
@Json(name = "settings")
110110
val settings: CallSettingsResponse,
111111

112+
@Json(name = "captioning")
113+
val captioning: kotlin.Boolean,
114+
112115
@Json(name = "transcribing")
113116
val transcribing: kotlin.Boolean,
114117

0 commit comments

Comments
 (0)