From bb7ff5193e11df20bbf3cb0337f158c57f83875e Mon Sep 17 00:00:00 2001 From: rahullohra Date: Thu, 12 Dec 2024 16:21:31 +0530 Subject: [PATCH 1/5] 1. Expose CC start and stop Rest Apis 2. Create its response model 3. Create WS events and map it 4. Add CaptionManager to handle cc relates properties 5. Update closedCaptionMode data type in TranscriptionSettingsResponse 6. Add closedCaptionUi argument 7. Update UT 8. Rename variables 9. Expose ClosedCaptionsSettings from Call class --- .../api/stream-video-android-core.api | 166 ++++++++++++++++-- .../io/getstream/video/android/core/Call.kt | 22 +++ .../getstream/video/android/core/CallState.kt | 10 ++ .../video/android/core/StreamVideoClient.kt | 14 ++ .../closedcaptions/ClosedCaptionManager.kt | 164 +++++++++++++++++ .../closedcaptions/ClosedCaptionsSettings.kt | 40 +++++ .../socket/common/parser2/MoshiVideoParser.kt | 5 + .../client/apis/ProductvideoApi.kt | 37 ++++ .../client/infrastructure/Serializer.kt | 1 + .../client/models/CallResponse.kt | 3 + .../client/models/ClosedCaptionEndedEvent.kt | 59 +++++++ .../models/ClosedCaptionStartedEvent.kt | 59 +++++++ .../models/StartClosedCaptionResponse.kt | 24 +++ .../models/StopClosedCaptionResponse.kt | 24 +++ .../models/TranscriptionSettingsResponse.kt | 40 ++++- .../openapitools/client/models/VideoEvent.kt | 2 + .../android/core/base/IntegrationTestBase.kt | 3 +- .../api/stream-video-android-ui-compose.api | 8 +- .../components/call/activecall/CallContent.kt | 4 + 19 files changed, 665 insertions(+), 20 deletions(-) create mode 100644 stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt create mode 100644 stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt create mode 100644 stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionEndedEvent.kt create mode 100644 stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionStartedEvent.kt create mode 100644 stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StartClosedCaptionResponse.kt create mode 100644 stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StopClosedCaptionResponse.kt diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index 5fec5db3146..dd47b5b8980 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -75,10 +75,12 @@ public final class io/getstream/video/android/core/Call { public final fun setSessionId (Ljava/lang/String;)V public final fun setVideoFilter (Lio/getstream/video/android/core/call/video/VideoFilter;)V public final fun setVisibility (Ljava/lang/String;Lstream/video/sfu/models/TrackType;Z)V + public final fun startClosedCaptions (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun startHLS (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun startRecording (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun startScreenSharing (Landroid/content/Intent;)V public final fun startTranscription (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun stopClosedCaptions (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun stopHLS (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun stopLive (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun stopRecording (Lkotlin/coroutines/Continuation;)Ljava/lang/Object; @@ -91,6 +93,7 @@ public final class io/getstream/video/android/core/Call { public final fun unpinForEveryone (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public final fun update (Ljava/util/Map;Lorg/openapitools/client/models/CallSettingsRequest;Lorg/threeten/bp/OffsetDateTime;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public static synthetic fun update$default (Lio/getstream/video/android/core/Call;Ljava/util/Map;Lorg/openapitools/client/models/CallSettingsRequest;Lorg/threeten/bp/OffsetDateTime;Lkotlin/coroutines/Continuation;ILjava/lang/Object;)Ljava/lang/Object; + public final fun updateClosedCaptionsSettings (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;)V public final fun updateMembers (Ljava/util/List;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; } @@ -2884,6 +2887,35 @@ public final class io/getstream/video/android/core/call/video/YuvFrame { public final fun bitmapFromVideoFrame (Lorg/webrtc/VideoFrame;)Landroid/graphics/Bitmap; } +public final class io/getstream/video/android/core/closedcaptions/ClosedCaptionManager { + public fun ()V + public fun (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;)V + public synthetic fun (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getCcMode ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getClosedCaptioning ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getClosedCaptions ()Lkotlinx/coroutines/flow/StateFlow; + public final fun handleCallUpdate (Lorg/openapitools/client/models/CallResponse;)V + public final fun handleEvent (Lorg/openapitools/client/models/VideoEvent;)V + public final fun updateClosedCaptionsSettings (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;)V +} + +public final class io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings { + public fun ()V + public fun (JZI)V + public synthetic fun (JZIILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()J + public final fun component2 ()Z + public final fun component3 ()I + public final fun copy (JZI)Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings; + public static synthetic fun copy$default (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;JZIILjava/lang/Object;)Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings; + public fun equals (Ljava/lang/Object;)Z + public final fun getAutoDismissCaptions ()Z + public final fun getMaxVisibleCaptions ()I + public final fun getVisibilityDurationMs ()J + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class io/getstream/video/android/core/dispatchers/DispatcherProvider { public static final field INSTANCE Lio/getstream/video/android/core/dispatchers/DispatcherProvider; public final fun getDefault ()Lkotlinx/coroutines/CoroutineDispatcher; @@ -6751,9 +6783,11 @@ public abstract interface class org/openapitools/client/apis/ProductvideoApi { public abstract fun requestPermission (Ljava/lang/String;Ljava/lang/String;Lorg/openapitools/client/models/RequestPermissionRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun sendCallEvent (Ljava/lang/String;Ljava/lang/String;Lorg/openapitools/client/models/SendCallEventRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun sendVideoReaction (Ljava/lang/String;Ljava/lang/String;Lorg/openapitools/client/models/SendReactionRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public abstract fun startClosedCaptions (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun startHLSBroadcasting (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun startRecording (Ljava/lang/String;Ljava/lang/String;Lorg/openapitools/client/models/StartRecordingRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun startTranscription (Ljava/lang/String;Ljava/lang/String;Lorg/openapitools/client/models/StartTranscriptionRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public abstract fun stopClosedCaptions (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun stopHLSBroadcasting (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun stopLive (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; public abstract fun stopRecording (Ljava/lang/String;Ljava/lang/String;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; @@ -7690,21 +7724,22 @@ public final class org/openapitools/client/models/CallRequest { } public final class org/openapitools/client/models/CallResponse { - public fun (ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;)V - public synthetic fun (ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;)V + public synthetic fun (ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun component1 ()Z public final fun component10 ()Lorg/openapitools/client/models/CallIngressResponse; public final fun component11 ()Z public final fun component12 ()Lorg/openapitools/client/models/CallSettingsResponse; public final fun component13 ()Z - public final fun component14 ()Ljava/lang/String; - public final fun component15 ()Lorg/threeten/bp/OffsetDateTime; + public final fun component14 ()Z + public final fun component15 ()Ljava/lang/String; public final fun component16 ()Lorg/threeten/bp/OffsetDateTime; - public final fun component17 ()Lorg/openapitools/client/models/CallSessionResponse; - public final fun component18 ()Lorg/threeten/bp/OffsetDateTime; - public final fun component19 ()Ljava/lang/String; + public final fun component17 ()Lorg/threeten/bp/OffsetDateTime; + public final fun component18 ()Lorg/openapitools/client/models/CallSessionResponse; + public final fun component19 ()Lorg/threeten/bp/OffsetDateTime; public final fun component2 ()Ljava/util/List; - public final fun component20 ()Lorg/openapitools/client/models/ThumbnailResponse; + public final fun component20 ()Ljava/lang/String; + public final fun component21 ()Lorg/openapitools/client/models/ThumbnailResponse; public final fun component3 ()Ljava/lang/String; public final fun component4 ()Lorg/threeten/bp/OffsetDateTime; public final fun component5 ()Lorg/openapitools/client/models/UserResponse; @@ -7712,11 +7747,12 @@ public final class org/openapitools/client/models/CallResponse { public final fun component7 ()Ljava/util/Map; public final fun component8 ()Lorg/openapitools/client/models/EgressResponse; public final fun component9 ()Ljava/lang/String; - public final fun copy (ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;)Lorg/openapitools/client/models/CallResponse; - public static synthetic fun copy$default (Lorg/openapitools/client/models/CallResponse;ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;ILjava/lang/Object;)Lorg/openapitools/client/models/CallResponse; + public final fun copy (ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;)Lorg/openapitools/client/models/CallResponse; + public static synthetic fun copy$default (Lorg/openapitools/client/models/CallResponse;ZLjava/util/List;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/UserResponse;Ljava/lang/String;Ljava/util/Map;Lorg/openapitools/client/models/EgressResponse;Ljava/lang/String;Lorg/openapitools/client/models/CallIngressResponse;ZLorg/openapitools/client/models/CallSettingsResponse;ZZLjava/lang/String;Lorg/threeten/bp/OffsetDateTime;Lorg/threeten/bp/OffsetDateTime;Lorg/openapitools/client/models/CallSessionResponse;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;Lorg/openapitools/client/models/ThumbnailResponse;ILjava/lang/Object;)Lorg/openapitools/client/models/CallResponse; public fun equals (Ljava/lang/Object;)Z public final fun getBackstage ()Z public final fun getBlockedUserIds ()Ljava/util/List; + public final fun getCaptioning ()Z public final fun getCid ()Ljava/lang/String; public final fun getCreatedAt ()Lorg/threeten/bp/OffsetDateTime; public final fun getCreatedBy ()Lorg/openapitools/client/models/UserResponse; @@ -8477,6 +8513,24 @@ public final class org/openapitools/client/models/ChannelResponse { public fun toString ()Ljava/lang/String; } +public final class org/openapitools/client/models/ClosedCaptionEndedEvent : org/openapitools/client/models/VideoEvent, org/openapitools/client/models/WSCallEvent { + public fun (Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/String; + public final fun component2 ()Lorg/threeten/bp/OffsetDateTime; + public final fun component3 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;)Lorg/openapitools/client/models/ClosedCaptionEndedEvent; + public static synthetic fun copy$default (Lorg/openapitools/client/models/ClosedCaptionEndedEvent;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;ILjava/lang/Object;)Lorg/openapitools/client/models/ClosedCaptionEndedEvent; + public fun equals (Ljava/lang/Object;)Z + public fun getCallCID ()Ljava/lang/String; + public final fun getCallCid ()Ljava/lang/String; + public final fun getCreatedAt ()Lorg/threeten/bp/OffsetDateTime; + public fun getEventType ()Ljava/lang/String; + public final fun getType ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/openapitools/client/models/ClosedCaptionEvent : org/openapitools/client/models/VideoEvent, org/openapitools/client/models/WSCallEvent { public fun (Ljava/lang/String;Lorg/openapitools/client/models/CallClosedCaption;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;)V public synthetic fun (Ljava/lang/String;Lorg/openapitools/client/models/CallClosedCaption;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V @@ -8497,6 +8551,24 @@ public final class org/openapitools/client/models/ClosedCaptionEvent : org/opena public fun toString ()Ljava/lang/String; } +public final class org/openapitools/client/models/ClosedCaptionStartedEvent : org/openapitools/client/models/VideoEvent, org/openapitools/client/models/WSCallEvent { + public fun (Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;)V + public synthetic fun (Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/String; + public final fun component2 ()Lorg/threeten/bp/OffsetDateTime; + public final fun component3 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;)Lorg/openapitools/client/models/ClosedCaptionStartedEvent; + public static synthetic fun copy$default (Lorg/openapitools/client/models/ClosedCaptionStartedEvent;Ljava/lang/String;Lorg/threeten/bp/OffsetDateTime;Ljava/lang/String;ILjava/lang/Object;)Lorg/openapitools/client/models/ClosedCaptionStartedEvent; + public fun equals (Ljava/lang/Object;)Z + public fun getCallCID ()Ljava/lang/String; + public final fun getCallCid ()Ljava/lang/String; + public final fun getCreatedAt ()Lorg/threeten/bp/OffsetDateTime; + public fun getEventType ()Ljava/lang/String; + public final fun getType ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/openapitools/client/models/CollectUserFeedbackRequest { public fun (ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/lang/String;)V public synthetic fun (ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V @@ -10359,6 +10431,17 @@ public final class org/openapitools/client/models/SortParam { public fun toString ()Ljava/lang/String; } +public final class org/openapitools/client/models/StartClosedCaptionResponse { + public fun (Ljava/lang/String;)V + public final fun component1 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;)Lorg/openapitools/client/models/StartClosedCaptionResponse; + public static synthetic fun copy$default (Lorg/openapitools/client/models/StartClosedCaptionResponse;Ljava/lang/String;ILjava/lang/Object;)Lorg/openapitools/client/models/StartClosedCaptionResponse; + public fun equals (Ljava/lang/Object;)Z + public final fun getDuration ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/openapitools/client/models/StartHLSBroadcastingResponse { public fun (Ljava/lang/String;Ljava/lang/String;)V public final fun component1 ()Ljava/lang/String; @@ -10444,6 +10527,17 @@ public final class org/openapitools/client/models/StatsOptions { public fun toString ()Ljava/lang/String; } +public final class org/openapitools/client/models/StopClosedCaptionResponse { + public fun (Ljava/lang/String;)V + public final fun component1 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;)Lorg/openapitools/client/models/StopClosedCaptionResponse; + public static synthetic fun copy$default (Lorg/openapitools/client/models/StopClosedCaptionResponse;Ljava/lang/String;ILjava/lang/Object;)Lorg/openapitools/client/models/StopClosedCaptionResponse; + public fun equals (Ljava/lang/Object;)Z + public final fun getDuration ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class org/openapitools/client/models/StopHLSBroadcastingResponse { public fun (Ljava/lang/String;)V public final fun component1 ()Ljava/lang/String; @@ -10635,20 +10729,62 @@ public final class org/openapitools/client/models/TranscriptionSettingsRequest$M } public final class org/openapitools/client/models/TranscriptionSettingsResponse { - public fun (Ljava/lang/String;Ljava/util/List;Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode;)V - public final fun component1 ()Ljava/lang/String; + public fun (Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode;Ljava/util/List;Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode;)V + public final fun component1 ()Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode; public final fun component2 ()Ljava/util/List; public final fun component3 ()Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode; - public final fun copy (Ljava/lang/String;Ljava/util/List;Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode;)Lorg/openapitools/client/models/TranscriptionSettingsResponse; - public static synthetic fun copy$default (Lorg/openapitools/client/models/TranscriptionSettingsResponse;Ljava/lang/String;Ljava/util/List;Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode;ILjava/lang/Object;)Lorg/openapitools/client/models/TranscriptionSettingsResponse; + public final fun copy (Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode;Ljava/util/List;Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode;)Lorg/openapitools/client/models/TranscriptionSettingsResponse; + public static synthetic fun copy$default (Lorg/openapitools/client/models/TranscriptionSettingsResponse;Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode;Ljava/util/List;Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode;ILjava/lang/Object;)Lorg/openapitools/client/models/TranscriptionSettingsResponse; public fun equals (Ljava/lang/Object;)Z - public final fun getClosedCaptionMode ()Ljava/lang/String; + public final fun getClosedCaptionMode ()Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode; public final fun getLanguages ()Ljava/util/List; public final fun getMode ()Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode; public fun hashCode ()I public fun toString ()Ljava/lang/String; } +public abstract class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode { + public static final field Companion Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Companion; + public synthetic fun (Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getValue ()Ljava/lang/String; + public fun toString ()Ljava/lang/String; +} + +public final class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$AutoOn : org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode { + public static final field INSTANCE Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$AutoOn; +} + +public final class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Available : org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode { + public static final field INSTANCE Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Available; +} + +public final class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$ClosedCaptionModeAdapter : com/squareup/moshi/JsonAdapter { + public fun ()V + public synthetic fun fromJson (Lcom/squareup/moshi/JsonReader;)Ljava/lang/Object; + public fun fromJson (Lcom/squareup/moshi/JsonReader;)Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode; + public synthetic fun toJson (Lcom/squareup/moshi/JsonWriter;Ljava/lang/Object;)V + public fun toJson (Lcom/squareup/moshi/JsonWriter;Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode;)V +} + +public final class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Companion { + public final fun fromString (Ljava/lang/String;)Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode; +} + +public final class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Disabled : org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode { + public static final field INSTANCE Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Disabled; +} + +public final class org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Unknown : org/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode { + public fun (Ljava/lang/String;)V + public final fun component1 ()Ljava/lang/String; + public final fun copy (Ljava/lang/String;)Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Unknown; + public static synthetic fun copy$default (Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Unknown;Ljava/lang/String;ILjava/lang/Object;)Lorg/openapitools/client/models/TranscriptionSettingsResponse$ClosedCaptionMode$Unknown; + public fun equals (Ljava/lang/Object;)Z + public final fun getUnknownValue ()Ljava/lang/String; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public abstract class org/openapitools/client/models/TranscriptionSettingsResponse$Mode { public static final field Companion Lorg/openapitools/client/models/TranscriptionSettingsResponse$Mode$Companion; public synthetic fun (Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt index 0838f4d4ba2..8c49b9194af 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt @@ -32,6 +32,8 @@ import io.getstream.video.android.core.call.audio.InputAudioFilter import io.getstream.video.android.core.call.utils.SoundInputProcessor import io.getstream.video.android.core.call.video.VideoFilter import io.getstream.video.android.core.call.video.YuvFrame +import io.getstream.video.android.core.closedcaptions.ClosedCaptionManager +import io.getstream.video.android.core.closedcaptions.ClosedCaptionsSettings import io.getstream.video.android.core.events.GoAwayEvent import io.getstream.video.android.core.events.JoinCallResponseEvent import io.getstream.video.android.core.events.VideoEventListener @@ -78,7 +80,9 @@ import org.openapitools.client.models.PinResponse import org.openapitools.client.models.RejectCallResponse import org.openapitools.client.models.SendCallEventResponse import org.openapitools.client.models.SendReactionResponse +import org.openapitools.client.models.StartClosedCaptionResponse import org.openapitools.client.models.StartTranscriptionResponse +import org.openapitools.client.models.StopClosedCaptionResponse import org.openapitools.client.models.StopLiveResponse import org.openapitools.client.models.StopTranscriptionResponse import org.openapitools.client.models.UnpinResponse @@ -265,6 +269,12 @@ public class Call( private var sfuListener: Job? = null private var sfuEvents: Job? = null + /** + * This [ClosedCaptionManager] is responsible for handling closed captions during the call. + * This includes processing events related to closed captions and maintaining their state. + */ + internal val closedCaptionManager = ClosedCaptionManager() + init { scope.launch { soundInputProcessor.currentAudioLevel.collect { @@ -1290,6 +1300,18 @@ public class Call( return clientImpl.listTranscription(type, id) } + suspend fun startClosedCaptions(): Result { + return clientImpl.startClosedCaptions(type, id) + } + + suspend fun stopClosedCaptions(): Result { + return clientImpl.stopClosedCaptions(type, id) + } + + fun updateClosedCaptionsSettings(closedCaptionsSettings: ClosedCaptionsSettings) { + closedCaptionManager.updateClosedCaptionsSettings(closedCaptionsSettings) + } + /** * Sets the preferred incoming video resolution. * diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt index 25e4325f7f3..32a17bccdf2 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt @@ -99,6 +99,9 @@ import org.openapitools.client.models.CallTranscriptionFailedEvent import org.openapitools.client.models.CallTranscriptionStartedEvent import org.openapitools.client.models.CallTranscriptionStoppedEvent import org.openapitools.client.models.CallUpdatedEvent +import org.openapitools.client.models.ClosedCaptionEndedEvent +import org.openapitools.client.models.ClosedCaptionEvent +import org.openapitools.client.models.ClosedCaptionStartedEvent import org.openapitools.client.models.ConnectedEvent import org.openapitools.client.models.CustomVideoEvent import org.openapitools.client.models.EgressHLSResponse @@ -949,6 +952,12 @@ public class CallState( is CallTranscriptionFailedEvent -> { _transcribing.value = false } + + is ClosedCaptionStartedEvent, + is ClosedCaptionEvent, + is ClosedCaptionEndedEvent, + -> + call.closedCaptionManager.handleEvent(event) } } @@ -1244,6 +1253,7 @@ public class CallState( _team.value = response.team updateRingingState() + call.closedCaptionManager.handleCallUpdate(response) } fun updateFromResponse(response: GetOrCreateCallResponse) { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/StreamVideoClient.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/StreamVideoClient.kt index 8c0e66eae53..cc63d81393d 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/StreamVideoClient.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/StreamVideoClient.kt @@ -114,10 +114,12 @@ import org.openapitools.client.models.SendCallEventRequest import org.openapitools.client.models.SendCallEventResponse import org.openapitools.client.models.SendReactionRequest import org.openapitools.client.models.SendReactionResponse +import org.openapitools.client.models.StartClosedCaptionResponse import org.openapitools.client.models.StartHLSBroadcastingResponse import org.openapitools.client.models.StartRecordingRequest import org.openapitools.client.models.StartTranscriptionRequest import org.openapitools.client.models.StartTranscriptionResponse +import org.openapitools.client.models.StopClosedCaptionResponse import org.openapitools.client.models.StopLiveResponse import org.openapitools.client.models.StopTranscriptionResponse import org.openapitools.client.models.UnblockUserRequest @@ -1115,6 +1117,18 @@ internal class StreamVideoClient internal constructor( coordinatorConnectionModule.api.listTranscriptions(type, id) } } + + suspend fun startClosedCaptions(type: String, id: String): Result { + return apiCall { + coordinatorConnectionModule.api.startClosedCaptions(type, id) + } + } + + suspend fun stopClosedCaptions(type: String, id: String): Result { + return apiCall { + coordinatorConnectionModule.api.stopClosedCaptions(type, id) + } + } } /** Extension function that makes it easy to use on kotlin, but keeps Java usable as well */ diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt new file mode 100644 index 00000000000..d1bef3b3a96 --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core.closedcaptions + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.openapitools.client.models.CallClosedCaption +import org.openapitools.client.models.CallResponse +import org.openapitools.client.models.ClosedCaptionEndedEvent +import org.openapitools.client.models.ClosedCaptionEvent +import org.openapitools.client.models.ClosedCaptionStartedEvent +import org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode +import org.openapitools.client.models.VideoEvent + +/** + * Manages the lifecycle, state, and configuration of closed captions for a video call. + * + * The [ClosedCaptionManager] is responsible for handling caption updates, maintaining caption states, + * and auto-removing captions based on the provided [ClosedCaptionsSettings]. It ensures thread-safe + * operations using a [Mutex] and manages jobs for scheduled caption removal using [CoroutineScope]. + * + * @property closedCaptionsSettings The configuration that defines how closed captions are managed, + * including auto-dismiss behavior, maximum number of captions to retain, and dismiss time. + */ + +class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSettings = ClosedCaptionsSettings()) { + + /** + * Holds the current list of closed captions. This list is updated dynamically + * and contains at most [ClosedCaptionsSettings.maxVisibleCaptions] captions. + */ + + private val _closedCaptions: MutableStateFlow> = + MutableStateFlow(emptyList()) + val closedCaptions: StateFlow> = _closedCaptions.asStateFlow() + + private val _ccMode = + MutableStateFlow(ClosedCaptionMode.Disabled) + val ccMode: StateFlow = _ccMode.asStateFlow() + + /** + * Tracks whether closed captioning is currently active for the call. + * True if captioning is ongoing, false otherwise. + */ + private val _closedCaptioning: MutableStateFlow = MutableStateFlow(false) + val closedCaptioning: StateFlow = _closedCaptioning + + /** + * Manages the job responsible for automatically removing closed captions after a delay. + */ + private var removalJob: Job? = null + private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + + /** + * Ensures thread-safe updates to the list of closed captions. + */ + private val mutex = Mutex() + + /** + * Updates the current configuration for the closed captions manager. + * + * @param closedCaptionsSettings The new configuration to apply. This affects behavior such as auto-dismiss + * and the number of captions retained. + */ + fun updateClosedCaptionsSettings(closedCaptionsSettings: ClosedCaptionsSettings) { + this.closedCaptionsSettings = closedCaptionsSettings + } + + /** + * Handles updates from the call response to determine the availability and state + * of closed captions. + * + * @param callResponse The response containing transcription and caption settings for the call. + */ + fun handleCallUpdate(callResponse: CallResponse) { + _closedCaptioning.value = callResponse.captioning + _ccMode.value = callResponse.settings.transcription.closedCaptionMode + } + + /** + * Processes incoming events related to closed captions, such as new captions being added, + * captioning starting, or captioning ending. + * + * @param videoEvent The event containing closed captioning information. + */ + fun handleEvent(videoEvent: VideoEvent) { + when (videoEvent) { + is ClosedCaptionEvent -> { + addCaption(videoEvent) + _closedCaptioning.value = true + } + + is ClosedCaptionStartedEvent -> { + _closedCaptioning.value = true + } + + is ClosedCaptionEndedEvent -> { + _closedCaptioning.value = false + } + } + } + + /** + * Adds a new caption to the list and manages the auto-dismiss logic. + * + * @param event The event containing the closed caption data to add. + */ + private fun addCaption(event: ClosedCaptionEvent) { + scope.launch { + mutex.withLock { + // Add the caption and keep the latest 3 + _closedCaptions.value = + (_closedCaptions.value + event.closedCaption).takeLast(closedCaptionsSettings.maxVisibleCaptions) + } + + if (closedCaptionsSettings.autoDismissCaptions) { + removalJob?.cancel() + scheduleRemoval() + } + } + } + + /** + * Schedules the removal of the oldest caption after the specified [ClosedCaptionsSettings.visibilityDurationMs]. + * + */ + private fun scheduleRemoval() { + removalJob = scope.launch { + delay(closedCaptionsSettings.visibilityDurationMs) + mutex.withLock { + if (_closedCaptions.value.isNotEmpty()) { + _closedCaptions.value = + _closedCaptions.value.drop(1) // Remove the oldest caption + } + } + if (_closedCaptions.value.isNotEmpty()) { + scheduleRemoval() // Continue scheduling removal for remaining captions + } + } + } +} diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt new file mode 100644 index 00000000000..929d6ea5ffb --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.getstream.video.android.core.closedcaptions + +private const val DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS = 2700L + +/** + * Configuration for managing closed captions in the [ClosedCaptionManager]. + * + * @param visibilityDurationMs The duration (in milliseconds) after which captions will be automatically removed. + * Set to [DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS] by default. + * + * @param autoDismissCaptions Determines whether closed captions should be automatically dismissed after a delay. + * If set to `false`, captions will remain visible indefinitely. + * + * @param maxVisibleCaptions The maximum number of closed captions to retain in the [ClosedCaptionManager.closedCaptions] flow. + * Must be greater than or equal to [io.getstream.video.android.compose.ui.components.closedcaptions.ClosedCaptionsThemeConfig.maxVisibleCaptions] + * to ensure the UI has sufficient data to render. + * + */ + +data class ClosedCaptionsSettings( + val visibilityDurationMs: Long = DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS, + val autoDismissCaptions: Boolean = true, + val maxVisibleCaptions: Int = 2, // Default to keep the latest 2 captions +) diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/socket/common/parser2/MoshiVideoParser.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/socket/common/parser2/MoshiVideoParser.kt index 344fbaf0bdc..557c0a797fd 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/socket/common/parser2/MoshiVideoParser.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/socket/common/parser2/MoshiVideoParser.kt @@ -108,6 +108,11 @@ internal class MoshiVideoParser : VideoParser { org.openapitools.client.models.TranscriptionSettingsResponse.Mode.ModeAdapter(), ), ) + .add( + lenientAdapter( + org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode.ClosedCaptionModeAdapter(), + ), + ) .add( lenientAdapter( org.openapitools.client.models.VideoSettingsRequest.CameraFacing.CameraFacingAdapter(), diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/apis/ProductvideoApi.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/apis/ProductvideoApi.kt index ed49d0218bc..472038428b6 100644 --- a/stream-video-android-core/src/main/kotlin/org/openapitools/client/apis/ProductvideoApi.kt +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/apis/ProductvideoApi.kt @@ -72,11 +72,13 @@ import org.openapitools.client.models.SendCallEventRequest import org.openapitools.client.models.SendCallEventResponse import org.openapitools.client.models.SendReactionRequest import org.openapitools.client.models.SendReactionResponse +import org.openapitools.client.models.StartClosedCaptionResponse import org.openapitools.client.models.StartHLSBroadcastingResponse import org.openapitools.client.models.StartRecordingRequest import org.openapitools.client.models.StartRecordingResponse import org.openapitools.client.models.StartTranscriptionRequest import org.openapitools.client.models.StartTranscriptionResponse +import org.openapitools.client.models.StopClosedCaptionResponse import org.openapitools.client.models.StopHLSBroadcastingResponse import org.openapitools.client.models.StopLiveResponse import org.openapitools.client.models.StopRecordingResponse @@ -854,4 +856,39 @@ interface ProductvideoApi { @Body unpinRequest: UnpinRequest ): UnpinResponse + + /** + * Start CC for a call + * Responses: + * - 201: Successful response + * - 400: Bad request + * - 429: Too many requests + * + * @param type + * @param id + * @return [StartClosedCaptionResponse] + */ + @POST("/video/call/{type}/{id}/start_closed_captions") + suspend fun startClosedCaptions( + @Path("type") type: String, + @Path("id") id: String, + ): StartClosedCaptionResponse + + + /** + * Stops CC for a call + * Responses: + * - 201: Successful response + * - 400: Bad request + * - 429: Too many requests + * + * @param type + * @param id + * @return [StopClosedCaptionResponse] + */ + @POST("/video/call/{type}/{id}/stop_closed_captions") + suspend fun stopClosedCaptions( + @Path("type") type: String, + @Path("id") id: String, + ): StopClosedCaptionResponse } diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/infrastructure/Serializer.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/infrastructure/Serializer.kt index 6cd2888d287..b837264b02f 100644 --- a/stream-video-android-core/src/main/kotlin/org/openapitools/client/infrastructure/Serializer.kt +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/infrastructure/Serializer.kt @@ -44,6 +44,7 @@ object Serializer { .add(org.openapitools.client.models.RecordSettingsRequest.Quality.QualityAdapter()) .add(org.openapitools.client.models.TranscriptionSettingsRequest.Mode.ModeAdapter()) .add(org.openapitools.client.models.TranscriptionSettingsResponse.Mode.ModeAdapter()) + .add(org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode.ClosedCaptionModeAdapter()) .add(org.openapitools.client.models.VideoSettingsRequest.CameraFacing.CameraFacingAdapter()) .add(org.openapitools.client.models.VideoSettingsResponse.CameraFacing.CameraFacingAdapter()) .add(BigDecimalAdapter()) diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/CallResponse.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/CallResponse.kt index b85d40dc652..b6fb4b3c5e3 100644 --- a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/CallResponse.kt +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/CallResponse.kt @@ -109,6 +109,9 @@ data class CallResponse ( @Json(name = "settings") val settings: CallSettingsResponse, + @Json(name = "captioning") + val captioning: kotlin.Boolean, + @Json(name = "transcribing") val transcribing: kotlin.Boolean, diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionEndedEvent.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionEndedEvent.kt new file mode 100644 index 00000000000..46993de2ec5 --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionEndedEvent.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:Suppress( + "ArrayInDataClass", + "EnumEntryName", + "RemoveRedundantQualifierName", + "UnusedImport" +) + +package org.openapitools.client.models + + +import com.squareup.moshi.Json + +/** + * This event is sent when closed captions are ended sent in a call, clients can use this to show the closed captions are stopped in the call screen + * + * @param callCid + * @param createdAt + * @param type The type of event: \"call.closed_caption_ended\" in this case + */ + + +data class ClosedCaptionEndedEvent ( + + @Json(name = "call_cid") + val callCid: kotlin.String, + + @Json(name = "created_at") + val createdAt: org.threeten.bp.OffsetDateTime, + + /* The type of event: \"call.closed_caption_ended\" in this case */ + @Json(name = "type") + val type: kotlin.String = "call.closed_caption_ended" + +) : VideoEvent(), WSCallEvent { + + override fun getCallCID(): String { + return callCid + } + + override fun getEventType(): String { + return type + } +} diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionStartedEvent.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionStartedEvent.kt new file mode 100644 index 00000000000..bd9eed71b22 --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/ClosedCaptionStartedEvent.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@file:Suppress( + "ArrayInDataClass", + "EnumEntryName", + "RemoveRedundantQualifierName", + "UnusedImport" +) + +package org.openapitools.client.models + + +import com.squareup.moshi.Json + +/** + * This event is sent when closed captions are started sent in a call, clients can use this to show the closed captions has been started + * + * @param callCid + * @param createdAt + * @param type The type of event: \"call.closed_captions_started\" in this case + */ + + +data class ClosedCaptionStartedEvent ( + + @Json(name = "call_cid") + val callCid: kotlin.String, + + @Json(name = "created_at") + val createdAt: org.threeten.bp.OffsetDateTime, + + /* The type of event: \"call.closed_captions_started\" in this case */ + @Json(name = "type") + val type: kotlin.String = "call.closed_captions_started" + +) : VideoEvent(), WSCallEvent { + + override fun getCallCID(): String { + return callCid + } + + override fun getEventType(): String { + return type + } +} diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StartClosedCaptionResponse.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StartClosedCaptionResponse.kt new file mode 100644 index 00000000000..8379af036b9 --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StartClosedCaptionResponse.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openapitools.client.models + +import com.squareup.moshi.Json + +data class StartClosedCaptionResponse(/* Duration of the request in human-readable format */ + @Json(name = "duration") + val duration: kotlin.String +) diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StopClosedCaptionResponse.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StopClosedCaptionResponse.kt new file mode 100644 index 00000000000..914a13c8e6d --- /dev/null +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/StopClosedCaptionResponse.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved. + * + * Licensed under the Stream License; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/GetStream/stream-video-android/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openapitools.client.models + +import com.squareup.moshi.Json + +data class StopClosedCaptionResponse(/* Duration of the request in human-readable format */ + @Json(name = "duration") + val duration: kotlin.String +) diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/TranscriptionSettingsResponse.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/TranscriptionSettingsResponse.kt index bb6cd009e8b..7acea589468 100644 --- a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/TranscriptionSettingsResponse.kt +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/TranscriptionSettingsResponse.kt @@ -47,7 +47,7 @@ import org.openapitools.client.infrastructure.Serializer data class TranscriptionSettingsResponse ( @Json(name = "closed_caption_mode") - val closedCaptionMode: kotlin.String, + val closedCaptionMode: TranscriptionSettingsResponse.ClosedCaptionMode, @Json(name = "languages") val languages: kotlin.collections.List, @@ -96,6 +96,44 @@ data class TranscriptionSettingsResponse ( } } + /** + * + * + * Values: available,disabled,autoOn + */ + + sealed class ClosedCaptionMode(val value: kotlin.String) { + override fun toString(): String = value + + companion object { + fun fromString(s: kotlin.String): ClosedCaptionMode = when (s) { + "available" -> Available + "disabled" -> Disabled + "auto-on" -> AutoOn + else -> Unknown(s) + } + } + + object Available : ClosedCaptionMode("available") + object Disabled : ClosedCaptionMode("disabled") + object AutoOn : ClosedCaptionMode("auto-on") + data class Unknown(val unknownValue: kotlin.String) : ClosedCaptionMode(unknownValue) + + class ClosedCaptionModeAdapter : JsonAdapter() { + @FromJson + override fun fromJson(reader: JsonReader): ClosedCaptionMode? { + val s = reader.nextString() ?: return null + return fromString(s) + } + + @ToJson + override fun toJson(writer: JsonWriter, value: ClosedCaptionMode?) { + writer.value(value?.value) + } + } + } + + } diff --git a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/VideoEvent.kt b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/VideoEvent.kt index a52e6bb28aa..a18df5c3d6b 100644 --- a/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/VideoEvent.kt +++ b/stream-video-android-core/src/main/kotlin/org/openapitools/client/models/VideoEvent.kt @@ -117,6 +117,8 @@ class VideoEventAdapter : JsonAdapter() { return when (type) { "call.accepted" -> CallAcceptedEvent::class.java "call.blocked_user" -> BlockedUserEvent::class.java + "call.closed_captions_started" -> ClosedCaptionStartedEvent::class.java + "call.closed_captions_stopped" -> ClosedCaptionEndedEvent::class.java "call.closed_caption" -> ClosedCaptionEvent::class.java "call.created" -> CallCreatedEvent::class.java "call.deleted" -> CallDeletedEvent::class.java diff --git a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/base/IntegrationTestBase.kt b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/base/IntegrationTestBase.kt index 4a52051030f..2ee2a770b0b 100644 --- a/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/base/IntegrationTestBase.kt +++ b/stream-video-android-core/src/test/kotlin/io/getstream/video/android/core/base/IntegrationTestBase.kt @@ -234,7 +234,7 @@ internal fun Call.toResponse(createdBy: UserResponse): CallResponse { ), screensharing = ScreensharingSettingsResponse(false, false), transcription = TranscriptionSettingsResponse( - "test", + TranscriptionSettingsResponse.ClosedCaptionMode.Available, emptyList(), TranscriptionSettingsResponse.Mode.Available, ), @@ -269,6 +269,7 @@ internal fun Call.toResponse(createdBy: UserResponse): CallResponse { settings = settings, egress = EgressResponse(false, emptyList(), null), updatedAt = now, + captioning = false, ) return response } diff --git a/stream-video-android-ui-compose/api/stream-video-android-ui-compose.api b/stream-video-android-ui-compose/api/stream-video-android-ui-compose.api index 017fbb75b1a..21f130adb82 100644 --- a/stream-video-android-ui-compose/api/stream-video-android-ui-compose.api +++ b/stream-video-android-ui-compose/api/stream-video-android-ui-compose.api @@ -1044,7 +1044,7 @@ public final class io/getstream/video/android/compose/ui/components/call/activec } public final class io/getstream/video/android/compose/ui/components/call/activecall/CallContentKt { - public static final fun CallContent (Lio/getstream/video/android/core/Call;Landroidx/compose/ui/Modifier;Lio/getstream/video/android/compose/ui/components/call/renderer/LayoutType;Lio/getstream/video/android/compose/permission/VideoPermissionsState;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function3;Lio/getstream/video/android/compose/ui/components/call/renderer/VideoRendererStyle;Lkotlin/jvm/functions/Function6;Lkotlin/jvm/functions/Function5;Lkotlin/jvm/functions/Function4;Lkotlin/jvm/functions/Function3;Lkotlin/jvm/functions/Function3;ZLkotlin/jvm/functions/Function3;ZLandroidx/compose/runtime/Composer;III)V + public static final fun CallContent (Lio/getstream/video/android/core/Call;Landroidx/compose/ui/Modifier;Lio/getstream/video/android/compose/ui/components/call/renderer/LayoutType;Lio/getstream/video/android/compose/permission/VideoPermissionsState;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function3;Lio/getstream/video/android/compose/ui/components/call/renderer/VideoRendererStyle;Lkotlin/jvm/functions/Function6;Lkotlin/jvm/functions/Function5;Lkotlin/jvm/functions/Function4;Lkotlin/jvm/functions/Function3;Lkotlin/jvm/functions/Function3;ZLkotlin/jvm/functions/Function3;ZLkotlin/jvm/functions/Function3;Landroidx/compose/runtime/Composer;III)V } public final class io/getstream/video/android/compose/ui/components/call/activecall/ComposableSingletons$AudioCallContentKt { @@ -1059,14 +1059,16 @@ public final class io/getstream/video/android/compose/ui/components/call/activec public static field lambda-1 Lkotlin/jvm/functions/Function6; public static field lambda-2 Lkotlin/jvm/functions/Function3; public static field lambda-3 Lkotlin/jvm/functions/Function3; - public static field lambda-4 Lkotlin/jvm/functions/Function2; + public static field lambda-4 Lkotlin/jvm/functions/Function3; public static field lambda-5 Lkotlin/jvm/functions/Function2; + public static field lambda-6 Lkotlin/jvm/functions/Function2; public fun ()V public final fun getLambda-1$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function6; public final fun getLambda-2$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function3; public final fun getLambda-3$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function3; - public final fun getLambda-4$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function2; + public final fun getLambda-4$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function3; public final fun getLambda-5$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function2; + public final fun getLambda-6$stream_video_android_ui_compose_release ()Lkotlin/jvm/functions/Function2; } public final class io/getstream/video/android/compose/ui/components/call/activecall/internal/ComposableSingletons$InviteUsersDialogKt { diff --git a/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/call/activecall/CallContent.kt b/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/call/activecall/CallContent.kt index bff570d7ab0..4ba97abea20 100644 --- a/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/call/activecall/CallContent.kt +++ b/stream-video-android-ui-compose/src/main/kotlin/io/getstream/video/android/compose/ui/components/call/activecall/CallContent.kt @@ -95,6 +95,7 @@ import io.getstream.video.android.mock.previewCall * @param controlsContent Content is shown that allows users to trigger different actions to control a joined call. * @param enableInPictureInPicture If the user has engaged in Picture-In-Picture mode. * @param pictureInPictureContent Content shown when the user enters Picture in Picture mode, if it's been enabled in the app. + * @param closedCaptionUi You can pass your composable lambda here to render Closed Captions */ @Composable public fun CallContent( @@ -149,6 +150,7 @@ public fun CallContent( enableInPictureInPicture: Boolean = true, pictureInPictureContent: @Composable (Call) -> Unit = { DefaultPictureInPictureContent(it) }, enableDiagnostics: Boolean = false, + closedCaptionUi: @Composable (Call) -> Unit = {}, ) { val context = LocalContext.current val orientation = LocalConfiguration.current.orientation @@ -231,6 +233,8 @@ public fun CallContent( showDiagnostics = false } } + + closedCaptionUi(call) }, ) } From 526c185eca0cb5c83528af1d415cd38dd49e01fd Mon Sep 17 00:00:00 2001 From: rahullohra Date: Mon, 23 Dec 2024 19:18:41 +0530 Subject: [PATCH 2/5] Change access modifier of closedCaptionManager to public --- stream-video-android-core/api/stream-video-android-core.api | 1 + .../src/main/kotlin/io/getstream/video/android/core/Call.kt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index dd47b5b8980..140a34d88ae 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -15,6 +15,7 @@ public final class io/getstream/video/android/core/Call { public final fun getAudioFilter ()Lio/getstream/video/android/core/call/audio/InputAudioFilter; public final fun getCamera ()Lio/getstream/video/android/core/CameraManager; public final fun getCid ()Ljava/lang/String; + public final fun getClosedCaptionManager ()Lio/getstream/video/android/core/closedcaptions/ClosedCaptionManager; public final fun getId ()Ljava/lang/String; public final fun getLocalMicrophoneAudioLevel ()Lkotlinx/coroutines/flow/StateFlow; public final fun getMicrophone ()Lio/getstream/video/android/core/MicrophoneManager; diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt index 8c49b9194af..39621563085 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt @@ -273,7 +273,7 @@ public class Call( * This [ClosedCaptionManager] is responsible for handling closed captions during the call. * This includes processing events related to closed captions and maintaining their state. */ - internal val closedCaptionManager = ClosedCaptionManager() + public val closedCaptionManager = ClosedCaptionManager() init { scope.launch { From 58f889c7516e22a1f577ae6a48f9e023c18522c2 Mon Sep 17 00:00:00 2001 From: rahullohra Date: Tue, 24 Dec 2024 11:27:51 +0530 Subject: [PATCH 3/5] Expose isClosedCaptioning and closedCaptions from Call State to follow SDK guidelines --- .../api/stream-video-android-core.api | 5 ++-- .../io/getstream/video/android/core/Call.kt | 9 +------ .../getstream/video/android/core/CallState.kt | 25 +++++++++++++++++-- .../closedcaptions/ClosedCaptionManager.kt | 15 +++++++++-- .../closedcaptions/ClosedCaptionsSettings.kt | 2 -- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index 140a34d88ae..3e5cec3c1ac 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -15,7 +15,6 @@ public final class io/getstream/video/android/core/Call { public final fun getAudioFilter ()Lio/getstream/video/android/core/call/audio/InputAudioFilter; public final fun getCamera ()Lio/getstream/video/android/core/CameraManager; public final fun getCid ()Ljava/lang/String; - public final fun getClosedCaptionManager ()Lio/getstream/video/android/core/closedcaptions/ClosedCaptionManager; public final fun getId ()Ljava/lang/String; public final fun getLocalMicrophoneAudioLevel ()Lkotlinx/coroutines/flow/StateFlow; public final fun getMicrophone ()Lio/getstream/video/android/core/MicrophoneManager; @@ -130,6 +129,7 @@ public final class io/getstream/video/android/core/CallState { public final fun getBlockedUsers ()Lkotlinx/coroutines/flow/StateFlow; public final fun getBroadcasting ()Lkotlinx/coroutines/flow/StateFlow; public final fun getCapabilitiesByRole ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getClosedCaptions ()Lkotlinx/coroutines/flow/StateFlow; public final fun getConnection ()Lkotlinx/coroutines/flow/StateFlow; public final fun getCreatedAt ()Lkotlinx/coroutines/flow/StateFlow; public final fun getCreatedBy ()Lkotlinx/coroutines/flow/StateFlow; @@ -180,6 +180,7 @@ public final class io/getstream/video/android/core/CallState { public final fun getUpdatedAt ()Lkotlinx/coroutines/flow/StateFlow; public final fun handleEvent (Lorg/openapitools/client/models/VideoEvent;)V public final fun hasPermission (Ljava/lang/String;)Lkotlinx/coroutines/flow/StateFlow; + public final fun isCaptioning ()Lkotlinx/coroutines/flow/StateFlow; public final fun isReconnecting ()Lkotlinx/coroutines/flow/StateFlow; public final fun markSpeakingAsMuted ()V public final fun pin (Ljava/lang/String;Ljava/lang/String;)V @@ -2895,9 +2896,7 @@ public final class io/getstream/video/android/core/closedcaptions/ClosedCaptionM public final fun getCcMode ()Lkotlinx/coroutines/flow/StateFlow; public final fun getClosedCaptioning ()Lkotlinx/coroutines/flow/StateFlow; public final fun getClosedCaptions ()Lkotlinx/coroutines/flow/StateFlow; - public final fun handleCallUpdate (Lorg/openapitools/client/models/CallResponse;)V public final fun handleEvent (Lorg/openapitools/client/models/VideoEvent;)V - public final fun updateClosedCaptionsSettings (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;)V } public final class io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt index 39621563085..2f67af8f84b 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/Call.kt @@ -32,7 +32,6 @@ import io.getstream.video.android.core.call.audio.InputAudioFilter import io.getstream.video.android.core.call.utils.SoundInputProcessor import io.getstream.video.android.core.call.video.VideoFilter import io.getstream.video.android.core.call.video.YuvFrame -import io.getstream.video.android.core.closedcaptions.ClosedCaptionManager import io.getstream.video.android.core.closedcaptions.ClosedCaptionsSettings import io.getstream.video.android.core.events.GoAwayEvent import io.getstream.video.android.core.events.JoinCallResponseEvent @@ -269,12 +268,6 @@ public class Call( private var sfuListener: Job? = null private var sfuEvents: Job? = null - /** - * This [ClosedCaptionManager] is responsible for handling closed captions during the call. - * This includes processing events related to closed captions and maintaining their state. - */ - public val closedCaptionManager = ClosedCaptionManager() - init { scope.launch { soundInputProcessor.currentAudioLevel.collect { @@ -1309,7 +1302,7 @@ public class Call( } fun updateClosedCaptionsSettings(closedCaptionsSettings: ClosedCaptionsSettings) { - closedCaptionManager.updateClosedCaptionsSettings(closedCaptionsSettings) + state.closedCaptionManager.updateClosedCaptionsSettings(closedCaptionsSettings) } /** diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt index 32a17bccdf2..6472db31b81 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt @@ -20,6 +20,8 @@ import android.util.Log import androidx.compose.runtime.Stable import io.getstream.log.taggedLogger import io.getstream.video.android.core.call.RtcSession +import io.getstream.video.android.core.closedcaptions.ClosedCaptionManager +import io.getstream.video.android.core.closedcaptions.ClosedCaptionsSettings import io.getstream.video.android.core.events.AudioLevelChangedEvent import io.getstream.video.android.core.events.ChangePublishQualityEvent import io.getstream.video.android.core.events.ConnectionQualityChangeEvent @@ -72,6 +74,7 @@ import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import org.openapitools.client.models.BlockedUserEvent import org.openapitools.client.models.CallAcceptedEvent +import org.openapitools.client.models.CallClosedCaption import org.openapitools.client.models.CallCreatedEvent import org.openapitools.client.models.CallEndedEvent import org.openapitools.client.models.CallIngressResponse @@ -576,6 +579,24 @@ public class CallState( internal var acceptedOnThisDevice: Boolean = false + /** + * This [ClosedCaptionManager] is responsible for handling closed captions during the call. + * This includes processing events related to closed captions and maintaining their state. + */ + internal val closedCaptionManager = ClosedCaptionManager() + + /** + * Tracks whether closed captioning is currently active for the call. + * True if captioning is ongoing, false otherwise. + */ + public val isCaptioning: StateFlow = closedCaptionManager.closedCaptioning + + /** + * Holds the current list of closed captions. This list is updated dynamically + * and contains at most [ClosedCaptionsSettings.maxVisibleCaptions] captions. + */ + public val closedCaptions: StateFlow> = closedCaptionManager.closedCaptions + fun handleEvent(event: VideoEvent) { logger.d { "Updating call state with event ${event::class.java}" } when (event) { @@ -957,7 +978,7 @@ public class CallState( is ClosedCaptionEvent, is ClosedCaptionEndedEvent, -> - call.closedCaptionManager.handleEvent(event) + closedCaptionManager.handleEvent(event) } } @@ -1253,7 +1274,7 @@ public class CallState( _team.value = response.team updateRingingState() - call.closedCaptionManager.handleCallUpdate(response) + closedCaptionManager.handleCallUpdate(response) } fun updateFromResponse(response: GetOrCreateCallResponse) { diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt index d1bef3b3a96..fe10e1697a8 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt @@ -57,6 +57,17 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet MutableStateFlow(emptyList()) val closedCaptions: StateFlow> = _closedCaptions.asStateFlow() + /** + * Holds the current closed caption mode for the video call. This object contains information about closed + * captioning feature availability. This state is updated dynamically based on the server's transcription + * setting which is [org.openapitools.client.models.TranscriptionSettingsResponse.closedCaptionMode] + * + * Possible values: + * - [ClosedCaptionMode.Available]: Closed captions are available and can be enabled. + * - [ClosedCaptionMode.Disabled]: Closed captions are explicitly disabled. + * - [ClosedCaptionMode.AutoOn]: Closed captions are automatically enabled as soon as user joins the call + * - [ClosedCaptionMode.Unknown]: Represents an unrecognized or unsupported mode. + */ private val _ccMode = MutableStateFlow(ClosedCaptionMode.Disabled) val ccMode: StateFlow = _ccMode.asStateFlow() @@ -85,7 +96,7 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet * @param closedCaptionsSettings The new configuration to apply. This affects behavior such as auto-dismiss * and the number of captions retained. */ - fun updateClosedCaptionsSettings(closedCaptionsSettings: ClosedCaptionsSettings) { + internal fun updateClosedCaptionsSettings(closedCaptionsSettings: ClosedCaptionsSettings) { this.closedCaptionsSettings = closedCaptionsSettings } @@ -95,7 +106,7 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet * * @param callResponse The response containing transcription and caption settings for the call. */ - fun handleCallUpdate(callResponse: CallResponse) { + internal fun handleCallUpdate(callResponse: CallResponse) { _closedCaptioning.value = callResponse.captioning _ccMode.value = callResponse.settings.transcription.closedCaptionMode } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt index 929d6ea5ffb..74f79f73d87 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt @@ -28,8 +28,6 @@ private const val DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS = 2700L * If set to `false`, captions will remain visible indefinitely. * * @param maxVisibleCaptions The maximum number of closed captions to retain in the [ClosedCaptionManager.closedCaptions] flow. - * Must be greater than or equal to [io.getstream.video.android.compose.ui.components.closedcaptions.ClosedCaptionsThemeConfig.maxVisibleCaptions] - * to ensure the UI has sufficient data to render. * */ From bb1b14f3e9d8dd6031b48ee5933b698bf4a224dd Mon Sep 17 00:00:00 2001 From: rahullohra Date: Tue, 24 Dec 2024 11:34:59 +0530 Subject: [PATCH 4/5] Expose ccMode from Call State --- .../api/stream-video-android-core.api | 1 + .../io/getstream/video/android/core/CallState.kt | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index 3e5cec3c1ac..bb50dd227a6 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -129,6 +129,7 @@ public final class io/getstream/video/android/core/CallState { public final fun getBlockedUsers ()Lkotlinx/coroutines/flow/StateFlow; public final fun getBroadcasting ()Lkotlinx/coroutines/flow/StateFlow; public final fun getCapabilitiesByRole ()Lkotlinx/coroutines/flow/StateFlow; + public final fun getCcMode ()Lkotlinx/coroutines/flow/StateFlow; public final fun getClosedCaptions ()Lkotlinx/coroutines/flow/StateFlow; public final fun getConnection ()Lkotlinx/coroutines/flow/StateFlow; public final fun getCreatedAt ()Lkotlinx/coroutines/flow/StateFlow; diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt index 6472db31b81..19e66398894 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/CallState.kt @@ -121,6 +121,7 @@ import org.openapitools.client.models.QueryCallMembersResponse import org.openapitools.client.models.ReactionResponse import org.openapitools.client.models.StartHLSBroadcastingResponse import org.openapitools.client.models.StopLiveResponse +import org.openapitools.client.models.TranscriptionSettingsResponse.ClosedCaptionMode import org.openapitools.client.models.UnblockedUserEvent import org.openapitools.client.models.UpdateCallResponse import org.openapitools.client.models.UpdatedCallPermissionsEvent @@ -597,6 +598,19 @@ public class CallState( */ public val closedCaptions: StateFlow> = closedCaptionManager.closedCaptions + /** + * Holds the current closed caption mode for the video call. This object contains information about closed + * captioning feature availability. This state is updated dynamically based on the server's transcription + * setting which is [org.openapitools.client.models.TranscriptionSettingsResponse.closedCaptionMode] + * + * Possible values: + * - [ClosedCaptionMode.Available]: Closed captions are available and can be enabled. + * - [ClosedCaptionMode.Disabled]: Closed captions are explicitly disabled. + * - [ClosedCaptionMode.AutoOn]: Closed captions are automatically enabled as soon as user joins the call + * - [ClosedCaptionMode.Unknown]: Represents an unrecognized or unsupported mode. + */ + val ccMode: StateFlow = closedCaptionManager.ccMode + fun handleEvent(event: VideoEvent) { logger.d { "Updating call state with event ${event::class.java}" } when (event) { From 3772869783518b10fc6e39f6131d4aed3235f0bb Mon Sep 17 00:00:00 2001 From: rahullohra Date: Tue, 24 Dec 2024 17:43:02 +0530 Subject: [PATCH 5/5] Add logic for deduplication of closed captions --- .../api/stream-video-android-core.api | 21 +++++- .../closedcaptions/ClosedCaptionManager.kt | 67 ++++++++++++++++--- .../closedcaptions/ClosedCaptionsSettings.kt | 19 ++++++ 3 files changed, 97 insertions(+), 10 deletions(-) diff --git a/stream-video-android-core/api/stream-video-android-core.api b/stream-video-android-core/api/stream-video-android-core.api index bb50dd227a6..39f0efa37ba 100644 --- a/stream-video-android-core/api/stream-video-android-core.api +++ b/stream-video-android-core/api/stream-video-android-core.api @@ -2890,10 +2890,27 @@ public final class io/getstream/video/android/core/call/video/YuvFrame { public final fun bitmapFromVideoFrame (Lorg/webrtc/VideoFrame;)Landroid/graphics/Bitmap; } +public final class io/getstream/video/android/core/closedcaptions/ClosedCaptionDeduplicationConfig { + public fun ()V + public fun (JZI)V + public synthetic fun (JZIILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()J + public final fun component2 ()Z + public final fun component3 ()I + public final fun copy (JZI)Lio/getstream/video/android/core/closedcaptions/ClosedCaptionDeduplicationConfig; + public static synthetic fun copy$default (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionDeduplicationConfig;JZIILjava/lang/Object;)Lio/getstream/video/android/core/closedcaptions/ClosedCaptionDeduplicationConfig; + public fun equals (Ljava/lang/Object;)Z + public final fun getAutoRemoveDuplicateCaptions ()Z + public final fun getCaptionSplitFactor ()I + public final fun getDuplicateCleanupFrequencyMs ()J + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + public final class io/getstream/video/android/core/closedcaptions/ClosedCaptionManager { public fun ()V - public fun (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;)V - public synthetic fun (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public fun (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;Lio/getstream/video/android/core/closedcaptions/ClosedCaptionDeduplicationConfig;)V + public synthetic fun (Lio/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings;Lio/getstream/video/android/core/closedcaptions/ClosedCaptionDeduplicationConfig;ILkotlin/jvm/internal/DefaultConstructorMarker;)V public final fun getCcMode ()Lkotlinx/coroutines/flow/StateFlow; public final fun getClosedCaptioning ()Lkotlinx/coroutines/flow/StateFlow; public final fun getClosedCaptions ()Lkotlinx/coroutines/flow/StateFlow; diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt index fe10e1697a8..d44bae30c7d 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionManager.kt @@ -39,14 +39,18 @@ import org.openapitools.client.models.VideoEvent * Manages the lifecycle, state, and configuration of closed captions for a video call. * * The [ClosedCaptionManager] is responsible for handling caption updates, maintaining caption states, - * and auto-removing captions based on the provided [ClosedCaptionsSettings]. It ensures thread-safe + * auto-removing and deduplicating captions based on the provided [ClosedCaptionsSettings] and [ClosedCaptionDeduplicationConfig]. It ensures thread-safe * operations using a [Mutex] and manages jobs for scheduled caption removal using [CoroutineScope]. * * @property closedCaptionsSettings The configuration that defines how closed captions are managed, * including auto-dismiss behavior, maximum number of captions to retain, and dismiss time. */ -class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSettings = ClosedCaptionsSettings()) { +class ClosedCaptionManager( + private var closedCaptionsSettings: ClosedCaptionsSettings = ClosedCaptionsSettings(), + private var closedCaptionDeduplicationConfig: ClosedCaptionDeduplicationConfig = + ClosedCaptionDeduplicationConfig(), +) { /** * Holds the current list of closed captions. This list is updated dynamically @@ -57,6 +61,16 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet MutableStateFlow(emptyList()) val closedCaptions: StateFlow> = _closedCaptions.asStateFlow() + /** + * A set to track unique keys for deduplication, preventing duplicate captions. + */ + private val seenKeys: MutableSet = mutableSetOf() + + /** + * A job to manage the periodic cleanup of outdated or excess keys in seenKeys. + */ + private var seenKeysCleanupJob: Job? = null + /** * Holds the current closed caption mode for the video call. This object contains information about closed * captioning feature availability. This state is updated dynamically based on the server's transcription @@ -82,7 +96,7 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet /** * Manages the job responsible for automatically removing closed captions after a delay. */ - private var removalJob: Job? = null + private var removeCaptionsJob: Job? = null private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) /** @@ -142,15 +156,25 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet private fun addCaption(event: ClosedCaptionEvent) { scope.launch { mutex.withLock { - // Add the caption and keep the latest 3 - _closedCaptions.value = - (_closedCaptions.value + event.closedCaption).takeLast(closedCaptionsSettings.maxVisibleCaptions) + val uniqueKey = "${event.closedCaption.speakerId}/${event.closedCaption.startTime.toEpochSecond()}" + + if (uniqueKey !in seenKeys) { + // Add the caption and keep the latest 2 + _closedCaptions.value = + (_closedCaptions.value + event.closedCaption).takeLast(closedCaptionsSettings.maxVisibleCaptions) + + seenKeys.add(uniqueKey) + } } if (closedCaptionsSettings.autoDismissCaptions) { - removalJob?.cancel() + removeCaptionsJob?.cancel() scheduleRemoval() } + + if (closedCaptionDeduplicationConfig.autoRemoveDuplicateCaptions) { + startCleanupTask() + } } } @@ -159,7 +183,7 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet * */ private fun scheduleRemoval() { - removalJob = scope.launch { + removeCaptionsJob = scope.launch { delay(closedCaptionsSettings.visibilityDurationMs) mutex.withLock { if (_closedCaptions.value.isNotEmpty()) { @@ -172,4 +196,31 @@ class ClosedCaptionManager(private var closedCaptionsSettings: ClosedCaptionsSet } } } + + /** + * Starts cleanup task to empty [seenKeys] it will run after [ClosedCaptionDeduplicationConfig.duplicateCleanupFrequencyMs] + */ + private fun startCleanupTask() { + if (seenKeysCleanupJob?.isActive == true) return + + seenKeysCleanupJob = scope.launch { + while (_closedCaptions.value.isNotEmpty()) { + delay(closedCaptionDeduplicationConfig.duplicateCleanupFrequencyMs) + mutex.withLock { + cleanUpSeenKeys() + } + } + seenKeysCleanupJob?.cancel() + } + } + + /** + * Remove the seen keys based on [ClosedCaptionDeduplicationConfig.captionSplitFactor] + */ + private fun cleanUpSeenKeys() { + if (seenKeys.size > 1) { + val itemsToRemove = seenKeys.size / closedCaptionDeduplicationConfig.captionSplitFactor + seenKeys.removeAll(seenKeys.asSequence().take(itemsToRemove).toSet()) + } + } } diff --git a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt index 74f79f73d87..58bbf2083e2 100644 --- a/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt +++ b/stream-video-android-core/src/main/kotlin/io/getstream/video/android/core/closedcaptions/ClosedCaptionsSettings.kt @@ -17,6 +17,7 @@ package io.getstream.video.android.core.closedcaptions private const val DEFAULT_CAPTIONS_AUTO_DISMISS_TIME_MS = 2700L +private const val DEFAULT_DUPLICATE_CAPTIONS_AUTO_CLEANUP_TIME_MS = 30_000L // Every 30 seconds /** * Configuration for managing closed captions in the [ClosedCaptionManager]. @@ -36,3 +37,21 @@ data class ClosedCaptionsSettings( val autoDismissCaptions: Boolean = true, val maxVisibleCaptions: Int = 2, // Default to keep the latest 2 captions ) + +/** + * Configuration for managing deduplication of captions in the [ClosedCaptionManager]. + * + * @param duplicateCleanupFrequencyMs The duration (in milliseconds) after which [ClosedCaptionManager.seenKeys] will be automatically removed. + * Set to [DEFAULT_DUPLICATE_CAPTIONS_AUTO_CLEANUP_TIME_MS] by default. + * + * @param autoRemoveDuplicateCaptions Determines whether client sdk should be deduplicate closed captions or not + * + * @param captionSplitFactor Factor to determine how many items to clean (e.g., 2 means clean half) + * + */ + +data class ClosedCaptionDeduplicationConfig( + val duplicateCleanupFrequencyMs: Long = DEFAULT_DUPLICATE_CAPTIONS_AUTO_CLEANUP_TIME_MS, + val autoRemoveDuplicateCaptions: Boolean = true, + val captionSplitFactor: Int = 2, +)