Switch video rooms to spotlight layout when in PiP mode (#8912)

* Switch video rooms to spotlight layout when in PiP mode

* Add some comments

Author: robintown

src/stores/VideoChannelStore.ts

@@ -25,6 +25,7 @@ import defaultDispatcher from "../dispatcher/dispatcher";
 import { ActionPayload } from "../dispatcher/payloads";
 import { ElementWidgetActions } from "./widgets/ElementWidgetActions";
 import { WidgetMessagingStore, WidgetMessagingStoreEvent } from "./widgets/WidgetMessagingStore";
+import ActiveWidgetStore, { ActiveWidgetStoreEvent } from "./ActiveWidgetStore";
 import { STUCK_DEVICE_TIMEOUT_MS, getVideoChannel, addOurDevice, removeOurDevice } from "../utils/VideoChannelUtils";
 import { timeout } from "../utils/promise";
 import WidgetUtils from "../utils/WidgetUtils";
@@ -234,6 +235,8 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
         }
 
         this.connected = true;
+        ActiveWidgetStore.instance.on(ActiveWidgetStoreEvent.Dock, this.onDock);
+        ActiveWidgetStore.instance.on(ActiveWidgetStoreEvent.Undock, this.onUndock);
         this.room.on(RoomEvent.MyMembership, this.onMyMembership);
         window.addEventListener("beforeunload", this.setDisconnected);
 
@@ -264,8 +267,14 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
         const roomId = this.roomId;
         const room = this.room;
 
-        this.activeChannel.off(`action:${ElementWidgetActions.HangupCall}`, this.onHangup);
         this.activeChannel.off(`action:${ElementWidgetActions.CallParticipants}`, this.onParticipants);
+        this.activeChannel.off(`action:${ElementWidgetActions.MuteAudio}`, this.onMuteAudio);
+        this.activeChannel.off(`action:${ElementWidgetActions.UnmuteAudio}`, this.onUnmuteAudio);
+        this.activeChannel.off(`action:${ElementWidgetActions.MuteVideo}`, this.onMuteVideo);
+        this.activeChannel.off(`action:${ElementWidgetActions.UnmuteVideo}`, this.onUnmuteVideo);
+        this.activeChannel.off(`action:${ElementWidgetActions.HangupCall}`, this.onHangup);
+        ActiveWidgetStore.instance.off(ActiveWidgetStoreEvent.Dock, this.onDock);
+        ActiveWidgetStore.instance.off(ActiveWidgetStoreEvent.Undock, this.onUndock);
         room.off(RoomEvent.MyMembership, this.onMyMembership);
         window.removeEventListener("beforeunload", this.setDisconnected);
         clearInterval(this.resendDevicesTimer);
@@ -324,4 +333,15 @@ export default class VideoChannelStore extends AsyncStoreWithClient<null> {
     private onMyMembership = (room: Room, membership: string) => {
         if (membership !== "join") this.setDisconnected();
     };
+
+    private onDock = async () => {
+        // The widget is no longer a PiP, so let's restore the default layout
+        await this.activeChannel.transport.send(ElementWidgetActions.TileLayout, {});
+    };
+
+    private onUndock = async () => {
+        // The widget has become a PiP, so let's switch Jitsi to spotlight mode
+        // to only show the active speaker and economize on space
+        await this.activeChannel.transport.send(ElementWidgetActions.SpotlightLayout, {});
+    };
 }

src/stores/widgets/ElementWidgetActions.ts

@@ -19,6 +19,8 @@ import { IWidgetApiRequest } from "matrix-widget-api";
 export enum ElementWidgetActions {
     ClientReady = "im.vector.ready",
     WidgetReady = "io.element.widget_ready",
+
+    // All of these actions are currently specific to Jitsi
     JoinCall = "io.element.join",
     HangupCall = "im.vector.hangup",
     ForceHangupCall = "io.element.force_hangup",
@@ -28,6 +30,10 @@ export enum ElementWidgetActions {
     MuteVideo = "io.element.mute_video",
     UnmuteVideo = "io.element.unmute_video",
     StartLiveStream = "im.vector.start_live_stream",
+    // Actions for switching layouts
+    TileLayout = "io.element.tile_layout",
+    SpotlightLayout = "io.element.spotlight_layout",
+
     OpenIntegrationManager = "integration_manager_open",
 
     /**

test/stores/VideoChannelStore-test.ts

@@ -14,14 +14,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { mocked } from "jest-mock";
-import { Widget, ClientWidgetApi, MatrixWidgetType, IWidgetApiRequest } from "matrix-widget-api";
+import { mocked, Mocked } from "jest-mock";
+import {
+    Widget,
+    ClientWidgetApi,
+    MatrixWidgetType,
+    WidgetApiAction,
+    IWidgetApiRequest,
+    IWidgetApiRequestData,
+} from "matrix-widget-api";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 
 import { stubClient, setupAsyncStoreWithClient, mkRoom } from "../test-utils";
 import { MatrixClientPeg } from "../../src/MatrixClientPeg";
 import WidgetStore, { IApp } from "../../src/stores/WidgetStore";
 import { WidgetMessagingStore } from "../../src/stores/widgets/WidgetMessagingStore";
+import ActiveWidgetStore, { ActiveWidgetStoreEvent } from "../../src/stores/ActiveWidgetStore";
 import { ElementWidgetActions } from "../../src/stores/widgets/ElementWidgetActions";
 import { VIDEO_CHANNEL_MEMBER, STUCK_DEVICE_TIMEOUT_MS } from "../../src/utils/VideoChannelUtils";
 import VideoChannelStore, { VideoChannelEvent } from "../../src/stores/VideoChannelStore";
@@ -43,22 +51,19 @@ describe("VideoChannelStore", () => {
     } as IApp;
 
     // Set up mocks to simulate the remote end of the widget API
-    let messageSent: Promise<void>;
-    let messageSendMock: () => void;
+    let sendMock: (action: WidgetApiAction, data: IWidgetApiRequestData) => void;
     let onMock: (action: string, listener: (ev: CustomEvent<IWidgetApiRequest>) => void) => void;
     let onceMock: (action: string, listener: (ev: CustomEvent<IWidgetApiRequest>) => void) => void;
     let messaging: ClientWidgetApi;
-    let cli: MatrixClient;
+    let cli: Mocked<MatrixClient>;
     beforeEach(() => {
         stubClient();
-        cli = MatrixClientPeg.get();
+        cli = mocked(MatrixClientPeg.get());
         setupAsyncStoreWithClient(WidgetMessagingStore.instance, cli);
         setupAsyncStoreWithClient(store, cli);
-        mocked(cli).getRoom.mockReturnValue(mkRoom(cli, "!1:example.org"));
+        cli.getRoom.mockReturnValue(mkRoom(cli, "!1:example.org"));
 
-        let resolveMessageSent: () => void;
-        messageSent = new Promise(resolve => resolveMessageSent = resolve);
-        messageSendMock = jest.fn().mockImplementation(() => resolveMessageSent());
+        sendMock = jest.fn();
         onMock = jest.fn();
         onceMock = jest.fn();
 
@@ -69,14 +74,19 @@ describe("VideoChannelStore", () => {
             stop: () => {},
             once: onceMock,
             transport: {
-                send: messageSendMock,
+                send: sendMock,
                 reply: () => {},
             },
         } as unknown as ClientWidgetApi;
     });
 
     afterEach(() => jest.useRealTimers());
 
+    const getRequest = <T extends IWidgetApiRequestData>(): Promise<[WidgetApiAction, T]> =>
+        new Promise<[WidgetApiAction, T]>(resolve => {
+            mocked(sendMock).mockImplementationOnce((action, data) => resolve([action, data as T]));
+        });
+
     const widgetReady = () => {
         // Tell the WidgetStore that the widget is ready
         const [, ready] = mocked(onceMock).mock.calls.find(([action]) =>
@@ -87,7 +97,7 @@ describe("VideoChannelStore", () => {
 
     const confirmConnect = async () => {
         // Wait for the store to contact the widget API
-        await messageSent;
+        await getRequest();
         // Then, locate the callback that will confirm the join
         const [, join] = mocked(onMock).mock.calls.find(([action]) =>
             action === `action:${ElementWidgetActions.JoinCall}`,
@@ -122,8 +132,9 @@ describe("VideoChannelStore", () => {
         expect(store.roomId).toBeFalsy();
         expect(store.connected).toEqual(false);
 
+        const connectConfirmed = confirmConnect();
         const connectPromise = store.connect("!1:example.org", null, null);
-        await confirmConnect();
+        await connectConfirmed;
         await expect(connectPromise).resolves.toBeUndefined();
         expect(store.roomId).toEqual("!1:example.org");
         expect(store.connected).toEqual(true);
@@ -135,7 +146,7 @@ describe("VideoChannelStore", () => {
             { devices: [cli.getDeviceId()], expires_ts: expect.any(Number) },
             cli.getUserId(),
         );
-        mocked(cli).sendStateEvent.mockClear();
+        cli.sendStateEvent.mockClear();
 
         // Our devices should be resent within the timeout period to prevent
         // the data from becoming stale
@@ -146,7 +157,7 @@ describe("VideoChannelStore", () => {
             { devices: [cli.getDeviceId()], expires_ts: expect.any(Number) },
             cli.getUserId(),
         );
-        mocked(cli).sendStateEvent.mockClear();
+        cli.sendStateEvent.mockClear();
 
         const disconnectPromise = store.disconnect();
         await confirmDisconnect();
@@ -165,10 +176,11 @@ describe("VideoChannelStore", () => {
     });
 
     it("waits for messaging when connecting", async () => {
+        const connectConfirmed = confirmConnect();
         const connectPromise = store.connect("!1:example.org", null, null);
         WidgetMessagingStore.instance.storeMessaging(widget, "!1:example.org", messaging);
         widgetReady();
-        await confirmConnect();
+        await connectConfirmed;
         await expect(connectPromise).resolves.toBeUndefined();
         expect(store.roomId).toEqual("!1:example.org");
         expect(store.connected).toEqual(true);
@@ -184,12 +196,30 @@ describe("VideoChannelStore", () => {
         expect(store.roomId).toBeFalsy();
         expect(store.connected).toEqual(false);
 
+        const requestPromise = getRequest();
         const connectPromise = store.connect("!1:example.org", null, null);
         // Wait for the store to contact the widget API, then stop the messaging
-        await messageSent;
+        await requestPromise;
         WidgetMessagingStore.instance.stopMessaging(widget, "!1:example.org");
         await expect(connectPromise).rejects.toBeDefined();
         expect(store.roomId).toBeFalsy();
         expect(store.connected).toEqual(false);
     });
+
+    it("switches to spotlight mode when the widget becomes a PiP", async () => {
+        WidgetMessagingStore.instance.storeMessaging(widget, "!1:example.org", messaging);
+        widgetReady();
+        confirmConnect();
+        await store.connect("!1:example.org", null, null);
+
+        const request = getRequest<IWidgetApiRequestData>();
+        ActiveWidgetStore.instance.emit(ActiveWidgetStoreEvent.Undock);
+        const [action, data] = await request;
+        expect(action).toEqual(ElementWidgetActions.SpotlightLayout);
+        expect(data).toEqual({});
+
+        store.disconnect();
+        await confirmDisconnect();
+        WidgetMessagingStore.instance.stopMessaging(widget, "!1:example.org");
+    });
 });