diff --git a/backend/dev_homeserver.yaml b/backend/dev_homeserver.yaml index 5abaf5196..fe89d95a4 100644 --- a/backend/dev_homeserver.yaml +++ b/backend/dev_homeserver.yaml @@ -38,6 +38,8 @@ experimental_features: # MSC4222 needed for syncv2 state_after. This allow clients to # correctly track the state of the room. msc4222_enabled: true + # sticky events for matrixRTC user state + msc4354_enabled: true # The maximum allowed duration by which sent events can be delayed, as # per MSC4140. Must be a positive value if set. Defaults to no diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 50498c7a4..e64ba80e9 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -88,7 +88,7 @@ services: synapse: hostname: homeserver - image: docker.io/matrixdotorg/synapse:latest + image: ghcr.io/element-hq/synapse:msc4354-5 pull_policy: always environment: - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml diff --git a/locales/en/app.json b/locales/en/app.json index 704f68ac0..71b087ac9 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -74,9 +74,12 @@ "matrix_id": "Matrix ID: {{id}}", "multi_sfu": "Multi-SFU media transport", "mute_all_audio": "Mute all audio (participants, reactions, join sounds)", + "prefer_sticky_events": { + "description": "Improves reliability of calls (requires homeserver support)", + "label": "Prefer sticky events" + }, "show_connection_stats": "Show connection statistics", "url_params": "URL parameters", - "use_new_membership_manager": "Use the new implementation of the call MembershipManager", "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key" }, "disconnected_banner": "Connectivity to the server has been lost.", diff --git a/package.json b/package.json index e02952a6a..cbcc5d036 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/home/useGroupCallRooms.ts b/src/home/useGroupCallRooms.ts index 8000e5857..977b59aba 100644 --- a/src/home/useGroupCallRooms.ts +++ b/src/home/useGroupCallRooms.ts @@ -20,7 +20,6 @@ import { MatrixRTCSessionManagerEvents, type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; -import { logger } from "matrix-js-sdk/lib/logger"; import { getKeyForRoom } from "../e2ee/sharedKeyManagement"; @@ -114,19 +113,49 @@ const roomIsJoinable = (room: Room): boolean => { } }; +/** + * Determines if a given room has call events in it, and therefore + * is likely to be a call room. + * @param room The Matrix room instance. + * @returns `true` if the room has call events. + */ const roomHasCallMembershipEvents = (room: Room): boolean => { - switch (room.getMyMembership()) { - case KnownMembership.Join: - return !!room - .getLiveTimeline() - .getState(EventTimeline.FORWARDS) - ?.events?.get(EventType.GroupCallMemberPrefix); - case KnownMembership.Knock: - // Assume that a room you've knocked on is able to hold calls + // Check our room membership first, to rule out any rooms + // we can't have a call in. + const myMembership = room.getMyMembership(); + if (myMembership === KnownMembership.Knock) { + // Assume that a room you've knocked on is able to hold calls + return true; + } else if (myMembership !== KnownMembership.Join) { + // Otherwise, non-joined rooms should never show up. + return false; + } + + // Legacy member state checks (cheaper to check.) + const timeline = room.getLiveTimeline(); + if ( + timeline + .getState(EventTimeline.FORWARDS) + ?.events?.has(EventType.GroupCallMemberPrefix) + ) { + return true; + } + + // Check for *active* calls using sticky events. + for (const sticky of room._unstable_getStickyEvents()) { + if (sticky.getType() === EventType.RTCMembership) { return true; - default: - return false; + } } + + // Otherwise, check recent event history to see if anyone had + // sent a call membership in here. + return timeline.getEvents().some( + (e) => + // Membership events only count if both of these are true + e.unstableStickyInfo && e.getType() === EventType.GroupCallMemberPrefix, + ); + // Otherwise, it's *unlikely* this room was ever a call. }; export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { @@ -140,24 +169,22 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] { .filter(roomHasCallMembershipEvents) .filter(roomIsJoinable); const sortedRooms = sortRooms(client, rooms); - Promise.all( - sortedRooms.map((room) => { - const session = client.matrixRTC.getRoomSession(room); - return { - roomAlias: room.getCanonicalAlias() ?? undefined, - roomName: room.name, - avatarUrl: room.getMxcAvatarUrl()!, - room, - session, - participants: session.memberships - .filter((m) => m.userId) - .map((m) => room.getMember(m.userId!)) - .filter((m) => m) as RoomMember[], - }; - }), - ) - .then((items) => setRooms(items)) - .catch(logger.error); + const items = sortedRooms.map((room) => { + const session = client.matrixRTC.getRoomSession(room); + return { + roomAlias: room.getCanonicalAlias() ?? undefined, + roomName: room.name, + avatarUrl: room.getMxcAvatarUrl()!, + room, + session, + participants: session.memberships + .filter((m) => m.sender) + .map((m) => room.getMember(m.sender!)) + .filter((m) => m) as RoomMember[], + }; + }); + + setRooms(items); } updateRooms(); diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index d229ec17d..0c0359700 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -70,10 +70,6 @@ import { UnknownCallError, } from "../utils/errors.ts"; import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx"; -import { - useNewMembershipManager as useNewMembershipManagerSetting, - useSetting, -} from "../settings/settings"; import { useTypedEventEmitter } from "../useEvents"; import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; import { useAppBarTitle } from "../AppBar.tsx"; @@ -186,7 +182,6 @@ export const GroupCallView: FC = ({ password: passwordFromUrl, } = useUrlParams(); const e2eeSystem = useRoomEncryptionSystem(room.roomId); - const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting); // Save the password once we start the groupCallView useEffect(() => { @@ -310,7 +305,6 @@ export const GroupCallView: FC = ({ mediaDevices, latestMuteStates, setJoined, - useNewMembershipManager, ]); // TODO refactor this + "joined" to just one callState diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index cf73dc1f1..543dc83f9 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -96,6 +96,7 @@ test("It joins the correct Session", async () => { { encryptMedia: true, useMultiSfu: USE_MUTI_SFU, + preferStickyEvents: false, }, ); @@ -111,7 +112,6 @@ test("It joins the correct Session", async () => { expect.objectContaining({ manageMediaKeys: true, useLegacyMemberEvents: false, - useNewMembershipManager: true, useExperimentalToDeviceTransport: false, }), ); @@ -197,6 +197,7 @@ test("It should not fail with configuration error if homeserver config has livek { encryptMedia: true, useMultiSfu: USE_MUTI_SFU, + preferStickyEvents: false, }, ); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index c50523394..ecbff79f0 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -99,12 +99,11 @@ export async function makeTransport( export interface EnterRTCSessionOptions { encryptMedia: boolean; - // TODO: remove this flag, the new membership manager is stable enough - useNewMembershipManager?: boolean; // TODO: remove this flag, to-device transport is stable enough now useExperimentalToDeviceTransport?: boolean; /** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */ - useMultiSfu?: boolean; + useMultiSfu: boolean; + preferStickyEvents: boolean; } /** @@ -116,20 +115,13 @@ export interface EnterRTCSessionOptions { export async function enterRTCSession( rtcSession: MatrixRTCSession, transport: LivekitTransport, - options: EnterRTCSessionOptions = { - encryptMedia: true, - useNewMembershipManager: true, - useExperimentalToDeviceTransport: false, - useMultiSfu: true, - }, -): Promise { - const { + { encryptMedia, - useNewMembershipManager = true, useExperimentalToDeviceTransport = false, - useMultiSfu = true, - } = options; - + useMultiSfu, + preferStickyEvents, + }: EnterRTCSessionOptions, +): Promise { PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); @@ -148,7 +140,6 @@ export async function enterRTCSession( { notificationType, callIntent, - useNewMembershipManager, manageMediaKeys: encryptMedia, ...(useDeviceSessionMemberEvents !== undefined && { useLegacyMemberEvents: !useDeviceSessionMemberEvents, @@ -164,6 +155,7 @@ export async function enterRTCSession( membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms, useExperimentalToDeviceTransport, + unstableSendStickyEvents: preferStickyEvents, }, ); if (widget) { diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 36c8a2e6c..63c5dabca 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -5,8 +5,20 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { type ChangeEvent, type FC, useCallback, useMemo } from "react"; +import { + type ChangeEvent, + type FC, + useCallback, + useEffect, + useMemo, + useState, +} from "react"; import { useTranslation } from "react-i18next"; +import { + UNSTABLE_MSC4354_STICKY_EVENTS, + type MatrixClient, +} from "matrix-js-sdk"; +import { logger } from "matrix-js-sdk/lib/logger"; import { FieldRow, InputField } from "../input/Input"; import { @@ -14,16 +26,16 @@ import { duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, showConnectionStats as showConnectionStatsSetting, - useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, multiSfu as multiSfuSetting, muteAllAudio as muteAllAudioSetting, alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, + preferStickyEvents as preferStickyEventsSetting, } from "./settings"; -import type { MatrixClient } from "matrix-js-sdk"; import type { Room as LivekitRoom } from "livekit-client"; import styles from "./DeveloperSettingsTab.module.css"; import { useUrlParams } from "../UrlParams"; + interface Props { client: MatrixClient; livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[]; @@ -36,12 +48,24 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { debugTileLayoutSetting, ); - const [showConnectionStats, setShowConnectionStats] = useSetting( - showConnectionStatsSetting, + const [stickyEventsSupported, setStickyEventsSupported] = useState(false); + useEffect(() => { + client + .doesServerSupportUnstableFeature(UNSTABLE_MSC4354_STICKY_EVENTS) + .then((result) => { + setStickyEventsSupported(result); + }) + .catch((ex) => { + logger.warn("Failed to check if sticky events are supported", ex); + }); + }, [client]); + + const [preferStickyEvents, setPreferStickyEvents] = useSetting( + preferStickyEventsSetting, ); - const [useNewMembershipManager, setNewMembershipManager] = useSetting( - useNewMembershipManagerSetting, + const [showConnectionStats, setShowConnectionStats] = useSetting( + showConnectionStatsSetting, ); const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting( @@ -128,29 +152,31 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { ): void => { - setShowConnectionStats(event.target.checked); + setPreferStickyEvents(event.target.checked); }, - [setShowConnectionStats], + [setPreferStickyEvents], )} /> ): void => { - setNewMembershipManager(event.target.checked); + setShowConnectionStats(event.target.checked); }, - [setNewMembershipManager], + [setShowConnectionStats], )} /> @@ -173,7 +199,9 @@ export const DeveloperSettingsTab: FC = ({ client, livekitRooms }) => { id="multiSfu" type="checkbox" label={t("developer_mode.multi_sfu")} - checked={multiSfu} + // If using sticky events we implicitly prefer use multi-sfu + checked={multiSfu || preferStickyEvents} + disabled={preferStickyEvents} onChange={useCallback( (event: ChangeEvent): void => { setMultiSfu(event.target.checked); diff --git a/src/settings/settings.ts b/src/settings/settings.ts index ef09162a2..790b7ae11 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -83,6 +83,11 @@ export const showConnectionStats = new Setting( false, ); +export const preferStickyEvents = new Setting( + "prefer-sticky-events", + false, +); + export const audioInput = new Setting( "audio-input", undefined, @@ -115,11 +120,6 @@ export const soundEffectVolume = new Setting( 0.5, ); -export const useNewMembershipManager = new Setting( - "new-membership-manager", - true, -); - export const useExperimentalToDeviceTransport = new Setting( "experimental-to-device-transport", true, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6b046b28b..8d4f3d234 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -91,6 +91,7 @@ import { duplicateTiles, multiSfu, playReactionsSound, + preferStickyEvents, showReactions, } from "../settings/settings"; import { isFirefox } from "../Platform"; @@ -262,23 +263,33 @@ export class CallViewModel extends ViewModel { /** * Lists the transports used by ourselves, plus all other MatrixRTC session * members. For completeness this also lists the preferred transport and - * whether we are in multi-SFU mode (because advertisedTransport$ wants to - * read them at the same time, and bundling data together when it might change - * together is what you have to do in RxJS to avoid reading inconsistent state - * or observing too many changes.) + * whether we are in multi-SFU mode or sticky events mode (because + * advertisedTransport$ wants to read them at the same time, and bundling data + * together when it might change together is what you have to do in RxJS to + * avoid reading inconsistent state or observing too many changes.) */ + // TODO-MULTI-SFU find a better name for this. with the addition of sticky events it's no longer just about transports. private readonly transports$: Behavior<{ local: Async; remote: { membership: CallMembership; transport: LivekitTransport }[]; preferred: Async; multiSfu: boolean; + preferStickyEvents: boolean; } | null> = this.scope.behavior( this.joined$.pipe( switchMap((joined) => joined ? combineLatest( - [this.preferredTransport$, this.memberships$, multiSfu.value$], - (preferred, memberships, multiSfu) => { + [ + this.preferredTransport$, + this.memberships$, + multiSfu.value$, + preferStickyEvents.value$, + ], + (preferred, memberships, preferMultiSfu, preferStickyEvents) => { + // Multi-SFU must be implicitly enabled when using sticky events + const multiSfu = preferStickyEvents || preferMultiSfu; + const oldestMembership = this.matrixRTCSession.getOldestMembership(); const remote = memberships.flatMap((m) => { @@ -289,6 +300,7 @@ export class CallViewModel extends ViewModel { ? [{ membership: m, transport: t }] : []; }); + let local = preferred; if (!multiSfu) { const oldest = this.matrixRTCSession.getOldestMembership(); @@ -299,6 +311,7 @@ export class CallViewModel extends ViewModel { local = ready(selection); } } + if (local.state === "error") { this._configError$.next( local.value instanceof ElementCallError @@ -306,7 +319,14 @@ export class CallViewModel extends ViewModel { : new UnknownCallError(local.value), ); } - return { local, remote, preferred, multiSfu }; + + return { + local, + remote, + preferred, + multiSfu, + preferStickyEvents, + }; }, ) : of(null), @@ -336,10 +356,11 @@ export class CallViewModel extends ViewModel { /** * The transport we should advertise in our MatrixRTC membership (plus whether - * it is a multi-SFU transport). + * it is a multi-SFU transport and whether we should use sticky events). */ private readonly advertisedTransport$: Behavior<{ multiSfu: boolean; + preferStickyEvents: boolean; transport: LivekitTransport; } | null> = this.scope.behavior( this.transports$.pipe( @@ -348,6 +369,7 @@ export class CallViewModel extends ViewModel { transports.preferred.state === "ready" ? { multiSfu: transports.multiSfu, + preferStickyEvents: transports.preferStickyEvents, // In non-multi-SFU mode we should always advertise the preferred // SFU to minimize the number of membership updates transport: transports.multiSfu @@ -358,6 +380,7 @@ export class CallViewModel extends ViewModel { ), distinctUntilChanged<{ multiSfu: boolean; + preferStickyEvents: boolean; transport: LivekitTransport; } | null>(deepCompare), ), @@ -1800,8 +1823,8 @@ export class CallViewModel extends ViewModel { await enterRTCSession(this.matrixRTCSession, advertised.transport, { encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE, useExperimentalToDeviceTransport: true, - useNewMembershipManager: true, useMultiSfu: advertised.multiSfu, + preferStickyEvents: advertised.preferStickyEvents, }); } catch (e) { logger.error("Error entering RTC session", e); diff --git a/yarn.lock b/yarn.lock index 8c0ce84b8..d9b9864f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7545,7 +7545,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -10335,9 +10335,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21": version: 38.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=fd949fe486038099ee111a72b57ce711e85bc352" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=e7f5bec51b6f70501a025b79fe5021c933385b21" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0" @@ -10353,7 +10353,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:13" - checksum: 10c0/d334811074726482b58089fef6c9e98a462fbc1d91c63798307648bdc1349d2e154aa31f391690e2c9cd90eee61a3be9fe8872873e7f828b529d9268d2a25b78 + checksum: 10c0/7adffdc183affd2d3ee1e8497cad6ca7904a37f98328ff7bc15aa6c1829dc9f9a92f8e1bd6260432a33626ff2a839644de938270163e73438b7294675cd954e4 languageName: node linkType: hard