diff --git a/src/@types/event.ts b/src/@types/event.ts index 230cb05039d..e7399a85d3e 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -42,6 +42,7 @@ export enum EventType { RoomMessage = "m.room.message", RoomMessageEncrypted = "m.room.encrypted", Sticker = "m.sticker", + CallInvite = "m.call.invite", CallCandidates = "m.call.candidates", CallAnswer = "m.call.answer", @@ -54,6 +55,10 @@ export enum EventType { CallReplaces = "m.call.replaces", CallAssertedIdentity = "m.call.asserted_identity", CallAssertedIdentityPrefix = "org.matrix.call.asserted_identity", + CallTrackSubscription = "m.call.track_subscription", + CallPing = "m.call.ping", + CallPong = "m.call.pong", + KeyVerificationRequest = "m.key.verification.request", KeyVerificationStart = "m.key.verification.start", KeyVerificationCancel = "m.key.verification.cancel", diff --git a/src/client.ts b/src/client.ts index b9de5b669ac..d3128483517 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1774,10 +1774,12 @@ export class MatrixClient extends TypedEventEmitter(); - private subscribedTracks: ISfuTrackDesc[] = []; + private subscribedTracks: FocusTrackDescription[] = []; private inviteOrAnswerSent = false; private waitForLocalAVStream = false; @@ -699,11 +695,11 @@ export class MatrixCall extends TypedEventEmitter { - await this.peerConn!.setLocalDescription(offer); - - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Publish, { - sdp: offer.sdp, - } as ISfuPublishDataChannelMessage); - this.emit(CallEvent.FeedsChanged, this.feeds); - }); - } else { - this.emit(CallEvent.FeedsChanged, this.feeds); - } + this.emit(CallEvent.FeedsChanged, this.feeds); } /** @@ -874,7 +858,7 @@ export class MatrixCall extends TypedEventEmitter { - await this.peerConn!.setLocalDescription(offer); - - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Unpublish, { - sdp: offer.sdp, - stop: tracksToUnpublish, - } as ISfuUnpublishDataChannelMessage); - }); - } } private deleteAllFeeds(): void { @@ -979,8 +952,9 @@ export class MatrixCall extends TypedEventEmitter { if (this.isFocus) { - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Metadata); + this.sendFocusEvent(EventType.CallSDPStreamMetadataChanged); } else { await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), @@ -1975,7 +1949,7 @@ export class MatrixCall extends TypedEventEmitter => { // TODO: play nice with application layer DC listeners - let json: ISfuBaseDataChannelMessage; + let json: FocusEvent; try { json = JSON.parse(event.data); } catch (e) { @@ -1983,52 +1957,59 @@ export class MatrixCall extends TypedEventEmitter { if (this.dataChannel?.readyState === 'connecting') { - const p = new Promise(resolve => { + const p = new Promise((resolve) => { this.dataChannel!.onopen = (): void => resolve(); this.dataChannel!.onclose = (): void => resolve(); }); @@ -2050,24 +2031,24 @@ export class MatrixCall extends TypedEventEmitter Boolean(info.tracks)) // Skip trackless feeds - .reduce((a: ISfuTrackDesc[], [s, i]) => ( + .reduce((a: FocusTrackDescription[], [s, i]) => ( [...a, ...Object.keys(i.tracks).map((t) => ({ stream_id: s, track_id: t }))] ), []) // Get array of tracks from feeds .filter((track) => !this.subscribedTracks.find((subscribed) => utils.deepCompare(track, subscribed))); // Filter out already subscribed tracks if (tracks.length === 0) { - logger.warn("Failed to find any new streams to subscribe to"); + logger.info("Failed to find any new streams to subscribe to"); return; } else { this.subscribedTracks.push(...tracks); - logger.warn("Subscribing to:", tracks); } - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Select, { - start: tracks, - } as ISfuSelectDataChannelMessage); + this.sendFocusEvent(EventType.CallTrackSubscription, { + subscribe: tracks, + unsubscribe: [], + } as FocusTrackSubscriptionEvent); } public updateRemoteSDPStreamMetadata(metadata: SDPStreamMetadata): void { @@ -2167,6 +2148,18 @@ export class MatrixCall extends TypedEventEmitter { - this.sendSFUDataChannelMessage(SFUDataChannelMessageOp.Alive); - }, SFU_KEEP_ALIVE_INTERVAL * 3 / 4); - } - } else if (this.peerConn?.iceConnectionState == 'failed') { + } else if (this.peerConn?.iceConnectionState == "failed") { // Firefox for Android does not yet have support for restartIce() // (the types say it's always defined though, so we have to cast // to prevent typescript from warning). @@ -2332,6 +2320,15 @@ export class MatrixCall extends TypedEventEmitter { if (stream.getTracks().length === 0) { + // FIXME: We should really be doing this per-track. Šimon + // leaves this for when we switch to mids for signalling + const getIndex = (): number => this.subscribedTracks.findIndex((t) => t.stream_id === stream.id); + let indexOfTrackToRemove = getIndex(); + while (indexOfTrackToRemove !== -1) { + this.subscribedTracks.splice(indexOfTrackToRemove, 1); + indexOfTrackToRemove = getIndex(); + } + logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); this.deleteFeedByStream(stream); stream.removeEventListener("removetrack", onRemoveTrack); @@ -2388,9 +2385,6 @@ export class MatrixCall extends TypedEventEmitter => { - // Other than the initial offer, we handle negotiation manually when calling with an SFU - if (this.isFocus && ![CallState.Fledgling, CallState.CreateOffer].includes(this.state)) return; - logger.info(`Call ${this.callId} Negotiation is needed!`); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { @@ -2508,18 +2502,19 @@ export class MatrixCall extends TypedEventEmitter ( - this.foci.find(f => pf.user_id === f.user_id && pf.device_id === pf.device_id) - ))); + const isUsingPreferredFocus = Boolean( + preferredFoci.find((pf) => + this.foci.find((f) => pf.user_id === f.user_id && pf.device_id === pf.device_id), + ), + ); return isUsingPreferredFocus ? [] : preferredFoci; } @@ -457,17 +459,14 @@ export class GroupCall extends TypedEventEmitter< this.activeSpeaker = undefined; this.onActiveSpeakerLoop(); - this.activeSpeakerLoopInterval = setInterval( - this.onActiveSpeakerLoop, - this.activeSpeakerInterval, - ); + this.activeSpeakerLoopInterval = setInterval(this.onActiveSpeakerLoop, this.activeSpeakerInterval); if (this.foci[0]) { const opponentDevice = { - "device_id": this.foci[0].device_id, + device_id: this.foci[0].device_id, // XXX: What if an SFU gets restarted? - "session_id": "sfu", - "feeds": [], + session_id: "sfu", + feeds: [], }; const onError = (e?: unknown): void => { @@ -481,17 +480,13 @@ export class GroupCall extends TypedEventEmitter< ); }; - const sfuCall = createNewMatrixCall( - this.client, - this.room.roomId, - { - invitee: this.foci[0].user_id, - opponentDeviceId: opponentDevice.device_id, - opponentSessionId: opponentDevice.session_id, - groupCallId: this.groupCallId, - isFocus: true, - }, - ); + const sfuCall = createNewMatrixCall(this.client, this.room.roomId, { + invitee: this.foci[0].user_id, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + isFocus: true, + }); if (!sfuCall) { onError(); return; @@ -517,10 +512,10 @@ export class GroupCall extends TypedEventEmitter< // Try to find a focus of another user to use let focusOfAnotherMember: IFocusInfo | undefined; for (const event of this.getMemberStateEvents()) { - const focus = event.getContent() - ?.["m.calls"] ?.[0] - ?.["m.devices"]?.[0] - ?.["m.foci.active"]?.[0]; + const focus = + event.getContent()?.["m.calls"]?.[0]?.["m.devices"]?.[0]?.[ + "m.foci.active" + ]?.[0]; if (focus) { focusOfAnotherMember = focus; break; @@ -759,10 +754,9 @@ export class GroupCall extends TypedEventEmitter< ); // TODO: handle errors - this.forEachCall(call => call.pushLocalFeed(call.isFocus - ? this.localScreenshareFeed! - : this.localScreenshareFeed!.clone(), - )); + this.forEachCall((call) => + call.pushLocalFeed(call.isFocus ? this.localScreenshareFeed! : this.localScreenshareFeed!.clone()), + ); return true; } catch (error) { @@ -1073,18 +1067,22 @@ export class GroupCall extends TypedEventEmitter< private onCallFeedsChanged = (call: MatrixCall): void => { // Find removed feeds - [...this.userMediaFeeds, ...this.screenshareFeeds].filter((gf) => gf.disposed).forEach((feed) => { - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); - else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); - }); + [...this.userMediaFeeds, ...this.screenshareFeeds] + .filter((gf) => gf.disposed) + .forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.removeUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.removeScreenshareFeed(feed); + }); // Find new feeds - call.getRemoteFeeds().filter((cf) => { - return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); - }).forEach((feed) => { - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); - else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); - }); + call.getRemoteFeeds() + .filter((cf) => { + return !this.userMediaFeeds.find((gf) => gf.stream.id === cf.stream.id); + }) + .forEach((feed) => { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia) this.addUserMediaFeed(feed); + else if (feed.purpose === SDPStreamMetadataPurpose.Screenshare) this.addScreenshareFeed(feed); + }); }; private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined): void => { @@ -1374,7 +1372,7 @@ export class GroupCall extends TypedEventEmitter< "device_id": this.client.getDeviceId()!, "session_id": this.client.getSessionId(), "expires_ts": Date.now() + DEVICE_TIMEOUT, - "feeds": this.getLocalFeeds().map(feed => ({ purpose: feed.purpose })), + "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose })), "m.foci.active": this.foci, "m.foci.preferred": this.getPreferredFoci(), // TODO: Add data channels