diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a18ccfae7..dd0043d25 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -141,7 +141,7 @@ export enum CallErrorCode { UnknownDevices = 'unknown_devices', /** - * Error code usewd when we fail to send the invite + * Error code used when we fail to send the invite * for some reason other than there being unknown devices */ SendInvite = 'send_invite', @@ -152,7 +152,7 @@ export enum CallErrorCode { CreateAnswer = 'create_answer', /** - * Error code usewd when we fail to send the answer + * Error code used when we fail to send the answer * for some reason other than there being unknown devices */ SendAnswer = 'send_answer', @@ -238,7 +238,7 @@ export class CallError extends Error { code: string; constructor(code: CallErrorCode, msg: string, err: Error) { - // Stil ldon't think there's any way to have proper nested errors + // Still don't think there's any way to have proper nested errors super(msg + ": " + err); this.code = code; @@ -278,16 +278,10 @@ export class MatrixCall extends EventEmitter { private sentEndOfCandidates: boolean; private peerConn: RTCPeerConnection; private feeds: Array; - private screenSharingStream: MediaStream; - // TODO: Rename to usermedia rather than AV for consistency - private localAVStream: MediaStream; private usermediaSenders: Array; private screensharingSenders: Array; private inviteOrAnswerSent: boolean; private waitForLocalAVStream: boolean; - // XXX: I don't know why this is called 'config'. - // XXX: Do we even needs this? Seems to be unused - private config: MediaStreamConstraints; private successor: MatrixCall; private opponentMember: RoomMember; private opponentVersion: number; @@ -301,9 +295,6 @@ export class MatrixCall extends EventEmitter { // This flag represents whether we want the other party to be on hold private remoteOnHold; - private micMuted; - private vidMuted; - // the stats for the call at the point it ended. We can't get these after we // tear the call down, so we just grab a snapshot before we stop the call. // The typescript definitions have this type as 'any' :( @@ -405,6 +396,18 @@ export class MatrixCall extends EventEmitter { return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); } + public get localScreensharingFeed(): CallFeed { + return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get localUsermediaStream(): MediaStream { + return this.localUsermediaFeed?.stream; + } + + private get localScreensharingStream(): MediaStream { + return this.localScreensharingFeed?.stream; + } + private getFeedByStreamId(streamId: string): CallFeed { return this.getFeeds().find((feed) => feed.stream.id === streamId); } @@ -638,7 +641,7 @@ export class MatrixCall extends EventEmitter { const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream; // According to previous comments in this file, firefox at some point did not - // add streams until media started ariving on them. Testing latest firefox + // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (!remoteStream || remoteStream.getTracks().length === 0) { logger.error("No remote stream or no tracks after setting remote description!"); @@ -687,7 +690,7 @@ export class MatrixCall extends EventEmitter { logger.debug(`Answering call ${this.callId} of type ${this.type}`); - if (!this.localAVStream && !this.waitForLocalAVStream) { + if (!this.localUsermediaStream && !this.waitForLocalAVStream) { const constraints = getUserMediaContraints( this.type == CallType.Video ? ConstraintsType.Video: @@ -705,8 +708,8 @@ export class MatrixCall extends EventEmitter { this.getUserMediaFailed(e); return; } - } else if (this.localAVStream) { - this.gotUserMediaForAnswer(this.localAVStream); + } else if (this.localUsermediaStream) { + this.gotUserMediaForAnswer(this.localUsermediaStream); } else if (this.waitForLocalAVStream) { this.setState(CallState.WaitLocalMedia); } @@ -724,12 +727,10 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if (this.state === CallState.CreateOffer) { logger.debug("Handing local stream to new call"); - newCall.gotUserMediaForAnswer(this.localAVStream); - delete(this.localAVStream); + newCall.gotUserMediaForAnswer(this.localUsermediaStream); } else if (this.state === CallState.InviteSent) { logger.debug("Handing local stream to new call"); - newCall.gotUserMediaForAnswer(this.localAVStream); - delete(this.localAVStream); + newCall.gotUserMediaForAnswer(this.localUsermediaStream); } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -791,7 +792,7 @@ export class MatrixCall extends EventEmitter { * @returns {boolean} is screensharing */ public isScreensharing(): boolean { - return Boolean(this.screenSharingStream); + return Boolean(this.localScreensharingStream); } /** @@ -821,9 +822,9 @@ export class MatrixCall extends EventEmitter { logger.debug(`Set screensharing enabled? ${enabled}`); if (enabled) { try { - this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); - if (!this.screenSharingStream) return false; - this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare); + const stream = await getScreensharingStream(selectDesktopCapturerSource); + if (!stream) return false; + this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { this.emit(CallEvent.Error, @@ -835,11 +836,10 @@ export class MatrixCall extends EventEmitter { for (const sender of this.screensharingSenders) { this.peerConn.removeTrack(sender); } - this.deleteFeedByStream(this.screenSharingStream); - for (const track of this.screenSharingStream.getTracks()) { + this.deleteFeedByStream(this.localScreensharingStream); + for (const track of this.localScreensharingStream.getTracks()) { track.stop(); } - this.screenSharingStream = null; return false; } } @@ -858,10 +858,10 @@ export class MatrixCall extends EventEmitter { logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); if (enabled) { try { - this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource); - if (!this.screenSharingStream) return false; + const stream = await getScreensharingStream(selectDesktopCapturerSource); + if (!stream) return false; - const track = this.screenSharingStream.getTracks().find((track) => { + const track = stream.getTracks().find((track) => { return track.kind === "video"; }); const sender = this.usermediaSenders.find((sender) => { @@ -869,7 +869,7 @@ export class MatrixCall extends EventEmitter { }); sender.replaceTrack(track); - this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare, false); + this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); return true; } catch (err) { @@ -879,7 +879,7 @@ export class MatrixCall extends EventEmitter { return false; } } else { - const track = this.localAVStream.getTracks().find((track) => { + const track = this.localUsermediaStream.getTracks().find((track) => { return track.kind === "video"; }); const sender = this.usermediaSenders.find((sender) => { @@ -887,11 +887,10 @@ export class MatrixCall extends EventEmitter { }); sender.replaceTrack(track); - this.deleteFeedByStream(this.screenSharingStream); - for (const track of this.screenSharingStream.getTracks()) { + this.deleteFeedByStream(this.localScreensharingStream); + for (const track of this.localScreensharingStream.getTracks()) { track.stop(); } - this.screenSharingStream = null; return false; } @@ -953,11 +952,11 @@ export class MatrixCall extends EventEmitter { if (this.isRemoteOnHold() === onHold) return; this.remoteOnHold = onHold; - for (const tranceiver of this.peerConn.getTransceivers()) { + for (const transceiver of this.peerConn.getTransceivers()) { // We don't send hold music or anything so we're not actually // sending anything, but sendrecv is fairly standard for hold and // it makes it a lot easier to figure out who's put who on hold. - tranceiver.direction = onHold ? 'sendonly' : 'sendrecv'; + transceiver.direction = onHold ? 'sendonly' : 'sendrecv'; } this.updateMuteStatus(); @@ -976,8 +975,8 @@ export class MatrixCall extends EventEmitter { // We consider a call to be on hold only if *all* the tracks are on hold // (is this the right thing to do?) - for (const tranceiver of this.peerConn.getTransceivers()) { - const trackOnHold = ['inactive', 'recvonly'].includes(tranceiver.currentDirection); + for (const transceiver of this.peerConn.getTransceivers()) { + const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection); if (!trackOnHold) callOnHold = false; } @@ -1008,8 +1007,8 @@ export class MatrixCall extends EventEmitter { const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; - setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted); - setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted); + setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); } /** @@ -1026,11 +1025,10 @@ export class MatrixCall extends EventEmitter { return; } - this.localAVStream = stream; this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); this.setState(CallState.CreateOffer); - logger.info("Got local AV stream with id " + this.localAVStream.id); + logger.info("Got local AV stream with id " + this.localUsermediaStream.id); logger.debug("gotUserMediaForInvite -> " + this.type); // Now we wait for the negotiationneeded event }; @@ -1051,7 +1049,7 @@ export class MatrixCall extends EventEmitter { 'm.call.dtmf': false, }; - // We have just taken the local description from the peerconnection which will + // We have just taken the local description from the peerConn which will // contain all the local candidates added so far, so we can discard any candidates // we had queued up because they'll be in the answer. logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); @@ -1089,8 +1087,7 @@ export class MatrixCall extends EventEmitter { this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - this.localAVStream = stream; - logger.info("Got local AV stream with id " + this.localAVStream.id); + logger.info("Got local AV stream with id " + this.localUsermediaStream.id); this.setState(CallState.CreateAnswer); @@ -1168,8 +1165,8 @@ export class MatrixCall extends EventEmitter { return; } - const cands = ev.getContent().candidates; - if (!cands) { + const candidates = ev.getContent().candidates; + if (!candidates) { logger.info("Ignoring candidates event with no candidates!"); return; } @@ -1178,10 +1175,10 @@ export class MatrixCall extends EventEmitter { if (this.opponentPartyId === undefined) { // we haven't picked an opponent yet so save the candidates - logger.info(`Bufferring ${cands.length} candidates until we pick an opponent`); - const bufferedCands = this.remoteCandidateBuffer.get(fromPartyId) || []; - bufferedCands.push(...cands); - this.remoteCandidateBuffer.set(fromPartyId, bufferedCands); + logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); + const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; + bufferedCandidates.push(...candidates); + this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); return; } @@ -1194,7 +1191,7 @@ export class MatrixCall extends EventEmitter { return; } - await this.addIceCandidates(cands); + await this.addIceCandidates(candidates); } /** @@ -1396,7 +1393,7 @@ export class MatrixCall extends EventEmitter { lifetime: CALL_TIMEOUT_MS, } as MCallOfferNegotiate; - // clunky because TypeScript can't folow the types through if we use an expression as the key + // clunky because TypeScript can't follow the types through if we use an expression as the key if (this.state === CallState.CreateOffer) { content.offer = this.peerConn.localDescription; } else { @@ -1561,7 +1558,7 @@ export class MatrixCall extends EventEmitter { } onNegotiationNeeded = async () => { - logger.info("Negotation is needed!"); + logger.info("Negotiation is needed!"); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); @@ -1649,12 +1646,12 @@ export class MatrixCall extends EventEmitter { // Don't send the ICE candidates yet if the call is in the ringing state: this // means we tried to pick (ie. started generating candidates) and then failed to // send the answer and went back to the ringing state. Queue up the candidates - // to send if we sucessfully send the answer. + // to send if we successfully send the answer. // Equally don't send if we haven't yet sent the answer because we can send the // first batch of candidates along with the answer if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return; - // MSC2746 reccomends these values (can be quite long when calling because the + // MSC2746 recommends these values (can be quite long when calling because the // callee will need a while to answer the call) const delay = this.direction === CallDirection.Inbound ? 500 : 2000; @@ -1670,7 +1667,7 @@ export class MatrixCall extends EventEmitter { */ async transfer(targetUserId: string) { // Fetch the target user's global profile info: their room avatar / displayname - // could be different in whatever room we shae with them. + // could be different in whatever room we share with them. const profileInfo = await this.client.getProfileInfo(targetUserId); const replacementId = genCallID(); @@ -1757,7 +1754,7 @@ export class MatrixCall extends EventEmitter { } private stopAllMedia() { - logger.debug(`stopAllMedia (stream=${this.localAVStream})`); + logger.debug(`stopAllMedia (stream=${this.localUsermediaStream})`); for (const feed of this.feeds) { for (const track of feed.stream.getTracks()) { @@ -1779,13 +1776,13 @@ export class MatrixCall extends EventEmitter { return; } - const cands = this.candidateSendQueue; + const candidates = this.candidateSendQueue; this.candidateSendQueue = []; ++this.candidateSendTries; const content = { - candidates: cands, + candidates: candidates, }; - logger.debug("Attempting to send " + cands.length + " candidates"); + logger.debug("Attempting to send " + candidates.length + " candidates"); try { await this.sendVoipEvent(EventType.CallCandidates, content); } catch (error) { @@ -1794,7 +1791,7 @@ export class MatrixCall extends EventEmitter { if (error.event) this.client.cancelPendingEvent(error.event); // put all the candidates we failed to send back in the queue - this.candidateSendQueue.push(...cands); + this.candidateSendQueue.push(...candidates); if (this.candidateSendTries > 5) { logger.debug( @@ -1826,7 +1823,6 @@ export class MatrixCall extends EventEmitter { this.client.callEventHandler.calls.set(this.callId, this); this.setState(CallState.WaitLocalMedia); this.direction = CallDirection.Outbound; - this.config = constraints; // make sure we have valid turn creds. Unless something's gone wrong, it should // poll and keep the credentials valid so this should be instant. @@ -1899,26 +1895,28 @@ export class MatrixCall extends EventEmitter { } private async addBufferedIceCandidates() { - const bufferedCands = this.remoteCandidateBuffer.get(this.opponentPartyId); - if (bufferedCands) { - logger.info(`Adding ${bufferedCands.length} buffered candidates for opponent ${this.opponentPartyId}`); - await this.addIceCandidates(bufferedCands); + const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); + if (bufferedCandidates) { + logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + await this.addIceCandidates(bufferedCandidates); } this.remoteCandidateBuffer = null; } - private async addIceCandidates(cands: RTCIceCandidate[]) { - for (const cand of cands) { + private async addIceCandidates(candidates: RTCIceCandidate[]) { + for (const candidate of candidates) { if ( - (cand.sdpMid === null || cand.sdpMid === undefined) && - (cand.sdpMLineIndex === null || cand.sdpMLineIndex === undefined) + (candidate.sdpMid === null || candidate.sdpMid === undefined) && + (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); continue; } - logger.debug("Call " + this.callId + " got remote ICE " + cand.sdpMid + " candidate: " + cand.candidate); + logger.debug( + "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, + ); try { - await this.peerConn.addIceCandidate(cand); + await this.peerConn.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { logger.info("Failed to add remote ICE candidate", err); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 96b233e0f..3e080c0eb 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -130,21 +130,19 @@ export class CallEventHandler { private handleCallEvent(event: MatrixEvent) { const content = event.getContent(); + const type = event.getType() as EventType; + const weSentTheEvent = event.getSender() === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; - //console.info("RECV %s content=%s", event.getType(), JSON.stringify(content)); + //console.info("RECV %s content=%s", type, JSON.stringify(content)); - if (event.getType() === EventType.CallInvite) { - if (event.getSender() === this.client.credentials.userId) { - return; // ignore invites you send - } + if (type === EventType.CallInvite) { + // ignore invites you send + if (weSentTheEvent) return; + // expired call + if (event.getLocalAge() > content.lifetime - RING_GRACE_PERIOD) return; + // stale/old invite event + if (call && call.state === CallState.Ended) return; - if (event.getLocalAge() > content.lifetime - RING_GRACE_PERIOD) { - return; // expired call - } - - if (call && call.state === CallState.Ended) { - return; // stale/old invite event - } if (call) { logger.log( `WARN: Already have a MatrixCall with id ${content.call_id} but got an ` + @@ -154,9 +152,11 @@ export class CallEventHandler { const timeUntilTurnCresExpire = this.client.getTurnServersExpiry() - Date.now(); logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms"); - call = createNewMatrixCall(this.client, event.getRoomId(), { - forceTURN: this.client.forceTURN, - }); + call = createNewMatrixCall( + this.client, + event.getRoomId(), + { forceTURN: this.client.forceTURN }, + ); if (!call) { logger.log( "Incoming call ID " + content.call_id + " but this client " + @@ -220,21 +220,9 @@ export class CallEventHandler { } else { this.client.emit("Call.incoming", call); } - } else if (event.getType() === EventType.CallAnswer) { - if (!call) { - return; - } - if (event.getSender() === this.client.credentials.userId) { - if (call.state === CallState.Ringing) { - call.onAnsweredElsewhere(content); - } - } else { - call.onAnswerReceived(event); - } - } else if (event.getType() === EventType.CallCandidates) { - if (event.getSender() === this.client.credentials.userId) { - return; - } + } else if (type === EventType.CallCandidates) { + if (weSentTheEvent) return; + if (!call) { // store the candidates; we may get a call eventually. if (!this.candidateEventsByCall.has(content.call_id)) { @@ -244,7 +232,7 @@ export class CallEventHandler { } else { call.onRemoteIceCandidatesReceived(event); } - } else if ([EventType.CallHangup, EventType.CallReject].includes(event.getType() as EventType)) { + } else if ([EventType.CallHangup, EventType.CallReject].includes(type)) { // Note that we also observe our own hangups here so we can see // if we've already rejected a call that would otherwise be valid if (!call) { @@ -259,7 +247,7 @@ export class CallEventHandler { } } else { if (call.state !== CallState.Ended) { - if (event.getType() === EventType.CallHangup) { + if (type === EventType.CallHangup) { call.onHangupReceived(content); } else { call.onRejectReceived(content); @@ -267,48 +255,40 @@ export class CallEventHandler { this.calls.delete(content.call_id); } } - } else if (event.getType() === EventType.CallSelectAnswer) { - if (!call) return; + } - if (event.getContent().party_id === call.ourPartyId) { - // Ignore remote echo - return; - } + // The following events need a call + if (!call) return; + // Ignore remote echo + if (event.getContent().party_id === call.ourPartyId) return; - call.onSelectAnswerReceived(event); - } else if (event.getType() === EventType.CallNegotiate) { - if (!call) return; + switch (type) { + case EventType.CallAnswer: + if (weSentTheEvent) { + if (call.state === CallState.Ringing) { + call.onAnsweredElsewhere(content); + } + } else { + call.onAnswerReceived(event); + } + break; + case EventType.CallSelectAnswer: + call.onSelectAnswerReceived(event); + break; - if (event.getContent().party_id === call.ourPartyId) { - // Ignore remote echo - return; - } + case EventType.CallNegotiate: + call.onNegotiateReceived(event); + break; - call.onNegotiateReceived(event); - } else if ( - event.getType() === EventType.CallAssertedIdentity || - event.getType() === EventType.CallAssertedIdentityPrefix - ) { - if (!call) return; + case EventType.CallAssertedIdentity: + case EventType.CallAssertedIdentityPrefix: + call.onAssertedIdentityReceived(event); + break; - if (event.getContent().party_id === call.ourPartyId) { - // Ignore remote echo (not that we send asserted identity, but still...) - return; - } - - call.onAssertedIdentityReceived(event); - } else if ( - event.getType() === EventType.CallSDPStreamMetadataChanged || - event.getType() === EventType.CallSDPStreamMetadataChangedPrefix - ) { - if (!call) return; - - if (event.getContent().party_id === call.ourPartyId) { - // Ignore remote echo - return; - } - - call.onSDPStreamMetadataChangedReceived(event); + case EventType.CallSDPStreamMetadataChanged: + case EventType.CallSDPStreamMetadataChangedPrefix: + call.onSDPStreamMetadataChangedReceived(event); + break; } } } diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 7adf640e8..009df5ed8 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -1,4 +1,4 @@ -// allow camelcase as these are events type that go onto the wire +// allow non-camelcase as these are events type that go onto the wire /* eslint-disable camelcase */ // TODO: Change to "sdp_stream_metadata" when MSC3077 is merged @@ -29,6 +29,12 @@ export interface CallCapabilities { 'm.call.dtmf': boolean; } +export interface CallReplacesTarget { + id: string; + display_name: string; + avatar_url: string; +} + export interface MCallAnswer { answer: CallOfferAnswer; capabilities: CallCapabilities; @@ -47,15 +53,9 @@ export interface MCallSDPStreamMetadataChanged { [SDPStreamMetadataKey]: SDPStreamMetadata; } -export interface MCallReplacesTarget { - id: string; - display_name: string; - avatar_url: string; -} - export interface MCallReplacesEvent { replacement_id: string; - target_user: MCallReplacesTarget; + target_user: CallReplacesTarget; create_call: string; await_call: string; target_room: string;