Dima/matrix-js-sdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1821 from SimonBrandner/task/cleanup

Browse Source 
Clean-up some VoIP code
This commit is contained in:
David Baker
2021-08-03 13:18:47 +01:00
committed by GitHub
parent aef89e4a26 775c3465bb
commit be8ecce995
3 changed files with 129 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/webrtc/call.ts
+72 -74
View File
@@ -141,7 +141,7 @@ export enum CallErrorCode {
UnknownDevices = 'unknown_devices',
/**
* Error code usewd when we fail to send the invite
* Error code used when we fail to send the invite
* for some reason other than there being unknown devices
*/
SendInvite = 'send_invite',
@@ -152,7 +152,7 @@ export enum CallErrorCode {
CreateAnswer = 'create_answer',
/**
* Error code usewd when we fail to send the answer
* Error code used when we fail to send the answer
* for some reason other than there being unknown devices
*/
SendAnswer = 'send_answer',
@@ -238,7 +238,7 @@ export class CallError extends Error {
code: string;
constructor(code: CallErrorCode, msg: string, err: Error) {
// Stil ldon't think there's any way to have proper nested errors
// Still don't think there's any way to have proper nested errors
super(msg + ": " + err);
this.code = code;
@@ -278,16 +278,10 @@ export class MatrixCall extends EventEmitter {
private sentEndOfCandidates: boolean;
private peerConn: RTCPeerConnection;
private feeds: Array<CallFeed>;
private screenSharingStream: MediaStream;
// TODO: Rename to usermedia rather than AV for consistency
private localAVStream: MediaStream;
private usermediaSenders: Array<RTCRtpSender>;
private screensharingSenders: Array<RTCRtpSender>;
private inviteOrAnswerSent: boolean;
private waitForLocalAVStream: boolean;
// XXX: I don't know why this is called 'config'.
// XXX: Do we even needs this? Seems to be unused
private config: MediaStreamConstraints;
private successor: MatrixCall;
private opponentMember: RoomMember;
private opponentVersion: number;
@@ -301,9 +295,6 @@ export class MatrixCall extends EventEmitter {
// This flag represents whether we want the other party to be on hold
private remoteOnHold;
private micMuted;
private vidMuted;
// the stats for the call at the point it ended. We can't get these after we
// tear the call down, so we just grab a snapshot before we stop the call.
// The typescript definitions have this type as 'any' :(
@@ -405,6 +396,18 @@ export class MatrixCall extends EventEmitter {
return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
}
public get localScreensharingFeed(): CallFeed {
return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare);
}
public get localUsermediaStream(): MediaStream {
return this.localUsermediaFeed?.stream;
}
private get localScreensharingStream(): MediaStream {
return this.localScreensharingFeed?.stream;
}
private getFeedByStreamId(streamId: string): CallFeed {
return this.getFeeds().find((feed) => feed.stream.id === streamId);
}
@@ -638,7 +641,7 @@ export class MatrixCall extends EventEmitter {
const remoteStream = this.feeds.find((feed) => !feed.isLocal())?.stream;
// According to previous comments in this file, firefox at some point did not
// add streams until media started ariving on them. Testing latest firefox
// add streams until media started arriving on them. Testing latest firefox
// (81 at time of writing), this is no longer a problem, so let's do it the correct way.
if (!remoteStream || remoteStream.getTracks().length === 0) {
logger.error("No remote stream or no tracks after setting remote description!");
@@ -687,7 +690,7 @@ export class MatrixCall extends EventEmitter {
logger.debug(`Answering call ${this.callId} of type ${this.type}`);
if (!this.localAVStream && !this.waitForLocalAVStream) {
if (!this.localUsermediaStream && !this.waitForLocalAVStream) {
const constraints = getUserMediaContraints(
this.type == CallType.Video ?
ConstraintsType.Video:
@@ -705,8 +708,8 @@ export class MatrixCall extends EventEmitter {
this.getUserMediaFailed(e);
return;
}
} else if (this.localAVStream) {
this.gotUserMediaForAnswer(this.localAVStream);
} else if (this.localUsermediaStream) {
this.gotUserMediaForAnswer(this.localUsermediaStream);
} else if (this.waitForLocalAVStream) {
this.setState(CallState.WaitLocalMedia);
}
@@ -724,12 +727,10 @@ export class MatrixCall extends EventEmitter {
newCall.waitForLocalAVStream = true;
} else if (this.state === CallState.CreateOffer) {
logger.debug("Handing local stream to new call");
newCall.gotUserMediaForAnswer(this.localAVStream);
delete(this.localAVStream);
newCall.gotUserMediaForAnswer(this.localUsermediaStream);
} else if (this.state === CallState.InviteSent) {
logger.debug("Handing local stream to new call");
newCall.gotUserMediaForAnswer(this.localAVStream);
delete(this.localAVStream);
newCall.gotUserMediaForAnswer(this.localUsermediaStream);
}
this.successor = newCall;
this.emit(CallEvent.Replaced, newCall);
@@ -791,7 +792,7 @@ export class MatrixCall extends EventEmitter {
* @returns {boolean} is screensharing
*/
public isScreensharing(): boolean {
return Boolean(this.screenSharingStream);
return Boolean(this.localScreensharingStream);
}
/**
@@ -821,9 +822,9 @@ export class MatrixCall extends EventEmitter {
logger.debug(`Set screensharing enabled? ${enabled}`);
if (enabled) {
try {
this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource);
if (!this.screenSharingStream) return false;
this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare);
const stream = await getScreensharingStream(selectDesktopCapturerSource);
if (!stream) return false;
this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare);
return true;
} catch (err) {
this.emit(CallEvent.Error,
@@ -835,11 +836,10 @@ export class MatrixCall extends EventEmitter {
for (const sender of this.screensharingSenders) {
this.peerConn.removeTrack(sender);
}
this.deleteFeedByStream(this.screenSharingStream);
for (const track of this.screenSharingStream.getTracks()) {
this.deleteFeedByStream(this.localScreensharingStream);
for (const track of this.localScreensharingStream.getTracks()) {
track.stop();
}
this.screenSharingStream = null;
return false;
}
}
@@ -858,10 +858,10 @@ export class MatrixCall extends EventEmitter {
logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`);
if (enabled) {
try {
this.screenSharingStream = await getScreensharingStream(selectDesktopCapturerSource);
if (!this.screenSharingStream) return false;
const stream = await getScreensharingStream(selectDesktopCapturerSource);
if (!stream) return false;
const track = this.screenSharingStream.getTracks().find((track) => {
const track = stream.getTracks().find((track) => {
return track.kind === "video";
});
const sender = this.usermediaSenders.find((sender) => {
@@ -869,7 +869,7 @@ export class MatrixCall extends EventEmitter {
});
sender.replaceTrack(track);
this.pushLocalFeed(this.screenSharingStream, SDPStreamMetadataPurpose.Screenshare, false);
this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false);
return true;
} catch (err) {
@@ -879,7 +879,7 @@ export class MatrixCall extends EventEmitter {
return false;
}
} else {
const track = this.localAVStream.getTracks().find((track) => {
const track = this.localUsermediaStream.getTracks().find((track) => {
return track.kind === "video";
});
const sender = this.usermediaSenders.find((sender) => {
@@ -887,11 +887,10 @@ export class MatrixCall extends EventEmitter {
});
sender.replaceTrack(track);
this.deleteFeedByStream(this.screenSharingStream);
for (const track of this.screenSharingStream.getTracks()) {
this.deleteFeedByStream(this.localScreensharingStream);
for (const track of this.localScreensharingStream.getTracks()) {
track.stop();
}
this.screenSharingStream = null;
return false;
}
@@ -953,11 +952,11 @@ export class MatrixCall extends EventEmitter {
if (this.isRemoteOnHold() === onHold) return;
this.remoteOnHold = onHold;
for (const tranceiver of this.peerConn.getTransceivers()) {
for (const transceiver of this.peerConn.getTransceivers()) {
// We don't send hold music or anything so we're not actually
// sending anything, but sendrecv is fairly standard for hold and
// it makes it a lot easier to figure out who's put who on hold.
tranceiver.direction = onHold ? 'sendonly' : 'sendrecv';
transceiver.direction = onHold ? 'sendonly' : 'sendrecv';
}
this.updateMuteStatus();
@@ -976,8 +975,8 @@ export class MatrixCall extends EventEmitter {
// We consider a call to be on hold only if *all* the tracks are on hold
// (is this the right thing to do?)
for (const tranceiver of this.peerConn.getTransceivers()) {
const trackOnHold = ['inactive', 'recvonly'].includes(tranceiver.currentDirection);
for (const transceiver of this.peerConn.getTransceivers()) {
const trackOnHold = ['inactive', 'recvonly'].includes(transceiver.currentDirection);
if (!trackOnHold) callOnHold = false;
}
@@ -1008,8 +1007,8 @@ export class MatrixCall extends EventEmitter {
const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold;
const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold;
setTracksEnabled(this.localAVStream.getAudioTracks(), !micShouldBeMuted);
setTracksEnabled(this.localAVStream.getVideoTracks(), !vidShouldBeMuted);
setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted);
setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted);
}
/**
@@ -1026,11 +1025,10 @@ export class MatrixCall extends EventEmitter {
return;
}
this.localAVStream = stream;
this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
this.setState(CallState.CreateOffer);
logger.info("Got local AV stream with id " + this.localAVStream.id);
logger.info("Got local AV stream with id " + this.localUsermediaStream.id);
logger.debug("gotUserMediaForInvite -> " + this.type);
// Now we wait for the negotiationneeded event
};
@@ -1051,7 +1049,7 @@ export class MatrixCall extends EventEmitter {
'm.call.dtmf': false,
};
// We have just taken the local description from the peerconnection which will
// We have just taken the local description from the peerConn which will
// contain all the local candidates added so far, so we can discard any candidates
// we had queued up because they'll be in the answer.
logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`);
@@ -1089,8 +1087,7 @@ export class MatrixCall extends EventEmitter {
this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
this.localAVStream = stream;
logger.info("Got local AV stream with id " + this.localAVStream.id);
logger.info("Got local AV stream with id " + this.localUsermediaStream.id);
this.setState(CallState.CreateAnswer);
@@ -1168,8 +1165,8 @@ export class MatrixCall extends EventEmitter {
return;
}
const cands = ev.getContent().candidates;
if (!cands) {
const candidates = ev.getContent().candidates;
if (!candidates) {
logger.info("Ignoring candidates event with no candidates!");
return;
}
@@ -1178,10 +1175,10 @@ export class MatrixCall extends EventEmitter {
if (this.opponentPartyId === undefined) {
// we haven't picked an opponent yet so save the candidates
logger.info(`Bufferring ${cands.length} candidates until we pick an opponent`);
const bufferedCands = this.remoteCandidateBuffer.get(fromPartyId) || [];
bufferedCands.push(...cands);
this.remoteCandidateBuffer.set(fromPartyId, bufferedCands);
logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`);
const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || [];
bufferedCandidates.push(...candidates);
this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates);
return;
}
@@ -1194,7 +1191,7 @@ export class MatrixCall extends EventEmitter {
return;
}
await this.addIceCandidates(cands);
await this.addIceCandidates(candidates);
}
/**
@@ -1396,7 +1393,7 @@ export class MatrixCall extends EventEmitter {
lifetime: CALL_TIMEOUT_MS,
} as MCallOfferNegotiate;
// clunky because TypeScript can't folow the types through if we use an expression as the key
// clunky because TypeScript can't follow the types through if we use an expression as the key
if (this.state === CallState.CreateOffer) {
content.offer = this.peerConn.localDescription;
} else {
@@ -1561,7 +1558,7 @@ export class MatrixCall extends EventEmitter {
}
onNegotiationNeeded = async () => {
logger.info("Negotation is needed!");
logger.info("Negotiation is needed!");
if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) {
logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event");
@@ -1649,12 +1646,12 @@ export class MatrixCall extends EventEmitter {
// Don't send the ICE candidates yet if the call is in the ringing state: this
// means we tried to pick (ie. started generating candidates) and then failed to
// send the answer and went back to the ringing state. Queue up the candidates
// to send if we sucessfully send the answer.
// to send if we successfully send the answer.
// Equally don't send if we haven't yet sent the answer because we can send the
// first batch of candidates along with the answer
if (this.state === CallState.Ringing || !this.inviteOrAnswerSent) return;
// MSC2746 reccomends these values (can be quite long when calling because the
// MSC2746 recommends these values (can be quite long when calling because the
// callee will need a while to answer the call)
const delay = this.direction === CallDirection.Inbound ? 500 : 2000;
@@ -1670,7 +1667,7 @@ export class MatrixCall extends EventEmitter {
*/
async transfer(targetUserId: string) {
// Fetch the target user's global profile info: their room avatar / displayname
// could be different in whatever room we shae with them.
// could be different in whatever room we share with them.
const profileInfo = await this.client.getProfileInfo(targetUserId);
const replacementId = genCallID();
@@ -1757,7 +1754,7 @@ export class MatrixCall extends EventEmitter {
}
private stopAllMedia() {
logger.debug(`stopAllMedia (stream=${this.localAVStream})`);
logger.debug(`stopAllMedia (stream=${this.localUsermediaStream})`);
for (const feed of this.feeds) {
for (const track of feed.stream.getTracks()) {
@@ -1779,13 +1776,13 @@ export class MatrixCall extends EventEmitter {
return;
}
const cands = this.candidateSendQueue;
const candidates = this.candidateSendQueue;
this.candidateSendQueue = [];
++this.candidateSendTries;
const content = {
candidates: cands,
candidates: candidates,
};
logger.debug("Attempting to send " + cands.length + " candidates");
logger.debug("Attempting to send " + candidates.length + " candidates");
try {
await this.sendVoipEvent(EventType.CallCandidates, content);
} catch (error) {
@@ -1794,7 +1791,7 @@ export class MatrixCall extends EventEmitter {
if (error.event) this.client.cancelPendingEvent(error.event);
// put all the candidates we failed to send back in the queue
this.candidateSendQueue.push(...cands);
this.candidateSendQueue.push(...candidates);
if (this.candidateSendTries > 5) {
logger.debug(
@@ -1826,7 +1823,6 @@ export class MatrixCall extends EventEmitter {
this.client.callEventHandler.calls.set(this.callId, this);
this.setState(CallState.WaitLocalMedia);
this.direction = CallDirection.Outbound;
this.config = constraints;
// make sure we have valid turn creds. Unless something's gone wrong, it should
// poll and keep the credentials valid so this should be instant.
@@ -1899,26 +1895,28 @@ export class MatrixCall extends EventEmitter {
}
private async addBufferedIceCandidates() {
const bufferedCands = this.remoteCandidateBuffer.get(this.opponentPartyId);
if (bufferedCands) {
logger.info(`Adding ${bufferedCands.length} buffered candidates for opponent ${this.opponentPartyId}`);
await this.addIceCandidates(bufferedCands);
const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId);
if (bufferedCandidates) {
logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`);
await this.addIceCandidates(bufferedCandidates);
}
this.remoteCandidateBuffer = null;
}
private async addIceCandidates(cands: RTCIceCandidate[]) {
for (const cand of cands) {
private async addIceCandidates(candidates: RTCIceCandidate[]) {
for (const candidate of candidates) {
if (
(cand.sdpMid === null || cand.sdpMid === undefined) &&
(cand.sdpMLineIndex === null || cand.sdpMLineIndex === undefined)
(candidate.sdpMid === null || candidate.sdpMid === undefined) &&
(candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined)
) {
logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex");
continue;
}
logger.debug("Call " + this.callId + " got remote ICE " + cand.sdpMid + " candidate: " + cand.candidate);
logger.debug(
"Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate,
);
try {
await this.peerConn.addIceCandidate(cand);
await this.peerConn.addIceCandidate(candidate);
} catch (err) {
if (!this.ignoreOffer) {
logger.info("Failed to add remote ICE candidate", err);
src/webrtc/callEventHandler.ts
+49 -69
View File
@@ -130,21 +130,19 @@ export class CallEventHandler {
private handleCallEvent(event: MatrixEvent) {
const content = event.getContent();
const type = event.getType() as EventType;
const weSentTheEvent = event.getSender() === this.client.credentials.userId;
let call = content.call_id ? this.calls.get(content.call_id) : undefined;
//console.info("RECV %s content=%s", event.getType(), JSON.stringify(content));
//console.info("RECV %s content=%s", type, JSON.stringify(content));
if (event.getType() === EventType.CallInvite) {
if (event.getSender() === this.client.credentials.userId) {
return; // ignore invites you send
}
if (type === EventType.CallInvite) {
// ignore invites you send
if (weSentTheEvent) return;
// expired call
if (event.getLocalAge() > content.lifetime - RING_GRACE_PERIOD) return;
// stale/old invite event
if (call && call.state === CallState.Ended) return;
if (event.getLocalAge() > content.lifetime - RING_GRACE_PERIOD) {
return; // expired call
}
if (call && call.state === CallState.Ended) {
return; // stale/old invite event
}
if (call) {
logger.log(
`WARN: Already have a MatrixCall with id ${content.call_id} but got an ` +
@@ -154,9 +152,11 @@ export class CallEventHandler {
const timeUntilTurnCresExpire = this.client.getTurnServersExpiry() - Date.now();
logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms");
call = createNewMatrixCall(this.client, event.getRoomId(), {
forceTURN: this.client.forceTURN,
});
call = createNewMatrixCall(
this.client,
event.getRoomId(),
{ forceTURN: this.client.forceTURN },
);
if (!call) {
logger.log(
"Incoming call ID " + content.call_id + " but this client " +
@@ -220,21 +220,9 @@ export class CallEventHandler {
} else {
this.client.emit("Call.incoming", call);
}
} else if (event.getType() === EventType.CallAnswer) {
if (!call) {
return;
}
if (event.getSender() === this.client.credentials.userId) {
if (call.state === CallState.Ringing) {
call.onAnsweredElsewhere(content);
}
} else {
call.onAnswerReceived(event);
}
} else if (event.getType() === EventType.CallCandidates) {
if (event.getSender() === this.client.credentials.userId) {
return;
}
} else if (type === EventType.CallCandidates) {
if (weSentTheEvent) return;
if (!call) {
// store the candidates; we may get a call eventually.
if (!this.candidateEventsByCall.has(content.call_id)) {
@@ -244,7 +232,7 @@ export class CallEventHandler {
} else {
call.onRemoteIceCandidatesReceived(event);
}
} else if ([EventType.CallHangup, EventType.CallReject].includes(event.getType() as EventType)) {
} else if ([EventType.CallHangup, EventType.CallReject].includes(type)) {
// Note that we also observe our own hangups here so we can see
// if we've already rejected a call that would otherwise be valid
if (!call) {
@@ -259,7 +247,7 @@ export class CallEventHandler {
}
} else {
if (call.state !== CallState.Ended) {
if (event.getType() === EventType.CallHangup) {
if (type === EventType.CallHangup) {
call.onHangupReceived(content);
} else {
call.onRejectReceived(content);
@@ -267,48 +255,40 @@ export class CallEventHandler {
this.calls.delete(content.call_id);
}
}
} else if (event.getType() === EventType.CallSelectAnswer) {
if (!call) return;
}
if (event.getContent().party_id === call.ourPartyId) {
// Ignore remote echo
return;
}
// The following events need a call
if (!call) return;
// Ignore remote echo
if (event.getContent().party_id === call.ourPartyId) return;
call.onSelectAnswerReceived(event);
} else if (event.getType() === EventType.CallNegotiate) {
if (!call) return;
switch (type) {
case EventType.CallAnswer:
if (weSentTheEvent) {
if (call.state === CallState.Ringing) {
call.onAnsweredElsewhere(content);
}
} else {
call.onAnswerReceived(event);
}
break;
case EventType.CallSelectAnswer:
call.onSelectAnswerReceived(event);
break;
if (event.getContent().party_id === call.ourPartyId) {
// Ignore remote echo
return;
}
case EventType.CallNegotiate:
call.onNegotiateReceived(event);
break;
call.onNegotiateReceived(event);
} else if (
event.getType() === EventType.CallAssertedIdentity ||
event.getType() === EventType.CallAssertedIdentityPrefix
) {
if (!call) return;
case EventType.CallAssertedIdentity:
case EventType.CallAssertedIdentityPrefix:
call.onAssertedIdentityReceived(event);
break;
if (event.getContent().party_id === call.ourPartyId) {
// Ignore remote echo (not that we send asserted identity, but still...)
return;
}
call.onAssertedIdentityReceived(event);
} else if (
event.getType() === EventType.CallSDPStreamMetadataChanged ||
event.getType() === EventType.CallSDPStreamMetadataChangedPrefix
) {
if (!call) return;
if (event.getContent().party_id === call.ourPartyId) {
// Ignore remote echo
return;
}
call.onSDPStreamMetadataChangedReceived(event);
case EventType.CallSDPStreamMetadataChanged:
case EventType.CallSDPStreamMetadataChangedPrefix:
call.onSDPStreamMetadataChangedReceived(event);
break;
}
}
}
src/webrtc/callEventTypes.ts
+8 -8
View File
@@ -1,4 +1,4 @@
// allow camelcase as these are events type that go onto the wire
// allow non-camelcase as these are events type that go onto the wire
/* eslint-disable camelcase */
// TODO: Change to "sdp_stream_metadata" when MSC3077 is merged
@@ -29,6 +29,12 @@ export interface CallCapabilities {
'm.call.dtmf': boolean;
}
export interface CallReplacesTarget {
id: string;
display_name: string;
avatar_url: string;
}
export interface MCallAnswer {
answer: CallOfferAnswer;
capabilities: CallCapabilities;
@@ -47,15 +53,9 @@ export interface MCallSDPStreamMetadataChanged {
[SDPStreamMetadataKey]: SDPStreamMetadata;
}
export interface MCallReplacesTarget {
id: string;
display_name: string;
avatar_url: string;
}
export interface MCallReplacesEvent {
replacement_id: string;
target_user: MCallReplacesTarget;
target_user: CallReplacesTarget;
create_call: string;
await_call: string;
target_room: string;