From e25a649e1a21eefdcff5ed26f0a9e338ad9a5224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 19 Aug 2021 11:48:37 +0200 Subject: [PATCH 1/5] Add webkitAudioContext to window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/@types/global.d.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/@types/global.d.ts b/src/@types/global.d.ts index d09d92f25..949630896 100644 --- a/src/@types/global.d.ts +++ b/src/@types/global.d.ts @@ -34,6 +34,7 @@ declare global { interface Window { electron?: Electron; + webkitAudioContext: typeof AudioContext; } interface Electron { From 6904f6b36fd514c0f8121a7937045bc825f08a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 19 Aug 2021 11:27:35 +0200 Subject: [PATCH 2/5] Emit volume changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 72 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index b82c68535..2f9386c72 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -19,12 +19,20 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; +const POLLING_INTERVAL = 250; // ms + export enum CallFeedEvent { NewStream = "new_stream", - MuteStateChanged = "mute_state_changed" + MuteStateChanged = "mute_state_changed", + VolumeChanged = "volume_changed", } export class CallFeed extends EventEmitter { + private measuringVolumeActivity = false; + private audioContext: AudioContext; + private analyser: AnalyserNode; + private frequencyBinCount: Float32Array; + constructor( public stream: MediaStream, public userId: string, @@ -35,6 +43,30 @@ export class CallFeed extends EventEmitter { private videoMuted: boolean, ) { super(); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } + } + + private get hasAudioTrack(): boolean { + return this.stream.getAudioTracks().length > 0; + } + + private initVolumeMeasuring(): void { + const AudioContext = window.AudioContext || window.webkitAudioContext; + if (!this.hasAudioTrack || !AudioContext) return; + + this.audioContext = new AudioContext(); + + this.analyser = this.audioContext.createAnalyser(); + this.analyser.fftSize = 512; + this.analyser.smoothingTimeConstant = 0.1; + + const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream); + mediaStreamAudioSourceNode.connect(this.analyser); + + this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); } /** @@ -81,6 +113,12 @@ export class CallFeed extends EventEmitter { public setNewStream(newStream: MediaStream) { this.stream = newStream; this.emit(CallFeedEvent.NewStream, this.stream); + + if (this.hasAudioTrack) { + this.initVolumeMeasuring(); + } else { + this.measureVolumeActivity(false); + } } public setAudioMuted(muted: boolean): void { @@ -92,4 +130,36 @@ export class CallFeed extends EventEmitter { this.videoMuted = muted; this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } + + public measureVolumeActivity(enabled: boolean) { + if (enabled) { + if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + + this.measuringVolumeActivity = true; + this.volumeLooper(); + } else { + this.measuringVolumeActivity = false; + this.emit(CallFeedEvent.VolumeChanged, -Infinity); + } + } + + private volumeLooper(): void { + if (!this.analyser) return; + + setTimeout(() => { + if (!this.measuringVolumeActivity) return; + + this.analyser.getFloatFrequencyData(this.frequencyBinCount); + + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; + } + } + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); + this.volumeLooper(); + }, POLLING_INTERVAL); + } } From 22cd475d22265fbb75b8f6d09de8392670699857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 19 Aug 2021 11:34:12 +0200 Subject: [PATCH 3/5] Add missing types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 2f9386c72..bcf4d1a3c 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -110,7 +110,7 @@ export class CallFeed extends EventEmitter { * This method should be only used by MatrixCall. * @param newStream new stream with which to replace the current one */ - public setNewStream(newStream: MediaStream) { + public setNewStream(newStream: MediaStream): void { this.stream = newStream; this.emit(CallFeedEvent.NewStream, this.stream); @@ -131,7 +131,7 @@ export class CallFeed extends EventEmitter { this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } - public measureVolumeActivity(enabled: boolean) { + public measureVolumeActivity(enabled: boolean): void { if (enabled) { if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; From 0746a64c83e26d08e66acf80c78be94e1d5049e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 19 Aug 2021 12:03:57 +0200 Subject: [PATCH 4/5] Add some docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index bcf4d1a3c..637e1996f 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -121,16 +121,28 @@ export class CallFeed extends EventEmitter { } } + /** + * Set feed's internal audio mute state + * @param muted is the feed's audio muted? + */ public setAudioMuted(muted: boolean): void { this.audioMuted = muted; this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } + /** + * Set feed's internal video mute state + * @param muted is the feed's video muted? + */ public setVideoMuted(muted: boolean): void { this.videoMuted = muted; this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } + /** + * Starts emitting volume_changed events where the emitter value is in decibels + * @param enabled emit volume changes + */ public measureVolumeActivity(enabled: boolean): void { if (enabled) { if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; From 425362edfe6502fb93c743d6e1d180a7feef4378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 19 Aug 2021 14:48:26 +0200 Subject: [PATCH 5/5] Emit Speaking events from CallFeed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callFeed.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 637e1996f..6ebadb273 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -20,11 +20,13 @@ import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; const POLLING_INTERVAL = 250; // ms +const SPEAKING_THRESHOLD = -60; // dB export enum CallFeedEvent { NewStream = "new_stream", MuteStateChanged = "mute_state_changed", VolumeChanged = "volume_changed", + Speaking = "speaking", } export class CallFeed extends EventEmitter { @@ -32,6 +34,8 @@ export class CallFeed extends EventEmitter { private audioContext: AudioContext; private analyser: AnalyserNode; private frequencyBinCount: Float32Array; + private speakingThreshold = SPEAKING_THRESHOLD; + private speaking = false; constructor( public stream: MediaStream, @@ -155,6 +159,10 @@ export class CallFeed extends EventEmitter { } } + public setSpeakingThreshold(threshold: number) { + this.speakingThreshold = threshold; + } + private volumeLooper(): void { if (!this.analyser) return; @@ -171,6 +179,12 @@ export class CallFeed extends EventEmitter { } this.emit(CallFeedEvent.VolumeChanged, maxVolume); + const newSpeaking = maxVolume > this.speakingThreshold; + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + this.volumeLooper(); }, POLLING_INTERVAL); }