Merge pull request #1865 from SimonBrandner/feature/voice-activity

Give `CallFeed` the capability to emit on volume changes

src/@types/global.d.ts

@@ -34,6 +34,7 @@ declare global {
 
     interface Window {
         electron?: Electron;
+        webkitAudioContext: typeof AudioContext;
     }
 
     interface Electron {

src/webrtc/callFeed.ts

@@ -19,12 +19,24 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes";
 import { MatrixClient } from "../client";
 import { RoomMember } from "../models/room-member";
 
+const POLLING_INTERVAL = 250; // ms
+const SPEAKING_THRESHOLD = -60; // dB
+
 export enum CallFeedEvent {
     NewStream = "new_stream",
-    MuteStateChanged = "mute_state_changed"
+    MuteStateChanged = "mute_state_changed",
+    VolumeChanged = "volume_changed",
+    Speaking = "speaking",
 }
 
 export class CallFeed extends EventEmitter {
+    private measuringVolumeActivity = false;
+    private audioContext: AudioContext;
+    private analyser: AnalyserNode;
+    private frequencyBinCount: Float32Array;
+    private speakingThreshold = SPEAKING_THRESHOLD;
+    private speaking = false;
+
     constructor(
         public stream: MediaStream,
         public userId: string,
@@ -35,6 +47,30 @@ export class CallFeed extends EventEmitter {
         private videoMuted: boolean,
     ) {
         super();
+
+        if (this.hasAudioTrack) {
+            this.initVolumeMeasuring();
+        }
+    }
+
+    private get hasAudioTrack(): boolean {
+        return this.stream.getAudioTracks().length > 0;
+    }
+
+    private initVolumeMeasuring(): void {
+        const AudioContext = window.AudioContext || window.webkitAudioContext;
+        if (!this.hasAudioTrack || !AudioContext) return;
+
+        this.audioContext = new AudioContext();
+
+        this.analyser = this.audioContext.createAnalyser();
+        this.analyser.fftSize = 512;
+        this.analyser.smoothingTimeConstant = 0.1;
+
+        const mediaStreamAudioSourceNode = this.audioContext.createMediaStreamSource(this.stream);
+        mediaStreamAudioSourceNode.connect(this.analyser);
+
+        this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
     }
 
     /**
@@ -78,18 +114,78 @@ export class CallFeed extends EventEmitter {
      * This method should be only used by MatrixCall.
      * @param newStream new stream with which to replace the current one
      */
-    public setNewStream(newStream: MediaStream) {
+    public setNewStream(newStream: MediaStream): void {
         this.stream = newStream;
         this.emit(CallFeedEvent.NewStream, this.stream);
+
+        if (this.hasAudioTrack) {
+            this.initVolumeMeasuring();
+        } else {
+            this.measureVolumeActivity(false);
+        }
     }
 
+    /**
+     * Set feed's internal audio mute state
+     * @param muted is the feed's audio muted?
+     */
     public setAudioMuted(muted: boolean): void {
         this.audioMuted = muted;
         this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
     }
 
+    /**
+     * Set feed's internal video mute state
+     * @param muted is the feed's video muted?
+     */
     public setVideoMuted(muted: boolean): void {
         this.videoMuted = muted;
         this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted);
     }
+
+    /**
+     * Starts emitting volume_changed events where the emitter value is in decibels
+     * @param enabled emit volume changes
+     */
+    public measureVolumeActivity(enabled: boolean): void {
+        if (enabled) {
+            if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return;
+
+            this.measuringVolumeActivity = true;
+            this.volumeLooper();
+        } else {
+            this.measuringVolumeActivity = false;
+            this.emit(CallFeedEvent.VolumeChanged, -Infinity);
+        }
+    }
+
+    public setSpeakingThreshold(threshold: number) {
+        this.speakingThreshold = threshold;
+    }
+
+    private volumeLooper(): void {
+        if (!this.analyser) return;
+
+        setTimeout(() => {
+            if (!this.measuringVolumeActivity) return;
+
+            this.analyser.getFloatFrequencyData(this.frequencyBinCount);
+
+            let maxVolume = -Infinity;
+            for (let i = 0; i < this.frequencyBinCount.length; i++) {
+                if (this.frequencyBinCount[i] > maxVolume) {
+                    maxVolume = this.frequencyBinCount[i];
+                }
+            }
+
+            this.emit(CallFeedEvent.VolumeChanged, maxVolume);
+            const newSpeaking = maxVolume > this.speakingThreshold;
+            if (this.speaking !== newSpeaking) {
+                this.speaking = newSpeaking;
+                this.emit(CallFeedEvent.Speaking, this.speaking);
+            }
+
+            this.volumeLooper();
+        }, POLLING_INTERVAL);
+    }
 }