diff --git a/src/client.js b/src/client.js
index d71aa6cb5..f1074f5b9 100644
--- a/src/client.js
+++ b/src/client.js
@@ -353,6 +353,10 @@ export function MatrixClient(opts) {
     if (call) {
         this._callEventHandler = new CallEventHandler(this);
         this._supportsVoip = true;
+        // Start listening for calls after the initial sync is done
+        // We do not need to backfill the call event buffer
+        // with encrypted events that might never get decrypted
+        this.on("sync", this._startCallEventHandler);
     } else {
         this._callEventHandler = null;
     }
@@ -4986,6 +4990,13 @@ MatrixClient.prototype.getOpenIdToken = function() {
 // VoIP operations
 // ===============
 
+MatrixClient.prototype._startCallEventHandler = function() {
+    if (this.isInitialSyncComplete()) {
+        this._callEventHandler.start();
+        this.off("sync", this._startCallEventHandler);
+    }
+};
+
 /**
  * @param {module:client.callback} callback Optional.
  * @return {Promise} Resolves: TODO
diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts
index 44815ee63..c6396bed7 100644
--- a/src/webrtc/callEventHandler.ts
+++ b/src/webrtc/callEventHandler.ts
@@ -43,6 +43,9 @@ export class CallEventHandler {
         // after loading and after we've been offline for a bit.
         this.callEventBuffer = [];
         this.candidateEventsByCall = new Map<string, Array<MatrixEvent>>();
+    }
+
+    public start() {
         this.client.on("sync", this.evaluateEventBuffer);
         this.client.on("event", this.onEvent);
     }
@@ -52,12 +55,11 @@ export class CallEventHandler {
         this.client.removeListener("event", this.onEvent);
     }
 
-    private evaluateEventBuffer = () => {
+    private evaluateEventBuffer = async () => {
         if (this.client.getSyncState() === "SYNCING") {
-            // don't process any events until they are all decrypted
-            if (this.callEventBuffer.some((e) => {
-                return e.isBeingDecrypted() || e.shouldAttemptDecryption()
-            })) return;
+            await Promise.all(this.callEventBuffer.map(event => {
+                this.client.decryptEventIfNeeded(event);
+            }));
 
             const ignoreCallIds = new Set<String>();
             // inspect the buffer and mark all calls which have been answered
@@ -88,20 +90,19 @@ export class CallEventHandler {
     }
 
     private onEvent = (event: MatrixEvent) => {
+        this.client.decryptEventIfNeeded(event);
         // any call events or ones that might be once they're decrypted
-        const isBeingDecrypted = event.isBeingDecrypted();
-        const shouldAttemptDecryption = event.shouldAttemptDecryption();
         if (
             event.getType().indexOf("m.call.") === 0 ||
             event.getType().indexOf("org.matrix.call.") === 0
-            || isBeingDecrypted || shouldAttemptDecryption
+            || event.isBeingDecrypted()
         ) {
             // queue up for processing once all events from this sync have been
             // processed (see above).
             this.callEventBuffer.push(event);
         }
 
-        if (event.isDecryptionFailure() || isBeingDecrypted || shouldAttemptDecryption) {
+        if (event.isBeingDecrypted() || event.isDecryptionFailure()) {
             // add an event listener for once the event is decrypted.
             event.once("Event.decrypted", () => {
                 if (event.getType().indexOf("m.call.") === -1) return;