diff --git a/crates/matrix-sdk-crypto/src/identities/device.rs b/crates/matrix-sdk-crypto/src/identities/device.rs
index 730433dd5..d30b96ce3 100644
--- a/crates/matrix-sdk-crypto/src/identities/device.rs
+++ b/crates/matrix-sdk-crypto/src/identities/device.rs
@@ -83,9 +83,10 @@ pub struct ReadOnlyDevice {
         deserialize_with = "local_trust_deserializer"
     )]
     trust_state: Arc<Atomic<LocalTrust>>,
-
+    /// Flag remembering if we successfully sent an `m.no_olm` withheld code to
+    /// this device.
     #[serde(default)]
-    no_olm_sent: Arc<AtomicBool>,
+    withheld_code_sent: Arc<AtomicBool>,
 }
 
 impl std::fmt::Debug for ReadOnlyDevice {
@@ -97,7 +98,7 @@ impl std::fmt::Debug for ReadOnlyDevice {
             .field("keys", self.keys())
             .field("deleted", &self.deleted.load(Ordering::SeqCst))
             .field("trust_state", &self.trust_state)
-            .field("no_olm_sent", &self.no_olm_sent)
+            .field("withheld_code_sent", &self.withheld_code_sent)
             .finish()
     }
 }
@@ -569,7 +570,7 @@ impl ReadOnlyDevice {
             inner: device_keys.into(),
             trust_state: Arc::new(Atomic::new(trust_state)),
             deleted: Arc::new(AtomicBool::new(false)),
-            no_olm_sent: Arc::new(AtomicBool::new(false)),
+            withheld_code_sent: Arc::new(AtomicBool::new(false)),
         }
     }
 
@@ -638,14 +639,14 @@ impl ReadOnlyDevice {
         self.trust_state.store(state, Ordering::Relaxed)
     }
 
-    pub(crate) fn set_no_olm_sent(&self, sent: bool) {
-        self.no_olm_sent.store(sent, Ordering::Relaxed)
+    pub(crate) fn mark_withheld_code_as_sent(&self) {
+        self.withheld_code_sent.store(true, Ordering::Relaxed)
     }
 
-    /// Returns true if a withheld no_olm code was already sent to this device.
-    /// Resets to false when a new olm session is created.
-    pub fn is_no_olm_sent(&self) -> bool {
-        self.no_olm_sent.load(Ordering::Relaxed)
+    /// Returns true if the `m.no_olm` withheld code was already sent to this
+    /// device.
+    pub fn was_withheld_code_sent(&self) -> bool {
+        self.withheld_code_sent.load(Ordering::Relaxed)
     }
 
     /// Get the list of algorithms this device supports.
@@ -882,7 +883,7 @@ impl TryFrom<&DeviceKeys> for ReadOnlyDevice {
             inner: device_keys.clone().into(),
             deleted: Arc::new(AtomicBool::new(false)),
             trust_state: Arc::new(Atomic::new(LocalTrust::Unset)),
-            no_olm_sent: Arc::new(AtomicBool::new(false)),
+            withheld_code_sent: Arc::new(AtomicBool::new(false)),
         };
 
         device.verify_device_keys(device_keys)?;
diff --git a/crates/matrix-sdk-crypto/src/machine.rs b/crates/matrix-sdk-crypto/src/machine.rs
index b72e7f9ca..3cb1bad0c 100644
--- a/crates/matrix-sdk-crypto/src/machine.rs
+++ b/crates/matrix-sdk-crypto/src/machine.rs
@@ -1046,12 +1046,6 @@ impl OlmMachine {
                 match decrypted.session {
                     SessionType::New(s) => {
                         changes.account = Some(self.account.inner.clone());
-                        // We have a new session with this device, clear the set_no_olm flag
-                        let device = self.store.get_device(&s.user_id, &s.device_id).await;
-                        if let Ok(Some(device)) = device {
-                            device.inner.set_no_olm_sent(false);
-                            changes.devices.changed.push(device.inner);
-                        }
                         changes.sessions.push(s);
                     }
                     SessionType::Existing(s) => {
diff --git a/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs b/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs
index d57030def..49fa1a6e6 100644
--- a/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs
+++ b/crates/matrix-sdk-crypto/src/session_manager/group_sessions.rs
@@ -167,7 +167,7 @@ impl GroupSessionManager {
                     let device = self.store.get_device(user_id, device_id).await;
 
                     if let Ok(Some(device)) = device {
-                        device.set_no_olm_sent(true);
+                        device.mark_withheld_code_as_sent();
                         changes.devices.changed.push(device.inner.clone());
                     } else {
                         error!(
@@ -572,7 +572,7 @@ impl GroupSessionManager {
         // `OutboundGroupSession` and the `Device` both interact with the flag we'll
         // leave it be.
         if code == &WithheldCode::NoOlm {
-            device.is_no_olm_sent()
+            device.was_withheld_code_sent()
                 || self.sessions.sessions.iter().any(|s| s.is_withheld_to(device, code))
         } else {
             group_session.is_withheld_to(device, code)
@@ -1311,7 +1311,7 @@ mod tests {
 
         // The device should be marked as having the `m.no_olm` code received only after
         // the request has been marked as sent.
-        assert!(!device.is_no_olm_sent());
+        assert!(!device.was_withheld_code_sent());
 
         for request in requests {
             machine.mark_request_as_sent(&request.txn_id, &response).await.unwrap();
@@ -1319,6 +1319,6 @@ mod tests {
 
         let device = machine.get_device(bob_id, "BOBDEVICE".into(), None).await.unwrap().unwrap();
 
-        assert!(device.is_no_olm_sent());
+        assert!(device.was_withheld_code_sent());
     }
 }
diff --git a/crates/matrix-sdk-crypto/src/session_manager/sessions.rs b/crates/matrix-sdk-crypto/src/session_manager/sessions.rs
index f621e7a19..eb0099e27 100644
--- a/crates/matrix-sdk-crypto/src/session_manager/sessions.rs
+++ b/crates/matrix-sdk-crypto/src/session_manager/sessions.rs
@@ -369,9 +369,6 @@ impl SessionManager {
                 };
 
                 changes.sessions.push(session);
-                device.set_no_olm_sent(false);
-                // We have a new session for that device, clear previous no_olm_sent
-                changes.devices.changed.push(device.clone());
                 new_sessions.entry(user_id).or_default().insert(device_id, session_info);
             }
         }