diff --git a/src/excalidraw-app/collab/Collab.tsx b/src/excalidraw-app/collab/Collab.tsx
index cb87a4bb1..4fb794838 100644
--- a/src/excalidraw-app/collab/Collab.tsx
+++ b/src/excalidraw-app/collab/Collab.tsx
@@ -882,17 +882,12 @@ class Collab extends PureComponent<Props, CollabState> {
   };
 
   setCollaborators(sockets: string[]) {
-    const collaborators: InstanceType<typeof Collab>["collaborators"] =
-      new Map();
-    for (const socketId of sockets) {
-      if (this.collaborators.has(socketId)) {
-        collaborators.set(socketId, this.collaborators.get(socketId)!);
-      } else {
-        collaborators.set(socketId, {});
+    this.collaborators.forEach((value, key) => {
+      if (value.socketId && !sockets.includes(value.socketId)) {
+        this.collaborators.delete(key);
       }
-    }
-    this.collaborators = collaborators;
-    this.excalidrawAPI.updateScene({ collaborators });
+    });
+    this.excalidrawAPI.updateScene({ collaborators: this.collaborators });
   }
 
   public setLastBroadcastedOrReceivedSceneVersion = (version: number) => {
diff --git a/src/excalidraw-app/collab/Portal.tsx b/src/excalidraw-app/collab/Portal.tsx
index 15ee62730..1d494d75e 100644
--- a/src/excalidraw-app/collab/Portal.tsx
+++ b/src/excalidraw-app/collab/Portal.tsx
@@ -74,6 +74,9 @@ class Portal {
         /* syncAll */ true,
       );
     });
+    this.socket.on("room-user-change", (clients: string[]) => {
+      this.collab.setCollaborators(clients);
+    });
   }
 
   isOpen() {
@@ -193,6 +196,7 @@ class Portal {
           userState,
           username: this.collab.state.username,
           userId: this.collab.state.userId,
+          socketId: this.socket.id,
         },
       };
       return this._broadcastSocketData(
@@ -216,6 +220,7 @@ class Portal {
             this.collab.excalidrawAPI.getAppState().selectedElementIds,
           username: this.collab.state.username,
           userId: this.collab.state.userId,
+          socketId: this.socket.id,
         },
       };
       return this._broadcastSocketData(
diff --git a/src/excalidraw-app/data/index.ts b/src/excalidraw-app/data/index.ts
index f7882827e..39efc24c4 100644
--- a/src/excalidraw-app/data/index.ts
+++ b/src/excalidraw-app/data/index.ts
@@ -111,6 +111,7 @@ export type SocketUpdateDataSource = {
       selectedElementIds: AppState["selectedElementIds"];
       username: string;
       userId: string;
+      socketId: string;
     };
   };
   IDLE_STATUS: {
@@ -119,6 +120,7 @@ export type SocketUpdateDataSource = {
       userState: UserIdleState;
       username: string;
       userId: string;
+      socketId: string;
     };
   };
 };
diff --git a/src/types.ts b/src/types.ts
index 411088b1b..d84ec7a9d 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -55,6 +55,7 @@ export type Collaborator = {
   avatarUrl?: string;
   // user id. If supplied, we'll filter out duplicates when rendering user avatars.
   id?: string;
+  socketId?: string;
 };
 
 export type DataURL = string & { _brand: "DataURL" };