diff --git a/src/Channel.ts b/src/Channel.ts
index 51ab8cf..fd0ce76 100644
--- a/src/Channel.ts
+++ b/src/Channel.ts
@@ -102,7 +102,9 @@ export class Channel {
         if (this.isSynced()) {
             if (this.matrixUsers.size === 2 && this.name === this.roomId) {
                 const otherUser = [...this.matrixUsers.values()].filter(m => m.nick !== this.ircUser.nick);
-                this.name = `&${otherUser[0].mxid.substring(1)}`
+                const directRoomsForUser = this.ircUser.directRooms.get(otherUser[0].mxid);
+                if (directRoomsForUser && directRoomsForUser.includes(this.roomId))
+                    this.name = `&${otherUser[0].mxid.substring(1)}`
             }
             this.ircUser.finishChannelSync(this);
             clearInterval(this.initialSyncID);
diff --git a/src/IRCUser.ts b/src/IRCUser.ts
index 49117f7..41736c0 100644
--- a/src/IRCUser.ts
+++ b/src/IRCUser.ts
@@ -8,6 +8,7 @@ export class IRCUser {
     private clients: Set<Client>
     public channels: Map<string, Channel>
     public roomIdToChannel: Map<string, Channel>
+    public directRooms: Map<string, string[]>
     private syncLocks: Set<Channel>
     public matrixUsers: Map<string, MatrixUser>
     public nickToMatrixUser: Map<string, MatrixUser>
@@ -27,6 +28,7 @@ export class IRCUser {
         this.clients = new Set();
         this.channels = new Map();
         this.roomIdToChannel = new Map();
+        this.directRooms = new Map();
         this.syncLocks = new Set();
         this.matrixUsers = new Map();
         this.nickToMatrixUser = new Map();
@@ -45,6 +47,16 @@ export class IRCUser {
         this.isSyncing = false;
         this.currentSyncTime = 0;
         this.syncIntervalID = setInterval(this.doSync.bind(this), 2000);
+        axios.get(`https://${this.homeserver}/_matrix/client/v3/user/${this.mxid}/account_data/m.direct?access_token=${this.accessToken}`).then(response => {
+            if (!response.data)
+                return;
+            //@ts-ignore
+            Object.entries(response.data).forEach(m => this.directRooms.set(m[0], m[1]));
+        }).catch(e => {
+            const errcode = e.response?.data?.errcode;
+            if (errcode !== "M_NOT_FOUND")
+                console.log(e);
+        })
     }
 
     getOrCreateMatrixUser(mxid: string): MatrixUser {