diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index ac9868d810528224354123d4be4d8ab4c4216896..0d6750f0e4567e40aff3ac5ab53a6cc4717f173b 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -338,7 +338,7 @@ class DeviceHandler(BaseHandler):
                         etype, state_key = key
                         if etype != EventTypes.Member:
                             continue
-                        possibly_changed.append(state_key)
+                        possibly_changed.add(state_key)
                     break
 
             # If there has been any change in membership, include them in the