diff --git a/changelog.d/3544.misc b/changelog.d/3544.misc
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py
index 258655349b62d3834422cc0f5fa9a8b68ea1607a..f2bde74dc58a53a1c5e4056b49377431e82321cf 100644
--- a/synapse/util/caches/stream_change_cache.py
+++ b/synapse/util/caches/stream_change_cache.py
@@ -74,12 +74,14 @@ class StreamChangeCache(object):
         assert type(stream_pos) is int
 
         if stream_pos >= self._earliest_known_stream_pos:
-            result = {
+            changed_entities = {
                 self._cache[k] for k in self._cache.islice(
                     start=self._cache.bisect_right(stream_pos),
                 )
             }
 
+            result = changed_entities.intersection(entities)
+
             self.metrics.inc_hits()
         else:
             result = set(entities)
diff --git a/tests/util/test_stream_change_cache.py b/tests/util/test_stream_change_cache.py
index fc45baaaa04c17ff647239618fd5ec48036d85cc..65b0f2e6fbc8ba4c9aab39a7c4e968a31561470f 100644
--- a/tests/util/test_stream_change_cache.py
+++ b/tests/util/test_stream_change_cache.py
@@ -178,6 +178,22 @@ class StreamChangeCacheTests(unittest.TestCase):
             ),
         )
 
+        # Query a subset of the entries mid-way through the stream. We should
+        # only get back the subset.
+        self.assertEqual(
+            cache.get_entities_changed(
+                [
+                    "bar@baz.net",
+                ],
+                stream_pos=2,
+            ),
+            set(
+                [
+                    "bar@baz.net",
+                ]
+            ),
+        )
+
     def test_max_pos(self):
         """
         StreamChangeCache.get_max_pos_of_last_change will return the most