diff --git a/changelog.d/5232.misc b/changelog.d/5232.misc
index 1cdc71f095113de41fb89457aadc21570da30149..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5232.misc
+++ b/changelog.d/5232.misc
@@ -1 +1 @@
-Run black on synapse.crypto.keyring.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5234.misc b/changelog.d/5234.misc
index 43fbd6f67c59a71c3f76fe4d00e4b3768775bac9..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5234.misc
+++ b/changelog.d/5234.misc
@@ -1 +1 @@
-Rewrite store_server_verify_key to store several keys at once.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5235.misc b/changelog.d/5235.misc
index 2296ad2a4f46da5f9d189d1b3a716a9c428f5eea..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5235.misc
+++ b/changelog.d/5235.misc
@@ -1 +1 @@
-Remove unused VerifyKey.expired and .time_added fields.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5236.misc b/changelog.d/5236.misc
index cb4417a9f45f9d767dc692eb22e0a9c07b719d29..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5236.misc
+++ b/changelog.d/5236.misc
@@ -1 +1 @@
-Simplify Keyring.process_v2_response.
\ No newline at end of file
+Preparatory work for key-validity features.
diff --git a/changelog.d/5237.misc b/changelog.d/5237.misc
index f4fe3b821bf6601c718615a9d95d67e617579106..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5237.misc
+++ b/changelog.d/5237.misc
@@ -1 +1 @@
-Store key validity time in the storage layer.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5244.misc b/changelog.d/5244.misc
index 9cc1fb869de07f5749207188fadfe5303b0f6cd5..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5244.misc
+++ b/changelog.d/5244.misc
@@ -1 +1 @@
-Refactor synapse.crypto.keyring to use a KeyFetcher interface.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5250.misc b/changelog.d/5250.misc
index 575a299a8214ccfe0ac742f50a71d6724710dc83..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5250.misc
+++ b/changelog.d/5250.misc
@@ -1 +1 @@
-Simplification to Keyring.wait_for_previous_lookups.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5284.misc b/changelog.d/5284.misc
new file mode 100644
index 0000000000000000000000000000000000000000..c4d42ca3d9e03a586d14b1a21235daac59b47a89
--- /dev/null
+++ b/changelog.d/5284.misc
@@ -0,0 +1 @@
+Improve sample config for monthly active user blocking.
diff --git a/changelog.d/5296.misc b/changelog.d/5296.misc
index a038a6f7f64b6e4ad489d65835d25916c80661b9..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5296.misc
+++ b/changelog.d/5296.misc
@@ -1 +1 @@
-Refactor keyring.VerifyKeyRequest to use attr.s.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5299.misc b/changelog.d/5299.misc
index 53297c768b9591cf682174b1b413e9979871a60d..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5299.misc
+++ b/changelog.d/5299.misc
@@ -1 +1 @@
-Rewrite get_server_verify_keys, again.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5317.bugfix b/changelog.d/5317.bugfix
new file mode 100644
index 0000000000000000000000000000000000000000..2709375214933d683746156cdb8b096254f320ae
--- /dev/null
+++ b/changelog.d/5317.bugfix
@@ -0,0 +1 @@
+Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails.
diff --git a/changelog.d/5343.misc b/changelog.d/5343.misc
index dbee0f71b9d8de7f073f31249d5db368d2014393..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5343.misc
+++ b/changelog.d/5343.misc
@@ -1 +1 @@
-Rename VerifyKeyRequest.deferred field.
+Preparatory work for key-validity features.
diff --git a/changelog.d/5347.misc b/changelog.d/5347.misc
index 436245fb11aa044501142c4bebd2c9901cfc2b1c..8336bc55dc4802b1c1249964a617e4aa2a3d0344 100644
--- a/changelog.d/5347.misc
+++ b/changelog.d/5347.misc
@@ -1,2 +1 @@
-Various improvements to debug logging.
-
+Preparatory work for key-validity features.
diff --git a/changelog.d/5352.bugfix b/changelog.d/5352.bugfix
new file mode 100644
index 0000000000000000000000000000000000000000..2ffefe5a6846e4ef52c96a73594b30529af82b95
--- /dev/null
+++ b/changelog.d/5352.bugfix
@@ -0,0 +1 @@
+Fix room stats and presence background updates to correctly handle missing events.
diff --git a/changelog.d/5356.misc b/changelog.d/5356.misc
new file mode 100644
index 0000000000000000000000000000000000000000..8336bc55dc4802b1c1249964a617e4aa2a3d0344
--- /dev/null
+++ b/changelog.d/5356.misc
@@ -0,0 +1 @@
+Preparatory work for key-validity features.
diff --git a/changelog.d/5357.doc b/changelog.d/5357.doc
new file mode 100644
index 0000000000000000000000000000000000000000..27cba49641ffd37aea0a88df6221088544bd6b68
--- /dev/null
+++ b/changelog.d/5357.doc
@@ -0,0 +1 @@
+Fix notes about ACME in the MSC1711 faq.
diff --git a/changelog.d/5360.feature b/changelog.d/5360.feature
new file mode 100644
index 0000000000000000000000000000000000000000..01fbb3b06d9e2b89a16c033acde12ef243b9a9aa
--- /dev/null
+++ b/changelog.d/5360.feature
@@ -0,0 +1 @@
+Update /_matrix/client/versions to reference support for r0.5.0.
diff --git a/changelog.d/5362.bugfix b/changelog.d/5362.bugfix
new file mode 100644
index 0000000000000000000000000000000000000000..1c8b19182cb60f17377ee74d3078a23a2d4becff
--- /dev/null
+++ b/changelog.d/5362.bugfix
@@ -0,0 +1 @@
+Fix `federation_custom_ca_list` configuration option.
diff --git a/docs/MSC1711_certificates_FAQ.md b/docs/MSC1711_certificates_FAQ.md
index ebfb20f5c86c2b62699a7fb954888a7c5a96dea7..37f7f669c9489dd709c61aeb86f36b0e6394bc01 100644
--- a/docs/MSC1711_certificates_FAQ.md
+++ b/docs/MSC1711_certificates_FAQ.md
@@ -145,12 +145,11 @@ You can do this with a `.well-known` file as follows:
1. Keep the SRV record in place - it is needed for backwards compatibility
with Synapse 0.34 and earlier.
- 2. Give synapse a certificate corresponding to the target domain
- (`customer.example.net` in the above example). Currently Synapse's ACME
- support [does not support
- this](https://github.com/matrix-org/synapse/issues/4552), so you will have
- to acquire a certificate yourself and give it to Synapse via
- `tls_certificate_path` and `tls_private_key_path`.
+ 2. Give Synapse a certificate corresponding to the target domain
+ (`customer.example.net` in the above example). You can either use Synapse's
+ built-in [ACME support](./ACME.md) for this (via the `domain` parameter in
+ the `acme` section), or acquire a certificate yourself and give it to
+ Synapse via `tls_certificate_path` and `tls_private_key_path`.
3. Restart Synapse to ensure the new certificate is loaded.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index d10a355d68c502d6ce084abc1f0a563ad7355b17..2f37e71601c7465d44b6f578c723b8b862821891 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -261,6 +261,22 @@ listeners:
# Monthly Active User Blocking
#
+# Used in cases where the admin or server owner wants to limit to the
+# number of monthly active users.
+#
+# 'limit_usage_by_mau' disables/enables monthly active user blocking. When
+# anabled and a limit is reached the server returns a 'ResourceLimitError'
+# with error type Codes.RESOURCE_LIMIT_EXCEEDED
+#
+# 'max_mau_value' is the hard limit of monthly active users above which
+# the server will start blocking user actions.
+#
+# 'mau_trial_days' is a means to add a grace period for active users. It
+# means that users must be active for this number of days before they
+# can be considered active and guards against the case where lots of users
+# sign up in a short space of time never to return after their initial
+# session.
+#
#limit_usage_by_mau: False
#max_mau_value: 50
#mau_trial_days: 2
diff --git a/synapse/config/server.py b/synapse/config/server.py
index e763e19e15a426d2fee5da3fac7d11db4494fdfa..334921d4210426a1af801f1aff8b413606915710 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -585,6 +585,22 @@ class ServerConfig(Config):
# Monthly Active User Blocking
#
+ # Used in cases where the admin or server owner wants to limit to the
+ # number of monthly active users.
+ #
+ # 'limit_usage_by_mau' disables/enables monthly active user blocking. When
+ # anabled and a limit is reached the server returns a 'ResourceLimitError'
+ # with error type Codes.RESOURCE_LIMIT_EXCEEDED
+ #
+ # 'max_mau_value' is the hard limit of monthly active users above which
+ # the server will start blocking user actions.
+ #
+ # 'mau_trial_days' is a means to add a grace period for active users. It
+ # means that users must be active for this number of days before they
+ # can be considered active and guards against the case where lots of users
+ # sign up in a short space of time never to return after their initial
+ # session.
+ #
#limit_usage_by_mau: False
#max_mau_value: 50
#mau_trial_days: 2
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 43712b8213e0a4ef0364030e01b8976460c2cc96..658f9dd3618dec9adb4c2f707b4e0bf21b7e5e5a 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -107,7 +107,7 @@ class TlsConfig(Config):
certs = []
for ca_file in custom_ca_list:
logger.debug("Reading custom CA certificate file: %s", ca_file)
- content = self.read_file(ca_file)
+ content = self.read_file(ca_file, "federation_custom_ca_list")
# Parse the CA certificates
try:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index cf4fad7de0c9a6a5db808b78e2942d999695d692..ac5ca791431a82e56eecbbaa82b516c164c53fcf 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -35,6 +35,7 @@ from synapse.api.errors import (
CodeMessageException,
FederationDeniedError,
FederationError,
+ RequestSendFailed,
StoreError,
SynapseError,
)
@@ -2027,9 +2028,21 @@ class FederationHandler(BaseHandler):
"""
room_version = yield self.store.get_room_version(event.room_id)
- yield self._update_auth_events_and_context_for_auth(
- origin, event, context, auth_events
- )
+ try:
+ yield self._update_auth_events_and_context_for_auth(
+ origin, event, context, auth_events
+ )
+ except Exception:
+ # We don't really mind if the above fails, so lets not fail
+ # processing if it does. However, it really shouldn't fail so
+ # let's still log as an exception since we'll still want to fix
+ # any bugs.
+ logger.exception(
+ "Failed to double check auth events for %s with remote. "
+ "Ignoring failure and continuing processing of event.",
+ event.event_id,
+ )
+
try:
self.auth.check(room_version, event, auth_events=auth_events)
except AuthError as e:
@@ -2042,6 +2055,15 @@ class FederationHandler(BaseHandler):
):
"""Helper for do_auth. See there for docs.
+ Checks whether a given event has the expected auth events. If it
+ doesn't then we talk to the remote server to compare state to see if
+ we can come to a consensus (e.g. if one server missed some valid
+ state).
+
+ This attempts to resovle any potential divergence of state between
+ servers, but is not essential and so failures should not block further
+ processing of the event.
+
Args:
origin (str):
event (synapse.events.EventBase):
@@ -2088,9 +2110,15 @@ class FederationHandler(BaseHandler):
missing_auth,
)
try:
- remote_auth_chain = yield self.federation_client.get_event_auth(
- origin, event.room_id, event.event_id
- )
+ try:
+ remote_auth_chain = yield self.federation_client.get_event_auth(
+ origin, event.room_id, event.event_id
+ )
+ except RequestSendFailed as e:
+ # The other side isn't around or doesn't implement the
+ # endpoint, so lets just bail out.
+ logger.info("Failed to get event auth from remote: %s", e)
+ return
seen_remotes = yield self.store.have_seen_events(
[e.event_id for e in remote_auth_chain]
@@ -2236,12 +2264,18 @@ class FederationHandler(BaseHandler):
try:
# 2. Get remote difference.
- result = yield self.federation_client.query_auth(
- origin,
- event.room_id,
- event.event_id,
- local_auth_chain,
- )
+ try:
+ result = yield self.federation_client.query_auth(
+ origin,
+ event.room_id,
+ event.event_id,
+ local_auth_chain,
+ )
+ except RequestSendFailed as e:
+ # The other side isn't around or doesn't implement the
+ # endpoint, so lets just bail out.
+ logger.info("Failed to query auth from remote: %s", e)
+ return
seen_remotes = yield self.store.have_seen_events(
[e.event_id for e in result["auth_chain"]]
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
index 6209858bbb9aafb7a78186f608eddfd28dc4b417..e49c8203efc6d6da3bd9a6cff91f227e24d52146 100644
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -828,14 +828,17 @@ class PresenceHandler(object):
# joins.
continue
- event = yield self.store.get_event(event_id)
- if event.content.get("membership") != Membership.JOIN:
+ event = yield self.store.get_event(event_id, allow_none=True)
+ if not event or event.content.get("membership") != Membership.JOIN:
# We only care about joins
continue
if prev_event_id:
- prev_event = yield self.store.get_event(prev_event_id)
- if prev_event.content.get("membership") == Membership.JOIN:
+ prev_event = yield self.store.get_event(prev_event_id, allow_none=True)
+ if (
+ prev_event
+ and prev_event.content.get("membership") == Membership.JOIN
+ ):
# Ignore changes to join events.
continue
diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py
index 0e92b405ba6f2279a721396ddeede6f0ef9e64f8..7ad16c85665ef65e0260ec3c414823b29f40028d 100644
--- a/synapse/handlers/stats.py
+++ b/synapse/handlers/stats.py
@@ -115,6 +115,7 @@ class StatsHandler(StateDeltasHandler):
event_id = delta["event_id"]
stream_id = delta["stream_id"]
prev_event_id = delta["prev_event_id"]
+ stream_pos = delta["stream_id"]
logger.debug("Handling: %r %r, %s", typ, state_key, event_id)
@@ -136,10 +137,15 @@ class StatsHandler(StateDeltasHandler):
event_content = {}
if event_id is not None:
- event_content = (yield self.store.get_event(event_id)).content or {}
+ event = yield self.store.get_event(event_id, allow_none=True)
+ if event:
+ event_content = event.content or {}
+
+ # We use stream_pos here rather than fetch by event_id as event_id
+ # may be None
+ now = yield self.store.get_received_ts_by_stream_pos(stream_pos)
# quantise time to the nearest bucket
- now = yield self.store.get_received_ts(event_id)
now = (now // 1000 // self.stats_bucket_size) * self.stats_bucket_size
if typ == EventTypes.Member:
@@ -149,9 +155,11 @@ class StatsHandler(StateDeltasHandler):
# compare them.
prev_event_content = {}
if prev_event_id is not None:
- prev_event_content = (
- yield self.store.get_event(prev_event_id)
- ).content
+ prev_event = yield self.store.get_event(
+ prev_event_id, allow_none=True,
+ )
+ if prev_event:
+ prev_event_content = prev_event.content
membership = event_content.get("membership", Membership.LEAVE)
prev_membership = prev_event_content.get("membership", Membership.LEAVE)
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index 27e7cbf3cc008332eca9ac9c8fa3cf49c68483dd..babbf6a23ce86437c741bb4b20399928dbad47c0 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -39,6 +39,7 @@ class VersionsRestServlet(RestServlet):
"r0.2.0",
"r0.3.0",
"r0.4.0",
+ "r0.5.0",
],
# as per MSC1497:
"unstable_features": {
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 17824280485a12ca39345c283e89a74738979ad0..cc7df5cf14dfed71e4eaf0d60f78d24f8ec1dce5 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -78,6 +78,43 @@ class EventsWorkerStore(SQLBaseStore):
desc="get_received_ts",
)
+ def get_received_ts_by_stream_pos(self, stream_ordering):
+ """Given a stream ordering get an approximate timestamp of when it
+ happened.
+
+ This is done by simply taking the received ts of the first event that
+ has a stream ordering greater than or equal to the given stream pos.
+ If none exists returns the current time, on the assumption that it must
+ have happened recently.
+
+ Args:
+ stream_ordering (int)
+
+ Returns:
+ Deferred[int]
+ """
+
+ def _get_approximate_received_ts_txn(txn):
+ sql = """
+ SELECT received_ts FROM events
+ WHERE stream_ordering >= ?
+ LIMIT 1
+ """
+
+ txn.execute(sql, (stream_ordering,))
+ row = txn.fetchone()
+ if row and row[0]:
+ ts = row[0]
+ else:
+ ts = self.clock.time_msec()
+
+ return ts
+
+ return self.runInteraction(
+ "get_approximate_received_ts",
+ _get_approximate_received_ts_txn,
+ )
+
@defer.inlineCallbacks
def get_event(
self,
diff --git a/tests/handlers/test_stats.py b/tests/handlers/test_stats.py
index 249aba3d598f7299d1f8350c7a10e0111963abff..2710c991cfec1c854cdeada0b3db7f68c2abaa30 100644
--- a/tests/handlers/test_stats.py
+++ b/tests/handlers/test_stats.py
@@ -204,7 +204,7 @@ class StatsRoomTests(unittest.HomeserverTestCase):
"a2": {"membership": "not a real thing"},
}
- def get_event(event_id):
+ def get_event(event_id, allow_none=True):
m = Mock()
m.content = events[event_id]
d = defer.Deferred()
@@ -224,7 +224,7 @@ class StatsRoomTests(unittest.HomeserverTestCase):
"room_id": "room",
"event_id": "a1",
"prev_event_id": "a2",
- "stream_id": "bleb",
+ "stream_id": 60,
}
]
@@ -241,7 +241,7 @@ class StatsRoomTests(unittest.HomeserverTestCase):
"room_id": "room",
"event_id": "a2",
"prev_event_id": "a1",
- "stream_id": "bleb",
+ "stream_id": 100,
}
]
@@ -249,3 +249,59 @@ class StatsRoomTests(unittest.HomeserverTestCase):
self.assertEqual(
f.value.args[0], "'not a real thing' is not a valid membership"
)
+
+ def test_redacted_prev_event(self):
+ """
+ If the prev_event does not exist, then it is assumed to be a LEAVE.
+ """
+ u1 = self.register_user("u1", "pass")
+ u1_token = self.login("u1", "pass")
+
+ room_1 = self.helper.create_room_as(u1, tok=u1_token)
+
+ # Do the initial population of the user directory via the background update
+ self._add_background_updates()
+
+ while not self.get_success(self.store.has_completed_background_updates()):
+ self.get_success(self.store.do_next_background_update(100), by=0.1)
+
+ events = {
+ "a1": None,
+ "a2": {"membership": Membership.JOIN},
+ }
+
+ def get_event(event_id, allow_none=True):
+ if events.get(event_id):
+ m = Mock()
+ m.content = events[event_id]
+ else:
+ m = None
+ d = defer.Deferred()
+ self.reactor.callLater(0.0, d.callback, m)
+ return d
+
+ def get_received_ts(event_id):
+ return defer.succeed(1)
+
+ self.store.get_received_ts = get_received_ts
+ self.store.get_event = get_event
+
+ deltas = [
+ {
+ "type": EventTypes.Member,
+ "state_key": "some_user:test",
+ "room_id": room_1,
+ "event_id": "a2",
+ "prev_event_id": "a1",
+ "stream_id": 100,
+ }
+ ]
+
+ # Handle our fake deltas, which has a user going from LEAVE -> JOIN.
+ self.get_success(self.handler._handle_deltas(deltas))
+
+ # One delta, with two joined members -- the room creator, and our fake
+ # user.
+ r = self.get_success(self.store.get_deltas_for_room(room_1, 0))
+ self.assertEqual(len(r), 1)
+ self.assertEqual(r[0]["joined_members"], 2)