From b31daac01c9cc757d7c73da2f23e1b7251c54b79 Mon Sep 17 00:00:00 2001
From: Eric Eastwood <erice@element.io>
Date: Fri, 11 Jun 2021 04:12:35 -0500
Subject: [PATCH] Add metrics to track how often events are `soft_failed`
(#10156)
Spawned from missing messages we were seeing on `matrix.org` from a
federated Gtiter bridged room, https://gitlab.com/gitterHQ/webapp/-/issues/2770.
The underlying issue in Synapse is tracked by https://github.com/matrix-org/synapse/issues/10066
where the message and join event race and the message is `soft_failed` before the
`join` event reaches the remote federated server.
Less soft_failed events = better and usually this should only trigger for events
where people are doing bad things and trying to fuzz and fake everything.
---
changelog.d/10156.misc | 1 +
synapse/handlers/federation.py | 7 +++++++
2 files changed, 8 insertions(+)
create mode 100644 changelog.d/10156.misc
diff --git a/changelog.d/10156.misc b/changelog.d/10156.misc
new file mode 100644
index 0000000000..92a188b87b
--- /dev/null
+++ b/changelog.d/10156.misc
@@ -0,0 +1 @@
+Add `synapse_federation_soft_failed_events_total` metric to track how often events are soft failed.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 6e40e2c216..6647063485 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -33,6 +33,7 @@ from typing import (
)
import attr
+from prometheus_client import Counter
from signedjson.key import decode_verify_key_bytes
from signedjson.sign import verify_signed_json
from unpaddedbase64 import decode_base64
@@ -101,6 +102,11 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+soft_failed_event_counter = Counter(
+ "synapse_federation_soft_failed_events_total",
+ "Events received over federation that we marked as soft_failed",
+)
+
@attr.s(slots=True)
class _NewEventInfo:
@@ -2498,6 +2504,7 @@ class FederationHandler(BaseHandler):
event_auth.check(room_version_obj, event, auth_events=current_auth_events)
except AuthError as e:
logger.warning("Soft-failing %r because %s", event, e)
+ soft_failed_event_counter.inc()
event.internal_metadata.soft_failed = True
async def on_get_missing_events(
--
GitLab