diff --git a/changelog.d/18124.misc b/changelog.d/18124.misc new file mode 100644 index 0000000000000000000000000000000000000000..8ac6a73a20dce481d7c9d1a70b41b247363983a5 --- /dev/null +++ b/changelog.d/18124.misc @@ -0,0 +1 @@ +Add log message when worker lock timeouts get large. diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index db998f6701a8ae0b7ed7d0698e114452d2e8a6c1..e58a4160264bb569c618c8d0d47cef4d7fcb623d 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -19,6 +19,7 @@ # # +import logging import random from types import TracebackType from typing import ( @@ -269,6 +270,10 @@ class WaitingLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) + if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes + logging.warning( + f"Lock timeout is getting excessive: {self._retry_interval}s. There may be a deadlock." + ) return next * random.uniform(0.9, 1.1) @@ -344,4 +349,8 @@ class WaitingMultiLock: def _get_next_retry_interval(self) -> float: next = self._retry_interval self._retry_interval = max(5, next * 2) + if self._retry_interval > 5 * 2 ^ 7: # ~10 minutes + logging.warning( + f"Lock timeout is getting excessive: {self._retry_interval}s. There may be a deadlock." + ) return next * random.uniform(0.9, 1.1)