From e2c300e7e4647500b0ff5ada8018cc984b78f91c Mon Sep 17 00:00:00 2001
From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com>
Date: Fri, 26 Nov 2021 14:05:20 +0000
Subject: [PATCH] Create healthcheck script for synapse-workers container
(#11429)
The intent is to iterate through all the worker ports and only
report healthy when all are healthy, starting with the main process.
---
changelog.d/11429.docker | 1 +
docker/Dockerfile-workers | 3 +++
docker/conf-workers/healthcheck.sh.j2 | 6 ++++++
docker/configure_workers_and_start.py | 13 +++++++++++++
4 files changed, 23 insertions(+)
create mode 100644 changelog.d/11429.docker
create mode 100644 docker/conf-workers/healthcheck.sh.j2
diff --git a/changelog.d/11429.docker b/changelog.d/11429.docker
new file mode 100644
index 0000000000..81db719ed6
--- /dev/null
+++ b/changelog.d/11429.docker
@@ -0,0 +1 @@
+Update `Dockerfile-workers` to healthcheck all workers in container.
diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers
index 969cf97286..46f2e17382 100644
--- a/docker/Dockerfile-workers
+++ b/docker/Dockerfile-workers
@@ -21,3 +21,6 @@ VOLUME ["/data"]
# files to run the desired worker configuration. Will start supervisord.
COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
ENTRYPOINT ["/configure_workers_and_start.py"]
+
+HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
+ CMD /bin/sh /healthcheck.sh
diff --git a/docker/conf-workers/healthcheck.sh.j2 b/docker/conf-workers/healthcheck.sh.j2
new file mode 100644
index 0000000000..79c621f89c
--- /dev/null
+++ b/docker/conf-workers/healthcheck.sh.j2
@@ -0,0 +1,6 @@
+#!/bin/sh
+# This healthcheck script is designed to return OK when every
+# host involved returns OK
+{%- for healthcheck_url in healthcheck_urls %}
+curl -fSs {{ healthcheck_url }} || exit 1
+{%- endfor %}
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index f4ac1c22a4..adbb551cee 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -474,10 +474,16 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
# Determine the load-balancing upstreams to configure
nginx_upstream_config = ""
+
+ # At the same time, prepare a list of internal endpoints to healthcheck
+ # starting with the main process which exists even if no workers do.
+ healthcheck_urls = ["http://localhost:8080/health"]
+
for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
body = ""
for port in upstream_worker_ports:
body += " server localhost:%d;\n" % (port,)
+ healthcheck_urls.append("http://localhost:%d/health" % (port,))
# Add to the list of configured upstreams
nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -510,6 +516,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
worker_config=supervisord_config,
)
+ # healthcheck config
+ convert(
+ "/conf/healthcheck.sh.j2",
+ "/healthcheck.sh",
+ healthcheck_urls=healthcheck_urls,
+ )
+
# Ensure the logging directory exists
log_dir = data_dir + "/logs"
if not os.path.exists(log_dir):
--
GitLab