diff --git a/changelog.d/4617.misc b/changelog.d/4617.misc
new file mode 100644
index 0000000000000000000000000000000000000000..6d751865c959e183dc41300f37a1e0841ab1c523
--- /dev/null
+++ b/changelog.d/4617.misc
@@ -0,0 +1 @@
+Don't create server contexts when TLS is disabled
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 6d72de1daad7cf29919f96ada126ed69cac4b154..6b3cb61ae9fc27a2457341d4e16d00954724ef82 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -214,6 +214,11 @@ def refresh_certificate(hs):
     disk and updating the TLS context factories to use them.
     """
     hs.config.read_certificate_from_disk()
+
+    if hs.config.no_tls:
+        # nothing else to do here
+        return
+
     hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
 
     if hs._listening_services:
diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py
index 286ad80100152ade5f13d85e8a91baac28f87347..85f2848fb183c3cde688e427eba4ce0737a3b0e8 100644
--- a/synapse/crypto/context_factory.py
+++ b/synapse/crypto/context_factory.py
@@ -43,9 +43,7 @@ class ServerContextFactory(ContextFactory):
             logger.exception("Failed to enable elliptic curve for TLS")
         context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
         context.use_certificate_chain_file(config.tls_certificate_file)
-
-        if not config.no_tls:
-            context.use_privatekey(config.tls_private_key)
+        context.use_privatekey(config.tls_private_key)
 
         # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
         context.set_cipher_list(