diff --git a/changelog.d/7799.misc b/changelog.d/7799.misc
new file mode 100644
index 0000000000000000000000000000000000000000..448b286df4002ea7e6e8d0b116cfe9bc46320157
--- /dev/null
+++ b/changelog.d/7799.misc
@@ -0,0 +1 @@
+Ensure that strings (not bytes) are passed into JSON serialization.
diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py
index 4ba004276890a01884696be50873d023f2a9451d..701233ebb4dfad9a38d2af10867c8a164e3934e8 100644
--- a/synapse/handlers/identity.py
+++ b/synapse/handlers/identity.py
@@ -251,10 +251,10 @@ class IdentityHandler(BaseHandler):
         # 'browser-like' HTTPS.
         auth_headers = self.federation_http_client.build_auth_headers(
             destination=None,
-            method="POST",
+            method=b"POST",
             url_bytes=url_bytes,
             content=content,
-            destination_is=id_server,
+            destination_is=id_server.encode("ascii"),
         )
         headers = {b"Authorization": auth_headers}
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 18f6a8fd292afa71df0c72d717b4024cc41ca76e..58aed5fd96cc6ecc7e99c1d6868488e2d0e7d48a 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -562,13 +562,17 @@ class MatrixFederationHttpClient(object):
         Returns:
             list[bytes]: a list of headers to be added as "Authorization:" headers
         """
-        request = {"method": method, "uri": url_bytes, "origin": self.server_name}
+        request = {
+            "method": method.decode("ascii"),
+            "uri": url_bytes.decode("ascii"),
+            "origin": self.server_name,
+        }
 
         if destination is not None:
-            request["destination"] = destination
+            request["destination"] = destination.decode("ascii")
 
         if destination_is is not None:
-            request["destination_is"] = destination_is
+            request["destination_is"] = destination_is.decode("ascii")
 
         if content is not None:
             request["content"] = content
diff --git a/synapse/rest/client/v1/voip.py b/synapse/rest/client/v1/voip.py
index 747d46eac20128319d06db9233487d6932694399..50277c6cf6c3bf623bba54e58c5f99494b6b63ba 100644
--- a/synapse/rest/client/v1/voip.py
+++ b/synapse/rest/client/v1/voip.py
@@ -50,7 +50,7 @@ class VoipRestServlet(RestServlet):
             # We need to use standard padded base64 encoding here
             # encode_base64 because we need to add the standard padding to get the
             # same result as the TURN server.
-            password = base64.b64encode(mac.digest())
+            password = base64.b64encode(mac.digest()).decode("ascii")
 
         elif turnUris and turnUsername and turnPassword and userLifetime:
             username = turnUsername