schema_versions.py

#!/usr/bin/env python
# Copyright 2023 The Matrix.org Foundation C.I.C.
# Copyright 2023 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A script to calculate which versions of Synapse have backwards-compatible
database schemas. It creates a Markdown table of Synapse versions and the earliest
compatible version.

It is compatible with the mdbook protocol for preprocessors (see
https://rust-lang.github.io/mdBook/for_developers/preprocessors.html#implementing-a-preprocessor-with-a-different-language):

Exit 0 to denote support for all renderers:

    ./scripts-dev/schema_versions.py supports <mdbook renderer>

Parse a JSON list from stdin and add the table to the proper documetnation page:

    ./scripts-dev/schema_versions.py

Additionally, the script supports dumping the table to stdout for debugging:

    ./scripts-dev/schema_versions.py dump
"""

import io
import json
import sys
from collections import defaultdict
from typing import Any, Dict, Iterator, Optional, Tuple

import git
from packaging import version

# The schema version has moved around over the years.
SCHEMA_VERSION_FILES = (
    "synapse/storage/schema/__init__.py",
    "synapse/storage/prepare_database.py",
    "synapse/storage/__init__.py",
    "synapse/app/homeserver.py",
)


# Skip versions of Synapse < v1.0, they're old and essentially not
# compatible with today's federation.
OLDEST_SHOWN_VERSION = version.parse("v1.0")


def get_schema_versions(tag: git.Tag) -> Tuple[Optional[int], Optional[int]]:
    """Get the schema and schema compat versions for a tag."""
    schema_version = None
    schema_compat_version = None

    for file in SCHEMA_VERSION_FILES:
        try:
            schema_file = tag.commit.tree / file
        except KeyError:
            continue

        # We (usually) can't execute the code since it might have unknown imports.
        if file != "synapse/storage/schema/__init__.py":
            with io.BytesIO(schema_file.data_stream.read()) as f:
                for line in f.readlines():
                    if line.startswith(b"SCHEMA_VERSION"):
                        schema_version = int(line.split()[2])

                        # Bail early.
                        break
        else:
            # SCHEMA_COMPAT_VERSION is sometimes across multiple lines, the easist
            # thing to do is exec the code. Luckily it has only ever existed in
            # a file which imports nothing else from Synapse.
            locals: Dict[str, Any] = {}
            exec(schema_file.data_stream.read().decode("utf-8"), {}, locals)
            schema_version = locals["SCHEMA_VERSION"]
            schema_compat_version = locals.get("SCHEMA_COMPAT_VERSION")

    return schema_version, schema_compat_version


def get_tags(repo: git.Repo) -> Iterator[git.Tag]:
    """Return an iterator of tags sorted by version."""
    tags = []
    for tag in repo.tags:
        # All "real" Synapse tags are of the form vX.Y.Z.
        if not tag.name.startswith("v"):
            continue

        # There's a weird tag from the initial react UI.
        if tag.name == "v0.1":
            continue

        try:
            tag_version = version.parse(tag.name)
        except version.InvalidVersion:
            # Skip invalid versions.
            continue

        # Skip pre- and post-release versions.
        if tag_version.is_prerelease or tag_version.is_postrelease or tag_version.local:
            continue

        # Skip old versions.
        if tag_version < OLDEST_SHOWN_VERSION:
            continue

        tags.append((tag_version, tag))

    # Sort based on the version number (not lexically).
    return (tag for _, tag in sorted(tags, key=lambda t: t[0]))


def calculate_version_chart() -> str:
    repo = git.Repo(path=".")

    # Map of schema version -> Synapse versions which are at that schema version.
    schema_versions = defaultdict(list)
    # Map of schema version -> Synapse versions which are compatible with that
    # schema version.
    schema_compat_versions = defaultdict(list)

    # Find ranges of versions which are compatible with a schema version.
    #
    # There are two modes of operation:
    #
    # 1. Pre-schema_compat_version (i.e. schema_compat_version of None), then
    #    Synapse is compatible up/downgrading to a version with
    #    schema_version >= its current version.
    #
    # 2. Post-schema_compat_version (i.e. schema_compat_version is *not* None),
    #    then Synapse is compatible up/downgrading to a version with
    #    schema version >= schema_compat_version.
    #
    #    This is more generous and avoids versions that cannot be rolled back.
    #
    # See https://github.com/matrix-org/synapse/pull/9933 which was included in v1.37.0.
    for tag in get_tags(repo):
        schema_version, schema_compat_version = get_schema_versions(tag)

        # If a schema compat version is given, prefer that over the schema version.
        schema_versions[schema_version].append(tag.name)
        schema_compat_versions[schema_compat_version or schema_version].append(tag.name)

    # Generate a table which maps the latest Synapse version compatible with each
    # schema version.
    result = f"| {'Versions': ^19} | Compatible version |\n"
    result += f"|{'-' * (19 + 2)}|{'-' * (18 + 2)}|\n"
    for schema_version, synapse_versions in schema_compat_versions.items():
        result += f"| {synapse_versions[0] + ' – ' + synapse_versions[-1]: ^19} | {schema_versions[schema_version][0]: ^18} |\n"

    return result


if __name__ == "__main__":
    if len(sys.argv) == 3 and sys.argv[1] == "supports":
        # We don't care about the renderer which is being used, which is the second argument.
        sys.exit(0)
    elif len(sys.argv) == 2 and sys.argv[1] == "dump":
        print(calculate_version_chart())
    else:
        # Expect JSON data on stdin.
        context, book = json.load(sys.stdin)

        for section in book["sections"]:
            if "Chapter" in section and section["Chapter"]["path"] == "upgrade.md":
                section["Chapter"]["content"] = section["Chapter"]["content"].replace(
                    "<!-- REPLACE_WITH_SCHEMA_VERSIONS -->", calculate_version_chart()
                )

        # Print the result back out to stdout.
        print(json.dumps(book))