diff --git a/.envrc b/.envrc index b9238c3bb2c0e4c82741e9bf44fa9e757222b6f4..403a9bdf34c58db239f159aed1f43bf2433d0f9e 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,5 @@ #!/usr/bin/env bash use flake + +PATH_add bin diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2015ee80795fc4d1f2a113721aae105e29ff2171..fb06a07cf6cafad9cd234749771b205989033442 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,14 @@ before_script: # Enable nix-command and flakes - if command -v nix > /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + # Add our own binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://nix.computer.surgery/conduit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:ZGAf6P6LhNvnoJJ3Me3PRg7tlLSrPxcQ2RiE5LIppjo=" >> /etc/nix/nix.conf; fi + + # Add crane binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi + # Add nix-community binary cache - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi @@ -34,20 +42,70 @@ ci: - target - .gitlab-ci.d -docker: +static:x86_64-unknown-linux-musl: + stage: artifacts + image: nixos/nix:2.19.2 + script: + # Push artifacts and build requirements to binary cache + - ./bin/nix-build-and-cache .#static-x86_64-unknown-linux-musl + + # Make the output less difficult to find + - cp result/bin/conduit conduit + artifacts: + paths: + - conduit + +static:aarch64-unknown-linux-musl: + stage: artifacts + image: nixos/nix:2.19.2 + script: + # Push artifacts and build requirements to binary cache + - ./bin/nix-build-and-cache .#static-aarch64-unknown-linux-musl + + # Make the output less difficult to find + - cp result/bin/conduit conduit + artifacts: + paths: + - conduit + +oci-image:x86_64-unknown-linux-gnu: + stage: artifacts + image: nixos/nix:2.19.2 + script: + # Push artifacts and build requirements to binary cache + # + # Since the OCI image package is based on the binary package, this has the + # fun side effect of uploading the normal binary too. Conduit users who are + # deploying with Nix can leverage this fact by adding our binary cache to + # their systems. + - ./bin/nix-build-and-cache .#oci-image + + # Make the output less difficult to find + - cp result oci-image.tar.gz + artifacts: + paths: + - oci-image.tar.gz + +oci-image:aarch64-unknown-linux-musl: stage: artifacts + needs: + # Wait for the static binary job to finish before starting so we don't have + # to build that twice for no reason + - static:aarch64-unknown-linux-musl image: nixos/nix:2.19.2 script: - - nix build .#oci-image + # Push artifacts and build requirements to binary cache + - ./bin/nix-build-and-cache .#oci-image-aarch64-unknown-linux-musl # Make the output less difficult to find - - cp result docker-image.tar.gz + - cp result oci-image.tar.gz artifacts: paths: - docker-image.tar.gz -debian: +debian:x86_64-unknown-linux-gnu: stage: artifacts + # See also `rust-toolchain.toml` image: rust:1.75.0 script: - apt-get update && apt-get install -y --no-install-recommends libclang-dev @@ -55,10 +113,10 @@ debian: - cargo deb # Make the output less difficult to find - - mv target/debian/*.deb . + - mv target/debian/*.deb conduit.deb artifacts: paths: - - "*.deb" + - conduit.deb cache: key: debian paths: diff --git a/Cargo.toml b/Cargo.toml index 026676c5cba6d4aeec75bd2b67111612e6b7d11c..937d5dd38c4bcc30dc850a89f021ab20522c7184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,10 +9,7 @@ readme = "README.md" version = "0.7.0-alpha+conduwuit-0.1.2" edition = "2021" -# When changing this, make sure to update the `flake.lock` file by running -# `nix flake update`. If you don't have Nix installed or otherwise don't know -# how to do this, ping `@charles:computer.surgery` or `@dusk:gaze.systems` in -# the matrix room. +# See also `rust-toolchain.toml` rust-version = "1.75.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache new file mode 100644 index 0000000000000000000000000000000000000000..b37ebd85f56cd559450c7d86d06b39c2ab5822c1 --- /dev/null +++ b/bin/nix-build-and-cache @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# The first argument must be the desired installable +INSTALLABLE="$1" + +# Build the installable and forward any other arguments too +nix build "$@" + +if [ ! -z ${ATTIC_TOKEN+x} ]; then + +nix run --inputs-from . attic -- login \ + conduit \ + https://nix.computer.surgery/conduit \ + "$ATTIC_TOKEN" + + push_args=( + # Attic and its build dependencies + "$(nix path-info --inputs-from . attic)" + "$(nix path-info --inputs-from . attic --derivation)" + + # The target installable and its build dependencies + "$(nix path-info "$INSTALLABLE" --derivation)" + "$(nix path-info "$INSTALLABLE")" + ) + + nix run --inputs-from . attic -- push conduit "${push_args[@]}" +else + echo "\$ATTIC_TOKEN is unset, skipping uploading to the binary cache" +fi diff --git a/flake.lock b/flake.lock index 092faeac744f89d77d13518a59db711f18db3934..5b670ef62ec6165590bf0fbf33636437f5c4dd57 100644 --- a/flake.lock +++ b/flake.lock @@ -1,17 +1,41 @@ { "nodes": { + "attic": { + "inputs": { + "crane": "crane", + "flake-compat": "flake-compat", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1705617092, + "narHash": "sha256-n9PK4O4X4S1JkwpkMuYm1wHZYJzRqif8g3RuVIPD+rY=", + "owner": "zhaofengli", + "repo": "attic", + "rev": "fbe252a5c21febbe920c025560cbd63b20e24f3b", + "type": "github" + }, + "original": { + "owner": "zhaofengli", + "ref": "main", + "repo": "attic", + "type": "github" + } + }, "crane": { "inputs": { "nixpkgs": [ + "attic", "nixpkgs" ] }, "locked": { - "lastModified": 1705974079, - "narHash": "sha256-HyC3C2esW57j6bG0MKwX4kQi25ltslRnr6z2uvpadJo=", + "lastModified": 1702918879, + "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=", "owner": "ipetkov", "repo": "crane", - "rev": "0b4e511fe6e346381e31d355e03de52aa43e8cb2", + "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb", "type": "github" }, "original": { @@ -20,6 +44,27 @@ "type": "github" } }, + "crane_2": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1706070683, + "narHash": "sha256-iQoXIFviH5QbSKDkWyFJKKUtTcSiCvl1tW18ax+pKfI=", + "owner": "CobaltCause", + "repo": "crane", + "rev": "8060686afabc5b042e8982200cb2cc3fd22cf416", + "type": "github" + }, + "original": { + "owner": "CobaltCause", + "ref": "crimes-for-cross", + "repo": "crane", + "type": "github" + } + }, "fenix": { "inputs": { "nixpkgs": [ @@ -28,11 +73,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1706077451, - "narHash": "sha256-kd8Mlh+4NIG/NIkXeEwSIlwQuvysKJM4BeLrt2nvcc8=", + "lastModified": 1706163833, + "narHash": "sha256-Vw+jTVtKceT+ScaIn7tHy8JjRZZpmg2fAdoInLAsW/M=", "owner": "nix-community", "repo": "fenix", - "rev": "77d5a2dd0b186c40953c435da6e0c2215d7e1dec", + "rev": "043f63f55e9c9b808852ea82edee1f2a1af37e91", "type": "github" }, "original": { @@ -41,7 +86,38 @@ "type": "github" } }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { "inputs": { "systems": "systems" }, @@ -75,6 +151,38 @@ } }, "nixpkgs": { + "locked": { + "lastModified": 1702539185, + "narHash": "sha256-KnIRG5NMdLIpEkZTnN5zovNYc0hhXjAgv6pfd5Z4c7U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "aa9d4729cbc99dabacb50e3994dcefb3ea0f7447", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1702780907, + "narHash": "sha256-blbrBBXjjZt6OKTcYX1jpe9SRof2P9ZYWPzq22tzXAA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1e2e384c5b7c50dbf8e9c441a9e58d85f408b01f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { "locked": { "lastModified": 1705856552, "narHash": "sha256-JXfnuEf5Yd6bhMs/uvM67/joxYKoysyE3M2k6T3eWbg=", @@ -92,21 +200,22 @@ }, "root": { "inputs": { - "crane": "crane", + "attic": "attic", + "crane": "crane_2", "fenix": "fenix", - "flake-utils": "flake-utils", + "flake-utils": "flake-utils_2", "nix-filter": "nix-filter", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs_2" } }, "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1705864945, - "narHash": "sha256-ZATChFWHToTZQFLlzrzDUX8fjEbMHHBIyPaZU1JGmjI=", + "lastModified": 1706106882, + "narHash": "sha256-31DivWu0cC50gR2CgbGtLCf77nuiw4kdiI7B8ioqzLw=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "d410d4a2baf9e99b37b03dd42f06238b14374bf7", + "rev": "0d52934d19d7addcafcfda92a1d547b51556beec", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1e81308be8e8e60c0fe8b604fd67bb80724e758b..4b456d24810ee0ec0d66038163e263b6ae39450a 100644 --- a/flake.nix +++ b/flake.nix @@ -10,9 +10,13 @@ }; crane = { - url = "github:ipetkov/crane"; + # TODO: Switch back to upstream after [this issue][0] is fixed + # + # [0]: https://github.com/ipetkov/crane/issues/497 + url = "github:CobaltCause/crane?ref=crimes-for-cross"; inputs.nixpkgs.follows = "nixpkgs"; }; + attic.url = "github:zhaofengli/attic?ref=main"; }; outputs = @@ -23,6 +27,7 @@ , fenix , crane + , ... }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { @@ -47,55 +52,119 @@ ]; }; - # Use mold on Linux - stdenv = if pkgs.stdenv.isLinux then - pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv - else - pkgs.stdenv; + pkgsHost = nixpkgs.legacyPackages.${system}; # Nix-accessible `Cargo.toml` cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); # The Rust toolchain to use - toolchain = fenix.packages.${system}.toolchainOf { - # Use the Rust version defined in `Cargo.toml` - channel = cargoToml.package.rust-version; + toolchain = fenix.packages.${system}.fromToolchainFile { + file = ./rust-toolchain.toml; - # THE rust-version HASH + # See also `rust-toolchain.toml` sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8="; }; - mkToolchain = fenix.packages.${system}.combine; + builder = pkgs: + ((crane.mkLib pkgs).overrideToolchain toolchain).buildPackage; - buildToolchain = mkToolchain (with toolchain; [ - cargo - rustc - ]); + nativeBuildInputs = pkgs: [ + # bindgen needs the build platform's libclang. Apparently due to + # "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't + # quite do the right thing here. + pkgs.buildPackages.rustPlatform.bindgenHook + ]; - devToolchain = mkToolchain (with toolchain; [ - cargo - clippy - rust-src - rustc - - # Always use nightly rustfmt because most of its options are unstable - fenix.packages.${system}.latest.rustfmt - ]); - - builder = - ((crane.mkLib pkgs).overrideToolchain buildToolchain).buildPackage; - - nativeBuildInputs = (with pkgs.rustPlatform; [ - bindgenHook - ]); - - env = { + env = pkgs: { ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include"; ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib"; - }; - in - { - packages.default = builder { + } + // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isStatic { + ROCKSDB_STATIC = ""; + } + // { + CARGO_BUILD_RUSTFLAGS = let inherit (pkgs) lib stdenv; in + lib.concatStringsSep " " ([] + ++ lib.optionals + # This disables PIE for static builds, which isn't great in terms + # of security. Unfortunately, my hand is forced because nixpkgs' + # `libstdc++.a` is built without `-fPIE`, which precludes us from + # leaving PIE enabled. + stdenv.hostPlatform.isStatic + ["-C" "relocation-model=static"] + ++ lib.optionals + (stdenv.buildPlatform.config != pkgs.stdenv.hostPlatform.config) + ["-l" "c"] + ++ lib.optionals + # This check has to match the one [here][0]. We only need to set + # these flags when using a different linker. Don't ask me why, + # though, because I don't know. All I know is it breaks otherwise. + # + # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L36-L39 + ( + pkgs.stdenv.hostPlatform.isAarch64 + && pkgs.stdenv.hostPlatform.isStatic + && !pkgs.stdenv.isDarwin + && !pkgs.stdenv.cc.bintools.isLLVM + ) + [ + "-l" + "stdc++" + "-L" + "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" + ] + ); + } + + # What follows is stolen from [here][0]. Its purpose is to properly + # configure compilers and linkers for various stages of the build, and + # even covers the case of build scripts that need native code compiled and + # run on the build platform (I think). + # + # [0]: https://github.com/NixOS/nixpkgs/blob/612f97239e2cc474c13c9dafa0df378058c5ad8d/pkgs/build-support/rust/lib/default.nix#L64-L78 + // ( + let + inherit (pkgs.rust.lib) envVars; + in + pkgs.lib.optionalAttrs + (pkgs.stdenv.targetPlatform.rust.rustcTarget + != pkgs.stdenv.hostPlatform.rust.rustcTarget) + ( + let + inherit (pkgs.stdenv.targetPlatform.rust) cargoEnvVarTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = + envVars.linkerForTarget; + } + ) + // ( + let + inherit (pkgs.stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForHost; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost; + CARGO_BUILD_TARGET = rustcTarget; + } + ) + // ( + let + inherit (pkgs.stdenv.buildPlatform.rust) cargoEnvVarTarget; + in + { + "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; + "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; + HOST_CC = "${pkgs.buildPackages.stdenv.cc}/bin/cc"; + HOST_CXX = "${pkgs.buildPackages.stdenv.cc}/bin/c++"; + } + )); + + package = pkgs: builder pkgs { src = nix-filter { root = ./.; include = [ @@ -108,46 +177,79 @@ # This is redundant with CI doCheck = false; - inherit - env - nativeBuildInputs - stdenv; + env = env pkgs; + nativeBuildInputs = nativeBuildInputs pkgs; meta.mainProgram = cargoToml.package.name; }; - packages.oci-image = - let - package = self.packages.${system}.default; - in - pkgs.dockerTools.buildImage { - name = package.pname; - tag = "latest"; - config = { - # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) are - # handled as expected - Entrypoint = [ - "${pkgs.lib.getExe' pkgs.tini "tini"}" - "--" - ]; - Cmd = [ - "${pkgs.lib.getExe package}" - ]; + mkOciImage = pkgs: package: + pkgs.dockerTools.buildImage { + name = package.pname; + tag = "latest"; + config = { + # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) + # are handled as expected + Entrypoint = [ + "${pkgs.lib.getExe' pkgs.tini "tini"}" + "--" + ]; + Cmd = [ + "${pkgs.lib.getExe package}" + ]; + }; }; - }; + in + { + packages = { + default = package pkgsHost; + + oci-image = mkOciImage pkgsHost self.packages.${system}.default; - devShells.default = (pkgs.mkShell.override { inherit stdenv; }) { - env = env // { + # Build an OCI image from the musl aarch64 build so we don't have to + # build for aarch64 twice (to make a gnu version specifically for the + # OCI image) + oci-image-aarch64-unknown-linux-musl = mkOciImage + pkgsHost + self.packages.${system}.static-aarch64-unknown-linux-musl; + + # Don't build a musl x86_64 OCI image because that would be pointless. + # Just use the gnu one (i.e. `self.packages."x86_64-linux".oci-image`). + } // builtins.listToAttrs ( + builtins.map + (crossSystem: { + name = "static-${crossSystem}"; + value = package (import nixpkgs { + inherit system; + crossSystem = { + config = crossSystem; + }; + }).pkgsStatic; + }) + [ + "x86_64-unknown-linux-musl" + "aarch64-unknown-linux-musl" + ] + ); + + devShells.default = pkgsHost.mkShell { + env = env pkgsHost // { # Rust Analyzer needs to be able to find the path to default crate # sources, and it can read this environment variable to do so. The # `rust-src` component is required in order for this to work. - RUST_SRC_PATH = "${devToolchain}/lib/rustlib/src/rust/library"; + RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library"; }; # Development tools - nativeBuildInputs = nativeBuildInputs ++ [ - devToolchain - ] ++ (with pkgs; [ + nativeBuildInputs = nativeBuildInputs pkgsHost ++ [ + # Always use nightly rustfmt because most of its options are unstable + # + # This needs to come before `toolchain` in this list, otherwise + # `$PATH` will have stable rustfmt instead. + fenix.packages.${system}.latest.rustfmt + + toolchain + ] ++ (with pkgsHost; [ engage ]); }; diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000000000000000000000000000000000000..8f59b2d7ea6bb331c40b98e93a8ed6660cca55f2 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,22 @@ +# This is the authoritiative configuration of this project's Rust toolchain. +# +# Other files that need upkeep when this changes: +# +# * `.gitlab-ci.yml` +# * `Cargo.toml` +# * `flake.nix` +# +# Search in those files for `rust-toolchain.toml` to find the relevant places. +# If you're having trouble making the relevant changes, bug a maintainer. + +[toolchain] +channel = "1.75.0" +components = [ + # For rust-analyzer + "rust-src", +] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", +] \ No newline at end of file