diff --git a/nix/lib/config/options.nix b/nix/lib/config/options.nix index 0ced1a9..6fb49cb 100644 --- a/nix/lib/config/options.nix +++ b/nix/lib/config/options.nix @@ -293,6 +293,11 @@ let type = with types; nullOr str; default = null; }; + diskSize = mkOption { + type = types.int; + default = 64; + description = "Root disk size for containers in MB"; + }; interfaces = mkOption { default = {}; type = with types; attrsOf (submodule interfaceOpts); diff --git a/nix/nixos-module/server/default.nix b/nix/nixos-module/server/default.nix index c232c57..ef242eb 100644 --- a/nix/nixos-module/server/default.nix +++ b/nix/nixos-module/server/default.nix @@ -3,6 +3,7 @@ imports = [ ./defaults.nix ./network.nix + ./drbd-utils.nix ./lxc-containers.nix ./qemu.nix ./pacemaker.nix diff --git a/nix/nixos-module/server/drbd-utils.nix b/nix/nixos-module/server/drbd-utils.nix new file mode 100644 index 000000000..1f57a0a --- /dev/null +++ b/nix/nixos-module/server/drbd-utils.nix @@ -0,0 +1,66 @@ +{ self, config, lib, pkgs, ... }: +with lib; + +let cfg = config.services.drbd-utils; in +{ + ###### interface + + options = { + + services.drbd-utils.enable = mkOption { + default = false; + type = types.bool; + description = '' + Whether to enable support for DRBD, the Distributed Replicated + Block Device. + ''; + }; + + services.drbd-utils.config = mkOption { + default = ""; + type = types.lines; + description = '' + Contents of the drbd.conf configuration file. + ''; + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + # use the latest drbd-utils from outside nixpkgs + nixpkgs.config.packageOverrides = pkgs: { + drbd = self.packages.${pkgs.system}.drbd-utils; + }; + + environment.systemPackages = [ pkgs.drbd ]; + services.udev.packages = [ pkgs.drbd ]; + boot.kernelModules = [ "drbd" ]; + + boot.extraModprobeConfig = + '' + options drbd usermode_helper=/run/current-system/sw/bin/drbdadm + ''; + + environment.etc."drbd.conf" = + { source = pkgs.writeText "drbd.conf" cfg.config; }; + + systemd.services.drbd = { + after = [ "systemd-udev.settle.service" "network-online.target" ]; + requires = [ "network-online.target" ]; + wants = [ "systemd-udev.settle.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.drbd ]; + script = '' + drbdadm up all + ''; + preStop = '' + drbdadm down all + ''; + serviceConfig.RemainAfterExit = true; + }; + }; +} diff --git a/nix/nixos-module/server/lxc-containers.nix b/nix/nixos-module/server/lxc-containers.nix index e371150..96945e4 100644 --- a/nix/nixos-module/server/lxc-containers.nix +++ b/nix/nixos-module/server/lxc-containers.nix @@ -1,6 +1,13 @@ { hostName, self, config, lib, pkgs, ... }: let + drbdPortBase = 17000; + + servers = + lib.filterAttrs (_: { role, ... }: + role == "server" + ) config.site.hosts; + # Containers that are run on this host containers = lib.filterAttrs (_: { role, model, ... }: @@ -109,14 +116,7 @@ let esac echo Installing $c - for d in \ - bin dev etc home mnt \ - nix/store nix/var \ - proc root run sys tmp var usr ; \ - do - mkdir -p /var/lib/lxc/$c/rootfs/$d - done - ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init + ln -fs $SYSTEM /var/lib/lxc/$c/system done # Activate all the desired container after all of them are @@ -150,6 +150,7 @@ in "net.ipv6.neigh.default.gc_thresh3" = 8192; "kernel.keys.maxkeys" = 2000; }; + boot.kernelModules = [ "loop" ]; virtualisation.lxc = lib.mkIf enabled { enable = true; @@ -180,8 +181,9 @@ in # Handled by lxc@.service lxc.start.auto = 0 lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs - lxc.init.cmd = "/init" + lxc.init.cmd = "/system/init" + lxc.mount.entry = /var/lib/lxc/${ctName}/system system none bind,ro 0 0 lxc.mount.entry = /nix/store nix/store none bind,ro 0 0 lxc.mount.entry = none tmp tmpfs defaults 0 0 lxc,mount.auto = proc:mixed sys:ro cgroup:mixed @@ -215,8 +217,9 @@ in systemd.services."lxc@" = { description = "LXC container '%i'"; after = [ "network.target" ]; + requires = [ "lxc-rootfs@%i.service" ]; unitConfig.ConditionPathExists = [ - "/var/lib/lxc/%i/rootfs/init" + "/var/lib/lxc/%i/system" ]; serviceConfig = with pkgs; { Type = "simple"; @@ -239,6 +242,124 @@ in }; }; + systemd.services."lxc-rootfs@" = { + wants = [ "drbd.service" ]; + unitConfig.ConditionPathExists = [ + "/dev/drbd/by-res/%i" + ]; + serviceConfig = + let + mkScript = name: content: + "${pkgs.writeShellScriptBin name '' + PATH=$PATH:/run/current-system/sw/bin + ${content} + ''}/bin/${name} %i"; + in { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = mkScript "lxc-rootfs-start" '' + drbdadm primary $1 + ROOT=/var/lib/lxc/$1/rootfs + mkdir -p $ROOT + mount /dev/drbd/by-res/$1 $ROOT + for DIR in \ + bin dev etc home mnt \ + nix/store nix/var \ + proc root run sys \ + system tmp var usr \ + ; do + mkdir -p $ROOT/$DIR + done + ''; + ExecStop = mkScript "lxc-rootfs-stop" '' + umount /var/lib/lxc/$1/rootfs + drbdadm secondary $1 + ''; + }; + }; + + # systemd.services.drbd.wants = [ "lxc-volumes.service" ]; + systemd.services.lxc-volumes = { + description = "Create LXC rootfs for drbd"; + wantedBy = [ "drbd.service" ]; + path = with pkgs; [ drbd coreutils util-linux e2fsprogs ]; + script = '' + create() { + file=$1 + size=$2 + [ -e $file ] || \ + dd if=/dev/zero of=$file bs=1024 seek=$(($size - 1)) count=1 + } + + ${lib.concatStrings (lib.imap0 (i: container: '' + mkdir -p /var/lib/lxc/${container} + cd /var/lib/lxc/${container} + + if [ ! -e rootfs.img ]; then + create rootfs.img ${toString (containers.${container}.diskSize * 1024)} + mkfs.ext4 rootfs.img + else + losetup -d /dev/loop${toString (2 * i)} 2> /dev/null || true + fi + # TODO: zfsPool + losetup /dev/loop${toString (2 * i)} rootfs.img + + if [ ! -e rootfs.meta ]; then + create rootfs.meta ${toString (((4095 + 36 + (containers.${container}.diskSize / 32)) / 4096) * 4096)} + losetup /dev/loop${toString (2 * i + 1)} rootfs.meta + drbdadm create-md ${container} + else + losetup /dev/loop${toString (2 * i + 1)} rootfs.meta + fi + '') (builtins.attrNames containers))} + ''; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + }; + + services.drbd-utils = { + enable = true; + config = '' + global { + # Do not participate in online survey + usage-count no; + } + + common { + # Protocol A: write IO is reported as completed, if it has + # reached local disk and local TCP send buffer. + protocol A; + syncer { + rate 60M; + } + } + + ${lib.concatStrings (lib.imap0 (i: container: '' + resource ${container} { + net { + cram-hmac-alg sha1; + shared-secret "TODO"; + } + device minor ${toString i}; + # /var/lib/lxc/${container}/rootfs.img + disk /dev/loop${toString (i * 2)}; + # /var/lib/lxc/${container}/rootfs.meta + meta-disk /dev/loop${toString (i * 2 + 1)}; + ${lib.concatMapStrings (server: '' + on ${server} { + address ${config.site.net.cluster.hosts4.${server}}:${toString (drbdPortBase + i)}; + } + '') (builtins.attrNames servers)} + } + '') (builtins.attrNames containers))} + ''; + }; + networking.firewall.allowedTCPPorts = + lib.imap0 (i: _server: drbdPortBase + i) + (builtins.attrNames servers); + services.pacemaker.services = map (ctName: "lxc@${ctName}.service") (builtins.attrNames containers); diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index bf518c0..5d9e059 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -120,6 +120,6 @@ in rootfs-packages // vm-packages // device-templates // network-graphs // starlink // subnetplans // { inherit all-rootfs export-openwrt-models export-config dns-slaves encrypt-secrets decrypt-secrets switch-to-production - pacemaker + drbd-utils pacemaker ; }