nixos-module/server/lxc-containers: add to drbd rootfs

nix/lib/config/options.nix

@@ -293,6 +293,11 @@ let
         type = with types; nullOr str;
         default = null;
       };
+      diskSize = mkOption {
+        type = types.int;
+        default = 64;
+        description = "Root disk size for containers in MB";
+      };
       interfaces = mkOption {
         default = {};
         type = with types; attrsOf (submodule interfaceOpts);

nix/nixos-module/server/default.nix

@@ -3,6 +3,7 @@
   imports = [
     ./defaults.nix
     ./network.nix
+    ./drbd-utils.nix
     ./lxc-containers.nix
     ./qemu.nix
     ./pacemaker.nix

nix/nixos-module/server/drbd-utils.nix

@@ -0,0 +1,66 @@
+{ self, config, lib, pkgs, ... }:
+with lib;
+
+let cfg = config.services.drbd-utils; in
+{
+  ###### interface
+
+  options = {
+
+    services.drbd-utils.enable = mkOption {
+      default = false;
+      type = types.bool;
+      description = ''
+        Whether to enable support for DRBD, the Distributed Replicated
+        Block Device.
+      '';
+    };
+
+    services.drbd-utils.config = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        Contents of the <filename>drbd.conf</filename> configuration file.
+      '';
+    };
+
+  };
+
+
+  ###### implementation
+
+  config = mkIf cfg.enable {
+
+    # use the latest drbd-utils from outside nixpkgs
+    nixpkgs.config.packageOverrides = pkgs: {
+      drbd = self.packages.${pkgs.system}.drbd-utils;
+    };
+
+    environment.systemPackages = [ pkgs.drbd ];
+    services.udev.packages = [ pkgs.drbd ];
+    boot.kernelModules = [ "drbd" ];
+
+    boot.extraModprobeConfig =
+      ''
+        options drbd usermode_helper=/run/current-system/sw/bin/drbdadm
+      '';
+
+    environment.etc."drbd.conf" =
+      { source = pkgs.writeText "drbd.conf" cfg.config; };
+
+    systemd.services.drbd = {
+      after = [ "systemd-udev.settle.service" "network-online.target" ];
+      requires = [ "network-online.target" ];
+      wants = [ "systemd-udev.settle.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.drbd ];
+      script = ''
+        drbdadm up all
+      '';
+      preStop = ''
+        drbdadm down all
+      '';
+      serviceConfig.RemainAfterExit = true;
+    };
+  };
+}

nix/nixos-module/server/lxc-containers.nix

@@ -1,6 +1,13 @@
 { hostName, self, config, lib, pkgs, ... }:
 
 let
+  drbdPortBase = 17000;
+
+  servers =
+    lib.filterAttrs (_: { role, ... }:
+      role == "server"
+    ) config.site.hosts;
+
   # Containers that are run on this host
   containers =
     lib.filterAttrs (_: { role, model, ... }:
@@ -109,14 +116,7 @@ let
       esac
 
       echo Installing $c
-      for d in \
-        bin dev etc home mnt \
-        nix/store nix/var \
-        proc root run sys tmp var usr ; \
-      do
-        mkdir -p /var/lib/lxc/$c/rootfs/$d
-      done
-      ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
+      ln -fs $SYSTEM /var/lib/lxc/$c/system
     done
 
     # Activate all the desired container after all of them are
@@ -150,6 +150,7 @@ in
     "net.ipv6.neigh.default.gc_thresh3" = 8192;
     "kernel.keys.maxkeys" = 2000;
   };
+  boot.kernelModules = [ "loop" ];
 
   virtualisation.lxc = lib.mkIf enabled {
     enable = true;
@@ -180,8 +181,9 @@ in
             # Handled by lxc@.service
             lxc.start.auto = 0
             lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
-            lxc.init.cmd = "/init"
+            lxc.init.cmd = "/system/init"
 
+            lxc.mount.entry = /var/lib/lxc/${ctName}/system system none bind,ro 0 0
             lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
             lxc.mount.entry = none tmp tmpfs defaults 0 0
             lxc,mount.auto = proc:mixed sys:ro cgroup:mixed
@@ -215,8 +217,9 @@ in
   systemd.services."lxc@" = {
     description = "LXC container '%i'";
     after = [ "network.target" ];
+    requires = [ "lxc-rootfs@%i.service" ];
     unitConfig.ConditionPathExists = [
-      "/var/lib/lxc/%i/rootfs/init"
+      "/var/lib/lxc/%i/system"
     ];
     serviceConfig = with pkgs; {
       Type = "simple";
@@ -239,6 +242,124 @@ in
     };
   };
 
+  systemd.services."lxc-rootfs@" = {
+    wants = [ "drbd.service" ];
+    unitConfig.ConditionPathExists = [
+      "/dev/drbd/by-res/%i"
+    ];
+    serviceConfig =
+      let
+        mkScript = name: content:
+          "${pkgs.writeShellScriptBin name ''
+            PATH=$PATH:/run/current-system/sw/bin
+            ${content}
+          ''}/bin/${name} %i";
+      in {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = mkScript "lxc-rootfs-start" ''
+          drbdadm primary $1
+          ROOT=/var/lib/lxc/$1/rootfs
+          mkdir -p $ROOT
+          mount /dev/drbd/by-res/$1 $ROOT
+          for DIR in \
+            bin dev etc home mnt \
+            nix/store nix/var \
+            proc root run sys \
+            system tmp var usr \
+          ; do
+            mkdir -p $ROOT/$DIR
+          done
+        '';
+        ExecStop = mkScript "lxc-rootfs-stop" ''
+          umount /var/lib/lxc/$1/rootfs
+          drbdadm secondary $1
+        '';
+      };
+  };
+
+  # systemd.services.drbd.wants = [ "lxc-volumes.service" ];
+  systemd.services.lxc-volumes = {
+    description = "Create LXC rootfs for drbd";
+    wantedBy = [ "drbd.service" ];
+    path = with pkgs; [ drbd coreutils util-linux e2fsprogs ];
+    script = ''
+      create() {
+        file=$1
+        size=$2
+        [ -e $file ] || \
+          dd if=/dev/zero of=$file bs=1024 seek=$(($size - 1)) count=1
+      }
+
+      ${lib.concatStrings (lib.imap0 (i: container: ''
+        mkdir -p /var/lib/lxc/${container}
+        cd /var/lib/lxc/${container}
+
+        if [ ! -e rootfs.img ]; then
+          create rootfs.img ${toString (containers.${container}.diskSize * 1024)}
+          mkfs.ext4 rootfs.img
+        else
+          losetup -d /dev/loop${toString (2 * i)} 2> /dev/null || true
+        fi
+        # TODO: zfsPool
+        losetup /dev/loop${toString (2 * i)} rootfs.img
+
+        if [ ! -e rootfs.meta ]; then
+          create rootfs.meta ${toString (((4095 + 36 + (containers.${container}.diskSize / 32)) / 4096) * 4096)}
+          losetup /dev/loop${toString (2 * i + 1)} rootfs.meta
+          drbdadm create-md ${container}
+        else
+          losetup /dev/loop${toString (2 * i + 1)} rootfs.meta
+        fi
+      '') (builtins.attrNames containers))}
+    '';
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+  };
+
+  services.drbd-utils = {
+    enable = true;
+    config = ''
+      global {
+        # Do not participate in online survey
+        usage-count no;
+      }
+
+      common {
+        # Protocol A: write IO is reported as completed, if it has
+        # reached local disk and local TCP send buffer.
+        protocol A;
+        syncer {
+          rate 60M;
+        }
+      }
+
+      ${lib.concatStrings (lib.imap0 (i: container: ''
+        resource ${container} {
+          net {
+            cram-hmac-alg sha1;
+            shared-secret "TODO";
+          }
+          device minor ${toString i};
+          # /var/lib/lxc/${container}/rootfs.img
+          disk /dev/loop${toString (i * 2)};
+          # /var/lib/lxc/${container}/rootfs.meta
+          meta-disk /dev/loop${toString (i * 2 + 1)};
+          ${lib.concatMapStrings (server: ''
+            on ${server} {
+              address ${config.site.net.cluster.hosts4.${server}}:${toString (drbdPortBase + i)};
+            }
+          '') (builtins.attrNames servers)}
+        }
+      '') (builtins.attrNames containers))}
+    '';
+  };
+  networking.firewall.allowedTCPPorts =
+    lib.imap0 (i: _server: drbdPortBase + i)
+      (builtins.attrNames servers);
+
   services.pacemaker.services =
     map (ctName: "lxc@${ctName}.service")
       (builtins.attrNames containers);

nix/pkgs/default.nix

@@ -120,6 +120,6 @@ in
 rootfs-packages // vm-packages // device-templates // network-graphs // starlink // subnetplans // {
   inherit all-rootfs export-openwrt-models export-config dns-slaves
     encrypt-secrets decrypt-secrets switch-to-production
-    pacemaker
+    drbd-utils pacemaker
     ;
 }