nix/nixos-module/pacemaker: start tinkering

nix/nixos-module/default.nix

@@ -15,6 +15,7 @@ in {
     ./network.nix
     ./firewall.nix
     ./collectd
+    ./pacemaker.nix
   ] ++
   optionals (hostConfig.role == "server") [
     ./server/default.nix

nix/nixos-module/pacemaker.nix

@@ -0,0 +1,186 @@
+{ self, pkgs, lib, config, ... }:
+let
+  description = "Pacemaker HA cluster manager";
+  cfg = config.services.pacemaker;
+in {
+  options.services = with lib; {
+    corosync = {
+      clusterName = mkOption {
+        type = types.str;
+        default = "nixcluster";
+      };
+
+
+      extraOptions = mkOption {
+        type = types.str;
+        default = "";
+      };
+
+      nodelist = mkOption {
+        default = [];
+        type = with types; listOf (submodule {
+          options = {
+            nodeid = mkOption {
+              type = int;
+            };
+            name = mkOption {
+              type = str;
+            };
+            ring_addrs = mkOption {
+              type = listOf str;
+            };
+          };
+        });
+      };
+    };
+
+    pacemaker = {
+      enable = mkEnableOption description;
+
+      package = mkOption {
+        type = types.package;
+        default = self.packages.${pkgs.system}.pacemaker;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    nixpkgs.config.packageOverrides = pkgs: {
+      corosync = pkgs.corosync.overrideAttrs ({ buildInputs, configureFlags, ... }: {
+        buildInputs = buildInputs ++ [
+          pkgs.systemd.dev
+        ];
+        configureFlags = configureFlags ++ [
+          # allows Type=notfiy in the systemd.service
+          "--enable-systemd"
+        ];
+      });
+    };
+
+    environment.systemPackages = with pkgs; [
+      corosync
+      cfg.package
+    ];
+
+    environment.etc."corosync/corosync.conf".text = ''
+      totem {
+        version: 2
+        secauth: off
+        cluster_name: ${config.services.corosync.clusterName}
+        transport: knet
+        # TODO: secauth
+      }
+
+      nodelist {
+        ${lib.concatMapStrings ({ nodeid, name, ring_addrs }: ''
+          node {
+                nodeid: ${toString nodeid}
+                name: ${name}
+                ${lib.concatStrings (lib.imap0 (i: addr: ''
+                  ring${toString i}_addr: ${addr}
+                '') ring_addrs)}
+          }
+        '') config.services.corosync.nodelist}
+      }
+
+      quorum {
+        # only corosync_votequorum is supported
+        provider: corosync_votequorum
+        wait_for_all: 0
+        ${lib.optionalString (builtins.length config.services.corosync.nodelist < 3) ''
+          two_node: 1
+        ''}
+      }
+
+      logging {
+        to_syslog: yes
+      }
+    '';
+    environment.etc."corosync/uidgid.d/root".text = ''
+      # allow pacemaker connection by root
+      uidgid {
+        uid: 0
+        gid: 0
+      }
+    '';
+
+    systemd.tmpfiles.rules = [
+      "d /var/lib/corosync 0700 root root -"
+      "d /var/lib/pacemaker 0700 hacluster pacemaker -"
+      "d /var/lib/pacemaker/cib 0700 hacluster pacemaker -"
+    ];
+
+    # used by pacemaker
+    users.users.hacluster = {
+      isSystemUser = true;
+      group = "pacemaker";
+    };
+    users.groups.pacemaker = {};
+
+    systemd.services = {
+      # see ${corosync.src}/init/corosync.service.in
+      corosync = {
+        requires = [ "network-online.target" ];
+        after= [ "network-online.target" ];
+        wantedBy = [ "multi-user.target" ];
+
+        serviceConfig = {
+          ExecStart = "${pkgs.corosync}/sbin/corosync -f ${config.services.corosync.extraOptions}";
+          ExecStop= "${pkgs.corosync}/sbin/corosync-cfgtool -H --force";
+          Type = "notify";
+          StandardError = "null";
+        };
+      };
+
+      # see ${pacemaker.src}/daemons/pacemakerd/pacemaker.service.in
+      pacemaker = {
+        inherit description;
+        after = [
+          "network.target" "time-sync.target" "dbus.service"
+          #"resource-agents-deps.target"
+          "corosync.service"
+        ];
+        wants = [ "dbus.service" ];
+        requires = [ "corosync.service" ];
+        wantedBy = [ "multi-user.target" ];
+
+        script =
+          let
+            cibXml = builtins.toFile "cib.xml" ''
+              <cib crm_feature_set="3.0.9" validate-with="pacemaker-2.1" have-quorum="1" dc-uuid="4">
+                <configuration>
+                  <crm_config/>
+                  <nodes>
+                    ${lib.concatMapStrings ({ nodeid, name, ring_addrs }: ''
+                      <node id="${toString nodeid}" name="${name}"/>
+                    '') config.services.corosync.nodelist}
+                  </nodes>
+                  <resources>
+                  </resources>
+                  <constraints>
+                  </constraints>
+                </configuration>
+              </cib>
+            '';
+          in ''
+            cp ${cibXml} /var/lib/pacemaker/cib/cib.xml
+            chown hacluster:pacemaker /var/lib/pacemaker/cib/cib.xml
+
+            exec ${cfg.package}/sbin/pacemakerd
+        '';
+        serviceConfig = {
+          Type = "simple";
+          KillMode = "process";
+          NotifyAccess = "main";
+          SuccessExitStatus = 100;
+          TasksMax = "infinity";
+          SendSIGKILL = false;
+          TimeoutStopSec = "30min";
+          TimeoutStartSec = "60s";
+          Restart = "on-failure";
+          StandardError = "null";
+        };
+      };
+    };
+  };
+}

nix/nixos-module/server/default.nix

@@ -5,6 +5,7 @@
     ./network.nix
     ./lxc-containers.nix
     ./qemu.nix
+    ./ha.nix
     # host-specific configuration
     (./. + "/${hostName}.nix")
   ];

nix/nixos-module/server/defaults.nix

@@ -11,6 +11,8 @@
     wget vim git screen
     ipmitool
   ];
+
+  systemd.services."systemd-networkd-wait-online".enable = false;
   services.openssh.enable = true;
   services.openssh.permitRootLogin = "prohibit-password";
 

nix/nixos-module/server/ha.nix

@@ -0,0 +1,19 @@
+{ lib, config, ... }:
+{
+  services.pacemaker.enable = true;
+  services.corosync = {
+    clusterName = "zentralwerk";
+    nodelist = lib.imap1 (i: hostName: {
+      nodeid = i;
+      name = hostName;
+      ring_addrs = map (net:
+        config.site.net.${net}.hosts6.dn42.${hostName}
+      ) [ "cluster" "mgmt" ];
+    }) (
+      builtins.attrNames (
+        lib.filterAttrs (_: { role, ... }: role == "server")
+          config.site.hosts
+      )
+    );
+  };
+}

nix/nixos-module/server/qemu.nix

@@ -1,5 +1,5 @@
 # Options for running under qemu (vm-packages)
-{ inputs, lib, options, ... }:
+{ inputs, lib, config, options, ... }:
 {
   # Get internet from qemu user networking
   systemd.network = lib.optionalAttrs (options.virtualisation ? qemu) {
@@ -23,6 +23,11 @@
     # keep the store paths built inside the VM across reboots
     writableStoreUseTmpfs = false;
     qemu.options = [ "-enable-kvm" ];
+    qemu.networkingOptions = [
+      # Useful for cluster dev
+      "-net nic,netdev=net.0,model=virtio"
+      "-netdev tap,id=net.0,ifname=${config.networking.hostName},script=no,downscript=no"
+    ];
   };
 
   # Let the nix registry point to the state of your local checkout