From bcf06cbbc97d2249b75bd9dd1c063b0dbcdff357 Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Wed, 2 Mar 2022 23:55:22 +0100
Subject: [PATCH] initial corosync/pacemaker setup with upstream modules

---
 config/secrets.nix                   |  4 ++
 flake.nix                            |  4 +-
 nix/lib/config/options.nix           |  4 ++
 nix/nixos-module/server/defaults.nix | 63 +++++++++++++++++++++++++---
 4 files changed, 68 insertions(+), 7 deletions(-)

diff --git a/config/secrets.nix b/config/secrets.nix
index b89a5c3..8615162 100644
--- a/config/secrets.nix
+++ b/config/secrets.nix
@@ -265,4 +265,8 @@
       ];
     } ];
   };
+
+  site.cluster = {
+    corosyncAuthKey = "8V82ry1A6Ki6EXWj2X8PJYC89xITLsgFteQbr6tiegUQLbbtMzWmT8ynyVn5cHiah52ANNfQk6yLrvAJrVDVlTFowG5D1GClOHQmmZi+Xv3nJ2fCUjCYa97/tSdV/1NnsNKkxMxJndef2TrknHAR4DBAM32USADBhP94nuv5FmdMOTLBDbvdlOrCGbdnaZKgIrhuN61atQ1iRexz0prHO+3WfOEx39N+Tzr4";
+  };
 }
diff --git a/flake.nix b/flake.nix
index fed8f7f..d9f52de 100644
--- a/flake.nix
+++ b/flake.nix
@@ -2,8 +2,8 @@
   description = "Zentralwerk network";
 
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/release-21.11";
-    nixpkgs-master.url = "github:NixOS/nixpkgs";
+    nixpkgs.url = "github:astro/nixpkgs/pacemaker";
+    nixpkgs-master.url = "github:astro/nixpkgs/pacemaker";
     openwrt.url = "git+https://git.openwrt.org/openwrt/openwrt.git?ref=openwrt-21.02";
     openwrt.flake = false;
   };
diff --git a/nix/lib/config/options.nix b/nix/lib/config/options.nix
index ae7345f..a85573a 100644
--- a/nix/lib/config/options.nix
+++ b/nix/lib/config/options.nix
@@ -577,6 +577,10 @@ in
     };
 
     vpn.wireguard = vpnOpts;
+
+    cluster.corosyncAuthKey = mkOption {
+      type = types.str;
+    };
   };
 
   config.warnings =
diff --git a/nix/nixos-module/server/defaults.nix b/nix/nixos-module/server/defaults.nix
index d18f1a9..64e7890 100644
--- a/nix/nixos-module/server/defaults.nix
+++ b/nix/nixos-module/server/defaults.nix
@@ -1,4 +1,4 @@
-{ pkgs, nixpkgs-master, ... }:
+{ config, lib, pkgs, nixpkgs-master, ... }:
 {
   boot.kernelModules = [ "kvm-intel" "pppoe" ];
   boot.kernelParams = [ "nomodeset" ];
@@ -7,10 +7,38 @@
 
   time.timeZone = "Europe/Berlin";
 
-  environment.systemPackages = with pkgs; [
-    wget vim git screen
-    ipmitool
-  ];
+  environment.systemPackages =
+    with pkgs;
+    let
+      containers = builtins.attrNames (
+        lib.filterAttrs (_: { role, ... }:
+          role == "container"
+        ) config.site.hosts
+      );
+      resources = builtins.toFile "cib-resources.xml" ''
+        <resources>
+          ${lib.concatMapStrings (container: ''
+            <primitive id="${container}" class="systemd" type="${container}">
+              <operations>
+                <op id=""${container}-start" name="start" interval="0" timeout="10s"/>
+                <op id=""${container}-stop" name="start" interval="0" timeout="10s"/>
+                <op id="${container}-monitor" name="monitor" interval="10s" timeout="10s"/>
+              </operations>
+            </primitive>
+          '') containers}
+        </resources>
+      '';
+      cib-set-resources = writeScriptBin "cib-set-resources" ''
+        !# ${runtimeShell} -e
+
+        crm_attribute -t crm_config -n stonith-enabled -v false
+        cibadmin --replace --scope resources --xml-file ${resources}
+      '';
+    in [
+      wget vim git screen
+      ipmitool
+      cib-set-resources
+    ];
   services.openssh.enable = true;
   services.openssh.permitRootLogin = "prohibit-password";
 
@@ -20,4 +48,29 @@
     # FIXME: IPMI is only available with nixpkgs-21.11 onwards
     package = nixpkgs-master.legacyPackages.${pkgs.system}.collectd;
   };
+
+  services.corosync = {
+    enable = true;
+    clusterName = "zentralwerk-network";
+    nodelist =
+      lib.imap (n: hostName: {
+        nodeid = n;
+        name = hostName;
+        ring_addrs = map (net:
+          config.site.net.${net}.hosts4.${hostName}
+        ) [ "cluster" "mgmt" ];
+      }) (
+        builtins.filter (hostName:
+          config.site.hosts.${hostName}.role == "server"
+        ) (builtins.attrNames config.site.hosts)
+      );
+  };
+  environment.etc."corosync/authkey" = {
+    source = builtins.toFile "authkey" config.site.cluster.corosyncAuthKey;
+    mode = "0400";
+  };
+
+  services.pacemaker = {
+    enable = true;
+  };
 }