{ config, lib, pkgs, ... }: let hostsWithRole = wantedRole: builtins.attrNames ( lib.filterAttrs (_: { role, ... }: role == wantedRole ) config.site.hosts ); in { environment.systemPackages = with pkgs; let setup-glusterfs = let in writeScriptBin "setup-glusterfs" '' #! ${runtimeShell} -e DIR="$1" if [ -z "$DIR" ]; then echo "Usage: $0 " exit 1 fi while ! systemctl is-active glusterd.service ; do echo "Wait for glusterd.service" sleep 1 done ${lib.concatMapStrings (server: '' gluster peer probe ${server} '') (hostsWithRole "server")} gluster peer status gluster volume create lxc ${lib.concatMapStringsSep " " (server: "\"${server}:$DIR\"" ) (hostsWithRole "server")} force gluster volume set shared network.ping-timeout 2 gluster volume start lxc systemctl start var-lib-lxc.mount mkdir /var/lib/lxc/rootfs ''; setup-pacemaker = let resources = builtins.toFile "cib-resources.xml" '' ${lib.concatMapStrings (container: '' '') (hostsWithRole "container")} ''; constraints = builtins.toFile "cib-constraints.xml" '' ${lib.optionalString (! config.virtualisation ? qemu) ( lib.concatMapStrings (server: '' '') (hostsWithRole "server") )} ''; in writeScriptBin "setup-pacemaker" '' #! ${runtimeShell} -e while ! systemctl is-active corosync.service ; do echo "Wait for corosync.service" sleep 1 done while ! systemctl is-active pacemaker.service ; do echo "Wait for pacemaker.service" sleep 1 done crm_attribute -t crm_config -n stonith-enabled -v false cibadmin --replace --scope resources --xml-file ${resources} cibadmin --replace --scope constraints --xml-file ${constraints} crm_attribute --name placement-strategy --update balanced ''; in [ setup-glusterfs setup-pacemaker ]; boot.supportedFilesystems = [ "glusterfs" ]; fileSystems."/var/lib/lxc" = { fsType = "glusterfs"; device = "localhost:/lxc"; options = [ "nofail" ]; }; services.corosync = { enable = true; clusterName = "zentralwerk-network"; nodelist = lib.imap (n: hostName: { nodeid = n; name = hostName; ring_addrs = map (net: config.site.net.${net}.hosts4.${hostName} ) [ "mgmt" ]; }) ( builtins.filter (hostName: config.site.hosts.${hostName}.role == "server" ) (builtins.attrNames config.site.hosts) ); }; environment.etc."corosync/authkey" = { source = builtins.toFile "authkey" config.site.cluster.corosync.authKey; mode = "0400"; }; services.pacemaker.enable = true; services.glusterfs.enable = true; networking.firewall.trustedInterfaces = [ "mgmt" ]; networking.hosts = lib.mkMerge ( map (hostName: builtins.foldl' (hosts: addr: hosts // { "${addr}" = [ hostName ]; }) {} ( [ config.site.net.mgmt.hosts4.${hostName} ] ++ map (hosts6: hosts6.${hostName}) ( builtins.attrValues config.site.net.mgmt.hosts6 ) )) (hostsWithRole "server") ); }