modules/cluster: init glusterfs and nomad

flake.nix

@@ -654,6 +654,7 @@
             ./hosts/server9
             microvm.nixosModules.host
             self.nixosModules.cluster-network
+            self.nixosModules.cluster
             { _module.args = { inherit self; }; }
           ];
         };
@@ -663,6 +664,7 @@
             ./hosts/server10
             microvm.nixosModules.host
             self.nixosModules.cluster-network
+            self.nixosModules.cluster
             { _module.args = { inherit self; }; }
           ];
         };
@@ -736,6 +738,7 @@
           c3d2.users = import ./users.nix;
           c3d2.nncp.neigh = import ./config/nncp-relays.nix;
         };
+        cluster = ./modules/cluster;
         cluster-network = ./modules/cluster-network.nix;
         microvm.imports = [
           microvm.nixosModules.microvm

hosts/hydra/default.nix

@@ -6,6 +6,7 @@
     ./network.nix
     ./hydra.nix
     ./updater.nix
+    ./nomad-server.nix
     ../../config/c3d2.nix
   ];
 

hosts/hydra/nomad-server.nix

@@ -0,0 +1,39 @@
+# Our bare-metal hydra is a server in the nomad cluster, it is not a
+# client and therefore does not run cluster jobs
+{ zentralwerk, config, pkgs, ... }:
+let
+  inherit (config.networking) hostName;
+  ipv6Addr = zentralwerk.lib.config.site.net.serv.hosts6.dn42.${hostName};
+in
+{
+  services.nomad = {
+    enable = true;
+    enableDocker = false;
+
+    settings = {
+      datacenter = "c3d2";
+      plugin.raw_exec.config.enabled = true;
+
+      addresses = {
+        http = "::1";
+        rpc = ipv6Addr;
+        serf = ipv6Addr;
+      };
+      server = {
+        enabled = true;
+        bootstrap_expect = 3;
+      };
+      server_join = {
+        retry_join = map (server:
+          zentralwerk.lib.config.site.net.cluster.hosts4.${server}
+        ) [ "server9" "server10" ];
+      };
+
+      client = {
+        enabled = true;
+        network_interface = "cluster";
+        servers = [ "localhost" ];
+      };
+    };
+  };
+}

modules/cluster/default.nix

@@ -0,0 +1,88 @@
+{ zentralwerk, config, lib, pkgs, ... }:
+let
+  inherit (config.networking) hostName;
+
+  # hydra does *not* use this module because it only runs a nomad
+  # server but no client and no microvms
+  servers = [ "server9" "server10" "hydra" ];
+
+  serverNet = server:
+    builtins.foldl' (result: net:
+      if result == null &&
+         zentralwerk.lib.config.site.net.${net}.hosts4 ? ${server}
+      then net
+      else result
+    ) null [ "cluster" "serv" ];
+
+  ipv6Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts6.dn42.${hostName};
+in {
+  imports = [
+  ];
+
+  # Open firewall between cluster members
+
+  networking.firewall.extraCommands = lib.concatMapStrings (server:
+    let
+      netConfig = zentralwerk.lib.config.site.net.${serverNet server};
+    in
+      lib.optionalString (server != hostName) ''
+        iptables -A nixos-fw --source ${netConfig.hosts4.${server}} -j ACCEPT
+        ${lib.concatMapStrings (hosts6: ''
+          ip6tables -A nixos-fw --source ${hosts6.${server}} -j ACCEPT
+        '') (builtins.attrValues netConfig.hosts6)}
+      '') servers;
+
+  # Storage
+
+  services.glusterfs.enable = true;
+
+  fileSystems."/glusterfs/fast" = {
+    fsType = "glusterfs";
+    device = "${config.networking.hostName}:/fast";
+  };
+
+  # Nomad
+
+  environment.systemPackages = with pkgs; [ nomad ];
+
+  services.nomad = {
+    enable = true;
+
+    enableDocker = false;
+    dropPrivileges = false;
+
+    extraPackages = with pkgs; [
+      systemd virtiofsd
+    ];
+
+    settings = {
+      datacenter = "c3d2";
+      plugin.raw_exec.config.enabled = true;
+
+      addresses = {
+        http = "::1";
+        rpc = ipv6Addr;
+        serf = ipv6Addr;
+      };
+
+      server = {
+        enabled = true;
+        bootstrap_expect = 3;
+      };
+      server_join = {
+        retry_join = map (server:
+          zentralwerk.lib.config.site.net.${serverNet server}.hosts6.dn42.${server}
+        ) (
+          builtins.filter (server: server != hostName)
+            servers
+        );
+      };
+
+      client = {
+        enabled = true;
+        network_interface = "cluster";
+        servers = [ "localhost" ];
+      };
+    };
+  };
+}