2022-07-04 00:21:29 +02:00
|
|
|
{ zentralwerk, config, lib, pkgs, ... }:
|
|
|
|
let
|
|
|
|
inherit (config.networking) hostName;
|
|
|
|
|
|
|
|
# hydra does *not* use this module because it only runs a nomad
|
|
|
|
# server but no client and no microvms
|
|
|
|
servers = [ "server9" "server10" "hydra" ];
|
|
|
|
|
|
|
|
serverNet = server:
|
|
|
|
builtins.foldl' (result: net:
|
|
|
|
if result == null &&
|
|
|
|
zentralwerk.lib.config.site.net.${net}.hosts4 ? ${server}
|
|
|
|
then net
|
|
|
|
else result
|
|
|
|
) null [ "cluster" "serv" ];
|
|
|
|
|
2022-07-04 00:31:23 +02:00
|
|
|
ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName};
|
2022-07-04 00:21:29 +02:00
|
|
|
in {
|
|
|
|
imports = [
|
|
|
|
];
|
|
|
|
|
|
|
|
# Open firewall between cluster members
|
|
|
|
|
|
|
|
networking.firewall.extraCommands = lib.concatMapStrings (server:
|
|
|
|
let
|
|
|
|
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
|
|
|
|
in
|
|
|
|
lib.optionalString (server != hostName) ''
|
|
|
|
iptables -A nixos-fw --source ${netConfig.hosts4.${server}} -j ACCEPT
|
|
|
|
${lib.concatMapStrings (hosts6: ''
|
|
|
|
ip6tables -A nixos-fw --source ${hosts6.${server}} -j ACCEPT
|
|
|
|
'') (builtins.attrValues netConfig.hosts6)}
|
|
|
|
'') servers;
|
|
|
|
|
|
|
|
# Storage
|
|
|
|
|
|
|
|
services.glusterfs.enable = true;
|
|
|
|
|
|
|
|
fileSystems."/glusterfs/fast" = {
|
|
|
|
fsType = "glusterfs";
|
|
|
|
device = "${config.networking.hostName}:/fast";
|
|
|
|
};
|
|
|
|
|
2022-07-07 01:30:58 +02:00
|
|
|
# stable uid is useful across glusterfs
|
|
|
|
users.users.microvm.uid = 997;
|
|
|
|
|
2022-07-04 00:21:29 +02:00
|
|
|
# Nomad
|
|
|
|
|
|
|
|
services.nomad = {
|
|
|
|
enable = true;
|
2022-07-05 00:08:03 +02:00
|
|
|
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
|
|
|
|
package = pkgs.nomad_1_3;
|
2022-07-04 00:21:29 +02:00
|
|
|
|
|
|
|
enableDocker = false;
|
|
|
|
dropPrivileges = false;
|
|
|
|
|
2022-09-29 18:23:36 +02:00
|
|
|
settings = rec {
|
2022-07-04 00:21:29 +02:00
|
|
|
datacenter = "c3d2";
|
|
|
|
plugin.raw_exec.config.enabled = true;
|
2022-08-15 23:38:20 +02:00
|
|
|
# no /dev/kvm otherwise. TODO: retry with nomad>1.3.3
|
2022-07-05 03:15:17 +02:00
|
|
|
plugin.raw_exec.config.no_cgroups = true;
|
2022-07-04 00:21:29 +02:00
|
|
|
|
|
|
|
server = {
|
|
|
|
enabled = true;
|
|
|
|
bootstrap_expect = 3;
|
2022-09-29 16:21:26 +02:00
|
|
|
server_join = {
|
|
|
|
retry_join = map (server:
|
|
|
|
zentralwerk.lib.config.site.net.${serverNet server}.hosts4.${server}
|
|
|
|
) (
|
|
|
|
builtins.filter (server: server != hostName)
|
|
|
|
servers
|
|
|
|
);
|
|
|
|
};
|
2022-07-04 00:21:29 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
client = {
|
|
|
|
enabled = true;
|
|
|
|
network_interface = "cluster";
|
2022-09-29 16:21:26 +02:00
|
|
|
inherit (server) server_join;
|
2022-07-04 00:21:29 +02:00
|
|
|
};
|
|
|
|
};
|
|
|
|
};
|
2022-07-12 01:16:46 +02:00
|
|
|
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
|
|
# nomad frontends
|
2022-07-19 20:59:53 +02:00
|
|
|
damon wander
|
2022-07-12 01:16:46 +02:00
|
|
|
];
|
2022-07-04 00:21:29 +02:00
|
|
|
}
|