nix-config/modules/cluster/default.nix

89 lines
2.3 KiB
Nix

{ zentralwerk, config, lib, pkgs, ... }:
let
inherit (config.networking) hostName;
# hydra does *not* use this module because it only runs a nomad
# server but no client and no microvms
servers = [ "server9" "server10" "hydra" ];
serverNet = server:
builtins.foldl' (result: net:
if result == null &&
zentralwerk.lib.config.site.net.${net}.hosts4 ? ${server}
then net
else result
) null [ "cluster" "serv" ];
ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName};
in {
imports = [
];
# Open firewall between cluster members
networking.firewall.extraCommands = lib.concatMapStrings (server:
let
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
in
lib.optionalString (server != hostName) ''
iptables -A nixos-fw --source ${netConfig.hosts4.${server}} -j ACCEPT
${lib.concatMapStrings (hosts6: ''
ip6tables -A nixos-fw --source ${hosts6.${server}} -j ACCEPT
'') (builtins.attrValues netConfig.hosts6)}
'') servers;
# Storage
services.glusterfs.enable = true;
fileSystems."/glusterfs/fast" = {
fsType = "glusterfs";
device = "${config.networking.hostName}:/fast";
};
# stable uid is useful across glusterfs
users.users.microvm.uid = 997;
# Nomad
services.nomad = {
enable = true;
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
package = pkgs.nomad_1_3;
enableDocker = false;
dropPrivileges = false;
settings = rec {
datacenter = "c3d2";
plugin.raw_exec.config.enabled = true;
# no /dev/kvm otherwise. TODO: retry with nomad>1.3.3
plugin.raw_exec.config.no_cgroups = true;
server = {
enabled = true;
bootstrap_expect = 3;
server_join = {
retry_join = map (server:
zentralwerk.lib.config.site.net.${serverNet server}.hosts4.${server}
) (
builtins.filter (server: server != hostName)
servers
);
};
};
client = {
enabled = true;
network_interface = "cluster";
inherit (server) server_join;
};
};
};
environment.systemPackages = with pkgs; [
# nomad frontends
damon wander
];
}