2023-11-12 02:54:42 +01:00
|
|
|
{ config, hostRegistry, lib, pkgs, ssh-public-keys, zentralwerk, ... }:
|
2023-01-17 00:38:26 +01:00
|
|
|
|
2022-07-04 00:21:29 +02:00
|
|
|
let
|
|
|
|
inherit (config.networking) hostName;
|
|
|
|
|
|
|
|
# hydra does *not* use this module because it only runs a nomad
|
|
|
|
# server but no client and no microvms
|
2023-11-12 02:54:42 +01:00
|
|
|
servers = [ "server9" "server10" "hydra" ];
|
|
|
|
microvmServers = [ "server9" "server10" ];
|
|
|
|
storageServers = [ "server9" ];
|
2022-07-04 00:21:29 +02:00
|
|
|
|
|
|
|
serverNet = server:
|
|
|
|
builtins.foldl' (result: net:
|
|
|
|
if result == null &&
|
|
|
|
zentralwerk.lib.config.site.net.${net}.hosts4 ? ${server}
|
|
|
|
then net
|
|
|
|
else result
|
|
|
|
) null [ "cluster" "serv" ];
|
2023-05-22 01:13:08 +02:00
|
|
|
|
|
|
|
skyflakeHome = "/var/lib/skyflake/home";
|
|
|
|
|
2022-07-04 00:21:29 +02:00
|
|
|
in {
|
|
|
|
# Open firewall between cluster members
|
2023-01-02 17:15:14 +01:00
|
|
|
networking.firewall.extraCommands = lib.concatMapStrings
|
|
|
|
(server:
|
|
|
|
let
|
|
|
|
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
|
|
|
|
in
|
|
|
|
lib.optionalString (server != hostName) ''
|
|
|
|
iptables -A nixos-fw --source ${netConfig.hosts4.${server}} -j ACCEPT
|
|
|
|
${lib.concatMapStrings (hosts6: ''
|
|
|
|
ip6tables -A nixos-fw --source ${hosts6.${server}} -j ACCEPT
|
|
|
|
'') (builtins.attrValues netConfig.hosts6)}
|
|
|
|
''
|
|
|
|
)
|
|
|
|
servers;
|
2022-07-04 00:21:29 +02:00
|
|
|
|
2022-11-01 02:36:25 +01:00
|
|
|
# Cluster configuration
|
|
|
|
skyflake = {
|
2022-12-18 03:20:13 +01:00
|
|
|
# debug = true;
|
2022-11-01 02:36:25 +01:00
|
|
|
nodes = builtins.listToAttrs (
|
|
|
|
map (name: {
|
|
|
|
inherit name;
|
2022-12-20 05:40:58 +01:00
|
|
|
value.address = hostRegistry.${name}.ip4;
|
2022-11-01 02:36:25 +01:00
|
|
|
}) servers
|
|
|
|
);
|
2023-01-08 03:05:55 +01:00
|
|
|
# Extra nomad configuration
|
2022-11-01 02:36:25 +01:00
|
|
|
nomad = {
|
2022-07-04 00:21:29 +02:00
|
|
|
datacenter = "c3d2";
|
2022-12-04 08:53:28 +01:00
|
|
|
inherit servers;
|
2022-11-01 02:36:25 +01:00
|
|
|
# run tasks only on these:
|
|
|
|
client.enable = builtins.elem hostName microvmServers;
|
2023-01-02 17:15:14 +01:00
|
|
|
client.meta = lib.optionalAttrs (builtins.elem hostName storageServers) {
|
|
|
|
"c3d2.storage" = "big";
|
|
|
|
};
|
2022-11-01 02:36:25 +01:00
|
|
|
};
|
2023-01-08 03:05:55 +01:00
|
|
|
# The user that runs skyflake MicroVMs
|
2022-11-01 02:36:25 +01:00
|
|
|
microvmUid = 997;
|
2022-07-04 00:21:29 +02:00
|
|
|
|
2023-01-02 17:15:14 +01:00
|
|
|
users = {
|
2023-01-08 03:05:55 +01:00
|
|
|
# Deployment user for hosts in this flake
|
2023-01-02 17:15:14 +01:00
|
|
|
c3d2 = {
|
|
|
|
uid = 1001;
|
|
|
|
sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
2023-05-22 01:13:08 +02:00
|
|
|
home = "${skyflakeHome}/c3d2";
|
2023-01-02 17:15:14 +01:00
|
|
|
};
|
2023-01-19 16:04:49 +01:00
|
|
|
# Deployment user for neighbour Andreas Lippmann <andreaslippmann@web.de>
|
|
|
|
luulaatsch = {
|
|
|
|
uid = 1003;
|
2023-05-22 01:13:08 +02:00
|
|
|
home = "${skyflakeHome}/luulaatsch";
|
2023-01-19 16:04:49 +01:00
|
|
|
sshKeys = [
|
|
|
|
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVipwWSzxgUgm0fMeTbOSCXDNkbUUp+k1tvWaJcBA1M+LTM21rhP3tQQP24smPx1b2V/by6vHakhsj5pIMLzSSW7he/mAnOtwHj2dCEk6VaMvozAfS6n777g8ujZg0N9ekrqoNLh0uNchs3DYFZWXAnKtfEYjzIdXqVgXMI4uDXM6ObEh+YR3iGaZSYKd6S5nufzJOR6EiYbv/z6ThRq16f+HqetVdBamqmpNK9NcWCtjMcww7nZTyUvTbd2AyrpTAXSnVg30fza428Kpc5ZdzDJFjn0B3MVnP2LavIo8VH3Y5sjpp0aMDKfbz6dPA/df/pR6LVMsxx0eCEXeGqo9sFdabu2AwIPi79QFANr3qJ/nZo2LR3o2LvcOLO06xVRFKReGTsy6WWTNOTAL03sdiIbqnlmkjXu5nb4hzxoHBgA6LZGMx3knoVcUMBlpVaxOoT2tzXzpYszsKpGSN1EBqUlDXzen3EkHg7TpjjTJSH7Q0AeeSCJsaoAepa6wcZeJlBWSRiJdqIn7YBmCBdjV1nNPVBYE5mm2x88ha9e4NbyFhjQvZ/BiLyKCFVyDs6lIvhV5QfAPWOkyPwbLXtXg6g1OvmuM6mGDhtEfZKvreeqmTKjB+x7OpG847JKxmU8eeYax63/WgRssbOfTpicjsdoKlFST3WpZHGT7mWzmB/w== andreas@luulaatschs-MBP.pub.zentralwerk.org"
|
2023-04-22 23:03:14 +02:00
|
|
|
] ++ ssh-public-keys.astro;
|
2023-01-19 16:04:49 +01:00
|
|
|
};
|
2023-02-01 21:12:17 +01:00
|
|
|
# Deployment user for marenz
|
|
|
|
marenz = {
|
|
|
|
uid = 1004;
|
2023-11-11 23:23:26 +01:00
|
|
|
sshKeys = config.users.users.root.openssh.authorizedKeys.keys ++ ssh-public-keys.marenz;
|
2023-05-22 01:13:08 +02:00
|
|
|
home = "${skyflakeHome}/marenz";
|
2023-02-01 21:12:17 +01:00
|
|
|
};
|
2022-11-21 04:00:54 +01:00
|
|
|
};
|
2023-01-02 17:15:14 +01:00
|
|
|
|
2023-01-19 16:42:43 +01:00
|
|
|
deploy.customizationModule = ./customization;
|
2022-07-04 00:21:29 +02:00
|
|
|
|
2023-01-08 03:05:55 +01:00
|
|
|
# Ceph storage cluster configuration
|
2023-05-23 20:09:55 +02:00
|
|
|
storage.ceph = rec {
|
2023-06-25 13:32:06 +02:00
|
|
|
package = pkgs.ceph_17_2;
|
2023-01-07 02:54:35 +01:00
|
|
|
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
2023-11-12 02:54:42 +01:00
|
|
|
mons = [ "server9" "server10" ];
|
2023-01-07 02:54:35 +01:00
|
|
|
mgrs = mons;
|
|
|
|
mdss = mons;
|
2023-05-19 21:48:48 +02:00
|
|
|
rbdPools.microvms = {
|
|
|
|
params = { size = 2; class = "ssd"; };
|
|
|
|
};
|
2023-07-07 00:29:10 +02:00
|
|
|
rbdPools.microvms-hdd = {
|
|
|
|
params = { size = 2; class = "hdd"; };
|
|
|
|
};
|
2023-05-22 01:13:08 +02:00
|
|
|
cephfs.home.mountPoint = skyflakeHome;
|
|
|
|
# Legacy: migration to rbd
|
|
|
|
cephfs.skyflake.mountPoint = "/storage/cephfs";
|
2023-01-07 02:54:35 +01:00
|
|
|
|
|
|
|
monKeyring = config.sops.secrets."ceph/monKeyring".path;
|
|
|
|
adminKeyring = config.sops.secrets."ceph/adminKeyring".path;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
2023-01-08 03:05:55 +01:00
|
|
|
# Ceph keyrings
|
2023-01-07 02:54:35 +01:00
|
|
|
sops.secrets = {
|
|
|
|
"ceph/monKeyring" = {
|
|
|
|
owner = "ceph";
|
2023-05-21 22:24:35 +02:00
|
|
|
sopsFile = ./ceph.yaml;
|
2023-01-07 02:54:35 +01:00
|
|
|
};
|
|
|
|
"ceph/adminKeyring" = {
|
|
|
|
owner = "ceph";
|
2023-05-21 22:24:35 +02:00
|
|
|
sopsFile = ./ceph.yaml;
|
2023-01-07 02:54:35 +01:00
|
|
|
};
|
2022-07-04 00:21:29 +02:00
|
|
|
};
|
2022-11-28 00:49:14 +01:00
|
|
|
|
2023-01-08 03:05:55 +01:00
|
|
|
# Collectd monitoring for ceph
|
|
|
|
services.collectd.plugins.ceph = ''
|
|
|
|
ConvertSpecialMetricTypes true
|
|
|
|
|
|
|
|
${lib.concatMapStrings (hostName: ''
|
|
|
|
<Daemon "mon.${hostName}">
|
|
|
|
SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok"
|
|
|
|
</Daemon>
|
|
|
|
'') config.services.ceph.mon.daemons}
|
|
|
|
${lib.concatMapStrings (hostName: ''
|
|
|
|
<Daemon "mgr.${hostName}">
|
|
|
|
SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok"
|
|
|
|
</Daemon>
|
|
|
|
'') config.services.ceph.mgr.daemons}
|
|
|
|
${lib.concatMapStrings (hostName: ''
|
|
|
|
<Daemon "mds.${hostName}">
|
|
|
|
SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok"
|
|
|
|
</Daemon>
|
|
|
|
'') config.services.ceph.mds.daemons}
|
|
|
|
${lib.concatMapStrings (id: ''
|
|
|
|
<Daemon "osd.${id}">
|
|
|
|
SocketPath "/var/run/ceph/ceph-osd.${id}.asok"
|
|
|
|
</Daemon>
|
|
|
|
'') config.services.ceph.osd.daemons}
|
|
|
|
'';
|
|
|
|
# HACK: let collectd access ceph sockets
|
|
|
|
systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph";
|
2022-07-04 00:21:29 +02:00
|
|
|
}
|