diff --git a/flake.nix b/flake.nix index 3eff36eb..cee41691 100644 --- a/flake.nix +++ b/flake.nix @@ -654,6 +654,7 @@ ./hosts/server9 microvm.nixosModules.host self.nixosModules.cluster-network + self.nixosModules.cluster { _module.args = { inherit self; }; } ]; }; @@ -663,6 +664,7 @@ ./hosts/server10 microvm.nixosModules.host self.nixosModules.cluster-network + self.nixosModules.cluster { _module.args = { inherit self; }; } ]; }; @@ -736,6 +738,7 @@ c3d2.users = import ./users.nix; c3d2.nncp.neigh = import ./config/nncp-relays.nix; }; + cluster = ./modules/cluster; cluster-network = ./modules/cluster-network.nix; microvm.imports = [ microvm.nixosModules.microvm diff --git a/hosts/hydra/default.nix b/hosts/hydra/default.nix index d65a7d1a..7d26376b 100644 --- a/hosts/hydra/default.nix +++ b/hosts/hydra/default.nix @@ -6,6 +6,7 @@ ./network.nix ./hydra.nix ./updater.nix + ./nomad-server.nix ../../config/c3d2.nix ]; diff --git a/hosts/hydra/nomad-server.nix b/hosts/hydra/nomad-server.nix new file mode 100644 index 00000000..e5a7affa --- /dev/null +++ b/hosts/hydra/nomad-server.nix @@ -0,0 +1,39 @@ +# Our bare-metal hydra is a server in the nomad cluster, it is not a +# client and therefore does not run cluster jobs +{ zentralwerk, config, pkgs, ... }: +let + inherit (config.networking) hostName; + ipv6Addr = zentralwerk.lib.config.site.net.serv.hosts6.dn42.${hostName}; +in +{ + services.nomad = { + enable = true; + enableDocker = false; + + settings = { + datacenter = "c3d2"; + plugin.raw_exec.config.enabled = true; + + addresses = { + http = "::1"; + rpc = ipv6Addr; + serf = ipv6Addr; + }; + server = { + enabled = true; + bootstrap_expect = 3; + }; + server_join = { + retry_join = map (server: + zentralwerk.lib.config.site.net.cluster.hosts4.${server} + ) [ "server9" "server10" ]; + }; + + client = { + enabled = true; + network_interface = "cluster"; + servers = [ "localhost" ]; + }; + }; + }; +} diff --git a/modules/cluster/default.nix b/modules/cluster/default.nix new file mode 100644 index 00000000..c076f6e4 --- /dev/null +++ b/modules/cluster/default.nix @@ -0,0 +1,88 @@ +{ zentralwerk, config, lib, pkgs, ... }: +let + inherit (config.networking) hostName; + + # hydra does *not* use this module because it only runs a nomad + # server but no client and no microvms + servers = [ "server9" "server10" "hydra" ]; + + serverNet = server: + builtins.foldl' (result: net: + if result == null && + zentralwerk.lib.config.site.net.${net}.hosts4 ? ${server} + then net + else result + ) null [ "cluster" "serv" ]; + + ipv6Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts6.dn42.${hostName}; +in { + imports = [ + ]; + + # Open firewall between cluster members + + networking.firewall.extraCommands = lib.concatMapStrings (server: + let + netConfig = zentralwerk.lib.config.site.net.${serverNet server}; + in + lib.optionalString (server != hostName) '' + iptables -A nixos-fw --source ${netConfig.hosts4.${server}} -j ACCEPT + ${lib.concatMapStrings (hosts6: '' + ip6tables -A nixos-fw --source ${hosts6.${server}} -j ACCEPT + '') (builtins.attrValues netConfig.hosts6)} + '') servers; + + # Storage + + services.glusterfs.enable = true; + + fileSystems."/glusterfs/fast" = { + fsType = "glusterfs"; + device = "${config.networking.hostName}:/fast"; + }; + + # Nomad + + environment.systemPackages = with pkgs; [ nomad ]; + + services.nomad = { + enable = true; + + enableDocker = false; + dropPrivileges = false; + + extraPackages = with pkgs; [ + systemd virtiofsd + ]; + + settings = { + datacenter = "c3d2"; + plugin.raw_exec.config.enabled = true; + + addresses = { + http = "::1"; + rpc = ipv6Addr; + serf = ipv6Addr; + }; + + server = { + enabled = true; + bootstrap_expect = 3; + }; + server_join = { + retry_join = map (server: + zentralwerk.lib.config.site.net.${serverNet server}.hosts6.dn42.${server} + ) ( + builtins.filter (server: server != hostName) + servers + ); + }; + + client = { + enabled = true; + network_interface = "cluster"; + servers = [ "localhost" ]; + }; + }; + }; +}