modules/cluster: init glusterfs and nomad
This commit is contained in:
parent
14d8d34112
commit
e8a66709b7
|
@ -654,6 +654,7 @@
|
|||
./hosts/server9
|
||||
microvm.nixosModules.host
|
||||
self.nixosModules.cluster-network
|
||||
self.nixosModules.cluster
|
||||
{ _module.args = { inherit self; }; }
|
||||
];
|
||||
};
|
||||
|
@ -663,6 +664,7 @@
|
|||
./hosts/server10
|
||||
microvm.nixosModules.host
|
||||
self.nixosModules.cluster-network
|
||||
self.nixosModules.cluster
|
||||
{ _module.args = { inherit self; }; }
|
||||
];
|
||||
};
|
||||
|
@ -736,6 +738,7 @@
|
|||
c3d2.users = import ./users.nix;
|
||||
c3d2.nncp.neigh = import ./config/nncp-relays.nix;
|
||||
};
|
||||
cluster = ./modules/cluster;
|
||||
cluster-network = ./modules/cluster-network.nix;
|
||||
microvm.imports = [
|
||||
microvm.nixosModules.microvm
|
||||
|
|
|
@ -6,6 +6,7 @@
|
|||
./network.nix
|
||||
./hydra.nix
|
||||
./updater.nix
|
||||
./nomad-server.nix
|
||||
../../config/c3d2.nix
|
||||
];
|
||||
|
||||
|
|
39
hosts/hydra/nomad-server.nix
Normal file
39
hosts/hydra/nomad-server.nix
Normal file
|
@ -0,0 +1,39 @@
|
|||
# Our bare-metal hydra is a server in the nomad cluster, it is not a
|
||||
# client and therefore does not run cluster jobs
|
||||
{ zentralwerk, config, pkgs, ... }:
|
||||
let
|
||||
inherit (config.networking) hostName;
|
||||
ipv6Addr = zentralwerk.lib.config.site.net.serv.hosts6.dn42.${hostName};
|
||||
in
|
||||
{
|
||||
services.nomad = {
|
||||
enable = true;
|
||||
enableDocker = false;
|
||||
|
||||
settings = {
|
||||
datacenter = "c3d2";
|
||||
plugin.raw_exec.config.enabled = true;
|
||||
|
||||
addresses = {
|
||||
http = "::1";
|
||||
rpc = ipv6Addr;
|
||||
serf = ipv6Addr;
|
||||
};
|
||||
server = {
|
||||
enabled = true;
|
||||
bootstrap_expect = 3;
|
||||
};
|
||||
server_join = {
|
||||
retry_join = map (server:
|
||||
zentralwerk.lib.config.site.net.cluster.hosts4.${server}
|
||||
) [ "server9" "server10" ];
|
||||
};
|
||||
|
||||
client = {
|
||||
enabled = true;
|
||||
network_interface = "cluster";
|
||||
servers = [ "localhost" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
88
modules/cluster/default.nix
Normal file
88
modules/cluster/default.nix
Normal file
|
@ -0,0 +1,88 @@
|
|||
{ zentralwerk, config, lib, pkgs, ... }:
|
||||
let
|
||||
inherit (config.networking) hostName;
|
||||
|
||||
# hydra does *not* use this module because it only runs a nomad
|
||||
# server but no client and no microvms
|
||||
servers = [ "server9" "server10" "hydra" ];
|
||||
|
||||
serverNet = server:
|
||||
builtins.foldl' (result: net:
|
||||
if result == null &&
|
||||
zentralwerk.lib.config.site.net.${net}.hosts4 ? ${server}
|
||||
then net
|
||||
else result
|
||||
) null [ "cluster" "serv" ];
|
||||
|
||||
ipv6Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts6.dn42.${hostName};
|
||||
in {
|
||||
imports = [
|
||||
];
|
||||
|
||||
# Open firewall between cluster members
|
||||
|
||||
networking.firewall.extraCommands = lib.concatMapStrings (server:
|
||||
let
|
||||
netConfig = zentralwerk.lib.config.site.net.${serverNet server};
|
||||
in
|
||||
lib.optionalString (server != hostName) ''
|
||||
iptables -A nixos-fw --source ${netConfig.hosts4.${server}} -j ACCEPT
|
||||
${lib.concatMapStrings (hosts6: ''
|
||||
ip6tables -A nixos-fw --source ${hosts6.${server}} -j ACCEPT
|
||||
'') (builtins.attrValues netConfig.hosts6)}
|
||||
'') servers;
|
||||
|
||||
# Storage
|
||||
|
||||
services.glusterfs.enable = true;
|
||||
|
||||
fileSystems."/glusterfs/fast" = {
|
||||
fsType = "glusterfs";
|
||||
device = "${config.networking.hostName}:/fast";
|
||||
};
|
||||
|
||||
# Nomad
|
||||
|
||||
environment.systemPackages = with pkgs; [ nomad ];
|
||||
|
||||
services.nomad = {
|
||||
enable = true;
|
||||
|
||||
enableDocker = false;
|
||||
dropPrivileges = false;
|
||||
|
||||
extraPackages = with pkgs; [
|
||||
systemd virtiofsd
|
||||
];
|
||||
|
||||
settings = {
|
||||
datacenter = "c3d2";
|
||||
plugin.raw_exec.config.enabled = true;
|
||||
|
||||
addresses = {
|
||||
http = "::1";
|
||||
rpc = ipv6Addr;
|
||||
serf = ipv6Addr;
|
||||
};
|
||||
|
||||
server = {
|
||||
enabled = true;
|
||||
bootstrap_expect = 3;
|
||||
};
|
||||
server_join = {
|
||||
retry_join = map (server:
|
||||
zentralwerk.lib.config.site.net.${serverNet server}.hosts6.dn42.${server}
|
||||
) (
|
||||
builtins.filter (server: server != hostName)
|
||||
servers
|
||||
);
|
||||
};
|
||||
|
||||
client = {
|
||||
enabled = true;
|
||||
network_interface = "cluster";
|
||||
servers = [ "localhost" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
Loading…
Reference in New Issue
Block a user