forked from c3d2/nix-config
add nomad scripting
This commit is contained in:
parent
cb4799d2e9
commit
5707cd896f
27
flake.nix
27
flake.nix
|
@ -318,6 +318,33 @@
|
|||
systemctl restart microvm@${name}.service
|
||||
END
|
||||
'';
|
||||
|
||||
"nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
|
||||
echo Copying Flakes
|
||||
nix copy --to ssh://root@hydra.serv.zentralwerk.org ${secrets} ${self}
|
||||
|
||||
echo Building on Hydra
|
||||
ssh root@hydra.serv.zentralwerk.org -- \
|
||||
nix build -L -o /tmp/microvm-${name}.job \
|
||||
${self}#nixosConfigurations.${name}.config.system.build.nomadJob
|
||||
|
||||
echo -n Built. Obtaining path...
|
||||
JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
|
||||
readlink /tmp/microvm-${name}.job)
|
||||
echo \ $JOB
|
||||
|
||||
for h in server9 server10 ; do
|
||||
echo Sharing with $h
|
||||
ssh root@$h.cluster.zentralwerk.org -- \
|
||||
nix copy --from https://hydra.hq.c3d2.de $JOB
|
||||
done
|
||||
|
||||
echo Now starting the job
|
||||
ssh root@hydra.serv.zentralwerk.org -- \
|
||||
nomad run -detach $JOB
|
||||
'';
|
||||
}) {} (builtins.attrNames self.nixosConfigurations) //
|
||||
|
||||
builtins.foldl' (result: host:
|
||||
|
|
|
@ -8,6 +8,8 @@ in
|
|||
{
|
||||
services.nomad = {
|
||||
enable = true;
|
||||
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
|
||||
package = pkgs.nomad_1_3;
|
||||
enableDocker = false;
|
||||
|
||||
settings = {
|
||||
|
|
|
@ -43,18 +43,14 @@ in {
|
|||
|
||||
# Nomad
|
||||
|
||||
environment.systemPackages = with pkgs; [ nomad ];
|
||||
|
||||
services.nomad = {
|
||||
enable = true;
|
||||
# nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
|
||||
package = pkgs.nomad_1_3;
|
||||
|
||||
enableDocker = false;
|
||||
dropPrivileges = false;
|
||||
|
||||
extraPackages = with pkgs; [
|
||||
systemd virtiofsd
|
||||
];
|
||||
|
||||
settings = {
|
||||
datacenter = "c3d2";
|
||||
plugin.raw_exec.config.enabled = true;
|
||||
|
|
|
@ -61,6 +61,93 @@ in
|
|||
|
||||
ssh root@${serverFQDN} -- $@
|
||||
'';
|
||||
|
||||
nomadJob =
|
||||
let
|
||||
stateDir = "/glusterfs/fast/microvms/${hostName}";
|
||||
# only create tuntap if not yet existing
|
||||
runTuntap = { id, ... }:
|
||||
pkgs.writeScript "tuntap-${hostName}-${id}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
if [ ! -d /sys/class/net/${id} ]; then
|
||||
ip tuntap add ${id} mode tap user microvm
|
||||
fi
|
||||
'';
|
||||
# change working directory before starting virtiofsd
|
||||
runVirtiofsd = { tag, socket, source, ... }:
|
||||
pkgs.writeScript "virtiofsd-${hostName}-${tag}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
cd ${stateDir}
|
||||
exec ${pkgs.virtiofsd}/bin/virtiofsd \
|
||||
--socket-path=${socket} \
|
||||
--socket-group=kvm \
|
||||
--shared-dir=${builtins.replaceStrings ["/var/lib/microvms/${hostName}"] [stateDir] source} \
|
||||
--sandbox=none
|
||||
'';
|
||||
# change working directory before starting hypervisor,
|
||||
runMicrovm =
|
||||
pkgs.writeScript "hypervisor-${hostName}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
cd ${stateDir}
|
||||
trap "echo TRAP; ${config.microvm.declaredRunner}/bin/microvm-shutdown" INT TERM
|
||||
${config.microvm.declaredRunner}/bin/microvm-run &
|
||||
wait $!
|
||||
'';
|
||||
in pkgs.writeText "${hostName}.job" ''
|
||||
job "${hostName}" {
|
||||
datacenters = ["c3d2"]
|
||||
type = "service"
|
||||
|
||||
group "microvm" {
|
||||
count = 1
|
||||
restart { attempts = 1 }
|
||||
${lib.concatMapStrings (interface@{ id, ... }: ''
|
||||
task "interface-${id}" {
|
||||
lifecycle {
|
||||
hook = "prestart"
|
||||
}
|
||||
driver = "raw_exec"
|
||||
user = "root"
|
||||
config {
|
||||
command = "${runTuntap interface}"
|
||||
}
|
||||
}
|
||||
'') config.microvm.interfaces}
|
||||
|
||||
${lib.concatMapStrings (share@{ tag, ... }: ''
|
||||
task "virtiofsd-${tag}" {
|
||||
lifecycle {
|
||||
hook = "prestart"
|
||||
sidecar = true
|
||||
}
|
||||
driver = "raw_exec"
|
||||
user = "root"
|
||||
config {
|
||||
command = "${runVirtiofsd share}"
|
||||
}
|
||||
|
||||
resources {
|
||||
memory = ${toString (config.microvm.vcpu * 32)}
|
||||
cpu = ${toString (config.microvm.vcpu * 10)}
|
||||
}
|
||||
}
|
||||
'') config.microvm.shares}
|
||||
|
||||
task "hypervisor" {
|
||||
driver = "raw_exec"
|
||||
user = "root"
|
||||
config {
|
||||
command = "${runMicrovm}"
|
||||
}
|
||||
|
||||
resources {
|
||||
memory = ${toString config.microvm.mem}
|
||||
cpu = ${toString (config.microvm.vcpu * 50)}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
config = {
|
||||
|
|
Loading…
Reference in New Issue