From 5707cd896fa6fcdb82c8ff271597b5d9b7fd51ab Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Tue, 5 Jul 2022 00:08:03 +0200
Subject: [PATCH] add nomad scripting

---
 flake.nix                    | 27 +++++++++++
 hosts/hydra/nomad-server.nix |  2 +
 modules/cluster/default.nix  |  8 +---
 modules/microvm.nix          | 87 ++++++++++++++++++++++++++++++++++++
 4 files changed, 118 insertions(+), 6 deletions(-)

diff --git a/flake.nix b/flake.nix
index cee41691..5c23e455 100644
--- a/flake.nix
+++ b/flake.nix
@@ -318,6 +318,33 @@
               systemctl restart microvm@${name}.service
               END
             '';
+
+            "nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
+              #!${pkgs.runtimeShell} -e
+
+              echo Copying Flakes
+              nix copy --to ssh://root@hydra.serv.zentralwerk.org ${secrets} ${self}
+
+              echo Building on Hydra
+              ssh root@hydra.serv.zentralwerk.org -- \
+                nix build -L -o /tmp/microvm-${name}.job \
+                ${self}#nixosConfigurations.${name}.config.system.build.nomadJob
+
+              echo -n Built. Obtaining path...
+              JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
+                readlink /tmp/microvm-${name}.job)
+              echo \ $JOB
+
+              for h in server9 server10 ; do
+                echo Sharing with $h
+                ssh root@$h.cluster.zentralwerk.org -- \
+                  nix copy --from https://hydra.hq.c3d2.de $JOB
+              done
+
+              echo Now starting the job
+              ssh root@hydra.serv.zentralwerk.org -- \
+                nomad run -detach $JOB
+            '';
           }) {} (builtins.attrNames self.nixosConfigurations) //
 
         builtins.foldl' (result: host:
diff --git a/hosts/hydra/nomad-server.nix b/hosts/hydra/nomad-server.nix
index 60a1ce7b..7dd0e69a 100644
--- a/hosts/hydra/nomad-server.nix
+++ b/hosts/hydra/nomad-server.nix
@@ -8,6 +8,8 @@ in
 {
   services.nomad = {
     enable = true;
+    # nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
+    package = pkgs.nomad_1_3;
     enableDocker = false;
 
     settings = {
diff --git a/modules/cluster/default.nix b/modules/cluster/default.nix
index d18040fa..67edd330 100644
--- a/modules/cluster/default.nix
+++ b/modules/cluster/default.nix
@@ -43,18 +43,14 @@ in {
 
   # Nomad
 
-  environment.systemPackages = with pkgs; [ nomad ];
-
   services.nomad = {
     enable = true;
+    # nomad<1.3 (default in nixos 22.05) is incompatible with cgroups-v2
+    package = pkgs.nomad_1_3;
 
     enableDocker = false;
     dropPrivileges = false;
 
-    extraPackages = with pkgs; [
-      systemd virtiofsd
-    ];
-
     settings = {
       datacenter = "c3d2";
       plugin.raw_exec.config.enabled = true;
diff --git a/modules/microvm.nix b/modules/microvm.nix
index e10e76a1..596785ab 100644
--- a/modules/microvm.nix
+++ b/modules/microvm.nix
@@ -61,6 +61,93 @@ in
 
       ssh root@${serverFQDN} -- $@
     '';
+
+    nomadJob =
+      let
+        stateDir = "/glusterfs/fast/microvms/${hostName}";
+        # only create tuntap if not yet existing
+        runTuntap = { id, ... }:
+          pkgs.writeScript "tuntap-${hostName}-${id}" ''
+            #!${pkgs.runtimeShell} -e
+            if [ ! -d /sys/class/net/${id} ]; then
+              ip tuntap add ${id} mode tap user microvm
+            fi
+          '';
+        # change working directory before starting virtiofsd
+        runVirtiofsd = { tag, socket, source, ... }:
+          pkgs.writeScript "virtiofsd-${hostName}-${tag}" ''
+            #!${pkgs.runtimeShell} -e
+            cd ${stateDir}
+            exec ${pkgs.virtiofsd}/bin/virtiofsd \
+              --socket-path=${socket} \
+              --socket-group=kvm \
+              --shared-dir=${builtins.replaceStrings ["/var/lib/microvms/${hostName}"] [stateDir] source} \
+              --sandbox=none
+          '';
+        # change working directory before starting hypervisor,
+        runMicrovm =
+          pkgs.writeScript "hypervisor-${hostName}" ''
+            #!${pkgs.runtimeShell} -e
+            cd ${stateDir}
+            trap "echo TRAP; ${config.microvm.declaredRunner}/bin/microvm-shutdown" INT TERM
+            ${config.microvm.declaredRunner}/bin/microvm-run &
+            wait $!
+          '';
+      in pkgs.writeText "${hostName}.job" ''
+        job "${hostName}" {
+          datacenters = ["c3d2"]
+          type = "service"
+
+          group "microvm" {
+            count = 1
+            restart { attempts = 1 }
+            ${lib.concatMapStrings (interface@{ id, ... }: ''
+              task "interface-${id}" {
+                lifecycle {
+                  hook = "prestart"
+                }
+                driver = "raw_exec"
+                user = "root"
+                config {
+                  command = "${runTuntap interface}"
+                }
+              }
+            '') config.microvm.interfaces}
+
+            ${lib.concatMapStrings (share@{ tag, ... }: ''
+              task "virtiofsd-${tag}" {
+                lifecycle {
+                  hook = "prestart"
+                  sidecar = true
+                }
+                driver = "raw_exec"
+                user = "root"
+                config {
+                  command = "${runVirtiofsd share}"
+                }
+
+                resources {
+                  memory = ${toString (config.microvm.vcpu * 32)}
+                  cpu = ${toString (config.microvm.vcpu * 10)}
+                }
+              }
+            '') config.microvm.shares}
+
+            task "hypervisor" {
+              driver = "raw_exec"
+              user = "root"
+              config {
+                command = "${runMicrovm}"
+              }
+
+              resources {
+                memory = ${toString config.microvm.mem}
+                cpu = ${toString (config.microvm.vcpu * 50)}
+              }
+            }
+          }
+        }
+      '';
   };
 
   config = {