modules/microvm: get hypervisors in nomad-jobs to shutdown gracefully

modules/microvm.nix

@@ -115,10 +115,25 @@ in
           pkgs.writeScript "hypervisor-${hostName}" ''
             #!${pkgs.runtimeShell} -e
             cd ${stateDir}
-            # hook SIGTERM for graceful shutdown (TODO: FIXME)
-            trap ${config.microvm.declaredRunner}/bin/microvm-shutdown TERM
             # start hypervisor
-            ${config.microvm.declaredRunner}/bin/microvm-run
+            exec ${config.microvm.declaredRunner}/bin/microvm-run
+          '';
+        stopMicrovm =
+          pkgs.writeScript "hypervisor-${hostName}-stop" ''
+            #!${pkgs.runtimeShell} -e
+            cd ${stateDir}
+            # stop hypervisor on signal
+            function handle_signal() {
+              ${config.microvm.declaredRunner}/bin/microvm-shutdown
+              exit
+            }
+            trap handle_signal TERM
+            # wait
+            while true; do
+              sleep 86400 &
+              # catch signals:
+              wait
+            done
           '';
       in pkgs.writeText "${hostName}.job" ''
         job "${hostName}" {
@@ -157,6 +172,8 @@ in
                 config {
                   command = "${runVirtiofsd share}"
                 }
+                kill_signal = "SIGCONT"
+                kill_timeout = "15s"
 
                 resources {
                   memory = ${toString (config.microvm.vcpu * 32)}
@@ -171,12 +188,24 @@ in
               config {
                 command = "${runMicrovm}"
               }
+              # don't get killed immediately but get shutdown by wait-shutdown
+              kill_signal = "SIGCONT"
+              kill_timeout = "15s"
 
               resources {
                 memory = ${toString config.microvm.mem}
                 cpu = ${toString (config.microvm.vcpu * 50)}
               }
             }
+
+            task "wait-shutdown" {
+              driver = "raw_exec"
+              user = "microvm"
+              config {
+                command = "${stopMicrovm}"
+              }
+              kill_signal = "SIGTERM"
+            }
           }
         }
       '';