From 08060dced92710bb23c01ba1d4028a01dc228253 Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Thu, 7 Jul 2022 01:21:39 +0200
Subject: [PATCH] improve nomad deployment

---
 flake.nix                           |  7 ++++++-
 hosts/containers/leon/default.nix   |  3 +--
 hosts/containers/mucbot/default.nix |  3 ++-
 modules/microvm.nix                 | 22 ++++++++++++++++++----
 4 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/flake.nix b/flake.nix
index d254f389..dbb88855 100644
--- a/flake.nix
+++ b/flake.nix
@@ -338,7 +338,12 @@
               for h in server9 server10 ; do
                 echo Sharing with $h
                 ssh root@$h.cluster.zentralwerk.org -- \
-                  nix copy --from https://hydra.hq.c3d2.de $JOB
+                  bash -e <<EOF
+                nix copy --from https://hydra.hq.c3d2.de $JOB
+                mkdir -p /glusterfs/fast/microvms/${name}
+                chown microvm:kvm /glusterfs/fast/microvms/${name}
+                chmod 0775 /glusterfs/fast/microvms/${name}
+              EOF
               done
 
               echo Now starting the job
diff --git a/hosts/containers/leon/default.nix b/hosts/containers/leon/default.nix
index 6ac46c3f..06c9dee4 100644
--- a/hosts/containers/leon/default.nix
+++ b/hosts/containers/leon/default.nix
@@ -5,13 +5,12 @@ let
 in
 {
   microvm = {
-    hypervisor = "qemu";
     mem = 2048;
   };
 
   c3d2.deployment = {
-    server = "server10";
     mounts = [ "etc" "home" "var"];
+    mountBase = "/glusterfs/fast/microvms/${config.networking.hostName}";
   };
 
   networking = {
diff --git a/hosts/containers/mucbot/default.nix b/hosts/containers/mucbot/default.nix
index 7c8b4156..65669ca6 100644
--- a/hosts/containers/mucbot/default.nix
+++ b/hosts/containers/mucbot/default.nix
@@ -3,7 +3,8 @@
 {
   c3d2.deployment = {
     server = "server10";
-    mounts = [ "etc" "var"];
+    mounts = [];
+    mountBase = "/glusterfs/fast/microvms/${config.networking.hostName}";
   };
 
   networking.hostName = "mucbot";
diff --git a/modules/microvm.nix b/modules/microvm.nix
index d80f8da3..570a12e8 100644
--- a/modules/microvm.nix
+++ b/modules/microvm.nix
@@ -29,7 +29,8 @@ in
 {
   options.c3d2.deployment = with lib; {
     server = mkOption {
-      type = types.enum [ "server9" "server10" ];
+      type = with types; nullOr (enum [ "server9" "server10" ]);
+      default = null;
       description = "Server that is supposed to host this MicroVM.";
     };
 
@@ -47,6 +48,14 @@ in
       type = with types; listOf str;
       default = [ "etc" ];
     };
+
+    mountBase = mkOption {
+      description = ''
+        Location (ZFS dataset, ...) where all the shares live.
+      '';
+      type = types.path;
+      default = "/var/lib/microvms/${hostName}";
+    };
   };
 
   config.system.build = with pkgs; {
@@ -78,10 +87,11 @@ in
           pkgs.writeScript "virtiofsd-${hostName}-${tag}" ''
             #!${pkgs.runtimeShell} -e
             cd ${stateDir}
+            mkdir -p ${source}
             exec ${pkgs.virtiofsd}/bin/virtiofsd \
               --socket-path=${socket} \
               --socket-group=kvm \
-              --shared-dir=${builtins.replaceStrings ["/var/lib/microvms/${hostName}"] [stateDir] source} \
+              --shared-dir=${source} \
               --sandbox=none
           '';
         # change working directory before starting hypervisor,
@@ -99,7 +109,7 @@ in
           datacenters = ["c3d2"]
           type = "service"
 
-          group "microvm" {
+          group "nixos-${config.system.nixos.label}" {
             count = 1
             restart { attempts = 1 }
             ${lib.concatMapStrings (interface@{ id, ... }: ''
@@ -157,6 +167,10 @@ in
       # No server/router runs any untrusted user code
       "mitigations=off"
     ];
+
+    # stable uid is useful across glusterfs
+    users.users.microvm.uid = 997;
+
     microvm = {
       hypervisor = lib.mkDefault "cloud-hypervisor";
       mem = lib.mkDefault 512;
@@ -183,7 +197,7 @@ in
         else let
           tag = builtins.replaceStrings ["/"] ["_"] dir;
         in {
-          source = "/var/lib/microvms/${hostName}/${dir}";
+          source = "${config.c3d2.deployment.mountBase}/${dir}";
           mountPoint = "/${dir}";
           inherit tag;
           proto = "virtiofs";