improve nomad deployment

flake.nix

@@ -338,7 +338,12 @@
               for h in server9 server10 ; do
                 echo Sharing with $h
                 ssh root@$h.cluster.zentralwerk.org -- \
-                  nix copy --from https://hydra.hq.c3d2.de $JOB
+                  bash -e <<EOF
+                nix copy --from https://hydra.hq.c3d2.de $JOB
+                mkdir -p /glusterfs/fast/microvms/${name}
+                chown microvm:kvm /glusterfs/fast/microvms/${name}
+                chmod 0775 /glusterfs/fast/microvms/${name}
+              EOF
               done
 
               echo Now starting the job

hosts/containers/leon/default.nix

@@ -5,13 +5,12 @@ let
 in
 {
   microvm = {
-    hypervisor = "qemu";
     mem = 2048;
   };
 
   c3d2.deployment = {
-    server = "server10";
     mounts = [ "etc" "home" "var"];
+    mountBase = "/glusterfs/fast/microvms/${config.networking.hostName}";
   };
 
   networking = {

hosts/containers/mucbot/default.nix

@@ -3,7 +3,8 @@
 {
   c3d2.deployment = {
     server = "server10";
-    mounts = [ "etc" "var"];
+    mounts = [];
+    mountBase = "/glusterfs/fast/microvms/${config.networking.hostName}";
   };
 
   networking.hostName = "mucbot";

modules/microvm.nix

@@ -29,7 +29,8 @@ in
 {
   options.c3d2.deployment = with lib; {
     server = mkOption {
-      type = types.enum [ "server9" "server10" ];
+      type = with types; nullOr (enum [ "server9" "server10" ]);
+      default = null;
       description = "Server that is supposed to host this MicroVM.";
     };
 
@@ -47,6 +48,14 @@ in
       type = with types; listOf str;
       default = [ "etc" ];
     };
+
+    mountBase = mkOption {
+      description = ''
+        Location (ZFS dataset, ...) where all the shares live.
+      '';
+      type = types.path;
+      default = "/var/lib/microvms/${hostName}";
+    };
   };
 
   config.system.build = with pkgs; {
@@ -78,10 +87,11 @@ in
           pkgs.writeScript "virtiofsd-${hostName}-${tag}" ''
             #!${pkgs.runtimeShell} -e
             cd ${stateDir}
+            mkdir -p ${source}
             exec ${pkgs.virtiofsd}/bin/virtiofsd \
               --socket-path=${socket} \
               --socket-group=kvm \
-              --shared-dir=${builtins.replaceStrings ["/var/lib/microvms/${hostName}"] [stateDir] source} \
+              --shared-dir=${source} \
               --sandbox=none
           '';
         # change working directory before starting hypervisor,
@@ -99,7 +109,7 @@ in
           datacenters = ["c3d2"]
           type = "service"
 
-          group "microvm" {
+          group "nixos-${config.system.nixos.label}" {
             count = 1
             restart { attempts = 1 }
             ${lib.concatMapStrings (interface@{ id, ... }: ''
@@ -157,6 +167,10 @@ in
       # No server/router runs any untrusted user code
       "mitigations=off"
     ];
+
+    # stable uid is useful across glusterfs
+    users.users.microvm.uid = 997;
+
     microvm = {
       hypervisor = lib.mkDefault "cloud-hypervisor";
       mem = lib.mkDefault 512;
@@ -183,7 +197,7 @@ in
         else let
           tag = builtins.replaceStrings ["/"] ["_"] dir;
         in {
-          source = "/var/lib/microvms/${hostName}/${dir}";
+          source = "${config.c3d2.deployment.mountBase}/${dir}";
           mountPoint = "/${dir}";
           inherit tag;
           proto = "virtiofs";