From d89ec28ff35288e556458d62fad16686b39687f2 Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Fri, 13 Jan 2023 01:35:20 +0100
Subject: [PATCH] modules/cluster: remove broken glusterfs setup

---
 README.md                              | 27 ++---------------
 modules/cluster/default.nix            | 19 ------------
 modules/cluster/deployment-options.nix |  2 +-
 packages.nix                           | 41 --------------------------
 4 files changed, 3 insertions(+), 86 deletions(-)

diff --git a/README.md b/README.md
index 8223fcfe..da572553 100644
--- a/README.md
+++ b/README.md
@@ -104,7 +104,7 @@ microvm -Ru $hostname
 [Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
 Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
 networking with one bridge per VLAN. Persistent storage is replicated
-with Glusterfs.
+with Cephfs.
 
 Recognize nixosConfiguration for our Skyflake deployment by the
 `self.nixosModules.cluster-options` module being included.
@@ -137,7 +137,7 @@ This will:
 1. Build the configuration on Hydra, refusing the branch update on
    broken builds (through a git hook)
 2. Copy the MicroVM package and its dependencies to the binary cache
-   that is accessible to all nodes with Glusterfs
+   that is accessible to all nodes with Cephfs
 3. Submit one job per MicroVM into the Nomad cluster
 
 *Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
@@ -154,29 +154,6 @@ ssh c3d2@hydra.serv.zentralwerk.org status
 
 ### Debugging for cluster admins
 
-#### Glusterfs
-
-Glusterfs holds our MicroVMs' state. They *must always be mounted* or
-brains are split.
-
-```bash
-gluster volume info
-gluster volume status
-```
-
-##### Restart glusterd
-
-```bash
-systemctl restart glusterd
-```
-
-##### Remount volumes
-
-```bash
-systemctl restart /glusterfs/fast
-systemctl restart /glusterfs/big
-```
-
 #### Nomad
 
 ##### Check the cluster state
diff --git a/modules/cluster/default.nix b/modules/cluster/default.nix
index 766d8bbd..712ad9c9 100644
--- a/modules/cluster/default.nix
+++ b/modules/cluster/default.nix
@@ -70,20 +70,6 @@ in {
 
     deploy.customizationModule = ./deployment.nix;
 
-    # Obsolete Glusterfs setup.
-    # TODO: Remove once all data has been moved off.
-    storage.glusterfs = {
-      fileSystems = [ {
-        servers = microvmServers;
-        mountPoint = "/glusterfs/fast";
-        source = "/var/glusterfs-fast";
-      } {
-        servers = storageServers;
-        mountPoint = "/glusterfs/big";
-        source = "/var/glusterfs-big";
-      } ];
-    };
-
     # Ceph storage cluster configuration
     storage.ceph = rec {
       fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
@@ -108,11 +94,6 @@ in {
     };
   };
 
-  systemd.tmpfiles.rules = [
-    # additional gcroots
-    "L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
-  ];
-
   # Collectd monitoring for ceph
   services.collectd.plugins.ceph = ''
     ConvertSpecialMetricTypes true
diff --git a/modules/cluster/deployment-options.nix b/modules/cluster/deployment-options.nix
index 0a68d9fe..517fbeaf 100644
--- a/modules/cluster/deployment-options.nix
+++ b/modules/cluster/deployment-options.nix
@@ -39,7 +39,7 @@
       type = types.enum [ "fast" "big" ];
       default = "fast";
       description = ''
-        Which glusterfs volume to use for persistedShares
+        Unused for now
       '';
     };
     extraShares = mkOption {
diff --git a/packages.nix b/packages.nix
index 7c197169..862d274d 100644
--- a/packages.nix
+++ b/packages.nix
@@ -224,47 +224,6 @@ lib.attrsets.mapAttrs
           systemctl restart microvm@${name}.service
           END
         '';
-
-        "nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
-          #!${pkgs.runtimeShell} -e
-
-          ${lib.optionalString (hostConfig.c3d2.deployment.server or null == "nomad") ''
-            echo "MicroVM must be configured for nomad" >&2
-            exit 1
-          ''}
-
-          echo Copying Flakes
-          nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}
-
-          echo Building on Hydra
-          ssh root@hydra.serv.zentralwerk.org -- \
-            nix build -L -o /tmp/microvm-${name}.job \
-            ${self}#nixosConfigurations.${name}.config.system.build.nomadJob
-
-          echo -n Built. Obtaining path...
-          JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
-            readlink /tmp/microvm-${name}.job)
-          echo \ $JOB
-
-          for h in server9 server10 ; do
-            echo Sharing with $h
-            ssh root@$h.cluster.zentralwerk.org -- \
-              bash -e <<EOF &
-            nix copy --from https://nix-serve.hq.c3d2.de $JOB
-            mkdir -p /glusterfs/fast/microvms/${name}
-            chown microvm:kvm /glusterfs/fast/microvms/${name}
-            chmod 0775 /glusterfs/fast/microvms/${name}
-            mkdir -p /nix/var/nix/gcroots/microvm
-            rm -f /nix/var/nix/gcroots/microvm/${name}
-            ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
-          EOF
-          done
-          wait
-
-          echo Now starting the job
-          ssh root@hydra.serv.zentralwerk.org -- \
-            nomad run -detach $JOB
-        '';
       })
     { }
     (builtins.attrNames self.nixosConfigurations) //