modules/cluster: remove broken glusterfs setup

This commit is contained in:
Astro 2023-01-13 01:35:20 +01:00
parent 972b562f78
commit d89ec28ff3
4 changed files with 3 additions and 86 deletions

View File

@ -104,7 +104,7 @@ microvm -Ru $hostname
[Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
networking with one bridge per VLAN. Persistent storage is replicated
with Glusterfs.
with Cephfs.
Recognize nixosConfiguration for our Skyflake deployment by the
`self.nixosModules.cluster-options` module being included.
@ -137,7 +137,7 @@ This will:
1. Build the configuration on Hydra, refusing the branch update on
broken builds (through a git hook)
2. Copy the MicroVM package and its dependencies to the binary cache
that is accessible to all nodes with Glusterfs
that is accessible to all nodes with Cephfs
3. Submit one job per MicroVM into the Nomad cluster
*Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
@ -154,29 +154,6 @@ ssh c3d2@hydra.serv.zentralwerk.org status
### Debugging for cluster admins
#### Glusterfs
Glusterfs holds our MicroVMs' state. They *must always be mounted* or
brains are split.
```bash
gluster volume info
gluster volume status
```
##### Restart glusterd
```bash
systemctl restart glusterd
```
##### Remount volumes
```bash
systemctl restart /glusterfs/fast
systemctl restart /glusterfs/big
```
#### Nomad
##### Check the cluster state

View File

@ -70,20 +70,6 @@ in {
deploy.customizationModule = ./deployment.nix;
# Obsolete Glusterfs setup.
# TODO: Remove once all data has been moved off.
storage.glusterfs = {
fileSystems = [ {
servers = microvmServers;
mountPoint = "/glusterfs/fast";
source = "/var/glusterfs-fast";
} {
servers = storageServers;
mountPoint = "/glusterfs/big";
source = "/var/glusterfs-big";
} ];
};
# Ceph storage cluster configuration
storage.ceph = rec {
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
@ -108,11 +94,6 @@ in {
};
};
systemd.tmpfiles.rules = [
# additional gcroots
"L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
];
# Collectd monitoring for ceph
services.collectd.plugins.ceph = ''
ConvertSpecialMetricTypes true

View File

@ -39,7 +39,7 @@
type = types.enum [ "fast" "big" ];
default = "fast";
description = ''
Which glusterfs volume to use for persistedShares
Unused for now
'';
};
extraShares = mkOption {

View File

@ -224,47 +224,6 @@ lib.attrsets.mapAttrs
systemctl restart microvm@${name}.service
END
'';
"nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
#!${pkgs.runtimeShell} -e
${lib.optionalString (hostConfig.c3d2.deployment.server or null == "nomad") ''
echo "MicroVM must be configured for nomad" >&2
exit 1
''}
echo Copying Flakes
nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}
echo Building on Hydra
ssh root@hydra.serv.zentralwerk.org -- \
nix build -L -o /tmp/microvm-${name}.job \
${self}#nixosConfigurations.${name}.config.system.build.nomadJob
echo -n Built. Obtaining path...
JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
readlink /tmp/microvm-${name}.job)
echo \ $JOB
for h in server9 server10 ; do
echo Sharing with $h
ssh root@$h.cluster.zentralwerk.org -- \
bash -e <<EOF &
nix copy --from https://nix-serve.hq.c3d2.de $JOB
mkdir -p /glusterfs/fast/microvms/${name}
chown microvm:kvm /glusterfs/fast/microvms/${name}
chmod 0775 /glusterfs/fast/microvms/${name}
mkdir -p /nix/var/nix/gcroots/microvm
rm -f /nix/var/nix/gcroots/microvm/${name}
ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
EOF
done
wait
echo Now starting the job
ssh root@hydra.serv.zentralwerk.org -- \
nomad run -detach $JOB
'';
})
{ }
(builtins.attrNames self.nixosConfigurations) //