modules/cluster: remove broken glusterfs setup
This commit is contained in:
parent
972b562f78
commit
d89ec28ff3
27
README.md
27
README.md
|
@ -104,7 +104,7 @@ microvm -Ru $hostname
|
|||
[Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
|
||||
Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
|
||||
networking with one bridge per VLAN. Persistent storage is replicated
|
||||
with Glusterfs.
|
||||
with Cephfs.
|
||||
|
||||
Recognize nixosConfiguration for our Skyflake deployment by the
|
||||
`self.nixosModules.cluster-options` module being included.
|
||||
|
@ -137,7 +137,7 @@ This will:
|
|||
1. Build the configuration on Hydra, refusing the branch update on
|
||||
broken builds (through a git hook)
|
||||
2. Copy the MicroVM package and its dependencies to the binary cache
|
||||
that is accessible to all nodes with Glusterfs
|
||||
that is accessible to all nodes with Cephfs
|
||||
3. Submit one job per MicroVM into the Nomad cluster
|
||||
|
||||
*Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
|
||||
|
@ -154,29 +154,6 @@ ssh c3d2@hydra.serv.zentralwerk.org status
|
|||
|
||||
### Debugging for cluster admins
|
||||
|
||||
#### Glusterfs
|
||||
|
||||
Glusterfs holds our MicroVMs' state. They *must always be mounted* or
|
||||
brains are split.
|
||||
|
||||
```bash
|
||||
gluster volume info
|
||||
gluster volume status
|
||||
```
|
||||
|
||||
##### Restart glusterd
|
||||
|
||||
```bash
|
||||
systemctl restart glusterd
|
||||
```
|
||||
|
||||
##### Remount volumes
|
||||
|
||||
```bash
|
||||
systemctl restart /glusterfs/fast
|
||||
systemctl restart /glusterfs/big
|
||||
```
|
||||
|
||||
#### Nomad
|
||||
|
||||
##### Check the cluster state
|
||||
|
|
|
@ -70,20 +70,6 @@ in {
|
|||
|
||||
deploy.customizationModule = ./deployment.nix;
|
||||
|
||||
# Obsolete Glusterfs setup.
|
||||
# TODO: Remove once all data has been moved off.
|
||||
storage.glusterfs = {
|
||||
fileSystems = [ {
|
||||
servers = microvmServers;
|
||||
mountPoint = "/glusterfs/fast";
|
||||
source = "/var/glusterfs-fast";
|
||||
} {
|
||||
servers = storageServers;
|
||||
mountPoint = "/glusterfs/big";
|
||||
source = "/var/glusterfs-big";
|
||||
} ];
|
||||
};
|
||||
|
||||
# Ceph storage cluster configuration
|
||||
storage.ceph = rec {
|
||||
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
||||
|
@ -108,11 +94,6 @@ in {
|
|||
};
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
# additional gcroots
|
||||
"L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
|
||||
];
|
||||
|
||||
# Collectd monitoring for ceph
|
||||
services.collectd.plugins.ceph = ''
|
||||
ConvertSpecialMetricTypes true
|
||||
|
|
|
@ -39,7 +39,7 @@
|
|||
type = types.enum [ "fast" "big" ];
|
||||
default = "fast";
|
||||
description = ''
|
||||
Which glusterfs volume to use for persistedShares
|
||||
Unused for now
|
||||
'';
|
||||
};
|
||||
extraShares = mkOption {
|
||||
|
|
41
packages.nix
41
packages.nix
|
@ -224,47 +224,6 @@ lib.attrsets.mapAttrs
|
|||
systemctl restart microvm@${name}.service
|
||||
END
|
||||
'';
|
||||
|
||||
"nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
|
||||
#!${pkgs.runtimeShell} -e
|
||||
|
||||
${lib.optionalString (hostConfig.c3d2.deployment.server or null == "nomad") ''
|
||||
echo "MicroVM must be configured for nomad" >&2
|
||||
exit 1
|
||||
''}
|
||||
|
||||
echo Copying Flakes
|
||||
nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}
|
||||
|
||||
echo Building on Hydra
|
||||
ssh root@hydra.serv.zentralwerk.org -- \
|
||||
nix build -L -o /tmp/microvm-${name}.job \
|
||||
${self}#nixosConfigurations.${name}.config.system.build.nomadJob
|
||||
|
||||
echo -n Built. Obtaining path...
|
||||
JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
|
||||
readlink /tmp/microvm-${name}.job)
|
||||
echo \ $JOB
|
||||
|
||||
for h in server9 server10 ; do
|
||||
echo Sharing with $h
|
||||
ssh root@$h.cluster.zentralwerk.org -- \
|
||||
bash -e <<EOF &
|
||||
nix copy --from https://nix-serve.hq.c3d2.de $JOB
|
||||
mkdir -p /glusterfs/fast/microvms/${name}
|
||||
chown microvm:kvm /glusterfs/fast/microvms/${name}
|
||||
chmod 0775 /glusterfs/fast/microvms/${name}
|
||||
mkdir -p /nix/var/nix/gcroots/microvm
|
||||
rm -f /nix/var/nix/gcroots/microvm/${name}
|
||||
ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
|
||||
EOF
|
||||
done
|
||||
wait
|
||||
|
||||
echo Now starting the job
|
||||
ssh root@hydra.serv.zentralwerk.org -- \
|
||||
nomad run -detach $JOB
|
||||
'';
|
||||
})
|
||||
{ }
|
||||
(builtins.attrNames self.nixosConfigurations) //
|
||||
|
|
Loading…
Reference in New Issue