modules/cluster: remove broken glusterfs setup
This commit is contained in:
parent
972b562f78
commit
d89ec28ff3
27
README.md
27
README.md
|
@ -104,7 +104,7 @@ microvm -Ru $hostname
|
||||||
[Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
|
[Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
|
||||||
Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
|
Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
|
||||||
networking with one bridge per VLAN. Persistent storage is replicated
|
networking with one bridge per VLAN. Persistent storage is replicated
|
||||||
with Glusterfs.
|
with Cephfs.
|
||||||
|
|
||||||
Recognize nixosConfiguration for our Skyflake deployment by the
|
Recognize nixosConfiguration for our Skyflake deployment by the
|
||||||
`self.nixosModules.cluster-options` module being included.
|
`self.nixosModules.cluster-options` module being included.
|
||||||
|
@ -137,7 +137,7 @@ This will:
|
||||||
1. Build the configuration on Hydra, refusing the branch update on
|
1. Build the configuration on Hydra, refusing the branch update on
|
||||||
broken builds (through a git hook)
|
broken builds (through a git hook)
|
||||||
2. Copy the MicroVM package and its dependencies to the binary cache
|
2. Copy the MicroVM package and its dependencies to the binary cache
|
||||||
that is accessible to all nodes with Glusterfs
|
that is accessible to all nodes with Cephfs
|
||||||
3. Submit one job per MicroVM into the Nomad cluster
|
3. Submit one job per MicroVM into the Nomad cluster
|
||||||
|
|
||||||
*Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
|
*Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
|
||||||
|
@ -154,29 +154,6 @@ ssh c3d2@hydra.serv.zentralwerk.org status
|
||||||
|
|
||||||
### Debugging for cluster admins
|
### Debugging for cluster admins
|
||||||
|
|
||||||
#### Glusterfs
|
|
||||||
|
|
||||||
Glusterfs holds our MicroVMs' state. They *must always be mounted* or
|
|
||||||
brains are split.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
gluster volume info
|
|
||||||
gluster volume status
|
|
||||||
```
|
|
||||||
|
|
||||||
##### Restart glusterd
|
|
||||||
|
|
||||||
```bash
|
|
||||||
systemctl restart glusterd
|
|
||||||
```
|
|
||||||
|
|
||||||
##### Remount volumes
|
|
||||||
|
|
||||||
```bash
|
|
||||||
systemctl restart /glusterfs/fast
|
|
||||||
systemctl restart /glusterfs/big
|
|
||||||
```
|
|
||||||
|
|
||||||
#### Nomad
|
#### Nomad
|
||||||
|
|
||||||
##### Check the cluster state
|
##### Check the cluster state
|
||||||
|
|
|
@ -70,20 +70,6 @@ in {
|
||||||
|
|
||||||
deploy.customizationModule = ./deployment.nix;
|
deploy.customizationModule = ./deployment.nix;
|
||||||
|
|
||||||
# Obsolete Glusterfs setup.
|
|
||||||
# TODO: Remove once all data has been moved off.
|
|
||||||
storage.glusterfs = {
|
|
||||||
fileSystems = [ {
|
|
||||||
servers = microvmServers;
|
|
||||||
mountPoint = "/glusterfs/fast";
|
|
||||||
source = "/var/glusterfs-fast";
|
|
||||||
} {
|
|
||||||
servers = storageServers;
|
|
||||||
mountPoint = "/glusterfs/big";
|
|
||||||
source = "/var/glusterfs-big";
|
|
||||||
} ];
|
|
||||||
};
|
|
||||||
|
|
||||||
# Ceph storage cluster configuration
|
# Ceph storage cluster configuration
|
||||||
storage.ceph = rec {
|
storage.ceph = rec {
|
||||||
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
||||||
|
@ -108,11 +94,6 @@ in {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
systemd.tmpfiles.rules = [
|
|
||||||
# additional gcroots
|
|
||||||
"L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
|
|
||||||
];
|
|
||||||
|
|
||||||
# Collectd monitoring for ceph
|
# Collectd monitoring for ceph
|
||||||
services.collectd.plugins.ceph = ''
|
services.collectd.plugins.ceph = ''
|
||||||
ConvertSpecialMetricTypes true
|
ConvertSpecialMetricTypes true
|
||||||
|
|
|
@ -39,7 +39,7 @@
|
||||||
type = types.enum [ "fast" "big" ];
|
type = types.enum [ "fast" "big" ];
|
||||||
default = "fast";
|
default = "fast";
|
||||||
description = ''
|
description = ''
|
||||||
Which glusterfs volume to use for persistedShares
|
Unused for now
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
extraShares = mkOption {
|
extraShares = mkOption {
|
||||||
|
|
41
packages.nix
41
packages.nix
|
@ -224,47 +224,6 @@ lib.attrsets.mapAttrs
|
||||||
systemctl restart microvm@${name}.service
|
systemctl restart microvm@${name}.service
|
||||||
END
|
END
|
||||||
'';
|
'';
|
||||||
|
|
||||||
"nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
|
|
||||||
#!${pkgs.runtimeShell} -e
|
|
||||||
|
|
||||||
${lib.optionalString (hostConfig.c3d2.deployment.server or null == "nomad") ''
|
|
||||||
echo "MicroVM must be configured for nomad" >&2
|
|
||||||
exit 1
|
|
||||||
''}
|
|
||||||
|
|
||||||
echo Copying Flakes
|
|
||||||
nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}
|
|
||||||
|
|
||||||
echo Building on Hydra
|
|
||||||
ssh root@hydra.serv.zentralwerk.org -- \
|
|
||||||
nix build -L -o /tmp/microvm-${name}.job \
|
|
||||||
${self}#nixosConfigurations.${name}.config.system.build.nomadJob
|
|
||||||
|
|
||||||
echo -n Built. Obtaining path...
|
|
||||||
JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
|
|
||||||
readlink /tmp/microvm-${name}.job)
|
|
||||||
echo \ $JOB
|
|
||||||
|
|
||||||
for h in server9 server10 ; do
|
|
||||||
echo Sharing with $h
|
|
||||||
ssh root@$h.cluster.zentralwerk.org -- \
|
|
||||||
bash -e <<EOF &
|
|
||||||
nix copy --from https://nix-serve.hq.c3d2.de $JOB
|
|
||||||
mkdir -p /glusterfs/fast/microvms/${name}
|
|
||||||
chown microvm:kvm /glusterfs/fast/microvms/${name}
|
|
||||||
chmod 0775 /glusterfs/fast/microvms/${name}
|
|
||||||
mkdir -p /nix/var/nix/gcroots/microvm
|
|
||||||
rm -f /nix/var/nix/gcroots/microvm/${name}
|
|
||||||
ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
|
|
||||||
EOF
|
|
||||||
done
|
|
||||||
wait
|
|
||||||
|
|
||||||
echo Now starting the job
|
|
||||||
ssh root@hydra.serv.zentralwerk.org -- \
|
|
||||||
nomad run -detach $JOB
|
|
||||||
'';
|
|
||||||
})
|
})
|
||||||
{ }
|
{ }
|
||||||
(builtins.attrNames self.nixosConfigurations) //
|
(builtins.attrNames self.nixosConfigurations) //
|
||||||
|
|
Loading…
Reference in New Issue