Only disable ceph config, not half the cluster
This commit is contained in:
parent
3d2a1d9fe5
commit
27930ce873
20
flake.nix
20
flake.nix
|
@ -384,7 +384,7 @@
|
||||||
|
|
||||||
gnunet = nixosSystem' {
|
gnunet = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
# self.nixosModules.cluster-options
|
self.nixosModules.cluster-options
|
||||||
self.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/gnunet
|
./hosts/gnunet
|
||||||
];
|
];
|
||||||
|
@ -413,7 +413,7 @@
|
||||||
|
|
||||||
hydra = nixosSystem' {
|
hydra = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
# self.nixosModules.cluster
|
self.nixosModules.cluster
|
||||||
# skyflake.nixosModules.default
|
# skyflake.nixosModules.default
|
||||||
./hosts/hydra
|
./hosts/hydra
|
||||||
];
|
];
|
||||||
|
@ -493,7 +493,7 @@
|
||||||
nixpkgs.overlays = [ secrets.overlays.mucbot ];
|
nixpkgs.overlays = [ secrets.overlays.mucbot ];
|
||||||
}
|
}
|
||||||
./hosts/mucbot
|
./hosts/mucbot
|
||||||
# self.nixosModules.cluster-options
|
self.nixosModules.cluster-options
|
||||||
self.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
@ -537,7 +537,7 @@
|
||||||
|
|
||||||
owncast = nixosSystem' {
|
owncast = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
# self.nixosModules.cluster-options
|
self.nixosModules.cluster-options
|
||||||
self.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/owncast
|
./hosts/owncast
|
||||||
];
|
];
|
||||||
|
@ -642,7 +642,7 @@
|
||||||
}
|
}
|
||||||
heliwatch.nixosModules.heliwatch
|
heliwatch.nixosModules.heliwatch
|
||||||
self.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
# self.nixosModules.cluster-options
|
self.nixosModules.cluster-options
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -650,7 +650,7 @@
|
||||||
modules = [
|
modules = [
|
||||||
./hosts/server8
|
./hosts/server8
|
||||||
self.nixosModules.cluster-network
|
self.nixosModules.cluster-network
|
||||||
# self.nixosModules.cluster
|
self.nixosModules.cluster
|
||||||
# skyflake.nixosModules.default
|
# skyflake.nixosModules.default
|
||||||
{ _module.args = { inherit self; }; }
|
{ _module.args = { inherit self; }; }
|
||||||
];
|
];
|
||||||
|
@ -661,7 +661,7 @@
|
||||||
./hosts/server9
|
./hosts/server9
|
||||||
self.nixosModules.microvm-host
|
self.nixosModules.microvm-host
|
||||||
self.nixosModules.cluster-network
|
self.nixosModules.cluster-network
|
||||||
# self.nixosModules.cluster
|
self.nixosModules.cluster
|
||||||
# skyflake.nixosModules.default
|
# skyflake.nixosModules.default
|
||||||
{ _module.args = { inherit self; }; }
|
{ _module.args = { inherit self; }; }
|
||||||
];
|
];
|
||||||
|
@ -672,7 +672,7 @@
|
||||||
./hosts/server10
|
./hosts/server10
|
||||||
self.nixosModules.microvm-host
|
self.nixosModules.microvm-host
|
||||||
self.nixosModules.cluster-network
|
self.nixosModules.cluster-network
|
||||||
# self.nixosModules.cluster
|
self.nixosModules.cluster
|
||||||
# skyflake.nixosModules.default
|
# skyflake.nixosModules.default
|
||||||
{ _module.args = { inherit self; }; }
|
{ _module.args = { inherit self; }; }
|
||||||
];
|
];
|
||||||
|
@ -688,7 +688,7 @@
|
||||||
|
|
||||||
sshlog = nixosSystem' {
|
sshlog = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
# self.nixosModules.cluster-options
|
self.nixosModules.cluster-options
|
||||||
self.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
sshlogd.nixosModule
|
sshlogd.nixosModule
|
||||||
./hosts/sshlog
|
./hosts/sshlog
|
||||||
|
@ -697,7 +697,7 @@
|
||||||
|
|
||||||
stream = nixosSystem' {
|
stream = nixosSystem' {
|
||||||
modules = [
|
modules = [
|
||||||
# self.nixosModules.cluster-options
|
self.nixosModules.cluster-options
|
||||||
self.nixosModules.microvm
|
self.nixosModules.microvm
|
||||||
./hosts/stream
|
./hosts/stream
|
||||||
];
|
];
|
||||||
|
|
|
@ -36,111 +36,111 @@ in {
|
||||||
servers;
|
servers;
|
||||||
|
|
||||||
# Cluster configuration
|
# Cluster configuration
|
||||||
skyflake = {
|
# skyflake = {
|
||||||
# debug = true;
|
# # debug = true;
|
||||||
nodes = builtins.listToAttrs (
|
# nodes = builtins.listToAttrs (
|
||||||
map (name: {
|
# map (name: {
|
||||||
inherit name;
|
# inherit name;
|
||||||
value.address = hostRegistry.${name}.ip4;
|
# value.address = hostRegistry.${name}.ip4;
|
||||||
}) servers
|
# }) servers
|
||||||
);
|
# );
|
||||||
# Extra nomad configuration
|
# # Extra nomad configuration
|
||||||
nomad = {
|
# nomad = {
|
||||||
datacenter = "c3d2";
|
# datacenter = "c3d2";
|
||||||
inherit servers;
|
# inherit servers;
|
||||||
# run tasks only on these:
|
# # run tasks only on these:
|
||||||
client.enable = builtins.elem hostName microvmServers;
|
# client.enable = builtins.elem hostName microvmServers;
|
||||||
client.meta = lib.optionalAttrs (builtins.elem hostName storageServers) {
|
# client.meta = lib.optionalAttrs (builtins.elem hostName storageServers) {
|
||||||
"c3d2.storage" = "big";
|
# "c3d2.storage" = "big";
|
||||||
};
|
# };
|
||||||
};
|
# };
|
||||||
# The user that runs skyflake MicroVMs
|
# # The user that runs skyflake MicroVMs
|
||||||
microvmUid = 997;
|
# microvmUid = 997;
|
||||||
|
|
||||||
users = {
|
# users = {
|
||||||
# Deployment user for hosts in this flake
|
# # Deployment user for hosts in this flake
|
||||||
c3d2 = {
|
# c3d2 = {
|
||||||
uid = 1001;
|
# uid = 1001;
|
||||||
sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
# sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
||||||
home = "${skyflakeHome}/c3d2";
|
# home = "${skyflakeHome}/c3d2";
|
||||||
};
|
# };
|
||||||
# Deployment user for neighbour Andreas Lippmann <andreaslippmann@web.de>
|
# # Deployment user for neighbour Andreas Lippmann <andreaslippmann@web.de>
|
||||||
luulaatsch = {
|
# luulaatsch = {
|
||||||
uid = 1003;
|
# uid = 1003;
|
||||||
home = "${skyflakeHome}/luulaatsch";
|
# home = "${skyflakeHome}/luulaatsch";
|
||||||
sshKeys = [
|
# sshKeys = [
|
||||||
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVipwWSzxgUgm0fMeTbOSCXDNkbUUp+k1tvWaJcBA1M+LTM21rhP3tQQP24smPx1b2V/by6vHakhsj5pIMLzSSW7he/mAnOtwHj2dCEk6VaMvozAfS6n777g8ujZg0N9ekrqoNLh0uNchs3DYFZWXAnKtfEYjzIdXqVgXMI4uDXM6ObEh+YR3iGaZSYKd6S5nufzJOR6EiYbv/z6ThRq16f+HqetVdBamqmpNK9NcWCtjMcww7nZTyUvTbd2AyrpTAXSnVg30fza428Kpc5ZdzDJFjn0B3MVnP2LavIo8VH3Y5sjpp0aMDKfbz6dPA/df/pR6LVMsxx0eCEXeGqo9sFdabu2AwIPi79QFANr3qJ/nZo2LR3o2LvcOLO06xVRFKReGTsy6WWTNOTAL03sdiIbqnlmkjXu5nb4hzxoHBgA6LZGMx3knoVcUMBlpVaxOoT2tzXzpYszsKpGSN1EBqUlDXzen3EkHg7TpjjTJSH7Q0AeeSCJsaoAepa6wcZeJlBWSRiJdqIn7YBmCBdjV1nNPVBYE5mm2x88ha9e4NbyFhjQvZ/BiLyKCFVyDs6lIvhV5QfAPWOkyPwbLXtXg6g1OvmuM6mGDhtEfZKvreeqmTKjB+x7OpG847JKxmU8eeYax63/WgRssbOfTpicjsdoKlFST3WpZHGT7mWzmB/w== andreas@luulaatschs-MBP.pub.zentralwerk.org"
|
# "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVipwWSzxgUgm0fMeTbOSCXDNkbUUp+k1tvWaJcBA1M+LTM21rhP3tQQP24smPx1b2V/by6vHakhsj5pIMLzSSW7he/mAnOtwHj2dCEk6VaMvozAfS6n777g8ujZg0N9ekrqoNLh0uNchs3DYFZWXAnKtfEYjzIdXqVgXMI4uDXM6ObEh+YR3iGaZSYKd6S5nufzJOR6EiYbv/z6ThRq16f+HqetVdBamqmpNK9NcWCtjMcww7nZTyUvTbd2AyrpTAXSnVg30fza428Kpc5ZdzDJFjn0B3MVnP2LavIo8VH3Y5sjpp0aMDKfbz6dPA/df/pR6LVMsxx0eCEXeGqo9sFdabu2AwIPi79QFANr3qJ/nZo2LR3o2LvcOLO06xVRFKReGTsy6WWTNOTAL03sdiIbqnlmkjXu5nb4hzxoHBgA6LZGMx3knoVcUMBlpVaxOoT2tzXzpYszsKpGSN1EBqUlDXzen3EkHg7TpjjTJSH7Q0AeeSCJsaoAepa6wcZeJlBWSRiJdqIn7YBmCBdjV1nNPVBYE5mm2x88ha9e4NbyFhjQvZ/BiLyKCFVyDs6lIvhV5QfAPWOkyPwbLXtXg6g1OvmuM6mGDhtEfZKvreeqmTKjB+x7OpG847JKxmU8eeYax63/WgRssbOfTpicjsdoKlFST3WpZHGT7mWzmB/w== andreas@luulaatschs-MBP.pub.zentralwerk.org"
|
||||||
] ++ ssh-public-keys.astro;
|
# ] ++ ssh-public-keys.astro;
|
||||||
};
|
# };
|
||||||
# Deployment user for marenz
|
# # Deployment user for marenz
|
||||||
marenz = {
|
# marenz = {
|
||||||
uid = 1004;
|
# uid = 1004;
|
||||||
sshKeys = config.users.users.root.openssh.authorizedKeys.keys ++ ssh-public-keys.marenz;
|
# sshKeys = config.users.users.root.openssh.authorizedKeys.keys ++ ssh-public-keys.marenz;
|
||||||
home = "${skyflakeHome}/marenz";
|
# home = "${skyflakeHome}/marenz";
|
||||||
};
|
# };
|
||||||
};
|
# };
|
||||||
|
|
||||||
deploy.customizationModule = ./customization;
|
# deploy.customizationModule = ./customization;
|
||||||
|
|
||||||
# Ceph storage cluster configuration
|
# # Ceph storage cluster configuration
|
||||||
storage.ceph = rec {
|
# storage.ceph = rec {
|
||||||
package = pkgs.ceph_17_2;
|
# package = pkgs.ceph_17_2;
|
||||||
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
# fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
||||||
mons = [ "server9" "server10" ];
|
# mons = [ "server9" "server10" ];
|
||||||
mgrs = mons;
|
# mgrs = mons;
|
||||||
mdss = mons;
|
# mdss = mons;
|
||||||
rbdPools.microvms = {
|
# rbdPools.microvms = {
|
||||||
params = { size = 2; class = "ssd"; };
|
# params = { size = 2; class = "ssd"; };
|
||||||
};
|
# };
|
||||||
rbdPools.microvms-hdd = {
|
# rbdPools.microvms-hdd = {
|
||||||
params = { size = 2; class = "hdd"; };
|
# params = { size = 2; class = "hdd"; };
|
||||||
};
|
# };
|
||||||
cephfs.home.mountPoint = skyflakeHome;
|
# cephfs.home.mountPoint = skyflakeHome;
|
||||||
# Legacy: migration to rbd
|
# # Legacy: migration to rbd
|
||||||
cephfs.skyflake.mountPoint = "/storage/cephfs";
|
# cephfs.skyflake.mountPoint = "/storage/cephfs";
|
||||||
|
|
||||||
monKeyring = config.sops.secrets."ceph/monKeyring".path;
|
# monKeyring = config.sops.secrets."ceph/monKeyring".path;
|
||||||
adminKeyring = config.sops.secrets."ceph/adminKeyring".path;
|
# adminKeyring = config.sops.secrets."ceph/adminKeyring".path;
|
||||||
};
|
# };
|
||||||
};
|
# };
|
||||||
|
|
||||||
# Ceph keyrings
|
# Ceph keyrings
|
||||||
sops.secrets = {
|
# sops.secrets = {
|
||||||
"ceph/monKeyring" = {
|
# "ceph/monKeyring" = {
|
||||||
owner = "ceph";
|
# owner = "ceph";
|
||||||
sopsFile = ./ceph.yaml;
|
# sopsFile = ./ceph.yaml;
|
||||||
};
|
# };
|
||||||
"ceph/adminKeyring" = {
|
# "ceph/adminKeyring" = {
|
||||||
owner = "ceph";
|
# owner = "ceph";
|
||||||
sopsFile = ./ceph.yaml;
|
# sopsFile = ./ceph.yaml;
|
||||||
};
|
# };
|
||||||
};
|
# };
|
||||||
|
|
||||||
# Collectd monitoring for ceph
|
# Collectd monitoring for ceph
|
||||||
services.collectd.plugins.ceph = ''
|
# services.collectd.plugins.ceph = ''
|
||||||
ConvertSpecialMetricTypes true
|
# ConvertSpecialMetricTypes true
|
||||||
|
|
||||||
${lib.concatMapStrings (hostName: ''
|
# ${lib.concatMapStrings (hostName: ''
|
||||||
<Daemon "mon.${hostName}">
|
# <Daemon "mon.${hostName}">
|
||||||
SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok"
|
# SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok"
|
||||||
</Daemon>
|
# </Daemon>
|
||||||
'') config.services.ceph.mon.daemons}
|
# '') config.services.ceph.mon.daemons}
|
||||||
${lib.concatMapStrings (hostName: ''
|
# ${lib.concatMapStrings (hostName: ''
|
||||||
<Daemon "mgr.${hostName}">
|
# <Daemon "mgr.${hostName}">
|
||||||
SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok"
|
# SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok"
|
||||||
</Daemon>
|
# </Daemon>
|
||||||
'') config.services.ceph.mgr.daemons}
|
# '') config.services.ceph.mgr.daemons}
|
||||||
${lib.concatMapStrings (hostName: ''
|
# ${lib.concatMapStrings (hostName: ''
|
||||||
<Daemon "mds.${hostName}">
|
# <Daemon "mds.${hostName}">
|
||||||
SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok"
|
# SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok"
|
||||||
</Daemon>
|
# </Daemon>
|
||||||
'') config.services.ceph.mds.daemons}
|
# '') config.services.ceph.mds.daemons}
|
||||||
${lib.concatMapStrings (id: ''
|
# ${lib.concatMapStrings (id: ''
|
||||||
<Daemon "osd.${id}">
|
# <Daemon "osd.${id}">
|
||||||
SocketPath "/var/run/ceph/ceph-osd.${id}.asok"
|
# SocketPath "/var/run/ceph/ceph-osd.${id}.asok"
|
||||||
</Daemon>
|
# </Daemon>
|
||||||
'') config.services.ceph.osd.daemons}
|
# '') config.services.ceph.osd.daemons}
|
||||||
'';
|
# '';
|
||||||
# HACK: let collectd access ceph sockets
|
# HACK: let collectd access ceph sockets
|
||||||
systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph";
|
# systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph";
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,10 +9,10 @@
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
assertions = [
|
assertions = [
|
||||||
{
|
# {
|
||||||
assertion = config.skyflake.storage.ceph.package != 17;
|
# assertion = config.skyflake.storage.ceph.package != 17;
|
||||||
message = "Please pin ceph to major version 17!";
|
# message = "Please pin ceph to major version 17!";
|
||||||
}
|
# }
|
||||||
];
|
];
|
||||||
|
|
||||||
microvm = {
|
microvm = {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user