katzenmann/nix-config
katzenmann/nix-config
1
0
Fork 0
forked from c3d2/nix-config
Code Pull Requests Activity

Only disable ceph config, not half the cluster

Browse Source
This commit is contained in:
Sandro - 2023-11-13 02:54:07 +01:00
parent 3d2a1d9fe5
commit 27930ce873
Signed by: sandro
GPG Key ID: 3AF5A43A3EECC2E5
3 changed files with 111 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
flake.nix
View File

@ -384,7 +384,7 @@
gnunet = nixosSystem' { gnunet = nixosSystem' {
modules = [ modules = [
# self.nixosModules.cluster-options self.nixosModules.cluster-options
self.nixosModules.microvm self.nixosModules.microvm
./hosts/gnunet ./hosts/gnunet
]; ];
@ -413,7 +413,7 @@
hydra = nixosSystem' { hydra = nixosSystem' {
modules = [ modules = [
# self.nixosModules.cluster self.nixosModules.cluster
# skyflake.nixosModules.default # skyflake.nixosModules.default
./hosts/hydra ./hosts/hydra
]; ];
@ -493,7 +493,7 @@
nixpkgs.overlays = [ secrets.overlays.mucbot ]; nixpkgs.overlays = [ secrets.overlays.mucbot ];
} }
./hosts/mucbot ./hosts/mucbot
# self.nixosModules.cluster-options self.nixosModules.cluster-options
self.nixosModules.microvm self.nixosModules.microvm
]; ];
}; };
@ -537,7 +537,7 @@
owncast = nixosSystem' { owncast = nixosSystem' {
modules = [ modules = [
# self.nixosModules.cluster-options self.nixosModules.cluster-options
self.nixosModules.microvm self.nixosModules.microvm
./hosts/owncast ./hosts/owncast
]; ];
@ -642,7 +642,7 @@
} }
heliwatch.nixosModules.heliwatch heliwatch.nixosModules.heliwatch
self.nixosModules.microvm self.nixosModules.microvm
# self.nixosModules.cluster-options self.nixosModules.cluster-options
]; ];
}; };
@ -650,7 +650,7 @@
modules = [ modules = [
./hosts/server8 ./hosts/server8
self.nixosModules.cluster-network self.nixosModules.cluster-network
# self.nixosModules.cluster self.nixosModules.cluster
# skyflake.nixosModules.default # skyflake.nixosModules.default
{ _module.args = { inherit self; }; } { _module.args = { inherit self; }; }
]; ];
@ -661,7 +661,7 @@
./hosts/server9 ./hosts/server9
self.nixosModules.microvm-host self.nixosModules.microvm-host
self.nixosModules.cluster-network self.nixosModules.cluster-network
# self.nixosModules.cluster self.nixosModules.cluster
# skyflake.nixosModules.default # skyflake.nixosModules.default
{ _module.args = { inherit self; }; } { _module.args = { inherit self; }; }
]; ];
@ -672,7 +672,7 @@
./hosts/server10 ./hosts/server10
self.nixosModules.microvm-host self.nixosModules.microvm-host
self.nixosModules.cluster-network self.nixosModules.cluster-network
# self.nixosModules.cluster self.nixosModules.cluster
# skyflake.nixosModules.default # skyflake.nixosModules.default
{ _module.args = { inherit self; }; } { _module.args = { inherit self; }; }
]; ];
@ -688,7 +688,7 @@
sshlog = nixosSystem' { sshlog = nixosSystem' {
modules = [ modules = [
# self.nixosModules.cluster-options self.nixosModules.cluster-options
self.nixosModules.microvm self.nixosModules.microvm
sshlogd.nixosModule sshlogd.nixosModule
./hosts/sshlog ./hosts/sshlog
@ -697,7 +697,7 @@
stream = nixosSystem' { stream = nixosSystem' {
modules = [ modules = [
# self.nixosModules.cluster-options self.nixosModules.cluster-options
self.nixosModules.microvm self.nixosModules.microvm
./hosts/stream ./hosts/stream
]; ];

194
modules/cluster/default.nix
View File

@ -36,111 +36,111 @@ in {
servers; servers;
# Cluster configuration # Cluster configuration
skyflake = { # skyflake = {
# debug = true; # # debug = true;
nodes = builtins.listToAttrs ( # nodes = builtins.listToAttrs (
map (name: { # map (name: {
inherit name; # inherit name;
value.address = hostRegistry.${name}.ip4; # value.address = hostRegistry.${name}.ip4;
}) servers # }) servers
); # );
# Extra nomad configuration # # Extra nomad configuration
nomad = { # nomad = {
datacenter = "c3d2"; # datacenter = "c3d2";
inherit servers; # inherit servers;
# run tasks only on these: # # run tasks only on these:
client.enable = builtins.elem hostName microvmServers; # client.enable = builtins.elem hostName microvmServers;
client.meta = lib.optionalAttrs (builtins.elem hostName storageServers) { # client.meta = lib.optionalAttrs (builtins.elem hostName storageServers) {
"c3d2.storage" = "big"; # "c3d2.storage" = "big";
}; # };
}; # };
# The user that runs skyflake MicroVMs # # The user that runs skyflake MicroVMs
microvmUid = 997; # microvmUid = 997;
users = { # users = {
# Deployment user for hosts in this flake # # Deployment user for hosts in this flake
c3d2 = { # c3d2 = {
uid = 1001; # uid = 1001;
sshKeys = config.users.users.root.openssh.authorizedKeys.keys; # sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
home = "${skyflakeHome}/c3d2"; # home = "${skyflakeHome}/c3d2";
}; # };
# Deployment user for neighbour Andreas Lippmann <andreaslippmann@web.de> # # Deployment user for neighbour Andreas Lippmann <andreaslippmann@web.de>
luulaatsch = { # luulaatsch = {
uid = 1003; # uid = 1003;
home = "${skyflakeHome}/luulaatsch"; # home = "${skyflakeHome}/luulaatsch";
sshKeys = [ # sshKeys = [
"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVipwWSzxgUgm0fMeTbOSCXDNkbUUp+k1tvWaJcBA1M+LTM21rhP3tQQP24smPx1b2V/by6vHakhsj5pIMLzSSW7he/mAnOtwHj2dCEk6VaMvozAfS6n777g8ujZg0N9ekrqoNLh0uNchs3DYFZWXAnKtfEYjzIdXqVgXMI4uDXM6ObEh+YR3iGaZSYKd6S5nufzJOR6EiYbv/z6ThRq16f+HqetVdBamqmpNK9NcWCtjMcww7nZTyUvTbd2AyrpTAXSnVg30fza428Kpc5ZdzDJFjn0B3MVnP2LavIo8VH3Y5sjpp0aMDKfbz6dPA/df/pR6LVMsxx0eCEXeGqo9sFdabu2AwIPi79QFANr3qJ/nZo2LR3o2LvcOLO06xVRFKReGTsy6WWTNOTAL03sdiIbqnlmkjXu5nb4hzxoHBgA6LZGMx3knoVcUMBlpVaxOoT2tzXzpYszsKpGSN1EBqUlDXzen3EkHg7TpjjTJSH7Q0AeeSCJsaoAepa6wcZeJlBWSRiJdqIn7YBmCBdjV1nNPVBYE5mm2x88ha9e4NbyFhjQvZ/BiLyKCFVyDs6lIvhV5QfAPWOkyPwbLXtXg6g1OvmuM6mGDhtEfZKvreeqmTKjB+x7OpG847JKxmU8eeYax63/WgRssbOfTpicjsdoKlFST3WpZHGT7mWzmB/w== andreas@luulaatschs-MBP.pub.zentralwerk.org" # "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDVipwWSzxgUgm0fMeTbOSCXDNkbUUp+k1tvWaJcBA1M+LTM21rhP3tQQP24smPx1b2V/by6vHakhsj5pIMLzSSW7he/mAnOtwHj2dCEk6VaMvozAfS6n777g8ujZg0N9ekrqoNLh0uNchs3DYFZWXAnKtfEYjzIdXqVgXMI4uDXM6ObEh+YR3iGaZSYKd6S5nufzJOR6EiYbv/z6ThRq16f+HqetVdBamqmpNK9NcWCtjMcww7nZTyUvTbd2AyrpTAXSnVg30fza428Kpc5ZdzDJFjn0B3MVnP2LavIo8VH3Y5sjpp0aMDKfbz6dPA/df/pR6LVMsxx0eCEXeGqo9sFdabu2AwIPi79QFANr3qJ/nZo2LR3o2LvcOLO06xVRFKReGTsy6WWTNOTAL03sdiIbqnlmkjXu5nb4hzxoHBgA6LZGMx3knoVcUMBlpVaxOoT2tzXzpYszsKpGSN1EBqUlDXzen3EkHg7TpjjTJSH7Q0AeeSCJsaoAepa6wcZeJlBWSRiJdqIn7YBmCBdjV1nNPVBYE5mm2x88ha9e4NbyFhjQvZ/BiLyKCFVyDs6lIvhV5QfAPWOkyPwbLXtXg6g1OvmuM6mGDhtEfZKvreeqmTKjB+x7OpG847JKxmU8eeYax63/WgRssbOfTpicjsdoKlFST3WpZHGT7mWzmB/w== andreas@luulaatschs-MBP.pub.zentralwerk.org"
] ++ ssh-public-keys.astro; # ] ++ ssh-public-keys.astro;
}; # };
# Deployment user for marenz # # Deployment user for marenz
marenz = { # marenz = {
uid = 1004; # uid = 1004;
sshKeys = config.users.users.root.openssh.authorizedKeys.keys ++ ssh-public-keys.marenz; # sshKeys = config.users.users.root.openssh.authorizedKeys.keys ++ ssh-public-keys.marenz;
home = "${skyflakeHome}/marenz"; # home = "${skyflakeHome}/marenz";
}; # };
}; # };
deploy.customizationModule = ./customization; # deploy.customizationModule = ./customization;
# Ceph storage cluster configuration # # Ceph storage cluster configuration
storage.ceph = rec { # storage.ceph = rec {
package = pkgs.ceph_17_2; # package = pkgs.ceph_17_2;
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf"; # fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
mons = [ "server9" "server10" ]; # mons = [ "server9" "server10" ];
mgrs = mons; # mgrs = mons;
mdss = mons; # mdss = mons;
rbdPools.microvms = { # rbdPools.microvms = {
params = { size = 2; class = "ssd"; }; # params = { size = 2; class = "ssd"; };
}; # };
rbdPools.microvms-hdd = { # rbdPools.microvms-hdd = {
params = { size = 2; class = "hdd"; }; # params = { size = 2; class = "hdd"; };
}; # };
cephfs.home.mountPoint = skyflakeHome; # cephfs.home.mountPoint = skyflakeHome;
# Legacy: migration to rbd # # Legacy: migration to rbd
cephfs.skyflake.mountPoint = "/storage/cephfs"; # cephfs.skyflake.mountPoint = "/storage/cephfs";
monKeyring = config.sops.secrets."ceph/monKeyring".path; # monKeyring = config.sops.secrets."ceph/monKeyring".path;
adminKeyring = config.sops.secrets."ceph/adminKeyring".path; # adminKeyring = config.sops.secrets."ceph/adminKeyring".path;
}; # };
}; # };
# Ceph keyrings # Ceph keyrings
sops.secrets = { # sops.secrets = {
"ceph/monKeyring" = { # "ceph/monKeyring" = {
owner = "ceph"; # owner = "ceph";
sopsFile = ./ceph.yaml; # sopsFile = ./ceph.yaml;
}; # };
"ceph/adminKeyring" = { # "ceph/adminKeyring" = {
owner = "ceph"; # owner = "ceph";
sopsFile = ./ceph.yaml; # sopsFile = ./ceph.yaml;
}; # };
}; # };
# Collectd monitoring for ceph # Collectd monitoring for ceph
services.collectd.plugins.ceph = '' # services.collectd.plugins.ceph = ''
ConvertSpecialMetricTypes true # ConvertSpecialMetricTypes true
${lib.concatMapStrings (hostName: '' # ${lib.concatMapStrings (hostName: ''
<Daemon "mon.${hostName}"> # <Daemon "mon.${hostName}">
SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok" # SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok"
</Daemon> # </Daemon>
'') config.services.ceph.mon.daemons} # '') config.services.ceph.mon.daemons}
${lib.concatMapStrings (hostName: '' # ${lib.concatMapStrings (hostName: ''
<Daemon "mgr.${hostName}"> # <Daemon "mgr.${hostName}">
SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok" # SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok"
</Daemon> # </Daemon>
'') config.services.ceph.mgr.daemons} # '') config.services.ceph.mgr.daemons}
${lib.concatMapStrings (hostName: '' # ${lib.concatMapStrings (hostName: ''
<Daemon "mds.${hostName}"> # <Daemon "mds.${hostName}">
SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok" # SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok"
</Daemon> # </Daemon>
'') config.services.ceph.mds.daemons} # '') config.services.ceph.mds.daemons}
${lib.concatMapStrings (id: '' # ${lib.concatMapStrings (id: ''
<Daemon "osd.${id}"> # <Daemon "osd.${id}">
SocketPath "/var/run/ceph/ceph-osd.${id}.asok" # SocketPath "/var/run/ceph/ceph-osd.${id}.asok"
</Daemon> # </Daemon>
'') config.services.ceph.osd.daemons} # '') config.services.ceph.osd.daemons}
''; # '';
# HACK: let collectd access ceph sockets # HACK: let collectd access ceph sockets
systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph"; # systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph";
} }

8
modules/microvm-host.nix
View File

@ -9,10 +9,10 @@
config = { config = {
assertions = [ assertions = [
{ # {
assertion = config.skyflake.storage.ceph.package != 17; # assertion = config.skyflake.storage.ceph.package != 17;
message = "Please pin ceph to major version 17!"; # message = "Please pin ceph to major version 17!";
} # }
]; ];
microvm = { microvm = {