modules/cluster/default: monitor ceph with collectd
This commit is contained in:
parent
8bc2b9d09b
commit
17cecf0049
|
@ -40,6 +40,7 @@ in {
|
||||||
value.address = hostRegistry.${name}.ip4;
|
value.address = hostRegistry.${name}.ip4;
|
||||||
}) servers
|
}) servers
|
||||||
);
|
);
|
||||||
|
# Extra nomad configuration
|
||||||
nomad = {
|
nomad = {
|
||||||
datacenter = "c3d2";
|
datacenter = "c3d2";
|
||||||
inherit servers;
|
inherit servers;
|
||||||
|
@ -49,13 +50,16 @@ in {
|
||||||
"c3d2.storage" = "big";
|
"c3d2.storage" = "big";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
# The user that runs skyflake MicroVMs
|
||||||
microvmUid = 997;
|
microvmUid = 997;
|
||||||
|
|
||||||
users = {
|
users = {
|
||||||
|
# Deployment user for hosts in this flake
|
||||||
c3d2 = {
|
c3d2 = {
|
||||||
uid = 1001;
|
uid = 1001;
|
||||||
sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
|
||||||
};
|
};
|
||||||
|
# Deployment user for leon who also uses this flake
|
||||||
leon = {
|
leon = {
|
||||||
uid = 1002;
|
uid = 1002;
|
||||||
sshKeys = with (import ../../ssh-public-keys.nix).users;
|
sshKeys = with (import ../../ssh-public-keys.nix).users;
|
||||||
|
@ -66,6 +70,8 @@ in {
|
||||||
|
|
||||||
deploy.customizationModule = ./deployment.nix;
|
deploy.customizationModule = ./deployment.nix;
|
||||||
|
|
||||||
|
# Obsolete Glusterfs setup.
|
||||||
|
# TODO: Remove once all data has been moved off.
|
||||||
storage.glusterfs = {
|
storage.glusterfs = {
|
||||||
fileSystems = [ {
|
fileSystems = [ {
|
||||||
servers = microvmServers;
|
servers = microvmServers;
|
||||||
|
@ -78,6 +84,7 @@ in {
|
||||||
} ];
|
} ];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Ceph storage cluster configuration
|
||||||
storage.ceph = rec {
|
storage.ceph = rec {
|
||||||
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
|
||||||
mons = [ "server8" "server9" "server10" ];
|
mons = [ "server8" "server9" "server10" ];
|
||||||
|
@ -89,6 +96,7 @@ in {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Ceph keyrings
|
||||||
sops.secrets = {
|
sops.secrets = {
|
||||||
"ceph/monKeyring" = {
|
"ceph/monKeyring" = {
|
||||||
owner = "ceph";
|
owner = "ceph";
|
||||||
|
@ -104,4 +112,32 @@ in {
|
||||||
# additional gcroots
|
# additional gcroots
|
||||||
"L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
|
"L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
|
||||||
];
|
];
|
||||||
|
|
||||||
|
# Collectd monitoring for ceph
|
||||||
|
services.collectd.plugins.ceph = ''
|
||||||
|
ConvertSpecialMetricTypes true
|
||||||
|
|
||||||
|
${lib.concatMapStrings (hostName: ''
|
||||||
|
<Daemon "mon.${hostName}">
|
||||||
|
SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok"
|
||||||
|
</Daemon>
|
||||||
|
'') config.services.ceph.mon.daemons}
|
||||||
|
${lib.concatMapStrings (hostName: ''
|
||||||
|
<Daemon "mgr.${hostName}">
|
||||||
|
SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok"
|
||||||
|
</Daemon>
|
||||||
|
'') config.services.ceph.mgr.daemons}
|
||||||
|
${lib.concatMapStrings (hostName: ''
|
||||||
|
<Daemon "mds.${hostName}">
|
||||||
|
SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok"
|
||||||
|
</Daemon>
|
||||||
|
'') config.services.ceph.mds.daemons}
|
||||||
|
${lib.concatMapStrings (id: ''
|
||||||
|
<Daemon "osd.${id}">
|
||||||
|
SocketPath "/var/run/ceph/ceph-osd.${id}.asok"
|
||||||
|
</Daemon>
|
||||||
|
'') config.services.ceph.osd.daemons}
|
||||||
|
'';
|
||||||
|
# HACK: let collectd access ceph sockets
|
||||||
|
systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph";
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue