From 17cecf0049f997e94fac4719dc5b5116eaba429c Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Sun, 8 Jan 2023 03:05:55 +0100
Subject: [PATCH] modules/cluster/default: monitor ceph with collectd

---
 modules/cluster/default.nix | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/modules/cluster/default.nix b/modules/cluster/default.nix
index 7c1e4115..766d8bbd 100644
--- a/modules/cluster/default.nix
+++ b/modules/cluster/default.nix
@@ -40,6 +40,7 @@ in {
         value.address = hostRegistry.${name}.ip4;
       }) servers
     );
+    # Extra nomad configuration
     nomad = {
       datacenter = "c3d2";
       inherit servers;
@@ -49,13 +50,16 @@ in {
         "c3d2.storage" = "big";
       };
     };
+    # The user that runs skyflake MicroVMs
     microvmUid = 997;
 
     users = {
+      # Deployment user for hosts in this flake
       c3d2 = {
         uid = 1001;
         sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
       };
+      # Deployment user for leon who also uses this flake
       leon = {
         uid = 1002;
         sshKeys = with (import ../../ssh-public-keys.nix).users;
@@ -66,6 +70,8 @@ in {
 
     deploy.customizationModule = ./deployment.nix;
 
+    # Obsolete Glusterfs setup.
+    # TODO: Remove once all data has been moved off.
     storage.glusterfs = {
       fileSystems = [ {
         servers = microvmServers;
@@ -78,6 +84,7 @@ in {
       } ];
     };
 
+    # Ceph storage cluster configuration
     storage.ceph = rec {
       fsid = "a06b1061-ef09-46d6-a15f-2f8ce4d7d1bf";
       mons = [ "server8" "server9" "server10" ];
@@ -89,6 +96,7 @@ in {
     };
   };
 
+  # Ceph keyrings
   sops.secrets = {
     "ceph/monKeyring" = {
       owner = "ceph";
@@ -104,4 +112,32 @@ in {
     # additional gcroots
     "L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
   ];
+
+  # Collectd monitoring for ceph
+  services.collectd.plugins.ceph = ''
+    ConvertSpecialMetricTypes true
+
+    ${lib.concatMapStrings (hostName: ''
+      <Daemon "mon.${hostName}">
+        SocketPath "/var/run/ceph/ceph-mon.${hostName}.asok"
+      </Daemon>
+    '') config.services.ceph.mon.daemons}
+    ${lib.concatMapStrings (hostName: ''
+      <Daemon "mgr.${hostName}">
+        SocketPath "/var/run/ceph/ceph-mgr.${hostName}.asok"
+      </Daemon>
+    '') config.services.ceph.mgr.daemons}
+    ${lib.concatMapStrings (hostName: ''
+      <Daemon "mds.${hostName}">
+        SocketPath "/var/run/ceph/ceph-mds.${hostName}.asok"
+      </Daemon>
+    '') config.services.ceph.mds.daemons}
+    ${lib.concatMapStrings (id: ''
+      <Daemon "osd.${id}">
+        SocketPath "/var/run/ceph/ceph-osd.${id}.asok"
+      </Daemon>
+    '') config.services.ceph.osd.daemons}
+  '';
+  # HACK: let collectd access ceph sockets
+  systemd.services.collectd.serviceConfig.User = lib.mkForce "ceph";
 }