mastodon: add collectd monitoring of sidekiq and postgresql

hosts/mastodon/default.nix

@@ -13,12 +13,19 @@
     };
     firewall.allowedTCPPorts = [ 80 443 ];
   };
-  c3d2.hq.statistics.enable = true;
   system.stateVersion = "22.11";
   nixpkgs.config.allowUnfreePredicate = pkg:
     lib.getName pkg == "elasticsearch";
   
-  services.postgresql.enable = true;
+  services.postgresql = {
+    enable = true;
+    ensureUsers = [ {
+      name = "collectd";
+      ensurePermissions = {
+        "DATABASE \"${config.services.mastodon.database.name}\"" = "ALL PRIVILEGES";
+      };
+    } ];
+  };
   services.elasticsearch = {
     enable = true;
     package = pkgs.elasticsearch7;
@@ -27,13 +34,14 @@
     enable = true;
     localDomain = "c3d2.social";
 
-    smtp.host = "c3d2.social";
-    smtp.fromAddress = "mail@c3d2.social";
-    # smtp.authenticate = true;
-    # smtp.user = secrets.email.smtp-user;
-    # smtp.passwordFile = "${pkgs.runCommand "smtp-password" {} ''
-    #     echo "${secrets.email.smtp-password}" > $out
-    #   ''}";
+    smtp = {
+      # TODO
+      host = "c3d2.social";
+      fromAddress = "mail@c3d2.social";
+      authenticate = false;
+      # user
+      # passwordFile
+    };
 
     elasticsearch.host = "127.0.0.1";
 
@@ -57,6 +65,7 @@
 
     configureNginx = true;
   };
+  # Inject LDAP secrets
   systemd.services.mastodon-init-dirs.script = lib.mkAfter ''
     cat ${config.sops.secrets."mastodon/env".path} >> /var/lib/mastodon/.secrets_env
   '';
@@ -65,4 +74,84 @@
   sops.secrets."mastodon/env" = {
     owner = "mastodon";
   };
+
+  # Sidekiq monitoring
+  c3d2.hq.statistics.enable = true;
+  services.collectd.plugins = {
+    redis =
+      let
+        queries = [ {
+          command = "GET stat:processed";
+          type = "counter";
+          instance = "sidekiq_stat_processed";
+        } {
+          command = "GET stat:failed";
+          type = "counter";
+          instance = "sidekiq_stat_failed";
+        } {
+          command = "LLEN queue:#default";
+          type = "queue_length";
+          instance = "sidekiq_default_queue_len";
+        } {
+          command = "LLEN queue:#ingress";
+          type = "queue_length";
+          instance = "sidekiq_ingress_queue_len";
+        } {
+          command = "LLEN queue:#mailers";
+          type = "queue_length";
+          instance = "sidekiq_mailers_queue_len";
+        } {
+          command = "LLEN queue:#pull";
+          type = "queue_length";
+          instance = "sidekiq_pull_queue_len";
+        } {
+          command = "LLEN queue:#push";
+          type = "queue_length";
+          instance = "sidekiq_push_queue_len";
+        } {
+          command = "LLEN queue:#scheduler";
+          type = "queue_length";
+          instance = "sidekiq_scheduler_queue_len";
+        } {
+          command = "ZCARD schedule";
+          type = "count";
+          instance = "sidekiq_scheduled";
+        } {
+          command = "ZCARD retry";
+          type = "count";
+          instance = "sidekiq_retries";
+        } {
+          command = "ZCARD dead";
+          type = "count";
+          instance = "sidekiq_dead";
+        } {
+          command = "SCARD processes";
+          type = "backends";
+          instance = "sidekiq_processes";
+        } ];
+      in ''
+        <Node "mastodon">
+          Host "${config.services.mastodon.redis.host}"
+          Port "${toString config.services.mastodon.redis.port}"
+          Timeout 3000
+
+          ${lib.concatMapStrings ({ command, type, instance }: ''
+            <Query "${command}">
+              Type "${type}"
+              Instance "${instance}"
+            </Query>
+          '') queries}
+        </Node>
+      '';
+    postgresql = ''
+      <Database "${config.services.mastodon.database.name}">
+        Param database "${config.services.mastodon.database.name}"
+        Query backends
+        Query transactions
+        Query queries
+        Query disk_io
+        Query disk_usage
+      </Database>
+    '';
+  };
 }