nix-config/hosts/prometheus/default.nix

{ zentralwerk, config, lib, ... }:

{
  deployment.mem = 1024;

  networking = {
    hostName = "prometheus";
    firewall.allowedUDPPorts = [
      # services.prometheus.exporters.collectd.collectdBinary
      25826
    ];
  };

  services.prometheus = {
    enable = true;
    retentionTime = "7d";

    alertmanager = {
      enable = true;
      webExternalUrl = "https://prometheus.serv.zentralwerk.org/alertmanager/";
      listenAddress = "[::1]";
      configuration = {
        "route" = {
          "group_by" = [ "instance" ];
          "group_wait" = "1m";
          "group_interval" = "1m";
          "repeat_interval" = "7d";
          "receiver" = "xmpp";
        };
        "receivers" = [{
          "name" = "xmpp";
          "webhook_configs" = [{
            "url" = "http://127.0.0.1:9022/alert";
          }];
        }];
      };
    };

    alertmanagers = [{
      static_configs = [{
        targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ];
      }];
      path_prefix = "/alertmanager";
    } {
      static_configs = [{
        # alert2muc
        targets = [ "localhost:9022" ];
      }];
    }];

    enableReload = true;
    ruleFiles = [ ./rules.yaml ];

    scrapeConfigs = [{
      # TODO: authorization?
      job_name = "node";
      scrape_interval = "1m";
      static_configs =
        let
          zwNets = zentralwerk.lib.config.site.net;
          fromNet = net: _:
            map
              (host:
                "${host}.${net}.zentralwerk.org:9100"
              )
              (builtins.attrNames zwNets.${net}.hosts4);
        in
        [ {
          targets = fromNet "serv" (_: true);
          labels.__meta_net = "net-serv";
        } {
          targets = fromNet "flpk" (host: host != "flpk-gw");
          labels.__meta_net = "net-flpk";
        } {
          targets = fromNet "cluster" (host: builtins.elem host [
            "server8"
            "server9"
            "server10"
          ]);
          labels.__meta_net = "net-flpk";
        } {
          targets = [ "localhost:${toString config.services.prometheus.exporters.collectd.port}" ];
        } {
          targets = [
            # caveman: caveman-hunter
            "${zwNets.flpk.hosts4.caveman}:9101"
            # caveman: caveman-gatherer
            "fedi.buzz"
            # buzzrelay: buzzrelay
            "relay.fedi.buzz"
          ];
        } ];
    }];

    exporters = {
      collectd = {
        enable = true;
        collectdBinary.enable = true;
      };
      # TODO: deploy with every nginx
      nginx = {
        enable = true;
        openFirewall = true;
      };
    };

    webExternalUrl = "https://prometheus.serv.zentralwerk.org/";
  };

  services.alert2muc = {
    enable = true;
    configFile = config.sops.secrets."alert2muc/config".path;
  };

  services.nginx = {
    enable = true;

    virtualHosts."prometheus.serv.zentralwerk.org" = {
      # serverAliases = [ "registry.serv.zentralwerk.org" ];
      enableACME = true;
      forceSSL = true;
      locations."/" = {
        proxyPass = "http://localhost:${toString config.services.prometheus.port}";
        extraConfig = ''
          auth_basic "Prometheus";
          auth_basic_user_file ${config.sops.secrets."nginx/httpAuth".path};
        '';
      };
      locations."/alertmanager" = {
        proxyPass = "http://localhost:${toString config.services.prometheus.alertmanager.port}";
        extraConfig = ''
          auth_basic "Prometheus";
          auth_basic_user_file ${config.sops.secrets."nginx/httpAuth".path};
        '';
      };
      locations."/alert2muc" = {
        proxyPass = "http://localhost:9022";
        extraConfig = ''
          rewrite ^/alert2muc/(.*) /$1  break;
        '';
      };
    };
  };

  sops = {
    defaultSopsFile = ./secrets.yaml;
    secrets."nginx/httpAuth".owner = config.systemd.services.nginx.serviceConfig.User;
    secrets."alertmanager/xmpp-password".owner = config.systemd.services.prometheus-xmpp-alerts.serviceConfig.User;
    secrets."alert2muc/config".owner = config.services.alert2muc.user;
  };

  system.stateVersion = "22.11";

  systemd.services.prometheus-xmpp-alerts.serviceConfig = {
    DynamicUser = lib.mkForce false;
    User = "prometheus";
  };
}