nix-config/hosts/prometheus/default.nix

{ zentralwerk, config, lib, ... }:

{
  networking = {
    hostName = "prometheus";
    firewall = {
      allowedTCPPorts = [ /*22*/ 80 443 /*9090 9091 9093 9094*/ ];
      enable = true;
    };
  };

  services.prometheus = {
    enable = true;
    retentionTime = "7d";

    alertmanager = {
      enable = true;
      openFirewall = true;
      webExternalUrl = "https://prometheus.serv.zentralwerk.org/alertmanager/";
      listenAddress = "127.0.0.1";
      configuration = {
        "global" = {
          # "smtp_smarthost" = "mail.serv.zentralwerk.org:587";
          # "smtp_from" = "alertmanager@prometheus.serv.zentralwerk.org";
        };
        "route" = {
          "group_by" = [ "alertname" "alias" ];
          "group_wait" = "30s";
          "group_interval" = "2m";
          "repeat_interval" = "4h";
          "receiver" = "team-admins";
        };
        "receivers" = [{
          "name" = "team-admins";
          # "email_configs" = [
          #   {
          #     "to" = "devnull@example.com";
          #     "send_resolved" = true;
          #   }
          # ];
          # "webhook_configs" = [
          #   {
          #     "url" = "https://example.com/prometheus-alerts";
          #     "send_resolved" = true;
          #   }
          # ];
        }];
      };
    };

    # pushgateway = {
    #   enable = true;
    #   web.external-url = "https://prometheus.serv.zentralwerk.org/push/";
    # };

    # rules = [{
    #   groups = [{
    #     alert = "oom_kills";
    #     expr = "increase(node_vmstat_oom_kill[7d]) > 0.999";
    #     for = "10m";
    #     labels = {
    #       severity = "error";
    #     };
    #     annotations = {
    #       summary = "Service gets oom killed";
    #       # description = "";
    #     };
    #   }];
    # }];

    rules = [
      ''
        groups:
          - name: default
            rules:
            - alert: oom_kills
              expr: increase(node_vmstat_oom_kill[7d]) > 0.999
              for: 10m
              labels:
                severity: error
              annotations:
                summary: service gets oom killed
      ''
    ];

    scrapeConfigs = [{
      # TODO: authorization?
      job_name = "node";
      scrape_interval = "1m";
      static_configs =
        let
          zwNets = zentralwerk.lib.config.site.net;
          fromNet = net: _:
            map
              (host:
                "${host}.${net}.zentralwerk.org:9100"
              )
              (builtins.attrNames zwNets.${net}.hosts4);
        in
        [{
          targets =
            fromNet "serv" (_: true);
          labels.__meta_net = "net-serv";
        }
          {
            targets =
              fromNet "flpk" (host: host != "flpk-gw");
            labels.__meta_net = "net-flpk";
          }
          {
            targets =
              fromNet "cluster" (host: builtins.elem host [
                "server8"
                "server9"
                "server10"
              ]);
            labels.__meta_net = "net-flpk";
          }];
    }];

    exporters = {
      collectd = {
        enable = true;
        openFirewall = true;
      };
      nginx = {
        enable = true;
        openFirewall = true;
      };
    };
  };

  services.nginx = {
    enable = true;

    virtualHosts."prometheus.serv.zentralwerk.org" = {
      # serverAliases = [ "registry.serv.zentralwerk.org" ];
      enableACME = true;
      forceSSL = true;
      locations."/" = {
        proxyPass = "http://localhost:${toString config.services.prometheus.port}";
        extraConfig = ''
          auth_basic "Prometheus";
          auth_basic_user_file ${config.sops.secrets."nginx/httpAuth".path};
        '';
      };
      locations."/alertmanager" = {
        proxyPass = "http://localhost:${toString config.services.prometheus.alertmanager.port}";
        extraConfig = ''
          auth_basic "Prometheus";
          auth_basic_user_file ${config.sops.secrets."nginx/httpAuth".path};
        '';
      };
    };
  };

  sops = {
    defaultSopsFile = ./secrets.yaml;
    secrets."nginx/httpAuth".owner = config.systemd.services.nginx.serviceConfig.User;
  };

  system.stateVersion = "22.11";
}