diff --git a/hosts/prometheus/default.nix b/hosts/prometheus/default.nix index 98f157db..c2ca5569 100644 --- a/hosts/prometheus/default.nix +++ b/hosts/prometheus/default.nix @@ -22,58 +22,32 @@ alertmanager = { enable = true; - openFirewall = true; webExternalUrl = "https://prometheus.serv.zentralwerk.org/alertmanager/"; - listenAddress = "127.0.0.1"; + listenAddress = "[::1]"; configuration = { - "global" = { - # "smtp_smarthost" = "mail.serv.zentralwerk.org:587"; - # "smtp_from" = "alertmanager@prometheus.serv.zentralwerk.org"; - }; + "global" = { }; "route" = { - "group_by" = [ "alertname" "alias" ]; - "group_wait" = "30s"; - "group_interval" = "2m"; + "group_by" = [ "instance" ]; + "group_wait" = "1m"; + "group_interval" = "1m"; "repeat_interval" = "4h"; - "receiver" = "team-admins"; + "receiver" = "xmpp"; }; "receivers" = [{ - "name" = "team-admins"; - # "email_configs" = [ - # { - # "to" = "devnull@example.com"; - # "send_resolved" = true; - # } - # ]; - # "webhook_configs" = [ - # { - # "url" = "https://example.com/prometheus-alerts"; - # "send_resolved" = true; - # } - # ]; + "name" = "xmpp"; + "webhook_configs" = with config.services.prometheus.xmpp-alerts.settings; [{ + "url" = "http://${listen_address}:${toString listen_port}/alert"; + }]; }]; }; }; - # pushgateway = { - # enable = true; - # web.external-url = "https://prometheus.serv.zentralwerk.org/push/"; - # }; - - # rules = [{ - # groups = [{ - # alert = "oom_kills"; - # expr = "increase(node_vmstat_oom_kill[7d]) > 0.999"; - # for = "10m"; - # labels = { - # severity = "error"; - # }; - # annotations = { - # summary = "Service gets oom killed"; - # # description = ""; - # }; - # }]; - # }]; + alertmanagers = [{ + static_configs = [{ + targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ]; + }]; + path_prefix = "/alertmanager"; + }]; rules = [ '' @@ -121,7 +95,7 @@ ]); labels.__meta_net = "net-flpk"; } { - targets = [ "http://localhost:${toString config.services.prometheus.exporters.collectd.port}/metrics" ]; + targets = [ "localhost:${toString config.services.prometheus.exporters.collectd.port}" ]; } ]; }]; @@ -130,11 +104,25 @@ enable = true; collectdBinary.enable = true; }; + # TODO: deploy with every nginx nginx = { enable = true; openFirewall = true; }; }; + + webExternalUrl = "https://prometheus.serv.zentralwerk.org/"; + + xmpp-alerts = { + enable = true; + settings = { + jid = "alerta@jabber.c3d2.de"; + password_command = "cat ${config.sops.secrets."alertmanager/xmpp-password".path}"; + to_jid = "admins@chat.c3d2.de"; + listen_address = "127.0.0.1"; + listen_port = 9199; + }; + }; }; services.nginx = { @@ -164,7 +152,13 @@ sops = { defaultSopsFile = ./secrets.yaml; secrets."nginx/httpAuth".owner = config.systemd.services.nginx.serviceConfig.User; + secrets."alertmanager/xmpp-password".owner = config.systemd.services.prometheus-xmpp-alerts.serviceConfig.User; }; system.stateVersion = "22.11"; + + systemd.services.prometheus-xmpp-alerts.serviceConfig = { + DynamicUser = lib.mkForce false; + User = "prometheus"; + }; } diff --git a/hosts/prometheus/secrets.yaml b/hosts/prometheus/secrets.yaml index c4aa3278..29bce232 100644 --- a/hosts/prometheus/secrets.yaml +++ b/hosts/prometheus/secrets.yaml @@ -1,3 +1,5 @@ +alertmanager: + xmpp-password: ENC[AES256_GCM,data:v+pRv/q4Z/ZT18PBxxKrq9P9QNbjN14edSeMCo5If6Y=,iv:oIdXFPdppgV7uTWY/eKSK9T2vDXW2Uur9iwnftQB+dQ=,tag:QD2MB3ZeMlvuGIMIpHI97Q==,type:str] nginx: httpAuth: ENC[AES256_GCM,data:PS7icDVNB4g7XBMP7mMSbalkvQ==,iv:0GOfGl97k1AjkRxm2x2f4LpeQOuJcFqAHgdRrbceW6U=,tag:GX5L0wI5zwHwuls7ZOPlOQ==,type:str] sops: @@ -24,8 +26,8 @@ sops: dkpOeVJIVnJtNDM5M0RQaWRudUcxOG8KZnHCLuyPFdx4j1WY6fk8nqMeACmpYZzU EpNqjoBswCkUnaRMVcj5lrHvHDjdbQ0Ypn3s/YvI4UBsXMnnv9UD7w== -----END AGE ENCRYPTED FILE----- - lastmodified: "2022-11-27T00:27:41Z" - mac: ENC[AES256_GCM,data:FsqddyIQqc0qZerOc6zXs39hBOOwh6Bnjd0gw+Kdq11NMxPFKd05/XGkpoHzVbAFioMc528XkpWubVO5rnCBsLKkwuCm+wtQbFU74oXXUbZKrF7Ucxk0bUSmCX1Y+YTsiO5SfUwWuO+YA5RZbdNekE19MYRnVQ4MDBnfWlrZERk=,iv:d8Rceuua4//ZEcDEoAziw70ySKv/PtPr46sM79s3Ex4=,tag:jfCwyfhjIrYlHgEyv8BaQA==,type:str] + lastmodified: "2022-12-10T21:16:19Z" + mac: ENC[AES256_GCM,data:yZyBuZE5Gb6TSViqZaugZeloGxbsspc1oDMVVaoNZ7fcDXT7ELg85qWTokLV5PcySeh2IUpDtaqt+QV8tqHlWbREOHHWi+gwBGfBWww7k+H6h9+Vj3hpydPBBohklMpYYWK0F8fAuJjLhE6Usv96N8/CnCv4NJUIjK33pca5vMY=,iv:s8jumWZMU52PKHq0FAdl8IDv7aSym9cOG0W/cSn9XUI=,tag:Rh3oCJ2jucubvqRs7A+abQ==,type:str] pgp: - created_at: "2022-11-27T00:27:03Z" enc: |-