From 1f7aea3a28eed2778e69dd0a47d1caa065a58bff Mon Sep 17 00:00:00 2001 From: Astro Date: Tue, 5 Oct 2021 00:12:02 +0200 Subject: [PATCH] logging: consolidate, flakify --- flake.nix | 8 ++ hosts/containers/elastic/configuration.nix | 75 -------------- hosts/containers/logging/configuration.nix | 73 ------------- hosts/containers/logging/default.nix | 113 +++++++++++++++++++++ hosts/containers/mongo/configuration.nix | 42 -------- 5 files changed, 121 insertions(+), 190 deletions(-) delete mode 100644 hosts/containers/elastic/configuration.nix delete mode 100644 hosts/containers/logging/configuration.nix create mode 100644 hosts/containers/logging/default.nix delete mode 100644 hosts/containers/mongo/configuration.nix diff --git a/flake.nix b/flake.nix index 58918785..62d120ba 100644 --- a/flake.nix +++ b/flake.nix @@ -355,6 +355,14 @@ system = "x86_64-linux"; }; + logging = nixosSystem' { + modules = [ + ./lib/lxc-container.nix + ./hosts/containers/logging + ]; + system = "x86_64-linux"; + }; + }; nixosModule = import ./lib; diff --git a/hosts/containers/elastic/configuration.nix b/hosts/containers/elastic/configuration.nix deleted file mode 100644 index 88cc6995..00000000 --- a/hosts/containers/elastic/configuration.nix +++ /dev/null @@ -1,75 +0,0 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running ‘nixos-help’). - -{ config, pkgs, lib, ... }: - -{ - imports = [ - ../../../lib - ../../../lib/lxc-container.nix - ../../../lib/shared.nix - ../../../lib/admins.nix - ]; - - networking = { - hostName = "elastic1"; - interfaces.eth0.ipv4.addresses = [{ - address = "172.20.73.15"; - prefixLength = 26; - }]; - defaultGateway = "172.20.73.1"; - firewall = { - allowedTCPPorts = [ 22 9200 9300 ]; - enable = true; - }; - }; - - nixpkgs.config.allowUnfree = true; - - services.openssh = { - enable = true; - }; - - services.elasticsearch = { - enable = true; - cluster_name = "zentralwerk"; - dataDir = "/srv/elasticsearch"; - listenAddress = "0.0.0.0"; - extraJavaOptions = [ "-Xms2g" "-Xmx2g" ]; - }; - systemd.services.elasticsearch.serviceConfig.Restart = "always"; - - # does not work, needs to be set on hv (done through ansible) - boot.kernel.sysctl = { "vm.max_map_count" = "262144"; }; - - services.elasticsearch-curator = { - enable = true; - actionYAML = '' - --- - actions: - 1: - action: delete_indices - description: >- - Delete indices older than 45 days (based on index name), for logstash- - prefixed indices. Ignore the error if the filter does not result in an - actionable list of indices (ignore_empty_list) and exit cleanly. - options: - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 45 - ''; - }; - - system.stateVersion = "20.09"; # Did you read the comment? - -} diff --git a/hosts/containers/logging/configuration.nix b/hosts/containers/logging/configuration.nix deleted file mode 100644 index a021d360..00000000 --- a/hosts/containers/logging/configuration.nix +++ /dev/null @@ -1,73 +0,0 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running ‘nixos-help’). - -{ config, pkgs, lib, ... }: - -{ - imports = [ - ../../../lib - ../../../lib/lxc-container.nix - ../../../lib/shared.nix - ../../../lib/admins.nix - ]; - - networking = { - hostName = "logging"; - interfaces.eth0.ipv4.addresses = [{ - address = "172.20.73.13"; - prefixLength = 26; - }]; - defaultGateway = "172.20.73.1"; - firewall = { - allowedTCPPorts = [ 22 9000 80 443 5044 12201 514 ]; - allowedUDPPorts = [ 514 ]; - enable = false; - }; - dhcpcd.denyInterfaces = [ "eth1" ]; - # interface for mgmt network - interfaces.eth1 = { - ipv4.addresses = [{ - address = "10.0.0.251"; - prefixLength = 24; - }]; - useDHCP = false; - }; - }; - - services.openssh = { - enable = true; - }; - - services.nginx = { - enable = true; - virtualHosts = { - default = { locations = { "/".proxyPass = "http://127.0.0.1:9000/"; }; }; - }; - }; - - services.graylog = { - enable = true; - passwordSecret = - "SDwK3ug9U4gYSVtj3h22i0l57QO6p5RE58sNehAgU3vXgqGa2HuNyhL19vhoUKFqy28rqGfDQkRD5834NqPi5wLsy8H1hz5V"; - # mongo.serv.zentralwerk. ? - elasticsearchHosts = [ "http://elastic1.serv.zentralwerk.org:9200" ]; - rootPasswordSha2 = - "3e784172684dcd89d66175b8719cd7894cc96b454ef1d5aa74bd92b3c57da7cd"; - # mongo.serv.zentralwerk. ? - mongodbUri = "mongodb://mongo.serv.zentralwerk.org/graylog"; - extraConfig = '' - http_bind_address = 0.0.0.0:9000 - http_publish_uri = http://logging.serv.zentralwerk.org/ - elasticsearch_shards = 1 - allow_highlighting = true - allow_leading_wildcard_searches = true - ''; - user = "root"; - }; - - systemd.services.graylog.serviceConfig.Restart = "always"; - - system.stateVersion = "20.09"; # Did you read the comment? - -} diff --git a/hosts/containers/logging/default.nix b/hosts/containers/logging/default.nix new file mode 100644 index 00000000..5f9ac2ba --- /dev/null +++ b/hosts/containers/logging/default.nix @@ -0,0 +1,113 @@ +{ hostRegistry, config, pkgs, lib, ... }: + +let + graylogPort = 9000; +in +{ + networking = { + hostName = "logging"; + interfaces.eth0.ipv4.addresses = [{ + address = hostRegistry.hosts.logging.ip4; + prefixLength = 26; + }]; + defaultGateway = "172.20.73.1"; + firewall = { + allowedTCPPorts = [ 22 80 443 5044 12201 514 ]; + allowedUDPPorts = [ 514 ]; + enable = false; + }; + dhcpcd.denyInterfaces = [ "eth1" ]; + # interface for mgmt network + interfaces.eth1 = { + ipv4.addresses = [{ + address = "10.0.0.251"; + prefixLength = 24; + }]; + useDHCP = false; + }; + }; + + # Don't loop + services.journalbeat.enable = lib.mkForce false; + + services.openssh = { + enable = true; + }; + + services.nginx = { + enable = true; + virtualHosts = { + "logging.serv.zentralwerk.org" = { + default = true; + enableACME = true; + forceSSL = true; + locations = { "/".proxyPass = "http://127.0.0.1:${toString graylogPort}/"; }; + }; + }; + }; + + services.graylog = { + enable = true; + passwordSecret = + "SDwK3ug9U4gYSVtj3h22i0l57QO6p5RE58sNehAgU3vXgqGa2HuNyhL19vhoUKFqy28rqGfDQkRD5834NqPi5wLsy8H1hz5V"; + elasticsearchHosts = [ "http://localhost:9200" ]; + rootPasswordSha2 = + "2bed7d6138c04098c05f492174c31d45d873f5146ad775e4c26a4863fa370d7d"; + mongodbUri = "mongodb://localhost/graylog"; + extraConfig = '' + http_bind_address = 127.0.0.1:${toString graylogPort} + http_publish_uri = https://logging.serv.zentralwerk.org/ + elasticsearch_shards = 1 + allow_highlighting = true + allow_leading_wildcard_searches = true + ''; + user = "root"; + }; + + services.mongodb = { + enable = true; + }; + + # noXlibs breaks cairo: + environment.noXlibs = false; + nixpkgs.config.allowUnfree = true; + services.elasticsearch = { + enable = true; + extraJavaOptions = [ "-Xms2g" "-Xmx2g" ]; + }; + systemd.services.elasticsearch.serviceConfig.Restart = "always"; + + # does not work, needs to be set on hv (done through ansible) + boot.kernel.sysctl = { "vm.max_map_count" = "262144"; }; + + services.elasticsearch-curator = { + enable = true; + actionYAML = '' + --- + actions: + 1: + action: delete_indices + description: >- + Delete indices older than 45 days (based on index name), for logstash- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: 45 + ''; + }; + + systemd.services.graylog.serviceConfig.Restart = "always"; + + system.stateVersion = "21.05"; +} diff --git a/hosts/containers/mongo/configuration.nix b/hosts/containers/mongo/configuration.nix deleted file mode 100644 index e6a80371..00000000 --- a/hosts/containers/mongo/configuration.nix +++ /dev/null @@ -1,42 +0,0 @@ -# Edit this configuration file to define what should be installed on -# your system. Help is available in the configuration.nix(5) man page -# and in the NixOS manual (accessible by running ‘nixos-help’). - -{ config, pkgs, lib, ... }: - -{ - imports = [ - ../../../lib - ../../../lib/lxc-container.nix - ../../../lib/shared.nix - ../../../lib/admins.nix - ]; - - environment.systemPackages = with pkgs; [ mongodb-tools ]; - - networking = { - hostName = "mongo"; - interfaces.eth0.ipv4.addresses = [{ - address = "172.20.73.21"; - prefixLength = 26; - }]; - defaultGateway = "172.20.73.1"; - firewall = { - allowedTCPPorts = [ 22 27017 ]; - enable = true; - }; - }; - - services.openssh = { - enable = true; - }; - - services.mongodb = { - enable = true; - bind_ip = "0.0.0.0"; - dbpath = "/srv/mongodb"; - }; - - system.stateVersion = "19.03"; # Did you read the comment? - -}