start implementing data-hoarder registry

flake.nix

@@ -241,6 +241,7 @@
       // (import ./pkgs/deployment.nix { inherit self pkgs lib; })
       // (lib.foldl (x: y: lib.mergeAttrs x { "${y.config.system.name}-vm" = y.config.system.build.vm; }) { } (lib.attrValues self.nixosConfigurations));
 
+      registry = import ./registry;
     in
     {
 
@@ -252,7 +253,7 @@
 
         data-hoarder = nixpkgs.lib.nixosSystem {
           system = "x86_64-linux";
-          specialArgs = { inherit inputs self; };
+          specialArgs = { inherit inputs self; registry = registry.data-hoarder; };
           modules = [
             microvm.nixosModules.microvm
             ./hosts/data-hoarder
@@ -261,7 +262,7 @@
 
         staging-data-hoarder = nixpkgs.lib.nixosSystem {
           system = "x86_64-linux";
-          specialArgs = { inherit inputs self; };
+          specialArgs = { inherit inputs self; registry = registry.data-hoarder; };
           modules = [
             ./hosts/staging-data-hoarder
             microvm.nixosModules.microvm

modules/data-hoarder/bureaucrat.nix

@@ -1,25 +1,15 @@
-{ config, ... }:
+{ config, registry, ... }: {
-let
-  service_number = 6;
-in
-{
   TLMS.bureaucrat = {
     enable = true;
-    grpc = {
+    grpc = registry.grpc-chemo-bureaucrat;
-      host = "127.0.0.1";
+    redis = registry.redis-bureaucrat-lizard;
-      port = 50050 + service_number;
-    };
-    redis = {
-      host = config.services.redis.servers."state".bind;
-      port = config.services.redis.servers."state".port;
-    };
   };
 
   services = {
-    redis.servers."state" = {
+    redis.servers."state" = with registry.redis-bureaucrat-lizard; {
+      inherit port;
       enable = true;
-      bind = "127.0.0.1";
+      bind = host;
-      port = 5314;
     };
   };
 }

modules/data-hoarder/chemo.nix

@@ -1,12 +1,7 @@
-{ config, ... }:
+{ config, registry, ... }: {
-let
-  service_number = 3;
-in
-{
   TLMS.chemo = {
+    inherit (registry.grpc-data_accumulator-chemo) host port;
     enable = true;
-    host = "127.0.0.1";
-    port = 50050 + service_number;
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
@@ -16,14 +11,12 @@ in
     };
     GRPC = [
       {
+        inherit (registry.grpc-chemo-bureaucrat) host port;
         name = "BUREAUCRAT";
-        host = config.TLMS.bureaucrat.grpc.host;
-        port = config.TLMS.bureaucrat.grpc.port;
       }
       {
+        inherit (registry.grpc-chemo-funnel) host port;
         name = "FUNNEL";
-        host = config.TLMS.funnel.GRPC.host;
-        port = config.TLMS.funnel.GRPC.port;
       }
     ];
   };

modules/data-hoarder/data-accumulator.nix

@@ -1,9 +1,7 @@
-{ config, ... }:
+{ config, registry, ... }: {
-{
   TLMS.dataAccumulator = {
+    inherit (registry.port-data_accumulator) host port;
     enable = true;
-    host = "0.0.0.0";
-    port = 8080;
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
@@ -11,13 +9,10 @@
       user = "tlms";
       database = "tlms";
     };
-    GRPC = [
+    GRPC = [{
-      {
+      inherit (registry.grpc-data_accumulator-chemo) host port;
-        name = "CHEMO";
+      name = "CHEMO";
-        host = config.TLMS.chemo.host;
+    }];
-        port = config.TLMS.chemo.port;
-      }
-    ];
   };
   systemd.services."data-accumulator" = {
     after = [ "postgresql.service" ];
@@ -29,7 +24,10 @@
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "dump.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "dump.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -41,7 +39,8 @@
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.dataAccumulator; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-data_accumulator;
+                "http://${host}:${toString port}/";
             };
           };
         };

modules/data-hoarder/datacare.nix

@@ -1,10 +1,7 @@
-{ config, ... }: {
+{ config, registry, ... }: {
   TLMS.datacare = {
     enable = true;
-    http = {
+    http = registry.port-datacare;
-      host = "127.0.0.1";
-      port = 8070;
-    };
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
@@ -22,13 +19,15 @@
     wants = [ "postgresql.service" ];
   };
 
-
   services = {
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "datacare.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "datacare.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -40,7 +39,8 @@
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.datacare.http; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-data_accumulator;
+                "http://${host}:${toString port}/";
               proxyWebsockets = true;
               extraConfig = ''
                 more_set_headers "Access-Control-Allow-Credentials: true";

modules/data-hoarder/documentation.nix

@@ -1,11 +1,13 @@
-{ pkgs, config, ... }:
+{ pkgs, config, ... }: {
-{
   services = {
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "docs.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "docs.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''

modules/data-hoarder/file_sharing.nix

@@ -4,7 +4,10 @@
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "files.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "files.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''

modules/data-hoarder/kindergarten.nix

@@ -2,7 +2,10 @@
   services.nginx = {
     enable = true;
     virtualHosts = {
-      "kid.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+      "kid.${
+        (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+          config.deployment-TLMS.domain)
+      }" = {
         enableACME = true;
         forceSSL = true;
         extraConfig = ''
@@ -13,7 +16,10 @@
         enableACME = true;
         forceSSL = true;
         locations."~ ^/(de|en)" = {
-          root = if (config.deployment-TLMS.domain == "tlm.solutions") then "${pkgs.kindergarten}" else "${pkgs.kindergarten-staging}";
+          root = if (config.deployment-TLMS.domain == "tlm.solutions") then
+            "${pkgs.kindergarten}"
+          else
+            "${pkgs.kindergarten-staging}";
           # index = "index.html";
           tryFiles = "$uri /$1/index.html =404";
           extraConfig = ''

modules/data-hoarder/lizard.nix

@@ -1,34 +1,22 @@
-{ config, ... }:
+{ config, registry, ... }: {
-let
-  service_number = 1;
-in
-{
   TLMS.lizard = {
     enable = true;
-    http = {
+    http = { inherit (registry.port-lizard) host port; };
-      host = "127.0.0.1";
-      port = 9000 + service_number;
-    };
 
-    redis = {
+    redis = registry.redis-bureaucrat-lizard;
-      host = config.services.redis.servers."state".bind;
-      port = config.services.redis.servers."state".port;
-    };
     logLevel = "debug";
     workerCount = 6;
   };
 
   services = {
-    redis.servers."state" = {
-      enable = true;
-      bind = "127.0.0.1";
-      port = 5314;
-    };
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "lizard.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "lizard.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -40,7 +28,8 @@ in
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.lizard.http; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-lizard;
+                "http://${host}:${toString port}/";
               proxyWebsockets = true;
             };
           };

modules/data-hoarder/map.nix

@@ -4,7 +4,10 @@
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "map.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "map.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''

modules/data-hoarder/nginx.nix

@@ -20,8 +20,7 @@ let
     # STS
     add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
   '';
-in
+in {
-{
   networking.firewall.allowedTCPPorts = [ 80 443 ];
 
   security.acme.acceptTerms = true;

modules/data-hoarder/postgres.nix

@@ -4,22 +4,19 @@
     enable = true;
     enableTCPIP = true;
     port = 5432;
-    authentication =
+    authentication = let
-      let
+      senpai-ip =
-        senpai-ip = self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4;
+        self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4;
-      in
+    in pkgs.lib.mkOverride 10 ''
-      pkgs.lib.mkOverride 10 ''
+      local	all	all	trust
-        local	all	all	trust
+      host	all	all	127.0.0.1/32	trust
-        host	all	all	127.0.0.1/32	trust
+      host	all	all	::1/128	trust
-        host	all	all	::1/128	trust
+      host	tlms	grafana	${senpai-ip}/32	scram-sha-256
-        host	tlms	grafana	${senpai-ip}/32	scram-sha-256
+    '';
-      '';
     package = pkgs.postgresql_14;
     ensureDatabases = [ "tlms" ];
     ensureUsers = [
-      {
+      { name = "grafana"; }
-        name = "grafana";
-      }
       {
         name = "tlms";
         ensurePermissions = {
@@ -30,15 +27,12 @@
     ];
   };
 
-  environment.systemPackages = [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ];
+  environment.systemPackages =
+    [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ];
 
   systemd.services.postgresql = {
-    unitConfig = {
+    unitConfig = { TimeoutStartSec = 3000; };
-      TimeoutStartSec = 3000;
+    serviceConfig = { TimeoutSec = lib.mkForce 3000; };
-    };
-    serviceConfig = {
-      TimeoutSec = lib.mkForce 3000;
-    };
     postStart = lib.mkAfter ''
       # set pw for the users
       $PSQL -c "ALTER ROLE tlms WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password.path})';"
@@ -63,9 +57,7 @@
 
   systemd.services.dump-csv = {
     path = [ config.services.postgresql.package ];
-    serviceConfig = {
+    serviceConfig = { User = "postgres"; };
-      User = "postgres";
-    };
     script = ''
       TMPFILE=$(mktemp)
       OUT_FOLDER=/var/lib/pub-files/postgres-dumps/$(date -d"$(date) - 1 day" +"%Y-%m")

modules/data-hoarder/secrets.nix

@@ -4,14 +4,19 @@ let
   data-accumulator-user = config.TLMS.dataAccumulator.user;
   trekkie-user = config.TLMS.trekkie.user;
   chemo-user = config.TLMS.chemo.user;
-in
+in {
-{
   sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
 
   users.groups = {
     postgres-tlms = {
       name = "postgres-tlms";
-      members = [ datacare-user data-accumulator-user trekkie-user chemo-user "postgres" ];
+      members = [
+        datacare-user
+        data-accumulator-user
+        trekkie-user
+        chemo-user
+        "postgres"
+      ];
     };
 
     password-salt = {
@@ -28,9 +33,7 @@ in
   };
 
   sops.secrets = {
-    wg-seckey = {
+    wg-seckey = { owner = config.users.users.systemd-network.name; };
-      owner = config.users.users.systemd-network.name;
-    };
     postgres_password_hash_salt = {
       group = config.users.groups.password-salt.name;
       mode = "0440";

modules/data-hoarder/socket.nix

@@ -1,20 +1,10 @@
-{ config, ... }:
+{ config, registry, ... }: {
-let
-  service_number = 2;
-in
-{
   TLMS.funnel = {
     enable = true;
-    GRPC = {
+    GRPC = registry.grpc-chemo-funnel;
-      host = "127.0.0.1";
+    defaultWebsocket = { inherit (registry.port-funnel) host port; };
-      port = 50050 + service_number;
-    };
-    defaultWebsocket = {
-      host = "127.0.0.1";
-      port = 9000 + service_number;
-    };
     metrics = {
-      port = 10010 + service_number;
+      inherit (registry.port-funnel-metrics) port;
       host = config.deployment-TLMS.net.wg.addr4;
     };
   };
@@ -23,12 +13,16 @@ in
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "socket.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "socket.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           locations."/" = {
             proxyWebsockets = true;
-            proxyPass = with config.TLMS.funnel.defaultWebsocket; "http://${host}:${toString port}/";
+            proxyPass = with registry.port-funnel;
+              "http://${host}:${toString port}/";
           };
         };
         "socket.${config.deployment-TLMS.domain}" = {
@@ -36,7 +30,8 @@ in
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.funnel.defaultWebsocket; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-funnel;
+                "http://${host}:${toString port}/";
               proxyWebsockets = true;
             };
           };

modules/data-hoarder/trekkie.nix

@@ -1,24 +1,16 @@
-{ config, ... }:
+{ config, registry, ... }: {
-{
   TLMS.trekkie = {
+    inherit (registry.port-trekkie) host port;
     enable = true;
-    host = "0.0.0.0";
     saltPath = config.sops.secrets.postgres_password_hash_salt.path;
-    port = 8060;
     database = {
       host = "127.0.0.1";
       port = config.services.postgresql.port;
       passwordFile = config.sops.secrets.postgres_password.path;
       user = "tlms";
     };
-    redis = {
+    redis = registry.redis-trekkie;
-      port = 6379;
+    grpc = registry.grpc-trekkie-chemo;
-      host = "localhost";
-    };
-    grpc = {
-      host = config.TLMS.chemo.host;
-      port = config.TLMS.chemo.port;
-    };
     logLevel = "info";
   };
   systemd.services."trekkie" = {
@@ -27,17 +19,20 @@
   };
 
   services = {
-    redis.servers."trekkie" = {
+    redis.servers."trekkie" = with registry.redis-trekkie; {
+      inherit port;
       enable = true;
-      bind = config.TLMS.trekkie.redis.host;
+      bind = host;
-      port = config.TLMS.trekkie.redis.port;
     };
 
     nginx = {
       enable = true;
       recommendedProxySettings = true;
       virtualHosts = {
-        "trekkie.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+        "trekkie.${
+          (builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
+            config.deployment-TLMS.domain)
+        }" = {
           enableACME = true;
           forceSSL = true;
           extraConfig = ''
@@ -49,7 +44,8 @@
           enableACME = true;
           locations = {
             "/" = {
-              proxyPass = with config.TLMS.trekkie; "http://${host}:${toString port}/";
+              proxyPass = with registry.port-trekkie;
+                "http://${host}:${toString port}/";
             };
           };
         };

modules/data-hoarder/website.nix

@@ -2,13 +2,14 @@
   services.nginx = {
     enable = true;
     virtualHosts = {
-      "${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
+      "${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
-        enableACME = true;
+        config.deployment-TLMS.domain)}" = {
-        forceSSL = true;
+          enableACME = true;
-        extraConfig = ''
+          forceSSL = true;
-          rewrite ^ https://kid.${config.deployment-TLMS.domain}/ permanent;
+          extraConfig = ''
-        '';
+            rewrite ^ https://kid.${config.deployment-TLMS.domain}/ permanent;
-      };
+          '';
+        };
       "${config.deployment-TLMS.domain}" = {
         enableACME = true;
         forceSSL = true;

registry/data-hoarder/default.nix

@@ -0,0 +1,55 @@
+rec {
+  redis-bureaucrat-lizard = {
+    host = "127.0.0.1";
+    port = 5314;
+  };
+
+  grpc-chemo-bureaucrat = {
+    host = "127.0.0.1";
+    port = 50056;
+  };
+
+  grpc-chemo-funnel = {
+    host = "127.0.0.1";
+    port = 50052;
+  };
+
+  grpc-data_accumulator-chemo = {
+    host = "127.0.0.1";
+    port = 50053;
+  };
+
+  grpc-trekkie-chemo = grpc-data_accumulator-chemo;
+
+  port-data_accumulator = {
+    host = "0.0.0.0";
+    port = 8080;
+  };
+
+  port-datacare = {
+    host = "127.0.0.1";
+    port = 8070;
+  };
+
+  port-lizard = {
+    host = "127.0.0.1";
+    port = 9001;
+  };
+
+  port-funnel = {
+    host = "127.0.0.1";
+    port = 9002;
+  };
+
+  port-funnel-metrics = { port = 10012; };
+
+  port-trekkie = {
+    host = "0.0.0.0";
+    port = 8060;
+  };
+
+  redis-trekkie = {
+    host = "localhost";
+    port = 6379;
+  };
+}

registry/default.nix

@@ -0,0 +1 @@
+{ data-hoarder = import ./data-hoarder; }