dump-dvb/nix-config
dump-dvb
/
nix-config
mirror of https://github.com/dump-dvb/nix-config.git
3
0
Fork 0
Code Issues Projects Releases Wiki Activity

start implementing data-hoarder registry

This commit is contained in:
Markus Schmidl 2023-11-25 20:56:46 +01:00
parent ed4107a1f9
commit 0c8b910ec5
18 changed files with 177 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
flake.nix
View File

@ -241,6 +241,7 @@
// (import ./pkgs/deployment.nix { inherit self pkgs lib; })
// (lib.foldl (x: y: lib.mergeAttrs x { "${y.config.system.name}-vm" = y.config.system.build.vm; }) { } (lib.attrValues self.nixosConfigurations));
registry = import ./registry;
in
{
@ -252,7 +253,7 @@
data-hoarder = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
specialArgs = { inherit inputs self; };
specialArgs = { inherit inputs self; registry = registry.data-hoarder; };
modules = [
microvm.nixosModules.microvm
./hosts/data-hoarder
@ -261,7 +262,7 @@
staging-data-hoarder = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
specialArgs = { inherit inputs self; };
specialArgs = { inherit inputs self; registry = registry.data-hoarder; };
modules = [
./hosts/staging-data-hoarder
microvm.nixosModules.microvm

22
modules/data-hoarder/bureaucrat.nix
View File

@ -1,25 +1,15 @@
{ config, ... }:
let
service_number = 6;
in
{
{ config, registry, ... }: {
TLMS.bureaucrat = {
enable = true;
grpc = {
host = "127.0.0.1";
port = 50050 + service_number;
};
redis = {
host = config.services.redis.servers."state".bind;
port = config.services.redis.servers."state".port;
};
grpc = registry.grpc-chemo-bureaucrat;
redis = registry.redis-bureaucrat-lizard;
};
services = {
redis.servers."state" = {
redis.servers."state" = with registry.redis-bureaucrat-lizard; {
inherit port;
enable = true;
bind = "127.0.0.1";
port = 5314;
bind = host;
};
};
}

15
modules/data-hoarder/chemo.nix
View File

@ -1,12 +1,7 @@
{ config, ... }:
let
service_number = 3;
in
{
{ config, registry, ... }: {
TLMS.chemo = {
inherit (registry.grpc-data_accumulator-chemo) host port;
enable = true;
host = "127.0.0.1";
port = 50050 + service_number;
database = {
host = "127.0.0.1";
port = config.services.postgresql.port;
@ -16,14 +11,12 @@ in
};
GRPC = [
{
inherit (registry.grpc-chemo-bureaucrat) host port;
name = "BUREAUCRAT";
host = config.TLMS.bureaucrat.grpc.host;
port = config.TLMS.bureaucrat.grpc.port;
}
{
inherit (registry.grpc-chemo-funnel) host port;
name = "FUNNEL";
host = config.TLMS.funnel.GRPC.host;
port = config.TLMS.funnel.GRPC.port;
}
];
};

25
modules/data-hoarder/data-accumulator.nix
View File

@ -1,9 +1,7 @@
{ config, ... }:
{
{ config, registry, ... }: {
TLMS.dataAccumulator = {
inherit (registry.port-data_accumulator) host port;
enable = true;
host = "0.0.0.0";
port = 8080;
database = {
host = "127.0.0.1";
port = config.services.postgresql.port;
@ -11,13 +9,10 @@
user = "tlms";
database = "tlms";
};
GRPC = [
{
name = "CHEMO";
host = config.TLMS.chemo.host;
port = config.TLMS.chemo.port;
}
];
GRPC = [{
inherit (registry.grpc-data_accumulator-chemo) host port;
name = "CHEMO";
}];
};
systemd.services."data-accumulator" = {
after = [ "postgresql.service" ];
@ -29,7 +24,10 @@
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"dump.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"dump.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
@ -41,7 +39,8 @@
enableACME = true;
locations = {
"/" = {
proxyPass = with config.TLMS.dataAccumulator; "http://${host}:${toString port}/";
proxyPass = with registry.port-data_accumulator;
"http://${host}:${toString port}/";
};
};
};

16
modules/data-hoarder/datacare.nix
View File

@ -1,10 +1,7 @@
{ config, ... }: {
{ config, registry, ... }: {
TLMS.datacare = {
enable = true;
http = {
host = "127.0.0.1";
port = 8070;
};
http = registry.port-datacare;
database = {
host = "127.0.0.1";
port = config.services.postgresql.port;
@ -22,13 +19,15 @@
wants = [ "postgresql.service" ];
};
services = {
nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"datacare.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"datacare.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
@ -40,7 +39,8 @@
enableACME = true;
locations = {
"/" = {
proxyPass = with config.TLMS.datacare.http; "http://${host}:${toString port}/";
proxyPass = with registry.port-data_accumulator;
"http://${host}:${toString port}/";
proxyWebsockets = true;
extraConfig = ''
more_set_headers "Access-Control-Allow-Credentials: true";

8
modules/data-hoarder/documentation.nix
View File

@ -1,11 +1,13 @@
{ pkgs, config, ... }:
{
{ pkgs, config, ... }: {
services = {
nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"docs.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"docs.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''

5
modules/data-hoarder/file_sharing.nix
View File

@ -4,7 +4,10 @@
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"files.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"files.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''

10
modules/data-hoarder/kindergarten.nix
View File

@ -2,7 +2,10 @@
services.nginx = {
enable = true;
virtualHosts = {
"kid.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"kid.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
@ -13,7 +16,10 @@
enableACME = true;
forceSSL = true;
locations."~ ^/(de|en)" = {
root = if (config.deployment-TLMS.domain == "tlm.solutions") then "${pkgs.kindergarten}" else "${pkgs.kindergarten-staging}";
root = if (config.deployment-TLMS.domain == "tlm.solutions") then
"${pkgs.kindergarten}"
else
"${pkgs.kindergarten-staging}";
# index = "index.html";
tryFiles = "$uri /$1/index.html =404";
extraConfig = ''

29
modules/data-hoarder/lizard.nix
View File

@ -1,34 +1,22 @@
{ config, ... }:
let
service_number = 1;
in
{
{ config, registry, ... }: {
TLMS.lizard = {
enable = true;
http = {
host = "127.0.0.1";
port = 9000 + service_number;
};
http = { inherit (registry.port-lizard) host port; };
redis = {
host = config.services.redis.servers."state".bind;
port = config.services.redis.servers."state".port;
};
redis = registry.redis-bureaucrat-lizard;
logLevel = "debug";
workerCount = 6;
};
services = {
redis.servers."state" = {
enable = true;
bind = "127.0.0.1";
port = 5314;
};
nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"lizard.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"lizard.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
@ -40,7 +28,8 @@ in
enableACME = true;
locations = {
"/" = {
proxyPass = with config.TLMS.lizard.http; "http://${host}:${toString port}/";
proxyPass = with registry.port-lizard;
"http://${host}:${toString port}/";
proxyWebsockets = true;
};
};

5
modules/data-hoarder/map.nix
View File

@ -4,7 +4,10 @@
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"map.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"map.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''

3
modules/data-hoarder/nginx.nix
View File

@ -20,8 +20,7 @@ let
# STS
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
'';
in
{
in {
networking.firewall.allowedTCPPorts = [ 80 443 ];
security.acme.acceptTerms = true;

38
modules/data-hoarder/postgres.nix
View File

@ -4,22 +4,19 @@
enable = true;
enableTCPIP = true;
port = 5432;
authentication =
let
senpai-ip = self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4;
in
pkgs.lib.mkOverride 10 ''
local all all trust
host all all 127.0.0.1/32 trust
host all all ::1/128 trust
host tlms grafana ${senpai-ip}/32 scram-sha-256
'';
authentication = let
senpai-ip =
self.nixosConfigurations.notice-me-senpai.config.deployment-TLMS.net.wg.addr4;
in pkgs.lib.mkOverride 10 ''
local all all trust
host all all 127.0.0.1/32 trust
host all all ::1/128 trust
host tlms grafana ${senpai-ip}/32 scram-sha-256
'';
package = pkgs.postgresql_14;
ensureDatabases = [ "tlms" ];
ensureUsers = [
{
name = "grafana";
}
{ name = "grafana"; }
{
name = "tlms";
ensurePermissions = {
@ -30,15 +27,12 @@
];
};
environment.systemPackages = [ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ];
environment.systemPackages =
[ inputs.tlms-rs.packages.x86_64-linux.run-migration-based ];
systemd.services.postgresql = {
unitConfig = {
TimeoutStartSec = 3000;
};
serviceConfig = {
TimeoutSec = lib.mkForce 3000;
};
unitConfig = { TimeoutStartSec = 3000; };
serviceConfig = { TimeoutSec = lib.mkForce 3000; };
postStart = lib.mkAfter ''
# set pw for the users
$PSQL -c "ALTER ROLE tlms WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password.path})';"
@ -63,9 +57,7 @@
systemd.services.dump-csv = {
path = [ config.services.postgresql.package ];
serviceConfig = {
User = "postgres";
};
serviceConfig = { User = "postgres"; };
script = ''
TMPFILE=$(mktemp)
OUT_FOLDER=/var/lib/pub-files/postgres-dumps/$(date -d"$(date) - 1 day" +"%Y-%m")

15
modules/data-hoarder/secrets.nix
View File

@ -4,14 +4,19 @@ let
data-accumulator-user = config.TLMS.dataAccumulator.user;
trekkie-user = config.TLMS.trekkie.user;
chemo-user = config.TLMS.chemo.user;
in
{
in {
sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
users.groups = {
postgres-tlms = {
name = "postgres-tlms";
members = [ datacare-user data-accumulator-user trekkie-user chemo-user "postgres" ];
members = [
datacare-user
data-accumulator-user
trekkie-user
chemo-user
"postgres"
];
};
password-salt = {
@ -28,9 +33,7 @@ in
};
sops.secrets = {
wg-seckey = {
owner = config.users.users.systemd-network.name;
};
wg-seckey = { owner = config.users.users.systemd-network.name; };
postgres_password_hash_salt = {
group = config.users.groups.password-salt.name;
mode = "0440";

29
modules/data-hoarder/socket.nix
View File

@ -1,20 +1,10 @@
{ config, ... }:
let
service_number = 2;
in
{
{ config, registry, ... }: {
TLMS.funnel = {
enable = true;
GRPC = {
host = "127.0.0.1";
port = 50050 + service_number;
};
defaultWebsocket = {
host = "127.0.0.1";
port = 9000 + service_number;
};
GRPC = registry.grpc-chemo-funnel;
defaultWebsocket = { inherit (registry.port-funnel) host port; };
metrics = {
port = 10010 + service_number;
inherit (registry.port-funnel-metrics) port;
host = config.deployment-TLMS.net.wg.addr4;
};
};
@ -23,12 +13,16 @@ in
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"socket.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"socket.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
locations."/" = {
proxyWebsockets = true;
proxyPass = with config.TLMS.funnel.defaultWebsocket; "http://${host}:${toString port}/";
proxyPass = with registry.port-funnel;
"http://${host}:${toString port}/";
};
};
"socket.${config.deployment-TLMS.domain}" = {
@ -36,7 +30,8 @@ in
enableACME = true;
locations = {
"/" = {
proxyPass = with config.TLMS.funnel.defaultWebsocket; "http://${host}:${toString port}/";
proxyPass = with registry.port-funnel;
"http://${host}:${toString port}/";
proxyWebsockets = true;
};
};

30
modules/data-hoarder/trekkie.nix
View File

@ -1,24 +1,16 @@
{ config, ... }:
{
{ config, registry, ... }: {
TLMS.trekkie = {
inherit (registry.port-trekkie) host port;
enable = true;
host = "0.0.0.0";
saltPath = config.sops.secrets.postgres_password_hash_salt.path;
port = 8060;
database = {
host = "127.0.0.1";
port = config.services.postgresql.port;
passwordFile = config.sops.secrets.postgres_password.path;
user = "tlms";
};
redis = {
port = 6379;
host = "localhost";
};
grpc = {
host = config.TLMS.chemo.host;
port = config.TLMS.chemo.port;
};
redis = registry.redis-trekkie;
grpc = registry.grpc-trekkie-chemo;
logLevel = "info";
};
systemd.services."trekkie" = {
@ -27,17 +19,20 @@
};
services = {
redis.servers."trekkie" = {
redis.servers."trekkie" = with registry.redis-trekkie; {
inherit port;
enable = true;
bind = config.TLMS.trekkie.redis.host;
port = config.TLMS.trekkie.redis.port;
bind = host;
};
nginx = {
enable = true;
recommendedProxySettings = true;
virtualHosts = {
"trekkie.${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
"trekkie.${
(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)
}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
@ -49,7 +44,8 @@
enableACME = true;
locations = {
"/" = {
proxyPass = with config.TLMS.trekkie; "http://${host}:${toString port}/";
proxyPass = with registry.port-trekkie;
"http://${host}:${toString port}/";
};
};
};

15
modules/data-hoarder/website.nix
View File

@ -2,13 +2,14 @@
services.nginx = {
enable = true;
virtualHosts = {
"${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ] config.deployment-TLMS.domain)}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
rewrite ^ https://kid.${config.deployment-TLMS.domain}/ permanent;
'';
};
"${(builtins.replaceStrings [ "tlm.solutions" ] [ "dvb.solutions" ]
config.deployment-TLMS.domain)}" = {
enableACME = true;
forceSSL = true;
extraConfig = ''
rewrite ^ https://kid.${config.deployment-TLMS.domain}/ permanent;
'';
};
"${config.deployment-TLMS.domain}" = {
enableACME = true;
forceSSL = true;

55
registry/data-hoarder/default.nix Normal file
View File

@ -0,0 +1,55 @@
rec {
redis-bureaucrat-lizard = {
host = "127.0.0.1";
port = 5314;
};
grpc-chemo-bureaucrat = {
host = "127.0.0.1";
port = 50056;
};
grpc-chemo-funnel = {
host = "127.0.0.1";
port = 50052;
};
grpc-data_accumulator-chemo = {
host = "127.0.0.1";
port = 50053;
};
grpc-trekkie-chemo = grpc-data_accumulator-chemo;
port-data_accumulator = {
host = "0.0.0.0";
port = 8080;
};
port-datacare = {
host = "127.0.0.1";
port = 8070;
};
port-lizard = {
host = "127.0.0.1";
port = 9001;
};
port-funnel = {
host = "127.0.0.1";
port = 9002;
};
port-funnel-metrics = { port = 10012; };
port-trekkie = {
host = "0.0.0.0";
port = 8060;
};
redis-trekkie = {
host = "localhost";
port = 6379;
};
}

1
registry/default.nix Normal file
View File

@ -0,0 +1 @@
{ data-hoarder = import ./data-hoarder; }