From 14a5b138826b92ebf29e05dafca24eba129251b3 Mon Sep 17 00:00:00 2001 From: Grigory Shipunov Date: Thu, 16 Mar 2023 19:06:29 +0100 Subject: [PATCH 1/5] fixup postgres perms --- modules/data-hoarder/postgres.nix | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/data-hoarder/postgres.nix b/modules/data-hoarder/postgres.nix index 5e3635f..841c9e6 100644 --- a/modules/data-hoarder/postgres.nix +++ b/modules/data-hoarder/postgres.nix @@ -29,23 +29,30 @@ TimeoutSec = lib.mkForce 3000; }; postStart = lib.mkAfter '' - # TODO: make shure grafana can't read tokens... - $PSQL -c "GRANT CONNECT ON DATABASE tlms TO grafana;" - $PSQL -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO grafana"; - + # set pw for the users $PSQL -c "ALTER ROLE tlms WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password.path})';" $PSQL -c "ALTER ROLE grafana WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password_grafana.path})';" + ##################### New DB ##################### + export DATABASE_URL=postgres:///tlms ${inputs.tlms-rs.packages.x86_64-linux.run-migration-based}/bin/run-migration # fixup permissions + # tlms is practically root, we need to FIXME something about it $PSQL -c "GRANT ALL ON DATABASE tlms TO tlms;" $PSQL -d tlms -c "GRANT ALL ON ALL TABLES IN SCHEMA public TO tlms;" $PSQL -d tlms -c "GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO tlms;" + # Get graphana to SELECT from tables that might be interesting for it + $PSQL -c "GRANT CONNECT ON DATABASE tlms TO grafana;" + $PSQL -c "GRANT SELECT ON r09_telegrams, raw_telegrams, gps_points, trekkie_runs, regions TO grafana;" + + unset DATABASE_URL + ##################### Old DB ##################### + # this is old, shit and legacy at this point export DATABASE_URL=postgres:///dvbdump ${inputs.tlms-rs.packages.x86_64-linux.run-migration}/bin/run-migration From cd713d773d8d6ed7b8bd3f06e9c495f63ed30480 Mon Sep 17 00:00:00 2001 From: Markus Schmidl Date: Thu, 16 Mar 2023 22:55:12 +0100 Subject: [PATCH 2/5] add follows for flake-utils --- flake.lock | 201 ++++++++++------------------------------------------- flake.nix | 13 ++++ 2 files changed, 50 insertions(+), 164 deletions(-) diff --git a/flake.lock b/flake.lock index 590c97f..bb65d70 100644 --- a/flake.lock +++ b/flake.lock @@ -8,7 +8,9 @@ "nixpkgs": [ "nixpkgs" ], - "utils": "utils" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678328499, @@ -35,7 +37,9 @@ "tlms-rs": [ "tlms-rs" ], - "utils": "utils_2" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678477361, @@ -89,21 +93,6 @@ "type": "github" } }, - "flake-utils": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "funnel": { "inputs": { "json-structs": "json-structs", @@ -111,7 +100,9 @@ "nixpkgs" ], "tlms-rust": "tlms-rust", - "utils": "utils_3" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678020126, @@ -132,7 +123,9 @@ "nixpkgs": [ "nixpkgs" ], - "utils": "utils_4" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678306615, @@ -169,7 +162,9 @@ "nixpkgs": [ "nixpkgs" ], - "utils": "utils_5" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678484240, @@ -188,7 +183,9 @@ "microvm": { "inputs": { "fenix": "fenix", - "flake-utils": "flake-utils", + "flake-utils": [ + "utils" + ], "nixpkgs": [ "nixpkgs" ] @@ -291,6 +288,7 @@ "telegram-decoder": "telegram-decoder", "tlms-rs": "tlms-rs", "trekkie": "trekkie", + "utils": "utils", "wartrammer": "wartrammer", "windshield": "windshield" } @@ -341,7 +339,9 @@ "nixpkgs": [ "nixpkgs" ], - "utils": "utils_6" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678547620, @@ -365,7 +365,9 @@ "nixpkgs": [ "nixpkgs" ], - "utils": "utils_7" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678328168, @@ -428,7 +430,9 @@ "tlms-rs": [ "tlms-rs" ], - "utils": "utils_8" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678485606, @@ -446,146 +450,11 @@ }, "utils": { "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", + "lastModified": 1678901627, + "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", "owner": "numtide", "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_10": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_2": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_3": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_4": { - "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_5": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_6": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_7": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_8": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "utils_9": { - "locked": { - "lastModified": 1676283394, - "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", + "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", "type": "github" }, "original": { @@ -603,7 +472,9 @@ "nixpkgs" ], "nixpkgs-unstable": "nixpkgs-unstable", - "utils": "utils_9" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1677985694, @@ -624,7 +495,9 @@ "nixpkgs": [ "nixpkgs" ], - "utils": "utils_10" + "utils": [ + "utils" + ] }, "locked": { "lastModified": 1678530041, diff --git a/flake.nix b/flake.nix index 8eb09ef..4d26138 100644 --- a/flake.nix +++ b/flake.nix @@ -9,9 +9,12 @@ inputs.nixpkgs.follows = "nixpkgs"; }; + utils.url = github:numtide/flake-utils; + microvm = { url = "github:astro/microvm.nix"; inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "utils"; }; sops-nix = { @@ -31,6 +34,7 @@ nixpkgs.follows = "nixpkgs"; naersk.follows = "naersk"; tlms-rs.follows = "tlms-rs"; + utils.follows = "utils"; }; }; @@ -40,12 +44,14 @@ nixpkgs.follows = "nixpkgs"; naersk.follows = "naersk"; tlms-rs.follows = "tlms-rs"; + utils.follows = "utils"; }; }; kindergarten = { url = "github:tlm-solutions/kindergarten"; inputs.nixpkgs.follows = "nixpkgs"; + inputs.utils.follows = "utils"; }; telegram-decoder = { @@ -53,40 +59,47 @@ inputs = { nixpkgs.follows = "nixpkgs"; naersk.follows = "naersk"; + utils.follows = "utils"; }; }; gnuradio-decoder = { url = "github:tlm-solutions/gnuradio-decoder"; inputs.nixpkgs.follows = "nixpkgs"; + inputs.utils.follows = "utils"; }; data-accumulator = { url = "github:tlm-solutions/data-accumulator"; inputs.nixpkgs.follows = "nixpkgs"; inputs.naersk.follows = "naersk"; + inputs.utils.follows = "utils"; }; state-api = { url = "github:tlm-solutions/state-api"; inputs.nixpkgs.follows = "nixpkgs"; inputs.naersk.follows = "naersk"; + inputs.utils.follows = "utils"; }; funnel = { url = "github:tlm-solutions/funnel"; inputs.nixpkgs.follows = "nixpkgs"; + inputs.utils.follows = "utils"; }; windshield = { url = "github:tlm-solutions/windshield"; inputs.nixpkgs.follows = "nixpkgs"; + inputs.utils.follows = "utils"; }; wartrammer = { url = "github:tlm-solutions/wartrammer-40k"; inputs.nixpkgs.follows = "nixpkgs"; inputs.naersk.follows = "naersk"; + inputs.utils.follows = "utils"; }; tlms-rs = { From 8bf0348143318a34465cb13ba728a1f59597c29d Mon Sep 17 00:00:00 2001 From: Markus Schmidl Date: Fri, 17 Mar 2023 19:51:49 +0100 Subject: [PATCH 3/5] fix utils input --- flake.lock | 54 +++++++++++++++++++++++++++--------------------------- flake.nix | 25 +++++++++++++------------ 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/flake.lock b/flake.lock index bb65d70..096fb4c 100644 --- a/flake.lock +++ b/flake.lock @@ -9,7 +9,7 @@ "nixpkgs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -38,7 +38,7 @@ "tlms-rs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -93,6 +93,21 @@ "type": "github" } }, + "flake-utils": { + "locked": { + "lastModified": 1678901627, + "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "funnel": { "inputs": { "json-structs": "json-structs", @@ -101,7 +116,7 @@ ], "tlms-rust": "tlms-rust", "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -124,7 +139,7 @@ "nixpkgs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -163,7 +178,7 @@ "nixpkgs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -184,7 +199,7 @@ "inputs": { "fenix": "fenix", "flake-utils": [ - "utils" + "flake-utils" ], "nixpkgs": [ "nixpkgs" @@ -277,6 +292,7 @@ "data-accumulator": "data-accumulator", "datacare": "datacare", "documentation-src": "documentation-src", + "flake-utils": "flake-utils", "funnel": "funnel", "gnuradio-decoder": "gnuradio-decoder", "kindergarten": "kindergarten", @@ -288,7 +304,6 @@ "telegram-decoder": "telegram-decoder", "tlms-rs": "tlms-rs", "trekkie": "trekkie", - "utils": "utils", "wartrammer": "wartrammer", "windshield": "windshield" } @@ -340,7 +355,7 @@ "nixpkgs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -366,7 +381,7 @@ "nixpkgs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -431,7 +446,7 @@ "tlms-rs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -448,21 +463,6 @@ "type": "github" } }, - "utils": { - "locked": { - "lastModified": 1678901627, - "narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "wartrammer": { "inputs": { "naersk": [ @@ -473,7 +473,7 @@ ], "nixpkgs-unstable": "nixpkgs-unstable", "utils": [ - "utils" + "flake-utils" ] }, "locked": { @@ -496,7 +496,7 @@ "nixpkgs" ], "utils": [ - "utils" + "flake-utils" ] }, "locked": { diff --git a/flake.nix b/flake.nix index 4d26138..fed3430 100644 --- a/flake.nix +++ b/flake.nix @@ -9,12 +9,13 @@ inputs.nixpkgs.follows = "nixpkgs"; }; - utils.url = github:numtide/flake-utils; + # DO NOT remame this to utils + flake-utils.url = github:numtide/flake-utils; microvm = { url = "github:astro/microvm.nix"; inputs.nixpkgs.follows = "nixpkgs"; - inputs.flake-utils.follows = "utils"; + inputs.flake-utils.follows = "flake-utils"; }; sops-nix = { @@ -34,7 +35,7 @@ nixpkgs.follows = "nixpkgs"; naersk.follows = "naersk"; tlms-rs.follows = "tlms-rs"; - utils.follows = "utils"; + utils.follows = "flake-utils"; }; }; @@ -44,14 +45,14 @@ nixpkgs.follows = "nixpkgs"; naersk.follows = "naersk"; tlms-rs.follows = "tlms-rs"; - utils.follows = "utils"; + utils.follows = "flake-utils"; }; }; kindergarten = { url = "github:tlm-solutions/kindergarten"; inputs.nixpkgs.follows = "nixpkgs"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; telegram-decoder = { @@ -59,47 +60,47 @@ inputs = { nixpkgs.follows = "nixpkgs"; naersk.follows = "naersk"; - utils.follows = "utils"; + utils.follows = "flake-utils"; }; }; gnuradio-decoder = { url = "github:tlm-solutions/gnuradio-decoder"; inputs.nixpkgs.follows = "nixpkgs"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; data-accumulator = { url = "github:tlm-solutions/data-accumulator"; inputs.nixpkgs.follows = "nixpkgs"; inputs.naersk.follows = "naersk"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; state-api = { url = "github:tlm-solutions/state-api"; inputs.nixpkgs.follows = "nixpkgs"; inputs.naersk.follows = "naersk"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; funnel = { url = "github:tlm-solutions/funnel"; inputs.nixpkgs.follows = "nixpkgs"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; windshield = { url = "github:tlm-solutions/windshield"; inputs.nixpkgs.follows = "nixpkgs"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; wartrammer = { url = "github:tlm-solutions/wartrammer-40k"; inputs.nixpkgs.follows = "nixpkgs"; inputs.naersk.follows = "naersk"; - inputs.utils.follows = "utils"; + inputs.utils.follows = "flake-utils"; }; tlms-rs = { From 84d53dd60be59065df565f91c173b6f21515a51b Mon Sep 17 00:00:00 2001 From: Markus Schmidl Date: Sun, 19 Mar 2023 21:22:51 +0100 Subject: [PATCH 4/5] update cachix --- modules/TLMS/binary-cache.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/TLMS/binary-cache.nix b/modules/TLMS/binary-cache.nix index 4395e4b..052ca7b 100644 --- a/modules/TLMS/binary-cache.nix +++ b/modules/TLMS/binary-cache.nix @@ -11,11 +11,11 @@ config = lib.mkIf config.TLMS.useBinaryCache { nix.settings = { substituters = [ - "https://dump-dvb.cachix.org" + "https://tlm-solutions.cachix.org" "https://hydra.hq.c3d2.de" ]; trusted-public-keys = [ - "dump-dvb.cachix.org-1:+Dq7gqpQG4YlLA2X3xJsG1v3BrlUGGpVtUKWk0dTyUU=" + "tlm-solutions.cachix.org-1:J7qT6AvoNWPSj+59ed5bNESj35DLJNaROqga1EjVIoA=" "nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps=" ]; }; From 3bca2e756f128dde669e9c633c4a710ca5b00f07 Mon Sep 17 00:00:00 2001 From: Grigory Shipunov Date: Mon, 20 Mar 2023 02:23:54 +0100 Subject: [PATCH 5/5] remove references to old db --- modules/data-hoarder/postgres.nix | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/modules/data-hoarder/postgres.nix b/modules/data-hoarder/postgres.nix index 841c9e6..b2d1a10 100644 --- a/modules/data-hoarder/postgres.nix +++ b/modules/data-hoarder/postgres.nix @@ -33,8 +33,6 @@ $PSQL -c "ALTER ROLE tlms WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password.path})';" $PSQL -c "ALTER ROLE grafana WITH PASSWORD '$(cat ${config.sops.secrets.postgres_password_grafana.path})';" - ##################### New DB ##################### - export DATABASE_URL=postgres:///tlms ${inputs.tlms-rs.packages.x86_64-linux.run-migration-based}/bin/run-migration @@ -46,24 +44,9 @@ # Get graphana to SELECT from tables that might be interesting for it $PSQL -c "GRANT CONNECT ON DATABASE tlms TO grafana;" - $PSQL -c "GRANT SELECT ON r09_telegrams, raw_telegrams, gps_points, trekkie_runs, regions TO grafana;" - + $PSQL -d tlms -c "GRANT SELECT ON r09_telegrams, raw_telegrams, gps_points, trekkie_runs, regions TO grafana;" unset DATABASE_URL - - ##################### Old DB ##################### - # this is old, shit and legacy at this point - export DATABASE_URL=postgres:///dvbdump - - ${inputs.tlms-rs.packages.x86_64-linux.run-migration}/bin/run-migration - - # fixup permissions - $PSQL -c "GRANT ALL ON DATABASE dvbdump TO dvbdump;" - $PSQL -d dvbdump -c "GRANT ALL ON ALL TABLES IN SCHEMA public TO dvbdump;" - $PSQL -d dvbdump -c "GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO dvbdump;" - - unset DATABASE_URL - ''; };