Merge remote-tracking branch 'origin/master' into mailtng

2022-12-18 19:56:04 +01:00 · 2022-12-18 19:56:04 +01:00 · 16351819ca
parent d76a4f1e7a e43ce3b29b
commit 16351819ca
111 changed files with 3126 additions and 1875 deletions
--- a/.sops.yaml
+++ b/.sops.yaml
@ -40,6 +40,7 @@ keys:
  - &leon age1cm0cjk2764s4pv5g7e67as34g9xtcltex96ga87wckndw62wqqlsvkscqc
  - &leoncloud age1aw9s4kcd6ys64ddzzfya9ajzln2tv8pm9uvz6d85v0r6eq4dudqq5vts86
  - &mailtngbert age1lgjvtszpds9flpwsstxdht00c7zlk3mz7nlc5qftyt8rhfdm330qqmhl72
+  - &mastodon age1dcpd6u4psq3hehjyjrt3s7kzmnvxd20vsc8urjcdv6anr5v7ky2sq9rhtt
  - &matemat age15vmz2evhnkn26fyt4vqvgztfrsr2s8qavd2m6zfjmkh84q2g75csnc5kr6
  - &mediawiki age1xjvep7hsnfefgxvuwall8nq0486qu8yknhzwhf0cskw5xlpm8qws9txc56
  - &mucbot age1cqeh03zq0hvz5l78r678q93ey5mlw49lqy4whvgqxgenudth7g6skee6kh
@ -58,6 +59,7 @@ keys:
  - &stream age1j5csp5v5s2g8am47dd85kcke8986e0qc88f0vfgd3kmvwu8azg3smslk92
  - &storage-ng age1qjvds58pedjdk9rj0yqfvad4xhpteapr9chvfucwcgwrsr8n7axqyhg2vu
  - &ticker age1kdrpaqsy7gdnf80fpq6qrrc98nqjuzzlqx955uk2pkky3xcxky8sw9cdjl
+  - &prometheus age13xhxqulvswuckmpkmy2fgeqd5jx0ar8e2hst33leljt69r6hsvnsrdw63k

 creation_rules:
  - path_regex: config/[^/]+\.yaml$
@ -86,11 +88,13 @@ creation_rules:
      - *leon
      - *leoncloud
      - *mailtngbert
+      - *mastodon
      - *matemat
      - *mediawiki
      - *mucbot
      - *nfsroot
      - *oparl
+      - *prometheus
      - *public-access-proxy
      - *pulsebert
      - *radiobert
@ -127,6 +131,12 @@ creation_rules:
      age:
      - *dn42
      - *polygon-snowflake
+  - path_regex: hosts/dacbert/[^/]+\.yaml$
+    key_groups:
+    - pgp: *admins
+      age:
+      - *dacbert
+      - *polygon-snowflake
  - path_regex: hosts/freifunk/[^/]+\.yaml$
    key_groups:
    - pgp: *admins
@ -163,6 +173,12 @@ creation_rules:
      age:
      - *mailtngbert
      - *polygon-snowflake
+  - path_regex: hosts/mastodon/[^/]+\.yaml$
+    key_groups:
+    - pgp: *admins
+      age:
+      - *mastodon
+      - *polygon-snowflake
  - path_regex: hosts/mediawiki/[^/]+\.yaml$
    key_groups:
    - pgp: *admins
@ -187,3 +203,9 @@ creation_rules:
      age:
      - *storage-ng
      - *polygon-snowflake
+  - path_regex: hosts/prometheus/[^/]+\.yaml$
+    key_groups:
+    - pgp: *admins
+      age:
+      - *prometheus
+      - *polygon-snowflake
--- a/README.md
+++ b/README.md
@ -139,14 +139,123 @@ so the following is all that is needed on a MicroVM-hosting server:
 microvm -Ru $hostname
 ```

-## High Availability Deployment on Nomad
+# Cluster deployment with Skyflake

-First, stop and delete `/var/lib/microvm/$NAME` where the
-systemd-managed MicroVMs live, or move the state to
-`/glusterfs/fast/microvms/$NAME`.
+## About

-```sh
-nix run .#nomad-$NAME
+[Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
+Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
+networking with one bridge per VLAN. Persistent storage is replicated
+with Glusterfs.
+
+Recognize nixosConfiguration for our Skyflake deployment by the
+`self.nixosModules.cluster-options` module being included.
+
+## User interface
+
+We use the less-privileged `c3d2@` user for deployment. This flake's
+name on the cluster is `config`. Other flakes can coexist in the same
+user so that we can run separately developed projects like
+*dump-dvb*. *leon* and potentially other users can deploy Flakes and
+MicroVMs without name clashes.
+
+### Deploying
+
+**git push** this repo to any machine in the cluster, preferably to
+Hydra because there building won't disturb any services.
+
+You don't deploy all MicroVMs at once. Instead, Skyflake allows you to
+select NixOS systems by the branches you push to. **You must commit
+before you push!**
+
+**Example:** deploy nixosConfigurations `mucbot` and `sdrweb` (`HEAD` is your
+current commit)
+
+```bash
+git push c3d2@hydra.serv.zentralwerk.org:config HEAD:mucbot HEAD:sdrweb
+```
+
+This will:
+1. Build the configuration on Hydra, refusing the branch update on
+   broken builds (through a git hook)
+2. Copy the MicroVM package and its dependencies to the binary cache
+   that is accessible to all nodes with Glusterfs
+3. Submit one job per MicroVM into the Nomad cluster
+
+*Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
+
+### Updating
+
+**TODO:** how would you like it?
+
+### MicroVM status
+
+```bash
+ssh c3d2@hydra.serv.zentralwerk.org status
+```
+
+## Debugging for cluster admins
+
+### Glusterfs
+
+Glusterfs holds our MicroVMs' state. They *must always be mounted* or
+brains are split.
+
+```bash
+gluster volume info
+gluster volume status
+```
+
+#### Restart glusterd
+
+```bash
+systemctl restart glusterd
+```
+
+#### Remount volumes
+
+```bash
+systemctl restart /glusterfs/fast
+systemctl restart /glusterfs/big
+```
+
+### Nomad
+
+#### Check the cluster state
+
+```shell
+nomad server members
+```
+
+Nomad *servers* **coordinate** the cluster.
+
+Nomad *clients* **run** the tasks.
+
+#### Browse in the terminal
+
+[wander](https://github.com/robinovitch61/wander) and
+[damon](https://github.com/hashicorp/damon) are nice TUIs that are
+preinstalled on our cluster nodes.
+
+#### Browse with a browser
+
+First, tunnel TCP port `:4646` from a cluster server:
+
+```bash
+ssh -L 4646:localhost:4646 root@server10.cluster.zentralwerk.org
+```
+
+Then, visit https://localhost:4646 for for full klickibunti.
+
+#### Reset the Nomad state on a node
+
+After upgrades, Nomad servers may fail rejoining the cluster. Do this
+to make a *Nomad server* behave like a newborn:
+
+```shell
+systemctl stop nomad
+rm -rf /var/lib/nomad/server/raft/
+systemctl start nomad
 ```

 # Secrets management
--- a/flake.lock
+++ b/flake.lock
@ -10,11 +10,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1663176622,
-        "narHash": "sha256-ahmQXwS2P34x7PxXt8Ve2ZVKJHW6yP1m/nZoo8sHwmE=",
-        "ref": "master",
-        "rev": "b56ed86e45b2a8cdf811f2659644192a69ab5818",
-        "revCount": 293,
+        "lastModified": 1669920985,
+        "narHash": "sha256-Ff9FxYqYNVovOCDcECGKHoiqpIOUvF6/q17H+k06iXw=",
+        "ref": "refs/heads/master",
+        "rev": "5bef189c308df9dda1449a8305a7092fb5c77827",
+        "revCount": 298,
        "type": "git",
        "url": "https://gitea.nek0.eu/nek0/affection"
      },
@ -23,9 +23,37 @@
        "url": "https://gitea.nek0.eu/nek0/affection"
      }
    },
+    "alert2muc": {
+      "inputs": {
+        "naersk": [
+          "naersk"
+        ],
+        "nixpkgs": [
+          "nixos"
+        ],
+        "utils": [
+          "flake-utils"
+        ]
+      },
+      "locked": {
+        "lastModified": 1671329943,
+        "narHash": "sha256-7gFF8z1ww+LoC9Pk5hflvnlrzIf8gp7UbL6o8Xyl4Dw=",
+        "ref": "refs/heads/main",
+        "rev": "0ae1024cc7bf45dceb03f089f3e5485c0a43b860",
+        "revCount": 14,
+        "type": "git",
+        "url": "https://gitea.c3d2.de/astro/alert2muc"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://gitea.c3d2.de/astro/alert2muc"
+      }
+    },
    "bevy-julia": {
      "inputs": {
-        "naersk": "naersk",
+        "naersk": [
+          "naersk"
+        ],
        "nixpkgs": [
          "nixos"
        ],
@ -91,11 +119,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1668535353,
-        "narHash": "sha256-cVe++C4wg2CG80qjjSVG4H1udcRi7sbdU/xMGC5WSxc=",
+        "lastModified": 1670028698,
+        "narHash": "sha256-JjmRgUg5004snQL03r5n4TjJmyD4qiwKdPDmx/1GeFw=",
        "ref": "main",
-        "rev": "a12dc753b3f3d145883e1de7a0c12407bbe97288",
-        "revCount": 111,
+        "rev": "908fb9c8d3d2b0fbe5a17895639ef478f81e3f8d",
+        "revCount": 153,
        "type": "git",
        "url": "https://gitea.c3d2.de/astro/caveman.git"
      },
@ -113,11 +141,11 @@
        "rust-analyzer-src": "rust-analyzer-src"
      },
      "locked": {
-        "lastModified": 1668493618,
-        "narHash": "sha256-Pw8dzHPI3My/nWthhWlD6nVLVPDAi9+NODvayVhKGKk=",
+        "lastModified": 1671171782,
+        "narHash": "sha256-G7FNiYVl/jOkvNEhLO8O7uq0MuhFLKGMsCxFjqATVc0=",
        "owner": "nix-community",
        "repo": "fenix",
-        "rev": "3e59a48c3171664ea0797f28273f9929a2335617",
+        "rev": "d3eaf97d81161bea9177cc80e07d26ba2d96569f",
        "type": "github"
      },
      "original": {
@ -144,11 +172,11 @@
    "harmonia": {
      "flake": false,
      "locked": {
-        "lastModified": 1668458039,
-        "narHash": "sha256-w4OCjMRPrPIY0dlUkiA1XPSmRstnNSRVmglw7hfSs3s=",
+        "lastModified": 1670319728,
+        "narHash": "sha256-7hsq6Sv06UcIjjlZTFlsYWDfGrc9u77OAr25SjnvZ4A=",
        "owner": "helsinki-systems",
        "repo": "harmonia",
-        "rev": "9b61645fce85a7abfe6e812f68df085d688f9711",
+        "rev": "f2476198fb8236c7c7eb432aab4472083cb9831d",
        "type": "github"
      },
      "original": {
@ -175,7 +203,7 @@
      "locked": {
        "lastModified": 1657923513,
        "narHash": "sha256-YzHPow09B9uSdybUxP5lQn2hXk90Q6oTDL6UXzD0/+k=",
-        "ref": "master",
+        "ref": "refs/heads/master",
        "rev": "f7cf04a7ad47e388121f0771651fec0df91407f3",
        "revCount": 61,
        "type": "git",
@ -237,11 +265,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1668518796,
-        "narHash": "sha256-DyteijJn0JZphJdQaHpPWxvkKrBPvCW3B53tCasDX8c=",
+        "lastModified": 1670897870,
+        "narHash": "sha256-nWLU48WlhVYm53cTUEDX8mZwZqdzObO6299hrCmdYcU=",
        "owner": "astro",
        "repo": "microvm.nix",
-        "rev": "5181933ca7bbaad37ceb82a848630ea3b30d522b",
+        "rev": "b12b4d426e1050479e9d571c81cda2b7ae0256da",
        "type": "github"
      },
      "original": {
@ -251,35 +279,17 @@
      }
    },
    "naersk": {
-      "inputs": {
-        "nixpkgs": "nixpkgs"
-      },
-      "locked": {
-        "lastModified": 1662220400,
-        "narHash": "sha256-9o2OGQqu4xyLZP9K6kNe1pTHnyPz0Wr3raGYnr9AIgY=",
-        "owner": "nix-community",
-        "repo": "naersk",
-        "rev": "6944160c19cb591eb85bbf9b2f2768a935623ed3",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "naersk",
-        "type": "github"
-      }
-    },
-    "naersk_2": {
      "inputs": {
        "nixpkgs": [
          "nixos"
        ]
      },
      "locked": {
-        "lastModified": 1662220400,
-        "narHash": "sha256-9o2OGQqu4xyLZP9K6kNe1pTHnyPz0Wr3raGYnr9AIgY=",
+        "lastModified": 1671096816,
+        "narHash": "sha256-ezQCsNgmpUHdZANDCILm3RvtO1xH8uujk/+EqNvzIOg=",
        "owner": "nix-community",
        "repo": "naersk",
-        "rev": "6944160c19cb591eb85bbf9b2f2768a935623ed3",
+        "rev": "d998160d6a076cfe8f9741e56aeec7e267e3e114",
        "type": "github"
      },
      "original": {
@ -288,28 +298,6 @@
        "type": "github"
      }
    },
-    "naersk_3": {
-      "inputs": {
-        "nixpkgs": [
-          "ticker",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1659610603,
-        "narHash": "sha256-LYgASYSPYo7O71WfeUOaEUzYfzuXm8c8eavJcel+pfI=",
-        "owner": "nix-community",
-        "repo": "naersk",
-        "rev": "c6a45e4277fa58abd524681466d3450f896dc094",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "naersk",
-        "rev": "c6a45e4277fa58abd524681466d3450f896dc094",
-        "type": "github"
-      }
-    },
    "newNixpkgs": {
      "locked": {
        "lastModified": 1647380550,
@ -329,7 +317,7 @@
    "nix": {
      "inputs": {
        "lowdown-src": "lowdown-src",
-        "nixpkgs": "nixpkgs_2",
+        "nixpkgs": "nixpkgs",
        "nixpkgs-regression": "nixpkgs-regression"
      },
      "locked": {
@ -349,43 +337,27 @@
    },
    "nixos": {
      "locked": {
-        "lastModified": 1668459637,
-        "narHash": "sha256-HqnWCKujmtu8v0CjzOT0sr7m2AR7+vpbZJOp1R0rodY=",
-        "owner": "nixos",
+        "lastModified": 1671217927,
+        "narHash": "sha256-VjoidSKImZGWYqL1Z+ntfEzqeMjNX4zxcBr9PIqORwM=",
+        "owner": "SuperSandro2000",
        "repo": "nixpkgs",
-        "rev": "16f4e04658c2ab10114545af2f39db17d51bd1bd",
+        "rev": "0a597dc83e81526de48488088c92349d7170445c",
        "type": "github"
      },
      "original": {
-        "owner": "nixos",
-        "ref": "nixos-22.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixos-armv6": {
-      "locked": {
-        "lastModified": 1664701736,
-        "narHash": "sha256-Va3NyZ+uyZztu506qM+sLxd69DBzN5CdoCAu1lzVk0U=",
-        "owner": "rnhmjoj",
-        "repo": "nixpkgs",
-        "rev": "10b75bee02bc7c25e596847357c70b277c534588",
-        "type": "github"
-      },
-      "original": {
-        "owner": "rnhmjoj",
-        "ref": "pr-fix-armv6",
+        "owner": "SuperSandro2000",
+        "ref": "nixos-22.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixos-hardware": {
      "locked": {
-        "lastModified": 1668334946,
-        "narHash": "sha256-omMbUj4r5DVBWh7KxkoO/Z/1V1shVR6Ls4jXNB4mr3U=",
+        "lastModified": 1671183612,
+        "narHash": "sha256-Q6so0tBGEb9Bhx++FP6cJQ+K83hOZ99ffmcdcWtDS14=",
        "owner": "nixos",
        "repo": "nixos-hardware",
-        "rev": "e0452b33ab0ef16ffe075e980644ed92a6a200bb",
+        "rev": "488931efb69a50307fa0d71e23e78c8706909416",
        "type": "github"
      },
      "original": {
@ -394,37 +366,7 @@
        "type": "github"
      }
    },
-    "nixos-unstable": {
-      "locked": {
-        "lastModified": 1668417584,
-        "narHash": "sha256-yeuEyxKPwsm5fIHN49L/syn9g5coxnPp3GsVquhrv5A=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "013fcdd106823416918004bb684c3c186d3c460f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1663264531,
-        "narHash": "sha256-2ncO5chPXlTxaebDlhx7MhL0gOEIWxzSyfsl0r0hxQk=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "454887a35de6317a30be284e8adc2d2f6d8a07c4",
-        "type": "github"
-      },
-      "original": {
-        "id": "nixpkgs",
-        "type": "indirect"
-      }
-    },
-    "nixpkgs-mobilizon": {
+    "nixos-mobilizon": {
      "locked": {
        "lastModified": 1664466500,
        "narHash": "sha256-FvEUAKkf0PDZ2j2qIbI4+3oPTnuQq4HdX00iqBkvKOU=",
@ -440,6 +382,53 @@
        "type": "github"
      }
    },
+    "nixos-unstable": {
+      "locked": {
+        "lastModified": 1671108576,
+        "narHash": "sha256-6ggOL6KoaELNA1562tnPjtAnQ9SwsKRTgeuaXvPzCwI=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "0f5996b524c91677891a432cc99c7567c7c402b1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixos-unstable-simd": {
+      "locked": {
+        "lastModified": 1671211489,
+        "narHash": "sha256-0AbRULr8+U70TPLiMz7PK8mTp1lD+Ct8VKElZ0WmPzE=",
+        "owner": "SuperSandro2000",
+        "repo": "nixpkgs",
+        "rev": "8ef6d44870fbc02b5c43922b8dbe95aa8b53091b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "SuperSandro2000",
+        "ref": "nixos-unstable-simd",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1645296114,
+        "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-21.05-small",
+        "type": "indirect"
+      }
+    },
    "nixpkgs-regression": {
      "locked": {
        "lastModified": 1643052045,
@ -455,21 +444,6 @@
        "type": "indirect"
      }
    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1645296114,
-        "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
-        "type": "github"
-      },
-      "original": {
-        "id": "nixpkgs",
-        "ref": "nixos-21.05-small",
-        "type": "indirect"
-      }
-    },
    "oparl-scraper": {
      "flake": false,
      "locked": {
@ -490,11 +464,11 @@
    "openwrt": {
      "flake": false,
      "locked": {
-        "lastModified": 1668297972,
-        "narHash": "sha256-i39KfzBoNiiScd1M8kV/5WrGzEgG6+PxsFxoW+DT9UQ=",
+        "lastModified": 1670552749,
+        "narHash": "sha256-ZhbNee6F+vJRirskdiYpXVRldJTHSbClLiYtJe4wgqE=",
        "ref": "openwrt-21.02",
-        "rev": "079ce0413a1e3c19dd00be1b90de737c2bc09223",
-        "revCount": 51164,
+        "rev": "784565b6a0ad482b19c4851b0ce8ffeddb7010c1",
+        "revCount": 51264,
        "type": "git",
        "url": "https://git.openwrt.org/openwrt/openwrt.git"
      },
@ -511,11 +485,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1668458369,
-        "narHash": "sha256-Mv4F2AgOIpGyvfoHQtw/sDg3hU7O0K1P9ercnfrwnEQ=",
+        "lastModified": 1671207552,
+        "narHash": "sha256-7C3QXWM615TJJqNQlKS27HaQh/m6vctKhZ1fw9ex7xE=",
        "owner": "astro",
        "repo": "nix-openwrt-imagebuilder",
-        "rev": "818db40dd8a0a22b029823ba477c5c3301bd7534",
+        "rev": "a53edbc364df74d467d4f1e8d45dca6ee4e92ced",
        "type": "github"
      },
      "original": {
@ -527,6 +501,7 @@
    "root": {
      "inputs": {
        "affection-src": "affection-src",
+        "alert2muc": "alert2muc",
        "bevy-julia": "bevy-julia",
        "bevy-mandelbrot": "bevy-mandelbrot",
        "caveman": "caveman",
@ -536,12 +511,12 @@
        "heliwatch": "heliwatch",
        "hydra-ca": "hydra-ca",
        "microvm": "microvm",
-        "naersk": "naersk_2",
+        "naersk": "naersk",
        "nixos": "nixos",
-        "nixos-armv6": "nixos-armv6",
        "nixos-hardware": "nixos-hardware",
+        "nixos-mobilizon": "nixos-mobilizon",
        "nixos-unstable": "nixos-unstable",
-        "nixpkgs-mobilizon": "nixpkgs-mobilizon",
+        "nixos-unstable-simd": "nixos-unstable-simd",
        "oparl-scraper": "oparl-scraper",
        "openwrt": "openwrt",
        "openwrt-imagebuilder": "openwrt-imagebuilder",
@ -562,11 +537,11 @@
    "rust-analyzer-src": {
      "flake": false,
      "locked": {
-        "lastModified": 1668182250,
-        "narHash": "sha256-PYGaOCiFvnJdVz+ZCaKF8geGdffXjJUNcMwaBHv0FT4=",
+        "lastModified": 1671029659,
+        "narHash": "sha256-2sqAgHcLWpNGVHG2LQIXgRMuqgCArOD9frQSrrXrwSA=",
        "owner": "rust-lang",
        "repo": "rust-analyzer",
-        "rev": "45ec315e01dc8dd1146dfeb65f0ef6e5c2efed78",
+        "rev": "95671d53ea3063da5316784af9ef69c71219f320",
        "type": "github"
      },
      "original": {
@ -586,11 +561,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1668479979,
-        "narHash": "sha256-UI+JUCBaMpn+5Y1hSePmndbYX5zu0+bavlfzrhPrGEk=",
+        "lastModified": 1671157233,
+        "narHash": "sha256-gvQaOKaV1UK6IzsFzkVLsEavGxnAsQFT3zUqcg0RXLU=",
        "owner": "oxalica",
        "repo": "rust-overlay",
-        "rev": "2342f70f7257046effc031333c4cfdea66c91d82",
+        "rev": "7da2f6b3a0c32f661cb2864d7fbd1d7e6f0c7543",
        "type": "github"
      },
      "original": {
@ -604,7 +579,7 @@
      "locked": {
        "lastModified": 1665446321,
        "narHash": "sha256-GuZr+BCAIe+UYmQrLHaVr8iRRajn5nSdWyqhjWDIX1Y=",
-        "ref": "master",
+        "ref": "refs/heads/master",
        "rev": "3700761dd06f271ef26261ed2a90dce8c22b6dca",
        "revCount": 61,
        "type": "git",
@ -624,7 +599,7 @@
      "locked": {
        "lastModified": 1659890996,
        "narHash": "sha256-xURgGoznCPmpX35dn5AXcyNYicVn5ruvUKxfIMMiu8o=",
-        "ref": "master",
+        "ref": "refs/heads/master",
        "rev": "5ca106f648bef15d9954d956bda336eea28e8d75",
        "revCount": 149,
        "type": "git",
@ -645,11 +620,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1668537992,
-        "narHash": "sha256-dktkznGkVUtOXyA19J4YoSiyhBkCinqH8LDnU2o/rmw=",
+        "lastModified": 1670188372,
+        "narHash": "sha256-lKjP3rYsmiXb9kEU+stcsm48dTEMC6Ed+rLEClEl+Vs=",
        "owner": "astro",
        "repo": "skyflake",
-        "rev": "4ccb72c616212ef2149458dade8d1199dc69477a",
+        "rev": "c78a3e8f64930bf5c48b0f75e577e4294d8750c6",
        "type": "github"
      },
      "original": {
@ -663,16 +638,16 @@
        "nixpkgs": [
          "nixos"
        ],
-        "nixpkgs-22_05": [
+        "nixpkgs-stable": [
          "nixos"
        ]
      },
      "locked": {
-        "lastModified": 1668311578,
-        "narHash": "sha256-nF6mwSbVyvnlIICWFZlADegWdTsgrk1pZnA/0VqByNw=",
+        "lastModified": 1670149631,
+        "narHash": "sha256-rwmtlxx45PvOeZNP51wql/cWjY3rqzIR3Oj2Y+V7jM0=",
        "owner": "Mic92",
        "repo": "sops-nix",
-        "rev": "39f0fe57f1ef78764c1abc1de145f091fee1bbbb",
+        "rev": "da98a111623101c64474a14983d83dad8f09f93d",
        "type": "github"
      },
      "original": {
@ -732,7 +707,9 @@
        "fenix": [
          "fenix"
        ],
-        "naersk": "naersk_3",
+        "naersk": [
+          "naersk"
+        ],
        "nixpkgs": [
          "nixos"
        ],
@ -743,7 +720,7 @@
      "locked": {
        "lastModified": 1666559258,
        "narHash": "sha256-m4f5QQbE+usnEnM6CJ+nCgXcsi9mm0cCwFm2BhJ0pvQ=",
-        "ref": "master",
+        "ref": "refs/heads/master",
        "rev": "22ecb2b375bebffdfb1af3435a4c4486e6dd923b",
        "revCount": 108,
        "type": "git",
@ -785,7 +762,7 @@
      "locked": {
        "lastModified": 1663279525,
        "narHash": "sha256-lUq4CY//ISplh/4i33nOU7cchpxKrw5V8mVdRnHMBaA=",
-        "ref": "master",
+        "ref": "refs/heads/master",
        "rev": "6d8d2cb1268d26add05baa3f21c325cfe051add3",
        "revCount": 342,
        "type": "git",
@ -830,11 +807,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1668554331,
-        "narHash": "sha256-QExvepHzH2QtHjeu1ZG2gIWh3bOxwsNsO2SBVmGIlRo=",
-        "ref": "master",
-        "rev": "9cb7dbc38595e7c213d193f2adf3cbeffcded74b",
-        "revCount": 1669,
+        "lastModified": 1671215340,
+        "narHash": "sha256-e1+KFpcOxdz0yvThbHszq9+sXhH7SsFGP/dsIpUrvZY=",
+        "ref": "refs/heads/master",
+        "rev": "4f090527d4cc64305add16ba68c6f72a275a507b",
+        "revCount": 1689,
        "type": "git",
        "url": "https://gitea.c3d2.de/zentralwerk/network.git"
      },
--- a/flake.nix
+++ b/flake.nix
@ -2,11 +2,12 @@
  description = "C3D2 NixOS configurations";

  inputs = {
-    nixos.url = "github:nixos/nixpkgs/nixos-22.05";
-    nixpkgs-mobilizon.url = "github:minijackson/nixpkgs/init-mobilizon";
-    nixos-unstable.url = "github:nixos/nixpkgs/nixos-unstable";
+    # use sandro's fork full with cherry-picked fixes
+    nixos.url = "github:SuperSandro2000/nixpkgs/nixos-22.11";
+    nixos-mobilizon.url = "github:minijackson/nixpkgs/init-mobilizon";
    nixos-hardware.url = "github:nixos/nixos-hardware";
-    nixos-armv6.url = "github:rnhmjoj/nixpkgs/pr-fix-armv6";
+    nixos-unstable.url = "github:nixos/nixpkgs/nixos-unstable";
+    nixos-unstable-simd.url = "github:SuperSandro2000/nixpkgs/nixos-unstable-simd";

    affection-src = {
      url = "git+https://gitea.nek0.eu/nek0/affection";
@ -15,6 +16,14 @@
        flake-utils.follows = "flake-utils";
      };
    };
+    alert2muc = {
+      url = "git+https://gitea.c3d2.de/astro/alert2muc";
+      inputs = {
+        naersk.follows = "naersk";
+        nixpkgs.follows = "nixos";
+        utils.follows = "flake-utils";
+      };
+    };
    bevy-mandelbrot = {
      # url = "github:matelab/bevy_mandelbrot";
      url = "git+https://gitea.c3d2.de/astro/bevy-mandelbrot.git?ref=main";
@ -29,9 +38,7 @@
      url = "git+https://gitea.c3d2.de/astro/bevy-julia.git?ref=main";
      inputs = {
        nixpkgs.follows = "nixos";
-        # breaks the build:
-        # naersk.follows = "naersk";
-        # naersk.inputs.nixpkgs.follows = "nixpkgs";
+        naersk.follows = "naersk";
        rust-overlay.follows = "rust-overlay";
      };
    };
@ -129,7 +136,7 @@
      url = "github:Mic92/sops-nix";
      inputs = {
        nixpkgs.follows = "nixos";
-        nixpkgs-22_05.follows = "nixos";
+        nixpkgs-stable.follows = "nixos";
      };
    };
    spacemsg = {
@ -140,8 +147,7 @@
      url = "git+https://gitea.c3d2.de/astro/ticker.git";
      inputs = {
        fenix.follows = "fenix";
-        # TODO: build regression in https://github.com/nix-community/naersk/commit/6944160c19cb591eb85bbf9b2f2768a935623ed3
-        # naersk.follows = "naersk";
+        naersk.follows = "naersk";
        nixpkgs.follows = "nixos";
        utils.follows = "flake-utils";
      };
@ -173,14 +179,9 @@
    };
  };

-  outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, caveman, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
+  outputs = inputs@{ self, alert2muc, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, caveman, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
    let
      inherit (nixos) lib;
-      forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ];
-
-      # all the input flakes for `nix copy` to the build machine,
-      # allowing --override-input
-      inputPaths = lib.escapeShellArgs (builtins.attrValues inputs);

      extractZwHosts = { hosts4, hosts6, ... }:
        lib.recursiveUpdate (
@ -207,348 +208,56 @@
      extraHostRegistry.hosts = import ./host-registry.nix;
      hostRegistry = lib.recursiveUpdate zwHostRegistry extraHostRegistry;

-      getHostAddr = name:
-        let
-          hostConf = hostRegistry.hosts."${name}";
-        in
-          if hostConf ? ip4
-          then hostConf.ip4
-          else if hostConf ? ip6
-          then hostConf.ip6
-          else throw "Host ${name} has no ip4 or ip6 address";
+      # Our custom NixOS builder
+      nixosSystem' =
+        { nixpkgs ? inputs.nixos
+        , modules
+        , extraArgs ? { }
+        , specialArgs ? { }
+        , system ? "x86_64-linux"
+      }@args:
+
+      { inherit args; } // nixpkgs.lib.nixosSystem {
+        inherit specialArgs system;
+
+        modules = [
+          ({ ... }: {
+            _module.args = extraArgs // {
+              inherit hostRegistry inputs zentralwerk;
+            };
+
+            nixpkgs = {
+              overlays = [ self.overlays ];
+            };
+          })
+
+          self.nixosModules.c3d2
+          ./modules/audio-server.nix
+          ./modules/c3d2.nix
+          ./modules/stats.nix
+          ./modules/pi-sensors.nix
+        ] ++ modules;
+      };
    in {
-      overlay = import ./overlay {
-        inherit nixos-unstable;
+      overlays = import ./overlays {
        inherit (inputs) tracer bevy-mandelbrot bevy-julia;
      };

-      legacyPackages = lib.attrsets.mapAttrs (system: pkgs:
+      legacyPackages = lib.attrsets.mapAttrs (_: pkgs:
        pkgs.appendOverlays [
-          fenix.overlay
+          fenix.overlays.default
          naersk.overlay
-          self.overlay
+          self.overlays
        ]) nixos.legacyPackages;

-      packages = lib.attrsets.mapAttrs (system: pkgs:
-        let overlayPkgs = builtins.intersectAttrs (self.overlay {} {}) pkgs;
-        in overlayPkgs //
-        {
-          host-registry = pkgs.runCommand "host-registry" {
-            src = builtins.toFile "host-registry.nix" (
-              lib.generators.toPretty {} hostRegistry
-            );
-          } ''
-            ln -s $src $out
-          '';
+      packages = import ./packages.nix { inherit hostRegistry inputs lib microvm secrets self; };

-          list-upgradable = pkgs.writeScriptBin "list-upgradable" ''
-            #! ${pkgs.runtimeShell}
-
-            NORMAL="\033[0m"
-            RED="\033[0;31m"
-            YELLOW="\033[0;33m"
-            GREEN="\033[0;32m"
-
-            ${pkgs.lib.concatMapStringsSep "\n" (name:
-              let
-                addr = getHostAddr name;
-              in nixos.lib.optionalString (addr != null) ''
-                echo -n -e "${name}: $RED"
-                RUNNING=$(ssh -o PreferredAuthentications=publickey -o StrictHostKeyChecking=accept-new root@"${addr}" "readlink /run/current-system")
-                if [ $? = 0 ] && [ -n "$RUNNING" ]; then
-                  CURRENT=$(nix eval --raw ".#nixosConfigurations.${name}.config.system.build.toplevel" 2>/dev/null)
-                  RUNNING_VER=$(basename $RUNNING|rev|cut -d - -f 1|rev)
-                  RUNNING_DATE=$(echo $RUNNING_VER|cut -d . -f 3)
-                  CURRENT_VER=$(basename $CURRENT|rev|cut -d - -f 1|rev)
-                  CURRENT_DATE=$(echo $CURRENT_VER|cut -d . -f 3)
-
-                  if [ "$RUNNING" = "$CURRENT" ]; then
-                    echo -e "$GREEN"current"$NORMAL $RUNNING_VER"
-                  elif [ $RUNNING_DATE -gt $CURRENT_DATE ]; then
-                    echo -e "$GREEN"newer"$NORMAL $RUNNING_VER > $CURRENT_VER"
-                  elif [ "$RUNNING_VER" = "$CURRENT_VER" ]; then
-                    echo -e "$YELLOW"modified"$NORMAL $RUNNING_VER"
-                  elif [ -n "$RUNNING_VER" ]; then
-                    echo -e "$RED"outdated"$NORMAL $RUNNING_VER < $CURRENT_VER"
-                  else
-                    echo -e "$RED"error"$NORMAL $RUNNING_VER"
-                  fi
-                fi
-                echo -n -e "$NORMAL"
-              '') (builtins.attrNames self.nixosConfigurations)}
-          '';
-
-          prebuild-all = pkgs.runCommand "prebuild-all" {
-            preferLocalBuild = true;
-          } ''
-            mkdir $out
-
-            ${pkgs.lib.concatMapStrings (name: ''
-              ln -s ${self.nixosConfigurations."${name}".config.system.build.toplevel} name
-            '') (builtins.attrNames self.nixosConfigurations)}
-          '';
-
-          prebuild-all-remote = pkgs.writeScriptBin "prebuild-all" ''
-            #!${pkgs.runtimeShell} -e
-
-            nix copy --no-check-sigs --to ssh-ng://$1 ${inputPaths}
-
-            set -x
-            ssh $1 -- nix build -L --no-link ${
-              pkgs.lib.concatMapStringsSep " " (name:
-                "${self}#nixosConfigurations.${name}.config.system.build.toplevel"
-              ) (builtins.attrNames self.nixosConfigurations)
-            }
-          '';
-
-        } //
-
-        builtins.foldl' (result: host: result // {
-          # TODO: check if the ethernet address is reachable and if not,
-          # execute wol on a machine in HQ.
-          "${host}-wake" = pkgs.writeScriptBin "${host}-wake" ''
-            #!${pkgs.runtimeShell}
-            exec ${pkgs.wol}/bin/wol ${hostRegistry.hosts."${host}".ether}
-          '';
-        }) {} (builtins.attrNames (nixos.lib.filterAttrs (_: { wol ? false, ... }: wol) hostRegistry.hosts)) //
-
-        builtins.foldl' (result: name:
-          let
-            host = getHostAddr name;
-            target = ''root@"${host}"'';
-            rebuildArg = "--flake ${self}#${name} --option extra-substituters https://nix-serve.hq.c3d2.de";
-            hostConfig = self.nixosConfigurations."${name}".config;
-            profile = hostConfig.system.build.toplevel;
-            # let /var/lib/microvm/*/flake point to the flake-update branch so that
-            # `microvm -u $NAME` updates to what hydra built today.
-            selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config?ref=flake-update";
-          in result // {
-            # Generate a small script for copying this flake to the
-            # remote machine and bulding and switching there.
-            # Can be run with `nix run c3d2#…-nixos-rebuild switch`
-            "${name}-nixos-rebuild" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
-              #!${pkgs.runtimeShell} -ex
-              [[ $(ssh ${target} cat /etc/hostname) == ${name} ]]
-              nix copy --no-check-sigs --to ssh-ng://${target} ${inputPaths}
-              ssh ${target} nixos-rebuild ${rebuildArg} "$@"
-            '';
-
-            "${name}-nixos-rebuild-hydra" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
-              #!${pkgs.runtimeShell} -e
-              echo Copying Flakes
-              nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${inputPaths}
-              echo Building on Hydra
-              ssh root@hydra.serv.zentralwerk.org -- \
-                nix build -L -o /tmp/nixos-system-${name} \
-                ${self}#nixosConfigurations.${name}.config.system.build.toplevel
-              echo Built. Obtaining link to data
-              TOPLEVEL=$(ssh root@hydra.serv.zentralwerk.org \
-                readlink /tmp/nixos-system-${name})
-              echo Checking target ${name}
-              ssh ${target} -- bash -e <<EOF
-              [[ \$(cat /etc/hostname) == ${name} ]]
-              echo Copying data from Hydra to ${name}
-              nix copy --from https://nix-serve.hq.c3d2.de \
-                $TOPLEVEL
-              echo Activation on ${name}: "$@"
-              nix-env -p /nix/var/nix/profiles/system --set $TOPLEVEL
-              $TOPLEVEL/bin/switch-to-configuration "$@"
-              EOF
-            '';
-
-            "${name}-nixos-rebuild-local" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
-               #!${pkgs.runtimeShell} -ex
-               [[ $1 == build || $(ssh ${target} cat /etc/hostname) == ${name} ]]
-               ${pkgs.nixos-rebuild}/bin/nixos-rebuild ${rebuildArg} --target-host ${target} --use-remote-sudo "$@"
-            '';
-
-            "${name}-cleanup" = pkgs.writeScriptBin "${name}-cleanup" ''
-              #!${pkgs.runtimeShell} -ex
-              ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
-            '';
-
-            "microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" ''
-              #!${pkgs.runtimeShell} -e
-
-              ${lib.optionalString (! builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]) ''
-                echo "MicroVM must be configured to proper server" >&2
-                exit 1
-              ''}
-
-              ${hostConfig.system.build.copyToServer} ${inputPaths}
-
-              ${hostConfig.system.build.runOnServer} bash -e <<END
-              mkdir -p /var/lib/microvms/${name}
-              cd /var/lib/microvms/${name}
-              chown root:kvm .
-              chmod 0775 .
-
-              rm -f old
-              [ -e current ] && cp --no-dereference current old
-
-              nix build -L \
-                -o current \
-                ${self}#nixosConfigurations.${name}.config.microvm.declaredRunner
-              echo '${selfRef}' > flake
-
-              [ -e old ] && nix store diff-closures ./old ./current
-              ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${name}
-              ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${name}
-              ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${name}
-
-              systemctl restart microvm@${name}.service
-              END
-            '';
-
-            "microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" ''
-              #!${pkgs.runtimeShell} -e
-
-              ${lib.optionalString (! builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]) ''
-                echo "MicroVM must be configured to proper server" >&2
-                exit 1
-              ''}
-
-              ${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner}
-
-              ${hostConfig.system.build.runOnServer} bash -e <<END
-              mkdir -p /var/lib/microvms/${name}
-              cd /var/lib/microvms/${name}
-              chown root:kvm .
-              chmod 0775 .
-
-              rm -f old
-              [ -e current ] && cp --no-dereference current old
-
-              ln -sfT ${hostConfig.microvm.declaredRunner} current
-              echo '${selfRef}' > flake
-
-              [ -e old ] && nix store diff-closures ./old ./current
-              ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${name}
-              ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${name}
-              ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${name}
-
-              systemctl restart microvm@${name}.service
-              END
-            '';
-
-            "nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
-              #!${pkgs.runtimeShell} -e
-
-              ${lib.optionalString (hostConfig.c3d2.deployment.server or null == "nomad") ''
-                echo "MicroVM must be configured for nomad" >&2
-                exit 1
-              ''}
-
-              echo Copying Flakes
-              nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}
-
-              echo Building on Hydra
-              ssh root@hydra.serv.zentralwerk.org -- \
-                nix build -L -o /tmp/microvm-${name}.job \
-                ${self}#nixosConfigurations.${name}.config.system.build.nomadJob
-
-              echo -n Built. Obtaining path...
-              JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
-                readlink /tmp/microvm-${name}.job)
-              echo \ $JOB
-
-              for h in server9 server10 ; do
-                echo Sharing with $h
-                ssh root@$h.cluster.zentralwerk.org -- \
-                  bash -e <<EOF &
-                nix copy --from https://nix-serve.hq.c3d2.de $JOB
-                mkdir -p /glusterfs/fast/microvms/${name}
-                chown microvm:kvm /glusterfs/fast/microvms/${name}
-                chmod 0775 /glusterfs/fast/microvms/${name}
-                mkdir -p /nix/var/nix/gcroots/microvm
-                rm -f /nix/var/nix/gcroots/microvm/${name}
-                ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
-              EOF
-              done
-              wait
-
-              echo Now starting the job
-              ssh root@hydra.serv.zentralwerk.org -- \
-                nomad run -detach $JOB
-            '';
-          }) {} (builtins.attrNames self.nixosConfigurations) //
-
-        builtins.foldl' (result: host:
-          let
-            inherit (self.nixosConfigurations.${host}) config;
-          in
-            result // {
-              # boot any machine in a microvm
-              "${host}-vm" = (self.nixosConfigurations.${host}
-                .extendModules {
-                  modules = [ {
-                    microvm = {
-                      mem = nixos.lib.mkForce 2048;
-                      hypervisor = nixos.lib.mkForce "qemu";
-                      socket = nixos.lib.mkForce null;
-                      shares = nixos.lib.mkForce [ {
-                        tag = "ro-store";
-                        source = "/nix/store";
-                        mountPoint = "/nix/.ro-store";
-                      } ];
-                      interfaces = nixos.lib.mkForce [ {
-                        type = "user";
-                        id = "eth0";
-                        mac = "02:23:de:ad:be:ef";
-                      } ];
-                    };
-                    boot.isContainer = lib.mkForce false;
-                    users.users.root.password = "";
-                    fileSystems."/".fsType = lib.mkForce "tmpfs";
-                    services.getty.helpLine = ''
-                        Log in as "root" with an empty password.
-                        Use "reboot" to shut qemu down.
-                      '';
-                  } ] ++ lib.optionals (! config ? microvm) [
-                    microvm.nixosModules.microvm
-                  ];
-                })
-                .config.microvm.declaredRunner;
-
-              "${host}-tftproot" =
-                if config.system.build ? tftproot
-                then config.system.build.tftproot
-                else throw "No tftproot for ${host}";
-            }
-        ) {} (builtins.attrNames self.nixosConfigurations)
-      ) self.legacyPackages;
-
-      nixosConfigurations = let
-        nixosSystem' =
-          # Our custom NixOS builder
-          { nixpkgs ? inputs.nixos, modules, extraArgs ? {}, specialArgs ? { }, system ? "x86_64-linux" }:
-          nixpkgs.lib.nixosSystem {
-            inherit specialArgs system;
-
-            modules = [
-              ({ pkgs, ... }: {
-                _module.args = extraArgs // {
-                  inherit hostRegistry inputs zentralwerk;
-                };
-                nixpkgs = {
-                  overlays = [ self.overlay ];
-                };
-              })
-
-              self.nixosModules.c3d2
-              ./modules/audio-server.nix
-              ./modules/c3d2.nix
-              ./modules/stats.nix
-              ./modules/pi-sensors.nix
-            ] ++ modules;
-          };
-      in {
+      nixosConfigurations = {
        auth = nixosSystem' {
          modules = [
            self.nixosModules.microvm
            ./hosts/auth
          ];
-          nixpkgs = inputs.nixos-unstable;
        };

        broker = nixosSystem' {
@ -590,7 +299,6 @@
            self.nixosModules.microvm
            ./hosts/gitea
          ];
-          nixpkgs = inputs.nixos-unstable;
        };

        glotzbert = nixosSystem' {
@ -608,7 +316,6 @@
            self.nixosModules.microvm
            ./hosts/hedgedoc
          ];
-          nixpkgs = inputs.nixos-unstable;
        };

        pulsebert = nixosSystem' {
@ -624,12 +331,11 @@
              };
            }
          ];
-          nixpkgs = inputs.nixos-unstable;
        };

        radiobert = nixosSystem' {
          modules = [
-            ({ modulesPath, ... }:
+            ({ ... }:
            {
              nixpkgs.overlays = [ heliwatch.overlay ];
            })
@ -674,7 +380,6 @@
        };

        schalter = nixosSystem' {
-          nixpkgs = inputs.nixos-armv6;
          modules = [
            "${nixos}/nixos/modules/installer/sd-card/sd-image-raspberrypi.nix"
            ({ lib, ... }: {
@ -751,7 +456,6 @@
            self.nixosModules.microvm
            ./hosts/grafana
          ];
-          nixpkgs = nixos-unstable;
        };

        hydra = nixosSystem' {
@ -767,7 +471,6 @@
              };
            }
          ];
-          nixpkgs = nixos-unstable;
        };

        mucbot = nixosSystem' {
@ -805,14 +508,14 @@

        stream = nixosSystem' {
          modules = [
-            self.nixosModules.microvm
+            self.nixosModules.cluster-options
            ./hosts/stream
          ];
        };

        mobilizon = nixosSystem' {
          # TODO: pending https://github.com/NixOS/nixpkgs/pull/119132
-          nixpkgs = inputs.nixpkgs-mobilizon;
+          nixpkgs = inputs.nixos-mobilizon;
          modules = [
            self.nixosModules.microvm
            ./hosts/mobilizon
@ -835,7 +538,7 @@
            heliwatch.nixosModules.heliwatch
            ./hosts/sdrweb
          ];
-          nixpkgs = nixos-unstable;
+          # nixpkgs = nixos-unstable;
        };

        bind = nixosSystem' {
@ -862,6 +565,16 @@
          ];
        };

+        server8 = nixosSystem' {
+          modules = [
+            ./hosts/server8
+            self.nixosModules.cluster-network
+            self.nixosModules.cluster
+            skyflake.nixosModules.default
+            { _module.args = { inherit self; }; }
+          ];
+        };
+
        server9 = nixosSystem' {
          modules = [
            ./hosts/server9
@ -872,7 +585,6 @@
            skyflake.nixosModules.default
            { _module.args = { inherit self; }; }
          ];
-          nixpkgs = nixos-unstable;
        };

        server10 = nixosSystem' {
@ -885,7 +597,6 @@
            skyflake.nixosModules.default
            { _module.args = { inherit self; }; }
          ];
-          nixpkgs = nixos-unstable;
        };

        oparl = nixosSystem' {
@ -900,14 +611,14 @@

        leon = nixosSystem' {
          modules = [
-            self.nixosModules.microvm
+            self.nixosModules.cluster-options
            ./hosts/leon
          ];
        };

        leoncloud = nixosSystem' {
          modules = [
-            self.nixosModules.microvm
+            self.nixosModules.cluster-options
            ./hosts/leoncloud
          ];
        };
@ -935,7 +646,7 @@
              };
            }
          ];
-          nixpkgs = nixos-unstable;
+          # nixpkgs = nixos-unstable;
          system = "x86_64-linux";
        };

@ -951,15 +662,15 @@
            self.nixosModules.microvm
            ./hosts/mediawiki
          ];
-          nixpkgs = nixos-unstable;
+          # nixpkgs = nixos-unstable;
        };

        gnunet = nixosSystem' {
          modules = [
-            self.nixosModules.microvm
+            self.nixosModules.cluster-options
            ./hosts/gnunet
          ];
-          nixpkgs = nixos-unstable;
+          # nixpkgs = nixos-unstable;
        };

        zengel = nixosSystem' {
@ -971,7 +682,7 @@

        sshlog = nixosSystem' {
          modules = [
-            self.nixosModules.microvm
+            self.nixosModules.cluster-options
            sshlogd.nixosModule
            ./hosts/sshlog
          ];
@ -992,9 +703,30 @@
          ];
        };

+        owncast = nixosSystem' {
+          modules = [
+            self.nixosModules.cluster-options
+            ./hosts/owncast
+          ];
+          # nixpkgs = nixos-unstable;
+        };
+
+        prometheus = nixosSystem' {
+          modules = [
+            self.nixosModules.cluster-options
+            alert2muc.nixosModules.default
+            ./hosts/prometheus
+          ];
+        };
+
+        mastodon = nixosSystem' {
+          modules = [
+            self.nixosModules.cluster-options
+            ./hosts/mastodon
+          ];
+        };
      };

-      nixosModule = self.nixosModules.c3d2;
      nixosModules = {
        c3d2 = {
          imports = [
@ -1004,7 +736,7 @@
            ./modules/backup.nix
            ./modules/c3d2.nix
            ./modules/nncp.nix
-            ./modules/logging.nix
+            ./modules/monitoring.nix
            ./modules/stats.nix
          ];
          c3d2.hosts = hostRegistry.hosts;
@ -1022,7 +754,7 @@
        nncp = ./modules/nncp.nix;
        plume = {
          imports = [ ./modules/plume.nix ];
-          nixpkgs.overlays = [ fenix.overlay naersk.overlay ];
+          nixpkgs.overlays = [ fenix.overlays.default naersk.overlay ];
        };
        rpi-netboot = ./modules/rpi-netboot.nix;
      };
@ -1039,15 +771,26 @@
      ) sops-nix.packages;

      hydraJobs =
-        builtins.mapAttrs (_: nixos.lib.hydraJob) (
-          builtins.mapAttrs (_: nixosSystem:
-            if nixosSystem.config ? microvm.declaredRunner
-            then nixosSystem.config.microvm.declaredRunner
-            else nixosSystem.config.system.build.toplevel
+        lib.mapAttrs (_: nixos.lib.hydraJob) (
+          let
+            getBuildEntryPoint = _: nixosSystem: nixosSystem.config.microvm.declaredRunner or nixosSystem.config.system.build.toplevel;
+          in
+          lib.mapAttrs getBuildEntryPoint self.nixosConfigurations
+          // lib.mapAttrs' (hostname: nixosSystem: lib.nameValuePair
+            # job display name
+            (hostname + "-nox")
+            (getBuildEntryPoint null (nixosSystem' (nixosSystem.args // (with nixosSystem.args; {
+              modules = modules ++ [
+                {
+                  # c3d2.simd.enable = lib.mkForce true;
+                  environment.noXlibs = true;
+                }
+              ];
+              # nixpkgs = nixos-unstable-simd;
+            }))))
          ) self.nixosConfigurations
-          //
-          nixos.lib.filterAttrs (name: _:
-            builtins.match ".+-tftproot" name != null
+          // nixos.lib.filterAttrs (name: attr:
+            (builtins.match ".+-tftproot" name != null && lib.isDerivation attr)
          ) self.packages.aarch64-linux
        );
    };
--- a/hosts/auth/default.nix
+++ b/hosts/auth/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ config, ... }:

 {
  c3d2 = {
@ -58,7 +58,7 @@
    age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
    defaultSopsFile = ./secrets.yaml;
    secrets."portunus/seed" = {
-      group = config.services.portunus.group;
+      inherit (config.services.portunus) group;
      owner = config.services.portunus.user;
    };
  };
--- a/hosts/blogs/default.nix
+++ b/hosts/blogs/default.nix
@ -1,4 +1,4 @@
-{ hostRegistry, zentralwerk, config, ... }:
+{ config, ... }:
 {
  microvm.mem = 2048;
  c3d2.deployment = {
--- a/hosts/broker/default.nix
+++ b/hosts/broker/default.nix
@ -1,4 +1,4 @@
-{ config, pkgs, lib, zentralwerk, ... }:
+{ config, pkgs, ... }:

 let
  mymqttui = pkgs.writeScriptBin "mqttui" ''
--- a/hosts/c3d2-web/default.nix
+++ b/hosts/c3d2-web/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, nixpkgs, config, lib, pkgs, ... }:
+{ config, pkgs, ... }:
 let
  webroot = "/var/www";
  geminiRoot = "/var/gemini";
@ -16,6 +16,8 @@ in
  # Network setup
  networking.hostName = "c3d2-web";
  networking.firewall.allowedTCPPorts = [
+    # telme10
+    23
    # http/https
    80 443
    # gemini
@ -48,11 +50,14 @@ in
          index portal.html index.html;
        '';
        locations = {
+          # Mastodon
+          "/.well-known/webfinger".return = "301 https://c3d2.social$request_uri";
+
          # SpaceAPI
          "/status.png".proxyPass = "http://[${config.c3d2.hosts.spaceapi.ip6}]:3000/status.png";
          "/spaceapi.json".proxyPass = "http://[${config.c3d2.hosts.spaceapi.ip6}]:3000/spaceapi.json";

-	      # WKD: Web Key Directory for PGP Keys
+          # WKD: Web Key Directory for PGP Keys
          "/openpgp" = {
            extraConfig = ''
              autoindex off;
@ -109,6 +114,15 @@ in
    contentDir = geminiRoot;
    language = "de";
  };
+
+  systemd.packages = with pkgs; [ telme10 ];
+
+  systemd.services.telme10 = {
+    serviceConfig = {
+      AmbientCapabilities="CAP_NET_BIND_SERVICE";
+    };
+  };
+
  # let agate access the tls certs
  systemd.services.agate = {
    requires = [ "agate-keys.service" ];
@ -147,11 +161,17 @@ in
    home = "/var/lib/c3d2-web";
  };

-  systemd.tmpfiles.rules = [
-    "d ${webroot}/c3d2 0755 c3d2-web ${config.users.users.c3d2-web.group} -"
-    "d ${webroot}/log 0755 c3d2-web ${config.users.users.c3d2-web.group} -"
-    "d ${geminiRoot} 0755 c3d2-web ${config.users.users.c3d2-web.group} -"
-    "d ${config.users.users.c3d2-web.home} 0700 c3d2-web ${config.users.users.c3d2-web.group} -"
+  users.groups.telme10 = {};
+  users.users.telme10 = {
+    isSystemUser = true;
+    group = "telme10";
+  };
+
+  systemd.tmpfiles.rules = with config.users.users.c3d2-web; [
+    "d ${webroot}/c3d2 0755 c3d2-web ${group} -"
+    "d ${webroot}/log 0755 c3d2-web ${group} -"
+    "d ${geminiRoot} 0755 c3d2-web ${group} -"
+    "d ${home} 0700 c3d2-web ${group} -"
  ];

  # Build script
--- a/hosts/caveman/default.nix
+++ b/hosts/caveman/default.nix
@ -10,7 +10,7 @@
  };
  microvm = {
    vcpu = 8;
-    mem = 8 * 1024;
+    mem = 16 * 1024;
  };

  networking = {
@ -23,7 +23,7 @@
  '';

  services.caveman = {
-    redis.maxmemory = config.microvm.mem * 1024 * 1024 * 3 / 2;
+    redis.maxmemory = config.microvm.mem * 1024 * 1024 / 2;

    hunter = {
      enable = true;
@ -54,6 +54,11 @@
        ];
        locations."/".proxyPass = "http://127.0.0.1:${toString config.services.caveman.gatherer.settings.listen_port}/";
      };
+      virtualHosts."relay.fedi.buzz" = {
+        forceSSL = true;
+        enableACME = true;
+        locations."/".proxyPass = "http://127.0.0.1:${toString 3000}/";
+      };
    };
  };
 }
--- a/hosts/dacbert/default.nix
+++ b/hosts/dacbert/default.nix
@ -1,5 +1,38 @@
-{ hostRegistry, config, lib, pkgs, modulesPath, ... }:
+{ hostRegistry, config, lib, pkgs, ... }:

+let
+  streamAudioToOwncastScript = pkgs.writeScriptBin "stream-audio-to-owncast" ''
+    #! ${pkgs.runtimeShell} -e
+
+    PATH=${with pkgs; lib.makeBinPath [ coreutils pavucontrol gnome.zenity ffmpeg ]}
+
+    MOVIE="$(zenity --file-selection --title 'Select a video to loop')"
+    MOVIE_PREENCODED="/tmp/`basename $MOVIE`.mp4"
+
+    echo
+    echo Preencoding the movie. Wait a bit...
+    echo
+
+    ffmpeg -i "$MOVIE" -an -s:v 480x320 -c:v libx264 -b:v 600k -f mp4 -y "$MOVIE_PREENCODED"
+
+    echo
+    echo "Starting stream. End with <q>"
+    echo
+
+    pavucontrol &
+
+    ffmpeg -f pulse -ac 2 -i default -stream_loop -1 -i "$MOVIE_PREENCODED" -c:a aac -c:v copy -f flv rtmp://owncast.serv.zentralwerk.org:1935/live/$(cat ${config.sops.secrets."owncast/authKey".path})
+  '';
+
+  streamAudioToOwncast = pkgs.makeDesktopItem rec {
+    name = "Stream to Owncast";
+    exec = "${streamAudioToOwncastScript}/bin/stream-audio-to-owncast";
+    icon = "media-record";
+    desktopName = name;
+    terminal = true;
+  };
+
+in
 {
  c3d2 = {
    isInHq = true;
@ -20,6 +53,11 @@
    } ];
  };

+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets."owncast/authKey".owner = "k-ot";
+  };
+
  hardware.enableRedistributableFirmware = true;
  powerManagement.cpuFreqGovernor = lib.mkDefault "performance";

@ -73,16 +111,16 @@
  };

  nix = {
-    buildCores = 4;
-    maxJobs = 1;
-    trustedUsers = [ "client" ];
-    extraOptions = ''
-      builders-use-substitutes = true
-      extra-platforms = armv6l-linux
-    '';
    daemonCPUSchedPolicy = "idle";
    daemonIOSchedClass = "idle";
-    settings.system-features = [];
+    settings = {
+      builders-use-substitutes = true;
+      cores = 4;
+      extra-platforms = "armv6l-linux";
+      max-jobs = 1;
+      system-features = [];
+      trusted-users = [ "client" ];
+    };
  };
  # kernel 32bit personality patch from Ubuntu
  boot.kernelPatches = [
@ -119,6 +157,7 @@
      libva-utils
      mpv
      vlc
+      ffmpeg
      yt-dlp
      ncpamixer
      pulseaudio # required for pactl
@ -130,6 +169,7 @@
      bevy_julia
      bevy_mandelbrot
      allcolors
+      streamAudioToOwncast
    ];
  };

--- a/hosts/dacbert/secrets.yaml
+++ b/hosts/dacbert/secrets.yaml
@ -0,0 +1,181 @@
+owncast:
+    authKey: ENC[AES256_GCM,data:Q0Zx9g==,iv:3RB7efC9t6SXEP4/yR5Or/f3GN820RiApqTJC0G5zaw=,tag:UaiJAdGefjuwVd3gGwTXTg==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1hg0mmua5y82ct7l6q9gpc8w940ce5seqcjhm4dgx7tlzvflznyas7v3hf4
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQ1l6MkNscFl3b0FReENC
+            cFozbUE4clBFSXlmSEtKaWFLV0I1aVFNVUg0ClhTTld1TkxsSWxWcDV1T2F6M0Ex
+            cHNGZnprMlljc1dBcG1iWEs5WTZGaFEKLS0tIHNrVk5uTDRwUlVkSVZxT0txUmxO
+            L0pDVlhTa0QyRWc5MHQ0dEJBOE1DMlkKXT0txgyo3DPYni8QKoZQqcmZckrKOehx
+            UJcHM6O7gwJ1cglkByFcAwhmMB+cPerUqsxoLt7RplILr2bET2gyyg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age12aukzah0pt2rck52hwn08kezyxueqz2f49ld7hpyuzmu847vavdqkunn5c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1OTFkbXNxWEYzZVZ0ZDlS
+            ak1VYlBzaThma3ZjaGVwZlNuaThXTHJFL2xjCkRMREdKdG9CTUNDZUZDY213cndI
+            ZVFBdEVVbDdwVUdyYzE0d3JVcHB0RnMKLS0tIFFvbVhsR1VKbWhQTzJsUXIzRzNI
+            K3I1KzNqT2RqL2I4alkrVGlBUWJjQjQKE8Icz22zUmZRlPLGBSVYK2O98jWTtsO2
+            sgEx3ZDqJi47bCqUKMhPz55TpIKsCx/bAAQ+qfV8g7DaoSz9UMk+/w==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-12-04T21:39:27Z"
+    mac: ENC[AES256_GCM,data:PQtTAEXBgp6MuPl5+6SE/zaOuYGvDRpl0w9XYgUJJHhDqiIEQfcAYIUEKv+Dvtxj94TQZ3v2kjT4Y3FF0ROYIcN8H1K1PtVXRLu+vptr2+wZ1Mahg6K4Ukk1s0eiBB3vsGrMONqaky5Jqy4My3+0NMnBuIvlqErpoUT3Gin37do=,iv:RUTAvD/hczbaX7w7ROPQZNII3kTRnEBG6aMMkTHuV/M=,tag:XLgfZnnDM1ffv6uBLYJXmg==,type:str]
+    pgp:
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA6j84+xkv3y7ARAAvea1x4I5nwUVUI5DHoflHivlVRauxetPBZoLIyfNklX3
+            oO2euRsEC66ZtuqDzSSoHpUM5NIdDPPgNf0sw6YJnbRH8F88ghNE87NXyevlvGnA
+            8IFOokn7ASjPk2DYLlh8xtZ+T7ZgAhIgv0lnZLa36n5h0tSL4t0OdGAnwWa9j9Mj
+            2gWGKymunpXMjRqEj/PGEuXEDRCdwpvmwi+8YKrh4D9dB67tepAA28MofSfJ5295
+            Sfyer+Hh41zPgYI2WeBRTS9Rn8Ea98Y5eWQsXIivG7+CnKl+uu53nzZevRJDk5NS
+            0qRsyIxeJvmX0fqLgHD1jZJ/UGeUoiRNNkifN9DVIikDj8coAyVCb3ebbQWs63nn
+            NL26fuONgZtfqF6DuIoFpGS+klQscWdmYQ729EIEvxzBo8jeSPnwf2ySyt8gA0J4
+            Y5lxnwoPl4b6nT4TAHSQ65HWxq2y2+KUT4oE3PrN883N5702Qlr6MxN3tFJ/V98x
+            7OyA9UjgkkHuFl7EnumkUOix3BqnBk5za4ABhWvYfBtm0nVVAnesEcjsIeT4ihJx
+            5lhi/CiNPth+PhsIDaqA2QVU2sjswgmOK3KbQcpktAROFoXTDayr2t4QNZUTmbWF
+            vwHxNIZ1CndzaPgl5WI4QJDerWErEPKVxf65HxrLtWft4MyhfevSfH63C0TO4YDS
+            UQHKQhF01t0ZXKPLnQDGAwZijyA7GD493Sr6pFMzenSipvwybvNEnfDOZuc7AU9b
+            4oXevCzGjpaub33hqorp1bcygwgl0aA1H3YvleyHBg5Mgw==
+            =sPOS
+            -----END PGP MESSAGE-----
+          fp: A5EE826D645DBE35F9B0993358512AE87A69900F
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA8zMZ+ak7y/zARAAuhnyVNytAgqL03jN9rXwwHuqgaqQ4SdjM7kTLe4WVCXJ
+            PwLtPoNNjlFqRKufXPQQuOBx2FMJl1yUQsI4P7OiWGak6/0f41U0bLFczTTGRgTh
+            vmEHwRtRGQAsWi5g+KqP0zAoiZNiBdk7s4THcA24tBBGViRtLS+swhV2SRvTT0NA
+            pLsOB/GpA94kEjfHhZ1RunKBMIxDKYaQvXUFlEPSkv/Q6ek8Tc3wgPMUp5wiN+cz
+            q628/KNM78nWCEyV0rQ3YAccnC6h1Wb6cHCOKE6jJJTfwm503tzZv1klQUkKrP0j
+            BvgFGIdn76XTER47A3sPweDbnHV8Z0qxnJ4z6YGN5zYW4HVAOWCSvI1OvM1q9oIj
+            SiwXVUIuhhGj1kb3LPstSWJvPmJR0bgmO4xSMYpB2d71iRTL7S82p9aWk3/Zzcr7
+            DNYWFxs0r7s0L0xJCDma0fM6krGof/ZcCdo1eRgSJZWbPrGL8ips++9/Fm0/xgzf
+            A8CxWpnWRE6S3dBRQZNfHOn9OyHJGKNVR85mF/gPI14viYmr9iJ4gqDzLcZF27V6
+            uasJrcnaKkSL8fBah4UiT3lRfNqX24HXZnDMhTHwJD1RQHn4qvmE+FiB4m8SEYS0
+            TCLTCxuWZV0p9MtzQbNyadmpCxb5JL2EugKaal2pW2cxhMuUryV8axj7QhD2UJ/S
+            UQE421UkjynmGgjRa8jYiS4sFYlrbNsHLAtRlinRyNya2je8vyOZNekQl59sVYA3
+            mb7bgu0XN1Y3cZtKAxughzSpkLjSGCR9GebNawKdUUcsPg==
+            =VokK
+            -----END PGP MESSAGE-----
+          fp: D4E89C6A0A58EE803EF708EFA9B23715F7AA3F1A
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcBMA45bZkLXmBFpAQf/XjbniHgbytyURYZ3qFCzijhLi7e99Q2jdsOQTQG4sOoE
+            jpPlOP4yU6lGjnYfQVYqH7CgyquqGrSB3M4ZHxUR2916xveG8vjWDUhURGWAUXwk
+            ho5dGm+HZGMJdETCmmCTALqxlrv/TgrUiUfSn1C9MKloR4PxYDFqupk9Bl5RyvQT
+            xdcIzVM6/7+xCasBErKoIQ3YoPKlxNWkr73S4Y/jhEVVaxEZWHoyCtK7SRDA4GOi
+            nIO2TMQ/eCbnrHbftwUsZNkeP1bee3+U7wiPOXOozftLrA2FYuT+SmCOK1/xaqHT
+            n4WwcPamFNqWqWMGyVNQd/rR0/Ry+hDYbY4jzm2+vtJRAXNtZLLw5BmnAbpZ0PY5
+            eBUYkDMJ81ua+I9fJIzg0WtbLTBWeIo50v6SrrUuCXdBbtvtsvMBPHuGDdwoqM61
+            kJoAZV8D9JDznjOnn0wUlFtF
+            =55R4
+            -----END PGP MESSAGE-----
+          fp: A4B0F5A80C2E2448A97BEC25BB829C4DECA6CCB9
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMAwMCBBrc/JA6AQ/+Ksp81JefhSw9XHfgtutZRbx/yhPi+OX2mMDNPjPpNykT
+            zSMVuZWA2Io/RxmVCOrEcJi1hHGZ989tZS8vPWyB+ddnnCx1h78PabJh0d/jqUN3
+            XTKli26QtGKvMPioVg89NQjD0pTqoHc6tTXdOP1XSS7R6lZCxz4hTn39jPwrhSgA
+            z5GocHUfTWoSYoG5GV8jsKrWNUqd6HwoKlucTku64SvquJQ8Ld+VeiwAbcxXtPAa
+            QuQ3cXkerYen5l86Hp8NkyqOub+DW5fgPfC5LqBdEhjp3tH0HUHsK9CWHv+3tQ+l
+            fDpmGt3RVQ5X5NJM1QKJeVkX33HI3ntqnhThnVbC6R1FBASaabQzyekdHpu1JWxM
+            YP5gh04+VZm0qdoPMgPin1aSKvNBCzyQqNK4f7/iwz/beX5V3wnq7G2hsfDQCzBt
+            W0y0bv7q0ToWlXmj/A2dytxHb0bWIRsbXU2p2Szl/ccoLDZYH5Mz/ZKrKH5xtD9Z
+            z4hq6p8WO2dAe2dkN48Bd8xbddNUs0Xem73T9FoHNKIcWMP/CA7Ir7p0agnYs7id
+            Dni/BB5bm6XIwqpwOPG2TqNl051MSVDLXB1ia88Z5WvRL+EYdm4o0xGzL1fPPxWX
+            S85Y0pe8QGxd8qikPHlHmFc0TLB05yN3LT17+CC1T3zjlNff0uMdFgiioLJHwz7S
+            UQGMPRKXvg1RZiU1GUfaXG/PFH/KAouKx+m97iCRzg2SNULNakq0tvWIV4Fp19yn
+            DpQs+g2eq7KYwW/udKzDpGOM2/IDY/s07YCipn48XEfaDw==
+            =7TJ4
+            -----END PGP MESSAGE-----
+          fp: 4F9F44A64CC2E438979329E1F122F05437696FCE
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA9XEenRNYVGHARAAkzr6nZaSnq88e+JHXvkKGBqxduAfWFlGvXYrp/2e1tyy
+            BCpiMCXrzoend43/Kz5pTCjV3KvcYOl22OYJMi2mYObsJKoi1ftInL5mc6NPMU/O
+            ixJncQ8NBa3lv3HGO1j9xETLLTt3220gUc4CKo65YkaIsJYumw2oCAbFhCdZy17g
+            qlzjFRy8VfWad48nAezymnfGFdTWKThxGzvOJLo5Jm98UIt2lC5T7vZjEwQSsjra
+            EnF/I6sAeT9kEVg17m2CEFK5LVb/4GtSr/rXFUDbihO3OVYRbdGk/7y4lo55pRi2
+            daZcwUFCqdj83QUUYimYpXQWhgyBSgIWCK7vb6Sr0rO7j0poqncpm3gX3dlIzmDk
+            bK8N+Heb7u7kZ24+p96jabgEBphLgPepE2fMoXFpnZ2P2e8QgiA8b/5Rqq2KomYD
+            4hbqUmSTpG1j+77+l5LMkbg/ugzRd3VviJO5dKsNvaUsKz7e5NNRV3mvtgaakP1s
+            ygQ3DBnJibKiR2QR6pJRvLAlNTy/bguKCZI8DCMO7SWlRuatBujV3htlG2AgzrKp
+            KLRSU4HuoyZXiOsqNA4tVH4ZklzXzhNAELt30+yYNJbS8evcd3ZZZnfsaZmr3jWE
+            jK2vXo+FUkbJHQrCLCHymURH+TijvvOgCYSYTmzvn7R+4Esfs+abrWugoXLwoznS
+            UQGIyCN+1XfmWTkHiiygdNWRb4HdB8OTTe75rsPnTJSmKbYhafE/Iqhf9rDEPNUk
+            e5l1I1sgwYq/bR2FKd1iWAfZlqa39dOi/C/tBD2KnuISOA==
+            =Ut63
+            -----END PGP MESSAGE-----
+          fp: 4B12EFA69166CA8C23FC47E49CD3A46248B660CA
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcBMA/Z87ylQaotQAQf9GF9yX0w0OwL251Ex76j3k3MF1Q0qOfgjsDBY3Ow56VO+
+            KhJ0Ugn0beKoYZrVn2ezvJZaNa7YAPry/oomul/Kvv3rTZDh2bs4lBu8qrjznj+m
+            bdX+dtKr3znJkkqqQkA+DF8bQ9DoZxDDz7FNQSuZocn+TpMn253cl0vBi7gyxxpb
+            Rfch55vZrhpGU2+QfH9euqSv8k3fOMbMEwVHL82uGybxN7a2fe/BtRwuyeSRjyGl
+            q1L4FrF1w1aUDp4M7bK+3eo2WpqCH9UxgLu74DW6SdVM3MV0B/IUXRU1rEqV4vja
+            bLMxnJ6xsByKsOVUP2ivalTnLyrtZPOcJ1wDe2SCKdJRAaegKPecQEqIuGuWBlng
+            t/W+RD/P0CqtDvly/2ALLZtAmyvXloCOtrMWkz/bnGpoqOlNL609FkY05RY3Quvm
+            C2OTSTvQ//MP7BPV99syTVTo
+            =xgMp
+            -----END PGP MESSAGE-----
+          fp: 9EA68B7F21204979645182E4287B083353C3241C
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA9qJIVK2WMV7AQ//c9HmDg63M7yWpeVOx7Eha/QVzPtXyJ7/G9dJrGltLbUj
+            B8xn/BeDT+ARVDRFnGJS053RxHJz5WqCIdcEQTZlD4GHp3Ez7vtXPEUJNymTmug7
+            VrCX4GhNjKdb55SlTkoeAbxdlUfyPUIhaLcdQ8pzHqyurcpRs8ZuYoSENbNoAbYY
+            NZbNJZoKJTc2vmI2zrO1HtVNc8HZYtnN5dawvlCn66Q3lhNLZ1wR4AFmmKvKgKIR
+            XFAjbsoEUDlYNBDw+yv8jkB179+wa6pLAhIaJ8NM2asimUz7GWxDumgesNbacTR0
+            ZJXrQWyuHruQ1FbZdVJ3+Ko2S2JnAiN7Ezmauqo3IQGt++ZXNOC5dl/FusY25TQ9
+            lpAW+klKkIqMfIZU0WsWWufhgv6cI8HqPY9z5TalSrGB5zyJeW2bnU2/ZAr5yNH8
+            /rOJVVF/1c4xkEqE0/Coy2ZqbE1IOTq5aiqaMuZIToSn+ftvg2KIxkdEvQ8gVigU
+            uZfnZ4HN0FQqJ2/9BHHFlhW8CdWdffTk8m2GEdfyjH+WIPCoRyvbf7RWAad8NFni
+            RHTf8+Uee5R2hth82UO0wSlMrLrQwdS7ju6ctpag1gm2YxEHRZBs6hnD8Ade//xR
+            RFkmcW5BICAPbBmJM4o1HSUjYqGhk46xU7y3UVNs4R144DtqEAMwjWE7U/IsvRTS
+            UQF2/UFLDd/LgVHt1uVhpzuWl5ot97FU6HIPSYrlkOPAzI49VQ0H+tLlywMbgjcD
+            0HoNDhCDgzTzp2/loq3wECS93QAx6L9+zHpUz+2db9LBFw==
+            =356k
+            -----END PGP MESSAGE-----
+          fp: 53B26AEDC08246715E15504B236B6291555E8401
+        - created_at: "2022-12-04T21:39:01Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA/YLzOYaRIJJAQ/8CFz+BjJA03Sgq6TeYzycAjdSFi3mTZ50L/sGvj2B5WBQ
+            WnVwA7tWSSQGmt91XZwSNgf7PTCgm3v6zMTHnYN5WMwk9vaDL/WIGbqD1LZw4zgK
+            fD+Lm/Yc/0vPrEahL9irCb9jrolWGagdPzQYqRLReD1aB5NnYp96FuELm6RF2YC9
+            AU6ZxrtrQkEZ61SsAP9ODTruRmCtmtpbJYweN1CmkatMj8yIu4I/Y939T5lx8V8F
+            ZgT89f/p795/2uFWJ4HpZk0Rkq78RJCO8HytKEw1CVS55P49PnjKKxqzyESZCaVU
+            bLI5MKUQowBSmgoHZWd+B4DO/7+4sgu30A4CuldZR2SZOqW+FwdEAuwdK8HrG9G8
+            d0gzujb4Iq7PtqbAMErr7h73hTXdBeFrUG8z5mgWmeEhP9o0Zlx/oXGpi2JOJYBb
+            VySPDLQxWBV65XpM0Qb8zxsKYAIrouy8YmbbrbNX8nzG/KLgUaNJcj3PYp3WI/cs
+            4MkBc2nh4s6OLGJ7YP5AoCayPSuFwsJ/+WZ+neQ6X1gkgSFtYmb/LoTSU4w35xPi
+            rbDwqxf2Lg1MbPP+hhW/Y0fZKVWUcLs9x4EbhS5LlmlKC66mRXgATKozLpivjDOa
+            kTTIwptGeo02NTIeEegP24AmMvXnvvFzUBZmN57ztCuj/Sckc7ICTMrFV8vo2fXS
+            UQHWb0zzNY1kk5Ui+kL2exUHdIdGDD8lc2Cpz40Fo6emfIGVlNgCCDK/upnu/Ims
+            GE35n8h33MapqihsFQpXJ2lNkzuckV/yIRUfRXTkP4miXw==
+            =RITE
+            -----END PGP MESSAGE-----
+          fp: 91EBE87016391323642A6803B966009D57E69CC6
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3
--- a/hosts/direkthilfe/default.nix
+++ b/hosts/direkthilfe/default.nix
@ -1,7 +1,6 @@
-{ pkgs, ... }:
-let
-  domain = "direkthilfe.c3d2.de";
-in {
+{ config, pkgs, ... }:
+
+{
  networking.hostName = "direkthilfe";
  microvm.mem = 1024;
  c3d2.deployment = {
@ -27,7 +26,7 @@ in {

  services.engelsystem = {
    enable = true;
-    domain = domain;
+    domain = "direkthilfe.c3d2.de";
    createDatabase = true;
    package = pkgs.engelsystem.override { php = pkgs.php74; };
    config = {
@ -64,7 +63,7 @@ in {
  services.phpfpm.phpPackage = pkgs.php74;
  services.nginx = {
    enable = true;
-    virtualHosts."${domain}" = {
+    virtualHosts."${config.services.engelsystem.domain}" = {
      default = true;
      forceSSL = true;
      enableACME = true;
--- a/hosts/dn42/default.nix
+++ b/hosts/dn42/default.nix
@ -140,7 +140,7 @@ in {
              }
            '' else
              "";
-            interface = if conf ? interface then conf.interface else name;
+            interface = conf.interface or name;
          in "${neighbor4}${neighbor6}") neighbors));
    in ''
      protocol kernel {
--- a/hosts/factorio/default.nix
+++ b/hosts/factorio/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ config, lib, ... }:

 {
  c3d2 = {
--- a/hosts/ftp/default.nix
+++ b/hosts/ftp/default.nix
@ -1,4 +1,4 @@
-{ config, pkgs, lib, zentralwerk, ... }:
+{ config, pkgs, ... }:

 {
  c3d2 = {
--- a/hosts/gitea/default.nix
+++ b/hosts/gitea/default.nix
@ -12,9 +12,9 @@

  networking = {
    hostName = "gitea";
-    hosts = {
-      "2a00:8180:2c00:282::48" = [ "auth.c3d2.de" ];
-      "172.20.73.72" = [ "auth.c3d2.de" ];
+    hosts = with zentralwerk.lib.config.site.net.serv; {
+      ${hosts6.up4.auth} = [ "auth.c3d2.de" ];
+      ${hosts4.auth} = [ "auth.c3d2.de" ];
    };
    firewall.allowedTCPPorts = [ 80 443 2222 ];
  };
@ -60,7 +60,7 @@
          ENABLED = true;
        };
        "cron.repo_health_check" = {
-          TIMEOUT = "120s";
+          TIMEOUT = "300s";
        };
        "cron.resync_all_sshkeys" = {
          ENABLED = true;
--- a/hosts/glotzbert/default.nix
+++ b/hosts/glotzbert/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, pkgs, ... }:
+{ config, pkgs, ... }:

 {
  imports = [ ./hardware-configuration.nix ];
@ -13,9 +13,9 @@

  nixpkgs.config.allowUnfree = true;

-  nix = {
-    buildCores = 4;
-    maxJobs = 4;
+  nix.settings = {
+    cores = 4;
+    max-jobs = 4;
  };

  sops = {
--- a/hosts/glotzbert/hardware-configuration.nix
+++ b/hosts/glotzbert/hardware-configuration.nix
@ -1,14 +1,14 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{ lib, modulesPath, ... }:

 {
  imports = [ (modulesPath + "/installer/scan/not-detected.nix") ];

  boot.initrd.availableKernelModules =
    [ "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ];
-  boot.kernelModules = [ "kvm-intel" ];
+  boot.kernelModules = [ "kvm-intel" "nct6775" ];
  boot.extraModulePackages = [ ];

  fileSystems."/" = {
@ -24,6 +24,5 @@

  swapDevices = [ ];

-  nix.maxJobs = lib.mkDefault 4;
  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
 }
--- a/hosts/gnunet/default.nix
+++ b/hosts/gnunet/default.nix
@ -2,11 +2,11 @@
  system.stateVersion = "22.05";

  c3d2.hq.statistics.enable = true;
-  c3d2.deployment = {
-    server = "server10";
-    mounts = [ "etc" "var" ];
+  deployment = {
+    # needs to keep just its ssh key for sops-nix
+    persistedShares = [ "/etc" "/var" ];
+    mem = 1024;
  };
-  microvm.mem = 1024;

  networking = {
    hostName = "gnunet";
--- a/hosts/grafana/dashboards/UX0rEpd7z.json
+++ b/hosts/grafana/dashboards/UX0rEpd7z.json
@ -3,30 +3,38 @@
    "list": [
      {
        "builtIn": 1,
-        "datasource": "-- Grafana --",
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
-  "gnetId": null,
+  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
-  "id": 21,
  "links": [],
+  "liveNow": false,
  "panels": [
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
-      "datasource": null,
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
+      "datasource": {
+        "type": "influxdb",
+        "uid": "P79EA3E036FD9FF91"
      },
      "fill": 6,
      "fillGradient": 0,
@ -55,7 +63,7 @@
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "7.5.11",
+      "pluginVersion": "9.3.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
@ -66,6 +74,10 @@
      "targets": [
        {
          "alias": "$tag_host C2S $tag_type_instance",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "P79EA3E036FD9FF91"
+          },
          "groupBy": [
            {
              "params": [
@ -121,9 +133,7 @@
        }
      ],
      "thresholds": [],
-      "timeFrom": null,
      "timeRegions": [],
-      "timeShift": null,
      "title": "Client to Server connections",
      "tooltip": {
        "shared": true,
@ -132,9 +142,7 @@
      },
      "type": "graph",
      "xaxis": {
-        "buckets": null,
        "mode": "time",
-        "name": null,
        "show": true,
        "values": []
      },
@ -142,75 +150,107 @@
        {
          "$$hashKey": "object:58",
          "format": "short",
-          "label": null,
          "logBase": 1,
-          "max": null,
-          "min": null,
          "show": true
        },
        {
          "$$hashKey": "object:59",
          "format": "short",
-          "label": null,
          "logBase": 1,
-          "max": null,
-          "min": null,
          "show": true
        }
      ],
      "yaxis": {
-        "align": false,
-        "alignLevel": null
+        "align": false
      }
    },
    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
+      "datasource": {
+        "type": "influxdb",
+        "uid": "P79EA3E036FD9FF91"
+      },
      "fieldConfig": {
-        "defaults": {},
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 60,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
        "overrides": []
      },
-      "fill": 6,
-      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 0
      },
-      "hiddenSeries": false,
      "id": 3,
      "interval": "180",
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null as zero",
      "options": {
-        "alertThreshold": true
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
      },
-      "percentage": false,
-      "pluginVersion": "7.5.11",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": true,
-      "steppedLine": false,
+      "pluginVersion": "9.3.1",
      "targets": [
        {
          "alias": "$tag_host S2S $tag_type_instance",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "P79EA3E036FD9FF91"
+          },
          "groupBy": [
            {
              "params": [
@ -266,98 +306,97 @@
          ]
        }
      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
      "title": "Incoming Server to Server connections",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:58",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:59",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "type": "timeseries"
    },
    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
+      "datasource": {
+        "type": "influxdb",
+        "uid": "P79EA3E036FD9FF91"
+      },
      "description": "",
      "fieldConfig": {
-        "defaults": {},
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 60,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "short"
+        },
        "overrides": []
      },
-      "fill": 6,
-      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 9
      },
-      "hiddenSeries": false,
      "id": 4,
      "interval": "180",
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null as zero",
      "options": {
-        "alertThreshold": true
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
      },
-      "percentage": false,
-      "pluginVersion": "7.5.11",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": true,
-      "steppedLine": false,
+      "pluginVersion": "9.3.1",
      "targets": [
        {
          "alias": "$tag_host S2S out $tag_type_instance",
+          "datasource": {
+            "type": "influxdb",
+            "uid": "P79EA3E036FD9FF91"
+          },
          "groupBy": [
            {
              "params": [
@ -412,64 +451,25 @@
          ]
        }
      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
      "title": "Outgoing Server to Server connections",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:58",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:59",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "type": "timeseries"
    }
  ],
  "refresh": "5s",
-  "schemaVersion": 27,
+  "schemaVersion": 37,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": []
  },
  "time": {
-    "from": "now-2d",
+    "from": "now-30d",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Prosody",
  "uid": "UX0rEpd7z",
-  "version": 3
-}
+  "version": 1,
+  "weekStart": ""
+}
--- a/hosts/grafana/dashboards/pVoyupoZk.json
+++ b/hosts/grafana/dashboards/pVoyupoZk.json
@ -26,7 +26,6 @@
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
-  "iteration": 1655678212481,
  "links": [],
  "liveNow": false,
  "panels": [
@ -41,6 +40,8 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
@ -99,7 +100,8 @@
        "legend": {
          "calcs": [],
          "displayMode": "list",
-          "placement": "bottom"
+          "placement": "bottom",
+          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
@ -199,7 +201,7 @@
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "8.5.6",
+      "pluginVersion": "9.3.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
@ -322,6 +324,8 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
@ -380,7 +384,8 @@
        "legend": {
          "calcs": [],
          "displayMode": "list",
-          "placement": "bottom"
+          "placement": "bottom",
+          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
@ -535,6 +540,8 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
            "axisLabel": "Temperature",
            "axisPlacement": "auto",
            "barAlignment": 0,
@ -621,8 +628,9 @@
      "options": {
        "legend": {
          "calcs": [],
-          "displayMode": "hidden",
-          "placement": "bottom"
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
        },
        "tooltip": {
          "mode": "single",
@ -862,7 +870,7 @@
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "8.5.6",
+      "pluginVersion": "9.3.1",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
@ -992,6 +1000,8 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
@ -1172,8 +1182,9 @@
      "options": {
        "legend": {
          "calcs": [],
-          "displayMode": "hidden",
-          "placement": "bottom"
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
        },
        "tooltip": {
          "mode": "multi",
@ -1269,6 +1280,8 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
@ -1362,7 +1375,8 @@
            "max"
          ],
          "displayMode": "table",
-          "placement": "bottom"
+          "placement": "bottom",
+          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
@ -1533,6 +1547,8 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
@ -1566,8 +1582,7 @@
            "mode": "absolute",
            "steps": [
              {
-                "color": "green",
-                "value": null
+                "color": "green"
              },
              {
                "color": "red",
@ -1591,7 +1606,8 @@
        "legend": {
          "calcs": [],
          "displayMode": "list",
-          "placement": "bottom"
+          "placement": "bottom",
+          "showLegend": true
        },
        "tooltip": {
          "mode": "multi",
@ -1748,7 +1764,7 @@
    }
  ],
  "refresh": false,
-  "schemaVersion": 36,
+  "schemaVersion": 37,
  "style": "dark",
  "tags": [],
  "templating": {
@ -1756,17 +1772,10 @@
      {
        "current": {
          "selected": false,
-          "text": "server10",
-          "value": "server10"
-        },
-        "datasource": {
-          "type": "influxdb",
-          "uid": "000000001"
+          "text": "hydra",
+          "value": "hydra"
        },
        "definition": "SHOW TAG VALUES WITH KEY = \"host\"",
-        "error": {
-          "message": "Datasource 000000001 was not found"
-        },
        "hide": 0,
        "includeAll": false,
        "multi": false,
@ -1807,4 +1816,4 @@
  "uid": "pVoyupoZk",
  "version": 1,
  "weekStart": ""
-}
+}
--- a/hosts/grafana/default.nix
+++ b/hosts/grafana/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, pkgs, lib, modulesPath, ... }:
+{ config, pkgs, ... }:

 let
  restartServices = [ "grafana" "influxdb" ];
@ -11,8 +11,6 @@ in {
  c3d2.isInHq = false;

  services.openssh.enable = true;
-  # noXlibs breaks cairo:
-  environment.noXlibs = false;

  networking.hostName = "grafana";

@ -41,7 +39,7 @@ in {
      datasources.settings.datasources = map (datasource: {
        inherit (datasource) name type access orgId url password user database isDefault jsonData;
      }) (with builtins; fromJSON (readFile ./datasources.json));
-      dashboards = [ {
+      dashboards.settings.providers = [ {
        settings = {
          apiVersion = 1;
          providers = [ {
@ -69,7 +67,7 @@ in {
  services.influxdb = let
    collectdTypes = pkgs.runCommand "collectd-types" {} ''
      mkdir -p $out/share/collectd
-      cat ${config.services.collectd.package}/share/collectd/types.db >> $out/share/collectd/types.db
+      cat ${pkgs.collectd-data}/share/collectd/types.db >> $out/share/collectd/types.db
      echo "stations  value:GAUGE:0:U" >> $out/share/collectd/types.db
    '';
  in {
--- a/hosts/hedgedoc/default.nix
+++ b/hosts/hedgedoc/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ config, pkgs, zentralwerk, ... }:

 {
  c3d2 = {
@ -18,9 +18,9 @@

  networking = {
    hostName = "hedgedoc";
-    hosts = {
-      "2a00:8180:2c00:282::48" = [ "auth.c3d2.de" ];
-      "172.20.73.72" = [ "auth.c3d2.de" ];
+    hosts = with zentralwerk.lib.config.site.net.serv; {
+      ${hosts6.up4.auth} = [ "auth.c3d2.de" ];
+      ${hosts4.auth} = [ "auth.c3d2.de" ];
    };
    firewall.allowedTCPPorts = [ 80 443 ];
  };
--- a/hosts/hydra/cache-pub.key
+++ b/hosts/hydra/cache-pub.key
@ -1 +0,0 @@
-nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps=
--- a/hosts/hydra/default.nix
+++ b/hosts/hydra/default.nix
@ -1,5 +1,8 @@
-{ zentralwerk, config, pkgs, lib, ... }:
+{ config, lib, pkgs, zentralwerk, ... }:

+let
+  cachePort = 5000;
+in
 {
  imports = [
    ./hardware-configuration.nix
@ -9,28 +12,9 @@
    ../../modules/c3d2.nix
  ];

-  nixpkgs.config.allowUnfree = true;
-
-  security.pam.enableSSHAgentAuth = true;
-
-  services.openssh = {
-    enable = true;
-    passwordAuthentication = false;
-  };
-
-  programs.mosh.enable = true;
-
-  nix = {
-    sshServe.enable = true;
-    trustedUsers = [ "@wheel" ];
-    settings = {
-      auto-optimise-store = true;
-      cores = 20;
-      keep-outputs = true;
-      max-jobs = 8;
-      # restore default overridden by harmonia
-      allowed-users = lib.mkForce [ "*" ];
-    };
+  c3d2 = {
+    hq.statistics.enable = true;
+    simd.arch = "ivybridge";
  };

  boot = {
@ -43,14 +27,265 @@
    binfmt.emulatedSystems = [ "armv6l-linux" "armv7l-linux" "aarch64-linux" "riscv32-linux" "riscv64-linux" ];
  };

+  nix = {
+    buildMachines = [{
+      hostName = "client@dacbert.hq.c3d2.de";
+      system = lib.concatStringsSep "," [
+        "aarch64-linux" "armv6l-linux" "armv7l-linux"
+      ];
+      supportedFeatures = [ "kvm" "nixos-test" ];
+      maxJobs = 1;
+    }];
+    daemonCPUSchedPolicy = "idle";
+    daemonIOSchedClass = "idle";
+    daemonIOSchedPriority = 7;
+    settings = {
+      allowed-uris = "http:// https:// ssh://";
+      auto-optimise-store = true;
+      builders-use-substitutes = true;
+      cores = 20;
+      keep-outputs = true;
+      max-jobs = 8;
+      trusted-users = [ "hydra" "root" "@wheel" ];
+    };
+  };
+
+  nixpkgs.config.allowUnfree = true;
+
+  # disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
+  # containers = {
+  #   hydra-ca = {
+  #     autoStart = true;
+  #     config = { ... }: {
+  #       imports = [
+  #         hydra-ca.nixosModules.hydra
+  #       ];
+
+  #       environment.systemPackages = with pkgs; [ git ];
+
+  #       networking.firewall.allowedTCPPorts = [ 3001 ];
+
+  #       nix = {
+  #         settings = {
+  #           allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
+  #           builders-use-substitutes = true;
+  #           experimental-features = "ca-derivations nix-command flakes";
+  #           extra-substituters = "https://cache.ngi0.nixos.org/";
+  #           extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
+  #           substituters = [
+  #             "https://cache.ngi0.nixos.org/"
+  #           ];
+  #           trusted-public-keys = [
+  #             "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
+  #           ];
+  #         };
+  #       };
+
+  #       nixpkgs = {
+  #         # config.contentAddressedByDefault = true;
+  #         overlays = [ self.overlay ];
+  #       };
+
+  #       services = {
+  #         hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
+  #           hydraURL = "https://hydra-ca.hq.c3d2.de";
+  #           port = 3001;
+  #         };
+  #       };
+
+  #       system.stateVersion = "22.05"; # Did you read the comment? No.
+  #     };
+  #     hostAddress = "192.168.100.1";
+  #     localAddress = "192.168.100.2";
+  #     privateNetwork = true;
+  #   };
+  # };
+
  networking = {
+    hosts = with zentralwerk.lib.config.site.net.serv; {
+      ${hosts6.up4.auth} = [ "auth.c3d2.de" ];
+      ${hosts4.auth} = [ "auth.c3d2.de" ];
+    };
    hostName = "hydra";
    firewall.enable = false;
    nameservers = [ "172.20.73.8" "9.9.9.9" ];
+    # nat = {
+    #   enable = true;
+    #   externalInterface = "serv";
+    #   internalInterfaces = [ "ve-hydra-ca" ];
+    # };
  };

-  c3d2.hq.statistics.enable = true;
-  services.smartd.enable = true;
+  services = {
+    hydra = {
+      enable = true;
+      buildMachinesFiles = [
+        "/etc/nix/machines"
+        "/var/lib/hydra/machines"
+      ];
+      hydraURL = "https://hydra.hq.c3d2.de";
+      logo = ./c3d2.svg;
+      minimumDiskFree = 50;
+      minimumDiskFreeEvaluator = 50;
+      notificationSender = "hydra@spam.works";
+      useSubstitutes = true;
+      extraConfig =
+        let
+          key = config.sops.secrets."nix-serve/secretKey".path;
+        in
+        ''
+          binary_cache_secret_key_file = ${key}
+          evaluator_workers = 4
+          evaluator_max_memory_size = 2048
+          max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
+          store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
+          upload_logs_to_binary_cache = true

-  system.stateVersion = "20.09"; # Did you read the comment?
+          # https://hydra.nixos.org/build/196107287/download/1/hydra/configuration.html#using-ldap-as-authentication-backend-optional
+          <ldap>
+            <config>
+              <credential>
+                class = Password
+                password_field = password
+                password_type = self_check
+              </credential>
+              <store>
+                class = LDAP
+                ldap_server = auth.c3d2.de
+                <ldap_server_options>
+                  scheme = ldaps
+                  timeout = 10
+                </ldap_server_options>
+                binddn = "uid=search,ou=users,dc=c3d2,dc=de"
+                include ldap-password.conf
+                start_tls = 0
+                <start_tls_options>
+                ciphers = TLS_AES_256_GCM_SHA384
+                sslversion = tlsv1_3
+                # verify = none
+                </start_tls_options>
+                user_basedn = "ou=users,dc=c3d2,dc=de"
+                user_filter = "(&(objectclass=person)(uid=%s))"
+                user_scope = one
+                user_field = uid
+                <user_search_options>
+                  deref = always
+                </user_search_options>
+                # Important for role mappings to work:
+                use_roles = 1
+                role_basedn = "ou=groups,dc=c3d2,dc=de"
+                role_filter = "(&(objectclass=group)(%s))"
+                role_scope = one
+                role_field = cn
+                role_value = dn
+                <role_search_options>
+                  deref = always
+                </role_search_options>
+              </store>
+            </config>
+            <role_mapping>
+              # maps directly to user roles
+              # Make all users in the hydra-admin group Hydra admins
+              hydra-admins = admin
+              # Allow all users in the dev group to restart jobs and cancel builds
+              #dev = restart-jobs
+              #dev = cancel-build
+            </role_mapping>
+          </ldap>
+        '';
+    };
+
+    # A rust nix binary cache
+    harmonia = {
+      enable = true;
+      settings = {
+        bind = "127.0.0.1:${toString cachePort}";
+        workers = 20;
+        max_connection_rate = 1024;
+        priority = 30;
+        sign_key_path = config.sops.secrets."nix-serve/secretKey".path;
+      };
+    };
+
+    nginx =
+      let
+        hydraVhost = {
+          forceSSL = true;
+          enableACME = true;
+          locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
+        };
+      in
+      {
+        enable = true;
+        virtualHosts = {
+          "hydra.hq.c3d2.de" = hydraVhost // {
+            default = true;
+          };
+          # "hydra-ca.hq.c3d2.de" = hydraVhost // {
+          #   locations."/".proxyPass = "http://192.168.100.2:3001";
+          # };
+          "hydra.serv.zentralwerk.org" = hydraVhost;
+          "nix-serve.hq.c3d2.de" = {
+            forceSSL = true;
+            enableACME = true;
+            locations."/".proxyPass = "http://localhost:${toString cachePort}";
+          };
+        };
+      };
+
+    smartd.enable = true;
+
+    resolved.enable = false;
+  };
+
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets."nix-serve/secretKey" = {
+      mode = "440";
+      owner = config.users.users.hydra-queue-runner.name;
+      inherit (config.users.users.hydra-queue-runner) group;
+    };
+    secrets."ldap/search-user-pw" = {
+      mode = "440";
+      owner = config.users.users.hydra-queue-runner.name;
+      inherit (config.users.users.hydra-queue-runner) group;
+      path = "/var/lib/hydra/ldap-password.conf";
+    };
+  };
+
+  system.stateVersion = "20.09";
+
+  systemd.services = {
+    hydra-evaluator.serviceConfig = {
+      CPUWeight = 2;
+      MemoryHigh = "64G";
+      MemoryMax = "64G";
+      MemorySwapMax = "64G";
+    };
+
+    hydra-init.preStart = let
+      makesSenseForQemuUser = feature:
+        ! (builtins.elem feature [ "kvm" "benchmark" ]);
+      # strips features that don't make sense on qemu-user
+      extraPlatformSystemFeatures =
+        builtins.filter makesSenseForQemuUser config.nix.settings.system-features;
+    in
+    # both entries cannot have localhost alone because then hydra would merge them together but we want explictily two to not allow benchmarkts for binfmt emulated arches
+    ''
+      cat << EOF > ~/machines
+      localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
+      hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
+      EOF
+    '';
+
+    nix-daemon.serviceConfig = {
+      CPUWeight = 5;
+      MemoryHigh = "64G";
+      MemoryMax = "64G";
+      MemorySwapMax = "64G";
+    };
+  };
+
+  # allow reading nix-serve secret
+  users.users.harmonia.extraGroups = [ "hydra" ];
 }
--- a/hosts/hydra/hardware-configuration.nix
+++ b/hosts/hydra/hardware-configuration.nix
@ -1,7 +1,7 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{ config, lib, modulesPath, ... }:

 {
  imports =
--- a/hosts/hydra/hydra.nix
+++ b/hosts/hydra/hydra.nix
@ -1,184 +1,2 @@
-{ self, hostRegistry, hydra-ca, config, lib, pkgs, ... }:
+{ config, lib, zentralwerk, ... }:

-let
-  cachePort = 5000;
-
-in
-{
-  # disabled because currently it display `ARRAY(0x4ec2040)` on the website and also uses a perl array in store paths instead of /nix/store
-  # containers = {
-  #   hydra-ca = {
-  #     autoStart = true;
-  #     config = { ... }: {
-  #       imports = [
-  #         hydra-ca.nixosModules.hydra
-  #       ];
-
-  #       environment.systemPackages = with pkgs; [ git ];
-
-  #       networking.firewall.allowedTCPPorts = [ 3001 ];
-
-  #       nix = {
-  #         settings = {
-  #           allowed-uris = "https://gitea.c3d2.de/ https://github.com/ https://gitlab.com/ ssh://gitea@gitea.c3d2.de/";
-  #           builders-use-substitutes = true;
-  #           experimental-features = "ca-derivations nix-command flakes";
-  #           extra-substituters = "https://cache.ngi0.nixos.org/";
-  #           extra-trusted-public-keys = "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA=";
-  #           substituters = [
-  #             "https://cache.ngi0.nixos.org/"
-  #           ];
-  #           trusted-public-keys = [
-  #             "cache.ngi0.nixos.org-1:KqH5CBLNSyX184S9BKZJo1LxrxJ9ltnY2uAs5c/f1MA="
-  #           ];
-  #         };
-  #       };
-
-  #       nixpkgs = {
-  #         # config.contentAddressedByDefault = true;
-  #         overlays = [ self.overlay ];
-  #       };
-
-  #       services = {
-  #         hydra-dev = lib.recursiveUpdate config.services.hydra-dev {
-  #           hydraURL = "https://hydra-ca.hq.c3d2.de";
-  #           port = 3001;
-  #         };
-  #       };
-
-  #       system.stateVersion = "22.05"; # Did you read the comment? No.
-  #     };
-  #     hostAddress = "192.168.100.1";
-  #     localAddress = "192.168.100.2";
-  #     privateNetwork = true;
-  #   };
-  # };
-
-  # networking.nat = {
-  #   enable = true;
-  #   externalInterface = "serv";
-  #   internalInterfaces = [ "ve-hydra-ca" ];
-  # };
-
-  nix = {
-    buildMachines = [{
-      hostName = "client@dacbert.hq.c3d2.de";
-      system = lib.concatStringsSep "," [
-        "aarch64-linux" "armv6l-linux" "armv7l-linux"
-      ];
-      supportedFeatures = [ "kvm" "nixos-test" ];
-      maxJobs = 1;
-    }];
-    daemonCPUSchedPolicy = "idle";
-    daemonIOSchedClass = "idle";
-    daemonIOSchedPriority = 7;
-    settings = {
-      allowed-uris = "http:// https:// ssh://";
-      builders-use-substitutes = true;
-      experimental-features = "ca-derivations nix-command flakes";
-      trusted-users = [ "hydra" "root" ];
-    };
-  };
-
-  services = {
-    hydra = {
-      enable = true;
-      buildMachinesFiles = [
-        "/etc/nix/machines"
-        "/var/lib/hydra/machines"
-      ];
-      hydraURL = "https://hydra.hq.c3d2.de";
-      logo = ./c3d2.svg;
-      minimumDiskFree = 50;
-      minimumDiskFreeEvaluator = 50;
-      notificationSender = "hydra@spam.works";
-      useSubstitutes = true;
-      extraConfig =
-        let
-          key = config.sops.secrets."nix-serve/secretKey".path;
-        in
-        ''
-          binary_cache_secret_key_file = ${key}
-          evaluator_workers = 4
-          evaluator_max_memory_size = 2048
-          max_output_size = ${toString (5*1024*1024*1024)} # sd card and raw images
-          store_uri = auto?secret-key=${key}&write-nar-listing=1&ls-compression=zstd&log-compression=zstd
-          upload_logs_to_binary_cache = true
-        '';
-    };
-
-    # A rust nix binary cache
-    harmonia = {
-      enable = true;
-
-      settings = {
-        bind = "127.0.0.1:${toString cachePort}";
-        workers = "20";
-        max_connection_rate = 1024;
-        priority = 30;
-        sign_key_path = config.sops.secrets."nix-serve/secretKey".path;
-      };
-    };
-
-    nginx =
-      let
-        hydraVhost = {
-          forceSSL = true;
-          enableACME = true;
-          locations."/".proxyPass = "http://localhost:${toString config.services.hydra.port}";
-        };
-      in
-      {
-        enable = true;
-        virtualHosts = {
-          "hydra.hq.c3d2.de" = hydraVhost // {
-            default = true;
-          };
-          # "hydra-ca.hq.c3d2.de" = hydraVhost // {
-          #   locations."/".proxyPass = "http://192.168.100.2:3001";
-          # };
-          "hydra.serv.zentralwerk.org" = hydraVhost;
-          "nix-serve.hq.c3d2.de" = {
-            forceSSL = true;
-            enableACME = true;
-            locations."/".proxyPass = "http://localhost:${toString cachePort}";
-          };
-        };
-      };
-    resolved.enable = false;
-  };
-
-  sops = {
-    defaultSopsFile = ./secrets.yaml;
-    secrets."nix-serve/secretKey".mode = "0444";
-  };
-
-  systemd.services = {
-    hydra-evaluator.serviceConfig = {
-      CPUWeight = 2;
-      MemoryHigh = "64G";
-      MemoryMax = "64G";
-      MemorySwapMax = "64G";
-    };
-
-    hydra-init.preStart = let
-      makesSenseForQemuUser = feature:
-        ! (builtins.elem feature [ "kvm" "benchmark" ]);
-      # strips features that don't make sense on qemu-user
-      extraPlatformSystemFeatures =
-        builtins.filter makesSenseForQemuUser config.nix.settings.system-features;
-    in ''
-      cat << EOF > ~/machines
-      localhost x86_64-linux - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," config.nix.settings.system-features} -
-      hydra@localhost ${lib.concatStringsSep "," config.nix.settings.extra-platforms} - ${toString config.nix.settings.max-jobs} 10 ${lib.concatStringsSep "," extraPlatformSystemFeatures} -
-      EOF
-    '';
-
-    nix-daemon.serviceConfig = {
-      CPUWeight = 5;
-      MemoryHigh = "64G";
-      MemoryMax = "64G";
-      MemorySwapMax = "64G";
-    };
-  };
-}
--- a/hosts/hydra/secrets.yaml
+++ b/hosts/hydra/secrets.yaml
@ -1,5 +1,8 @@
 nix-serve:
+    publicKey: ENC[AES256_GCM,data:sR5wk7yvH5+lLpSIP0zNqCLvDRRvR8ws4Q8rVcVJx8YkrywwPcvIsJ1h6mVEu3nc6SLoZlQsuxOGCyNGD98CBNY=,iv:fFV2D27hWoxGtqVt3EnS4hMlrqW5LnIZ3LB5k4xmFWs=,tag:g7R1ossy2On6B2nVfKC9iA==,type:str]
    secretKey: ENC[AES256_GCM,data:cm84sA7E6AnzpVoYuaYepbHGWkRigLdD2RxN21UsXCe7FXQxeTQTxxbzVxJ3G9Lt3kRXuZnODntOo5EQKhs46+wzpO8YLKQxkJXrdluXoGVIWl3/6QFVq66XLJ2i6G4eBK9IH0DYJ+anj8/i8Q==,iv:GEM8Vmx0A8LfJo7QOl0N67Cgk+JqHpp7r+41VivmTg4=,tag:O4Kq4WKgbyt354HSa/7eQQ==,type:str]
+ldap:
+    search-user-pw: ENC[AES256_GCM,data:tSWin/QPIow2P5Aps/XaT42J+MXb8+a24SEri1QjF1O3bDlCxcR8RHqSX8d4Vg==,iv:P5qMaE2cdKxTaXuKO2nh+LDhKkY3psSlWf+JckmUYt4=,tag:eq8XW7P6FNlkviY5PydkZg==,type:str]
 sops:
    kms: []
    gcp_kms: []
@ -24,8 +27,8 @@ sops:
            K3Bpb0svZ1YvVm9ha1ArdVBlN3NHM0kKM6CEQ+dStjEsgppQZYjb1zwyzfwAc0FI
            O5+vi2x8/N/1OH5jeVzLnLjOhXRXrYcR9EDsjT+KDo0ykYh+NjB0DA==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2022-09-25T21:00:25Z"
-    mac: ENC[AES256_GCM,data:eqaN9WFcKAl7Y0HW9liiUyn9eZmLjWOGcNGfu5CQbvQvBXq89mCDyb05gHyQmDm0AsAXI4bU0DUgmdCc846NfOT2kujPQWwiofmTQxlTwxfqt+AVqpwejVqxO3VApCSnkhDrt0jiO9WeyDYUbeVwgnL5CZoJGyYBmmU1LZ2twMo=,iv:tIQpTh0V9qiJsIQ6y0b1+rh+oLRCDrenOixi0GG1Y/M=,tag:J6QLNSH5gQpCAy+P1UAdeg==,type:str]
+    lastmodified: "2022-12-18T17:49:41Z"
+    mac: ENC[AES256_GCM,data:oQGsk4gJ4dGCr3zCanOpSsKQyx/OuWiYe0ef8pvWhqFq0+YvwUzoiF3jQmqahLWGLHUl1yb1kb91GEr7q47G83Z9YXVVtTTzJUWpKZQL1WD12f5ERPSlKnT251fvoMtqNqKrRirGA9ao8OxVpeh3/SQCAGTqWAcpz1J7dxvL7E4=,iv:td9t1MKcXH5RYZxsD1wA4oG61AfLLGSiVIEq/sPRZG4=,tag:UqHGkKqXeeBYXQYtvplCdA==,type:str]
    pgp:
        - created_at: "2022-07-15T23:31:58Z"
          enc: |
--- a/hosts/leon/default.nix
+++ b/hosts/leon/default.nix
@ -26,27 +26,14 @@
 #                  `...-'                                       `...-'
 #--------------------------------------------------------------------------------

-
-{ config, lib, pkgs, ... }:
-
+{ lib, pkgs, ... }:
 {
-  microvm = {
+  deployment = {
+    persistedShares = [ "/etc" "/home" "/var" ];
    mem = 2048;
-    writableStoreOverlay = "/nix/.rw-store";
-    volumes = [ {
-      image = "nix-store-overlay.img";
-      mountPoint = config.microvm.writableStoreOverlay;
-      size = 3048;
-    } ];
  };
  nix.settings.auto-optimise-store = lib.mkForce false;

-  c3d2.deployment = {
-    server = "server10";
-    mounts = [ "etc" "home" "var"];
-    autoNetSetup = true;
-  };
-
  networking = {
    hostName = "leon";
    firewall.enable = true;
@ -65,26 +52,23 @@
  # List packages installed in system profile. To search, run:
  # $ nix search wget
  environment.systemPackages = with pkgs; [
-     wget vim python3Full   python310Packages.pip nmap htop wireguard-tools docker-compose
+     wget vim python3Full   python310Packages.pip python310Packages.flask nmap htop wireguard-tools docker-compose
   ];

  virtualisation.docker.enable = true;

-
  users.users.leon = {
    isNormalUser = true;
    extraGroups = [ "wheel" "docker" ];
    createHome = true;
-    openssh.authorizedKeys.keys = [
-	    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIANupx+diz5N8sGZOc7ZXopyPh9HaML8M7Qh70aVVIaJ leon@leons-Air"
-	  ];
+    openssh.authorizedKeys.keys = with import ../../users.nix;
+      leon.sshKeys;
  };
-
+  networking.nameservers = ["172.20.73.8" "9.9.9.9"];
 	networking.firewall = {
-    allowedTCPPorts = [ 5000 22 53 80 8080 ];
-    allowedUDPPorts = [ 5000 22 53 80 8080 18900 19900 ];  
+    allowedTCPPorts = [ 5000 22 53 80 443 8080 12000 ];
+    allowedUDPPorts = [ 53 80 8080 18900 19900 ];  
  };
-
 #_______________________________Begin-VPN1-Server____________________________________
 
 networking.wireguard.interfaces = {
@ -172,17 +156,7 @@
          allowedIPs = [ "10.10.11.30" ];
 	    }
 	   #---------------------black----------------------------.
-	    {
-		  #black-phone
-		  publicKey= "3GEocBTOztOKWtWQFOmI5hynPiPcMMe54kVaHECAG1A=";
-          allowedIPs = [ "10.10.11.90" ];
-	    }
-	    {
-		  #black-desktop
-		  publicKey= "24TAa1HMXTPaFNfbXLkuxVy305NjSAegMSYjKLH5Kjc=";
-          allowedIPs = [ "10.10.11.91" ];
-	    }
-
+	   
 	  ];
  };
 };
@ -219,20 +193,45 @@ networking.wireguard.interfaces = {

 #-----------------------------ngin-X--------------------------------

-#  services.nginx = {
-#    listen 80;
-#	 enable = true;
-#    virtualHosts."bicospacetech.c3d2.de" = {
-#      default = true;
-#      forceSSL = true;
-#      enableACME = true;
-#      locations."/login" = {
-#        proxyPass = "http://10.10.11.1:80";
-#        proxyWebsockets = true;
-#      };
-#    };
+  services.nginx = {
+    enable = true;
+	virtualHosts."bicospacetech.c3d2.de" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://45.158.40.162:12000";
+        proxyWebsockets = true;
+      };
+    };
+	virtualHosts."bicospacetech.cloud.c3d2.de" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+	    proxyPass = "http://45.158.40.165:80";   
+	    proxyWebsockets = true;
+      };
+	};
+	virtualHosts."bicospacetech.stream.c3d2.de" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://10.10.11.4:14500";
+        proxyWebsockets = true;
+      };
+	};
+	virtualHosts."portainer.netlab.c3d2.de" = {
+      forceSSL = true;
+      enableACME = true;
+      locations."/" = {
+        proxyPass = "http://10.10.11.60:9000";
+        proxyWebsockets = true;
+      };
+	};

+
+};
 #-----------------------------ngin-X--------------------------------

 system.stateVersion = "22.05";
+
 }
--- a/hosts/leoncloud/default.nix
+++ b/hosts/leoncloud/default.nix
@ -1,24 +1,14 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ config, lib, pkgs, ... }:

 {
-  microvm = {
-    mem = 4024;
-    writableStoreOverlay = "/nix/.rw-store";
-    volumes = [ {
-      image = "nix-store-overlay.img";
-      mountPoint = config.microvm.writableStoreOverlay;
-      size = 32 * 1024;
-    } ];
+  deployment = {
+    persistedShares = [ "/etc" "/home" "/var" ];
+    storage = "big";
+    mem = 2048;
  };

  nix.settings.auto-optimise-store = lib.mkForce false;

-  c3d2.deployment = {
-    server = "server9";
-    mounts = [ "etc" "home" "var" ];
-    autoNetSetup = true;
-  };
-
  networking = {
    hostName = "leoncloud";
    firewall.enable = true;
@ -38,6 +28,7 @@
    vim
    python3Full
    python310Packages.pip
+    python310Packages.flask
    nmap
    htop
    wireguard-tools
@ -48,19 +39,15 @@
    isNormalUser = true;
    extraGroups = [ "wheel" "docker" ];
    createHome = true;
-    openssh.authorizedKeys.keys = [
-	    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPZoT83l0ogbJpviBs4VmO+NdF4NPtYAnyf8RRSoXsv leon@leon"
-		"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIANupx+diz5N8sGZOc7ZXopyPh9HaML8M7Qh70aVVIaJ leon@leons-Air"
-	  ];
-  };
+    openssh.authorizedKeys.keys = with import ../../users.nix;
+      leon.sshKeys;
+};


 # enable IP routing
-  boot.kernel.sysctl."net.ipv4.conf.all.forwarding" = 1;
-  boot.kernel.sysctl."net.ipv4.conf.default.forwarding" = 1;

  networking.firewall = {
-    allowedTCPPorts = [ 80 443 22 53 14000 14500 15000  ];
+    allowedTCPPorts = [ 80 443 8080 22 53 14000 14500 15000  ];
    allowedUDPPorts = [ 18900 53 ];  
  };

@ -79,7 +66,7 @@
      privateKeyFile = "/etc/nixos/wireguard-keys/private-key";
      peers = [
        {
-          publicKey = "w3qegSSuqFTFrGk4XJaWBSwGDOiqbnnAIR9MzwjYVA8=";
+          publicKey = "iEVq4lvvKFfqjcoYYyNkA0MS8rcSGaDfPwQGN3C7+D0=";
          allowedIPs = [ "10.10.11.0/24" ];
          endpoint = "45.158.40.162:18900";
          persistentKeepalive = 25;
@ -102,8 +89,8 @@ services.nextcloud = {
      dbname = "nextcloud";
      adminpassFile = "/etc/nixos/next-cloud/pass";
      adminuser = "root";
-      extraTrustedDomains = [ "45.158.40.162" "172.20.79.254" "10.10.11.4" "10.10.11.1" ];
-    };
+      extraTrustedDomains = ["10.10.11.4" "10.10.11.1" "45.158.40.165" "bicospacetech.cloud.c3d2.de"]; 
+   };
  };

  services.postgresql = {
@ -123,6 +110,8 @@ services.nextcloud = {
    after = ["postgresql.service"];
  };

+
+
 #<-----------------nextcloud----------------------

  system.stateVersion = "22.05";
--- a/hosts/mastodon/default.nix
+++ b/hosts/mastodon/default.nix
@ -0,0 +1,168 @@
+{ zentralwerk, config, lib, pkgs, ... }:
+{
+  deployment = {
+    mem = 16 * 1024;
+    vcpu = 16;
+    needForSpeed = true;
+  };
+
+  networking = {
+    hostName = "mastodon";
+    hosts = with zentralwerk.lib.config.site.net.serv; {
+      ${hosts6.up4.auth} = [ "auth.c3d2.de" ];
+      ${hosts4.auth} = [ "auth.c3d2.de" ];
+    };
+    firewall.allowedTCPPorts = [ 80 443 ];
+  };
+
+  system.stateVersion = "22.11";
+  nixpkgs.config.allowUnfreePredicate = pkg:
+    lib.getName pkg == "elasticsearch";
+  
+  services = {
+    elasticsearch = {
+      enable = true;
+      package = pkgs.elasticsearch7;
+    };
+
+    mastodon = {
+      enable = true;
+      localDomain = "c3d2.social";
+
+      smtp = {
+        host = "mail.c3d2.de";
+        port = 587;
+        fromAddress = "mail@c3d2.social";
+        authenticate = false;
+      };
+
+      elasticsearch.host = "127.0.0.1";
+
+      extraConfig = {
+        ALTERNATE_DOMAINS = lib.concatStringsSep "," config.services.nginx.virtualHosts.${config.services.mastodon.localDomain}.serverAliases;
+        DEFAULT_LOCALE = "de";
+        WEB_CONCURRENCY = toString config.deployment.vcpu;
+        # MAX_THREADS = toString 5;
+        LOG_LEVEL = "debug";
+
+        LDAP_ENABLED = "true";
+        LDAP_METHOD = "simple_tls";
+        LDAP_HOST = "auth.c3d2.de";
+        LDAP_PORT = "636";
+        LDAP_BIND_DN = "uid=search,ou=users,dc=c3d2,dc=de";
+        LDAP_BASE = "ou=users,dc=c3d2,dc=de";
+        LDAP_SEARCH_FILTER = "(&(objectclass=person)(|(%{uid}=%{email})(%{mail}=%{email})))";
+        LDAP_UID = "uid";
+      };
+
+      configureNginx = true;
+    };
+
+    nginx.virtualHosts.${config.services.mastodon.localDomain}.serverAliases = [
+      "${config.networking.hostName}.flpk.zentralwerk.org"
+      "social.c3d2.de"
+    ];
+
+    postgresql = {
+      enable = true;
+      ensureUsers = [ {
+        name = "collectd";
+        ensurePermissions = {
+          "DATABASE \"${config.services.mastodon.database.name}\"" = "ALL PRIVILEGES";
+        };
+      } ];
+    };
+  };
+
+  # Inject LDAP secrets
+  systemd.services.mastodon-init-dirs.script = lib.mkAfter ''
+    cat ${config.sops.secrets."mastodon/env".path} >> /var/lib/mastodon/.secrets_env
+  '';
+
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets."mastodon/env" = {
+      owner = "mastodon";
+    };
+  };
+
+  # Sidekiq monitoring
+  c3d2.hq.statistics.enable = true;
+  services.collectd.plugins = {
+    redis =
+      let
+        queries = [ {
+          command = "GET stat:processed";
+          type = "counter";
+          instance = "sidekiq_stat_processed";
+        } {
+          command = "GET stat:failed";
+          type = "counter";
+          instance = "sidekiq_stat_failed";
+        } {
+          command = "LLEN queue:#default";
+          type = "queue_length";
+          instance = "sidekiq_default_queue_len";
+        } {
+          command = "LLEN queue:#ingress";
+          type = "queue_length";
+          instance = "sidekiq_ingress_queue_len";
+        } {
+          command = "LLEN queue:#mailers";
+          type = "queue_length";
+          instance = "sidekiq_mailers_queue_len";
+        } {
+          command = "LLEN queue:#pull";
+          type = "queue_length";
+          instance = "sidekiq_pull_queue_len";
+        } {
+          command = "LLEN queue:#push";
+          type = "queue_length";
+          instance = "sidekiq_push_queue_len";
+        } {
+          command = "LLEN queue:#scheduler";
+          type = "queue_length";
+          instance = "sidekiq_scheduler_queue_len";
+        } {
+          command = "ZCARD schedule";
+          type = "count";
+          instance = "sidekiq_scheduled";
+        } {
+          command = "ZCARD retry";
+          type = "count";
+          instance = "sidekiq_retries";
+        } {
+          command = "ZCARD dead";
+          type = "count";
+          instance = "sidekiq_dead";
+        } {
+          command = "SCARD processes";
+          type = "backends";
+          instance = "sidekiq_processes";
+        } ];
+      in ''
+        <Node "mastodon">
+          Host "${config.services.mastodon.redis.host}"
+          Port "${toString config.services.mastodon.redis.port}"
+          Timeout 3000
+
+          ${lib.concatMapStrings ({ command, type, instance }: ''
+            <Query "${command}">
+              Type "${type}"
+              Instance "${instance}"
+            </Query>
+          '') queries}
+        </Node>
+      '';
+    postgresql = ''
+      <Database "${config.services.mastodon.database.name}">
+        Param database "${config.services.mastodon.database.name}"
+        Query backends
+        Query transactions
+        Query queries
+        Query disk_io
+        Query disk_usage
+      </Database>
+    '';
+  };
+}
--- a/hosts/mastodon/secrets.yaml
+++ b/hosts/mastodon/secrets.yaml
@ -0,0 +1,181 @@
+mastodon:
+    env: ENC[AES256_GCM,data:m7NvIAydlGvvNEShlqH8GngjPb6z3TIGkZNcFcBoAWYHCimcp+0c8NNVf4cP7sq3Xg==,iv:PMC4vVN4felWaa7FDUyoYzNk4Eiy56pxK1cOxbAfZ9c=,tag:NQXqWljloBTxXC1tlxylpQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1dcpd6u4psq3hehjyjrt3s7kzmnvxd20vsc8urjcdv6anr5v7ky2sq9rhtt
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBPUmZnVjR3SGs3MnJSaUxT
+            aVJJZDRVK2hXdDZqeTBRT2ovVmxWMm5SMGdvCm5LSGtKVkprQmlUaDdGT3krZnVl
+            aE5kTG5QZ1JhbjdiWVNwRUp5dkRKbk0KLS0tIFdhSnd5OU02ZTlrMGNMemdYbVNO
+            YlRrbklFV3lmWGRYVXhvUkNLSzdkRUEKWppsjTBRljJnyXTgMSnq4eSlXdzjF0Bc
+            LN6oLoylo3zeT+pWjDFG7A9+fIpCiXsIMcZsHiRR7kfX8lHGi5rv2A==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age12aukzah0pt2rck52hwn08kezyxueqz2f49ld7hpyuzmu847vavdqkunn5c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBdmNOUUpxOE03UFVoN2NY
+            dFMrUUwyVHZjeEZYWWhCMFM4YkVFVkpQa2x3CmoyUkhYRXZORnhXVUd2N0plZEdZ
+            UVUvS0dudkdCRkx0YXdMbWVGV3p4dmMKLS0tIHBCQ2RSRDgyclMzeGJwNndVU3dk
+            ZHQwTTZVNWRRTUR5STRQb3VIODdPcEEK7emzmECdUobG5/N3TMRcUaZSTsgTF6m/
+            5f8v8Tpf0Ve6veQhC/5+MYtHJOWDkN+nD8gFgRkPn/IlNz9IAMGx2A==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-11-30T00:54:21Z"
+    mac: ENC[AES256_GCM,data:Ys/exz5C1UO3FZ5XXSx5wolzO+yJ7p59LFOwAOQM5h5wsSd7T43gql9ecjzYZkBub238+uSJYU8kDCaxUFOwLPBIZ6rmA/MS9wRUVr8z5NB9usQFgtD53NNdcRgMWZcM3GEkCBrjB0grIEA6/Btc2klYzLdYS0Glmnn5IMNwkvE=,iv:vJRinUplCiM6m53Yz5TObXaPI2cAcGVBlfvJAgJcd1U=,tag:7DhGp4KyfaOAFG3UT7PWEw==,type:str]
+    pgp:
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA6j84+xkv3y7AQ//THu/1WoBYLiEjUWSgMXej4xbmN5sUCSxqnyaMaekcQXK
+            MsUODOaqLCo/c1mfpbZ/Pv5CLHah2R92idqShogaiB5tDqxq8MTk8VCR2bzGcg4f
+            CJHf4dvEBzTWIygUM/TH0AaLB277C9WvLHK1ANE2GolSls6hIN5YYiMmAJboImux
+            QYcwZIk5DUtk8CjMMdXoWXS3Nf5LyoV8TnqwGc8KyIK/l+PSLuOX6KqG5TArCCT5
+            BTCbXqpk5IPGVNiflWZfZBSrGFQ/7L+mHYABBR8bj1cCHKOtYJqVCKNQOkApRsc9
+            Z8UtGHedcArwZUNilFUNYMZjK0Iyrme3Fvjt7ztY//ZvgRgBzIfeUlMMYNCD0stT
+            OhjlclAkh7az5LPyhKW7FxEPhr+XsJCiAaaNpHntSwv/5yjPCDLoCbKF8pkZY7xW
+            oaYVJk2/BuNPG8wQAHl9N15XHN5bPK0JIqT8/aSwRx6rjI2JlBk8tOo5EpmdJrLa
+            xPJ7cS8KEpY+mlWO+GkmB3EomjbdgRtFqx4DEjbLYV/olN4+s2RLqK53WprNjqdv
+            fltEZvY8vWcg6UHc3DhAy9Oa6QnR8ifraPkpKlMUZyIiQFOWcrDs9XCpKx/ATmXO
+            +wpQHGZXq6b47Gfp/XA4HibA7YY9qDz2PDtu4PrKg00j6wWBujdc5+HTfBn05JvS
+            UQG2+OgjrHFdYKIK0+vMovt3WFzbrCAeIx9AU84dUMiUZMOtyHZvL+hU3ynVZYfu
+            Z/yvMM78Q1qEdcKWBtgEJORXQIwWTK3JyYRC0N4/3COViw==
+            =ULdx
+            -----END PGP MESSAGE-----
+          fp: A5EE826D645DBE35F9B0993358512AE87A69900F
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA8zMZ+ak7y/zAQ//bSruuL/XKnFtewxG3weWTIUzLjW2d4b0WffSzRogs3BM
+            FE0vbb6atEQVImnr8CFl8r6R1jE3lz6VEVirCAHdycPFxHFnIuzbzc5WYVgTC00E
+            v5L3UKJSccjzWMONdL0XaXUDHWE78vfiMKwjNMrg/SmLElkjJUvBVz1y9lih/Fwp
+            V/CWV/S4/v5WoO53hldoQ3jCDWLLVoJHB5rBO140hpmrLOkwTvXQNW1pK0GwJb6r
+            dKqzUQzJgmdQcMswNDcmtHQV59HGk7/anaWGMD5FvVz6FQdXA2bBH6WD9OmqggUw
+            xN5sWON3yYEkWZ7L0fmLIoNmmsH6gMnPVIpCK40g2nyFCjiqRdxf2gn5iasq9fzJ
+            LAZkUHt6LSNikRPsm9p/SKl5EfIWSmQfUGWYMRqwkHi+a85NfkDPgbOae2gQhaGi
+            TjdKy3wS5SQ1KZU1+YV19JJx8RnzevcIHv/SV8C+z4UosEDBGuncda3S8VRDvyOe
+            hI/ZbASGBwWXni3UIGQfLiDp0nGZz3NAGKcmbWN4DHN7tj36Xc0NkLe9AuWs+2fq
+            rhnfYCLl8tn8gdBQ2/sMC2K8AwpRLdcoOhZPz9CCY3Qv4edxXeSMMotQwhB3jglh
+            GBNFEKd/XKHMfwZsfWITxHBhtX0yL6CEG8A+OHHrYzXcphAcy981T07ThfTkU/DS
+            UQGAruf+SPfBRKj+jHdIXHtrVlYFtI1SHLueZzAw8vMPI4vXbvbZq2qNdWk5uLIx
+            AJHARp7tFLZjeVM8O3fR13HM5Ho50cJLnkCBqqDLXFKtSw==
+            =ZSmW
+            -----END PGP MESSAGE-----
+          fp: D4E89C6A0A58EE803EF708EFA9B23715F7AA3F1A
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcBMA45bZkLXmBFpAQf7BQARnoKk+Dl65Far++0lF3CvvOeiBttUCQoV8aNmMq2V
+            Sy1CfGGhg53xN6qeqh0GE5gXQU6Qio86KCxKZ6yh0i6wicv0G1Ld19omKGFrD6y+
+            ne6ognWNVXzOxp6jzT6bzTxN4KkIrtKwuFb6796E1N0haVzCOjxQaTqUKig7Vy8P
+            0cCR1K/HiVVDs/XdK3QL47Ty4oSCAZDD/Vj9r40w/fKykTw5PBwRsBMCPL1ZxoeI
+            WEMhxxKdCpp5OprMHtsRkulPoArUkyFOKtsCvCRAnLtwoJaJTBb4I0RcTy+wjwEv
+            ONdpobMPntiXY9P8L/ovmYX+GlQKi3AEHflmPmLPFdJRAdzGII2ZnF77Z+e7WpJK
+            SXkWRdSRwJzatIs2TfnEYKAMevqEpzYTIu2Am0NxbTj7MgmfGw7oUc0n+oGh0gLE
+            Jd5EMV7E8k8nHqyCLoEa3q+3
+            =poTN
+            -----END PGP MESSAGE-----
+          fp: A4B0F5A80C2E2448A97BEC25BB829C4DECA6CCB9
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMAwMCBBrc/JA6AQ/9Hz/H90Ab0cQ0mBmFWdgc+zsPVhHXLlKaTszaMbrQ0Hlh
+            Db/1ap7tSQviLZMJFh1O9TjMfRrbCyGE8NRRr1jM2z1KxRSENHMywE+5wR+QlNE7
+            s5v5tNOYyLA5rlkyAL7tezKLnsi4CD3hj27ZVU3igtJ4GXW51rHABiwMQ1FOfVeo
+            vGCQdXAxcFo4TucdIjMWo9hAV8ncVYokw5jq4ufHKjb1keBFq4Ob82dkc3DWwaq7
+            sbUMPGzDkgqwVS2aFk2uoaBjceZeOjY1INNtxtSrmQvjrnJTjoKKRErU11OZ19Om
+            gal7gIUhHb3kJKLkXDZD0l2AD4NUeeFLI7SyVv7PmHD/dI58SQ3+ofsC+Cukvqeo
+            KI9Xyg6+pJCF8wer0tYD1uX4avF9TsdFwVjrrOW2iXvX7ePqP9+ZdwodPOmvB5g6
+            ofxgbX5clek2LCUO/Q8hytx3W/6oBHfqZgoiLitfT+Ss0NoEusGnSDnGlgUAbEyJ
+            KEWEMiRcj8OARw4/jbJrrBjvbIJJaWmdtHBSsqV+3/yY4GqTw8pvUYp3feocLu1Z
+            FwIx/TcvyOmhZ6MFu3sz+xWntyvOnDZoXfnzTLWt7XM2pNHwyWmG62fhlhAIo7jR
+            ENu1Zet7lpj/eUUJAbcx2sWtoClSq8rYqX4PBR4rFnLJBqyPZlVGLzFzlw/kr57S
+            UQGZijKUYvlcDtFjUTiVmUtYagb94Fm4v20P1N1IgwdZp5l00MYY1GT0s7+4QR7Y
+            rRRPLRu4wCvPsYeSpT3xf0bx5tZF7YuzuU0bcr13NcJADw==
+            =44mD
+            -----END PGP MESSAGE-----
+          fp: 4F9F44A64CC2E438979329E1F122F05437696FCE
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA9XEenRNYVGHARAAjqd4Vk98BuS7EGmWe9asgpmuF1hBpi6EGbfIJHy/QdMu
+            Ug27yGxrhVoxP4mX0KcdL0GES6zpcqAuSjKuqnjRIQaBqRmLImvlwpRELQyzJCsj
+            fkhTe3nG14Ew1ukuAPA4LiOpydpefdHb0KiCR4GYggR+P9nPo4CIUVDXIkWdCJHF
+            29b2febhwUZuxAQF0otzvFXd8VpFxP5aqiqjdVQ6wm9R0r/l5XyU8ZwtgN4TjcbO
+            DxvjS73i+CZhi2gPR4Q1pTOFunxQC/shyeOLHFpGnWZ/OkYO+93P9PHPIX9BSUz8
+            PVrBPtLLXm8rnnklqViBTBt+AfwjzyhWTC9BGoKfJ3gmfA//oVJNraQJRXS3AtlY
+            awvIWtLDGSDwzR68cpdoG9DzIES5rJ3gN9u+b/IV6pwvu35E0X3CKA4FN66wdNag
+            VXA6RsJVM1+pCxeKHAyMwD26Udq77XJ1nAxvUaOn0+MGdXD3KKF+WQ+4wv4LW5vl
+            mRKMVuI884B8Qm349TZxUalEy6v7ioC6pJMj13Yn0eWx91z/Kzj8IP06nZ2y486B
+            b8aDmbQUNIF4g3s/LN4y7ENYRVtvL4yhL4JQfDCMeA7wn4Ne/XNIX6/mgcN70JI5
+            ywS4udq+8NX6ZTDoiLNVwZwO3j2V61p1ISx5Q1vPrevCUHHRvUisFL/OzyYYDVXS
+            UQEfC+uXFz7eX15F0esLCcMfDZFWc73IWbGfLL+eXatw5R4Ii2nN+Q8eXLFolmg2
+            DQu+qK0ovO5/9I3VmQM9Y9n0b9CFsfcBLFLA8pfNtj6DOg==
+            =vjG5
+            -----END PGP MESSAGE-----
+          fp: 4B12EFA69166CA8C23FC47E49CD3A46248B660CA
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcBMA/Z87ylQaotQAQgAki4625tAoV04bTPWkawsYck88PbAVctHuwWVi/YprVLy
+            VwXM4nMao+fg16UrRd1sc0Dh1TdPq3E+4oIf9gYJXH8GLcjkj5v3L8jWdaWhC+2G
+            M6IrXdn0kM5uXx3PTFQA6TYkfs/BdNsjTnspfr44EBvWG/OwLmTJo3QC6MDDHZk3
+            JF4cMXXhHe1gRMF5DznexxCHfrod5NYYIzGpqCo8qBtBnY5QnTf7wXwtWnFeRtQk
+            fA/EDgM1Xghyqx7oDxQ7ubaCWPyNREhiB6vKvF8ZnUyWW4oQmx7gmttb+Gm/Pb/G
+            HB9GMixRzmlo4dVGlTF9llYCxcIrvxJNasFEXnuHsdJRAXBwOvBH4JsYgjIYlo0a
+            l+INJ9JBJqvSqbrAadmhmtg7v6fJVaWQiwamI/N50ifHAQcKNxH/OiInuh80Qvtf
+            9utJNqgbFFnzPxx5028FriVd
+            =rqgj
+            -----END PGP MESSAGE-----
+          fp: 9EA68B7F21204979645182E4287B083353C3241C
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA9qJIVK2WMV7AQ/8Ca/FQ2eZdaIerMMiGL0wP8uClgzDqW6Xku4sdGI9nO4s
+            oBzmj3amCgp588PV5/TbyawVrfbMQFvr78GF2T3MQS1KOzFePITQPXceuPia6KzV
+            bFNpElCP80AlYZKWkPQ26Uz1gB+Dk75F+Ws5ttP+A4bgSBUpW7R3CQr4eIM9rN7G
+            TXgTWlbcYZK5IyVZkueOXN+vwT5N/C+rf3Ockv77Yhrasbc/OiRd3kPvwYINrBk5
+            HRpaebK+/1Ku5SHIFhx/TGtyCcwS2ciPBCX1NYvcI+Kh1I2MlHKJqpgY6C0e6QHo
+            sByoi/cy6IIPXupuVMxs/dMzkaZJYMzM3te8D6aakO+yaR/xBlJgGZqzWVgrDZaR
+            7ioncUPcSbR28cGoV4l2bYGu+Z4bQ0oAE8CSV4IYLJeInwRed45E9G5zyTU9//Zd
+            C4dxATU17FXeom15LSoZkFnRLbWWPi6q2SwwidG7+oK5FcMCyc4zarR65/i/02Tm
+            mEUc533y+t0pud9GgdQKrjNyQ654mrsPrXOYviTrBVzy3BU6dkbEjPEmfw253a0Z
+            SDV88+WOSVzwVK9K0EY9NR+sRYlQtBkoKDYrLgthCN+Ucl/tOBIKjwv5lnBUihG/
+            ouTUDJEN5phT2tf8MDchNea31Cm6AT0ph6Iz/Pr0hqUNQOz6UmNgf1ZNH3RCehjS
+            UQGdPPcXxEG5JhmiltTOph024yMOkgmt+5xHyCZ6L1QaaYMjIV08Xq0I2YvJXUse
+            QlZRHAcOAsSI7mYd0jqFxOfj8ZrclTqn15peXweoEUzKIg==
+            =vX2f
+            -----END PGP MESSAGE-----
+          fp: 53B26AEDC08246715E15504B236B6291555E8401
+        - created_at: "2022-11-30T00:52:41Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA/YLzOYaRIJJAQ//RPrikrIQUJ6qVHdEQVtH9CKnNBDCk/ipP4CU1sdHFx7F
+            S4ssLv/sTDofJzo+LFB0QG3JRWEx1+IgtRF+6JCw0Mm7ed2STMPh6/emaX0MIcgS
+            x1t0LMsEmGa6Iuyjm24x5I8qdLpYtOjZd4r797GNeGEXUzRAUe9zbFEY45lSrzn+
+            H/mbohDAK/xEuvY0w1qYW2NgTkUqjxgpK7fx9df5GpiEE9XPHsQindpPYh7BNcau
+            fIqN7d+A+nMw3DI8YcmC2yYnr/5UWBe2UweQuFWyBxyI6548kKbjKOJcAboo1YVL
+            MPyf+Z3eySNG32MyE8vq2/d32/Sh+Cwk/YG6o4mpjwdSTT65JDNZs2eu2alOXnsi
+            uvGDMTlKwvUP89ynB8tDQ8HdV9bxFT3/9yOFAYa+yU9AguvV13zl6ublpO1pPY1q
+            iJMDc51iyRsH9jv9ALHqQ+FIptQDX1yjoIobHx64zrBaiOKeqkdWgZGUfnldbOyw
+            in8Ooa1qgG70h0T+tearHB0BDTCQx7FvEgUN+6eAdGLncoV9a2bGbtb2IBzm7pA3
+            YHV4Ab308WyO/8MkSwd9uGqlsxE772TN5XYLuWo+7KgcSCGght6vnlLWIDhbQ+YR
+            /ZLMik2Gbsl5bV6cpkbzI9VBEx8vHulpQazkHSAfsxr6/aiqmz0TJM8I0oe+d7TS
+            UQHZoPhsqIYDwOqNDi8erCeeQzkDxSVPlBICv8+YGXd5qSAAte7Fbo/uQvn4fGdg
+            9YAyhaZ/XAWnb3rL3VpKlzGXq6LAaog9OyQ+e+bnP/7x8A==
+            =76ZY
+            -----END PGP MESSAGE-----
+          fp: 91EBE87016391323642A6803B966009D57E69CC6
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3
--- a/hosts/matemat/default.nix
+++ b/hosts/matemat/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ pkgs, ... }:

 {
  c3d2 = {
--- a/hosts/mediawiki/default.nix
+++ b/hosts/mediawiki/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ config, lib, pkgs, ... }:

 {
  networking.hostName = "mediawiki";
--- a/hosts/mobilizon/default.nix
+++ b/hosts/mobilizon/default.nix
@ -1,11 +1,11 @@
-{ zentralwerk, config, pkgs, ... }:
+{ config, pkgs, ... }:
 {
-  microvm.mem = 1024;
  c3d2.isInHq = false;
  c3d2.deployment = {
    server = "server10";
    mounts = [ "etc" "home" "var"];
  };
+  microvm.mem = 2048;
  system.stateVersion = "22.05";
  networking = {
    hostName = "mobilizon";
--- a/hosts/mucbot/default.nix
+++ b/hosts/mucbot/default.nix
@ -1,4 +1,4 @@
-{ config, pkgs, tigger, ... }:
+{ pkgs, tigger, ... }:

 {
  deployment = {
--- a/hosts/network-homepage/default.nix
+++ b/hosts/network-homepage/default.nix
@ -1,6 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
-
-with lib;
+{ zentralwerk, pkgs, ... }:

 {
  system.stateVersion = "22.05";
--- a/hosts/nfsroot/default.nix
+++ b/hosts/nfsroot/default.nix
@ -1,7 +1,5 @@
-{ zentralwerk, config, lib, pkgs, ... }:
+{ lib, ... }:
 let
-  netConfig = zentralwerk.lib.config.site.net.serv;
-
  nfsExports = [
    "var/lib/nfsroot/dacbert"
    "var/lib/nfsroot/riscbert"
--- a/hosts/nncp/default.nix
+++ b/hosts/nncp/default.nix
@ -37,7 +37,7 @@
            xx = "tx"; # transmit only
            when-tx-exists = true;
          };
-        in lib.mapAttrs (name: value:
+        in lib.mapAttrs (_: value:
          value // {
            via = lib.lists.remove "c3d2" value.via;
          } // (lib.attrsets.optionalAttrs (value.addrs or { } != { }) {
--- a/hosts/nncp/neighbours.nix
+++ b/hosts/nncp/neighbours.nix
@ -1,7 +1,7 @@
-{ config, lib, pkgs, ... }:
+{ config, pkgs, ... }:

-let
-  exec = {
+{
+  programs.nncp.settings.neigh.emery.exec = {
    # A command to asynchronously export store paths to a remote caller.
    nix-store-export = with pkgs;
      [
@ -16,4 +16,4 @@ let
    # A command to import store paths from a remote caller.
    nix-store-import = [ "${config.nix.package}/bin/nix-store" "--import" ];
  };
-in { programs.nncp.settings.neigh = { emery = { inherit exec; }; }; }
+}
--- a/hosts/oparl/default.nix
+++ b/hosts/oparl/default.nix
@ -1,15 +1,14 @@
-{ zentralwerk, oparl-scraper, config, pkgs, ... }:
+{ oparl-scraper, config, pkgs, ... }:
+
 let
  ratsinfo-scraper = import oparl-scraper { inherit pkgs; };
-
-  netConfig = zentralwerk.lib.config.site.net.serv;
-
 in
 {
  c3d2.deployment = {
    server = "server10";
    mounts = [ "etc" "home" "var"];
  };
+  microvm.mem = 1024;

  networking.hostName = "oparl";

@ -41,7 +40,7 @@ in
    path = with pkgs; [
      git openssh poppler_utils ratsinfo-scraper
    ];
-    script = ''
+    script = /* bash */ ''
      if [ -d data ]; then
        pushd data
        git pull
--- a/hosts/owncast/default.nix
+++ b/hosts/owncast/default.nix
@ -0,0 +1,58 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    ./owncast-archiver.nix
+  ];
+
+  deployment = {
+    vcpu = 8;
+    mem = 2048;
+    persistedShares = [ "/etc" "/home" "/var" ];
+    extraShares = [ {
+      source = "/glusterfs/big/microvms/c3d2/config/owncast/archive";
+      mountPoint = config.services.owncast-archiver.targetDir;
+    } ];
+  };
+  c3d2.hq.statistics.enable = true;
+
+  networking = {
+    hostName = "owncast";
+    firewall.allowedTCPPorts = [ 80 443 ];
+  };
+
+  services.owncast = {
+    enable = true;
+    openFirewall = true;
+  };
+
+  services.nginx = {
+    enable = true;
+    recommendedOptimisation = true;
+    recommendedTlsSettings = true;
+    recommendedGzipSettings = true;
+    additionalModules = [ pkgs.nginxModules.fancyindex ];
+    virtualHosts."owncast.c3d2.de" = {
+      default = true;
+      forceSSL = true;
+      enableACME = true;
+
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${toString config.services.owncast.port}";
+        proxyWebsockets = true;
+      };
+      locations."/archive/" = {
+        alias = "${config.services.owncast-archiver.targetDir}/";
+        extraConfig = ''
+          fancyindex on;
+          fancyindex_exact_size off;
+        '';
+      };
+    };
+  };
+  systemd.services.nginx.serviceConfig.ReadWritePaths = [
+    config.services.owncast-archiver.targetDir
+  ];
+
+  services.owncast-archiver.enable = true;
+}
--- a/hosts/owncast/owncast-archiver.nix
+++ b/hosts/owncast/owncast-archiver.nix
@ -0,0 +1,61 @@
+{ config, lib, pkgs, ... }:
+
+let
+
+  owncastArchiver = with pkgs; writeScript "owncast-archiver.sh" ''
+    #! ${runtimeShell} -e
+
+    PATH=${lib.makeBinPath [ coreutils curl jq ffmpeg ]}
+
+    while true; do
+
+      STATUS="$(curl -s https://owncast.c3d2.de/api/status)"
+      ONLINE="$(echo "$STATUS" | jq -r .online)"
+      if [ "$ONLINE" = true ]; then
+        TITLE="$(echo "$STATUS" | jq -r .streamTitle)"
+        ffmpeg -i https://owncast.c3d2.de/hls/0/stream.m3u8 -c copy "$(echo "$(date -Iseconds)_$TITLE.mkv"|tr " +<>:/" "____\\-\\-")"
+      fi
+
+      sleep ${toString cfg.pollInterval}
+    done
+  '';
+
+  cfg = config.services.owncast-archiver;
+
+in
+
+{
+  options.services.owncast-archiver = with lib; {
+    enable = mkEnableOption "owncast archiver";
+
+    targetDir = mkOption {
+      type = types.str;
+      default = "/mnt/archive";
+    };
+
+    pollInterval = mkOption {
+      type = types.int;
+      default = 10;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users.archiver = {
+      isSystemUser = true;
+      group = "nginx";
+    };
+
+    systemd.services.owncast-archiver = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "owncast.service" ];
+      serviceConfig = {
+        ReadWritePaths = cfg.targetDir;
+        WorkingDirectory = cfg.targetDir;
+        User = "archiver";
+        ExecStart = owncastArchiver;
+        Restart = "always";
+        RestartSec = 60;
+      };
+    };
+  };
+}
--- a/hosts/prometheus/configuration.nix
+++ b/hosts/prometheus/configuration.nix
@ -1,83 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-{
-  imports = [
-    ../../../modules/lxc-container.nix
-  ];
-
-  networking = {
-    hostName = "prometheus";
-    firewall = {
-      allowedTCPPorts = [ 22 80 443 9090 9091 9093 9094 ];
-      enable = true;
-    };
-  };
-
-  services.prometheus = {
-    enable = true;
-
-    alertmanager = {
-      enable = true;
-      openFirewall = true;
-      webExternalUrl = "http://prometheus.serv.zentralwerk.org/alertmanager/";
-      listenAddress = "0.0.0.0";
-      configuration = {
-        "global" = {
-          "smtp_smarthost" = "mail.serv.zentralwerk.org:587";
-          "smtp_from" = "alertmanager@prometheus.serv.zentralwerk.org";
-        };
-        "route" = {
-          "group_by" = [ "alertname" "alias" ];
-          "group_wait" = "30s";
-          "group_interval" = "2m";
-          "repeat_interval" = "4h";
-          "receiver" = "team-admins";
-        };
-        "receivers" = [{
-          "name" = "team-admins";
-          # "email_configs" = [
-          #   {
-          #     "to" = "devnull@example.com";
-          #     "send_resolved" = true;
-          #   }
-          # ];
-          # "webhook_configs" = [
-          #   {
-          #     "url" = "https://example.com/prometheus-alerts";
-          #     "send_resolved" = true;
-          #   }
-          # ];
-        }];
-      };
-    };
-
-    # alertmanagerURL = [ "http://prometheus.serv.zentralwerk.org/alertmanager/" ];
-
-    pushgateway = {
-      enable = true;
-      web.external-url = "http://prometheus.serv.zentralwerk.org/push/";
-    };
-
-    exporters.collectd.enable = true;
-    exporters.collectd.openFirewall = true;
-
-    exporters.nginx.enable = true;
-    exporters.nginx.openFirewall = true;
-  };
-
-  services.nginx = {
-    enable = true;
-
-    virtualHosts."prometheus.serv.zentralwerk.org" = {
-      # serverAliases = [ "registry.serv.zentralwerk.org" ];
-      enableACME = true;
-      onlySSL = true;
-      locations.".well-known/acme-challenge/" = {
-        root = "/var/lib/acme/acme-challenge/.well-known/acme-challenge/";
-      };
-      locations."/" = { proxyPass = "http://localhost:9090"; };
-    };
-  };
-
-  system.stateVersion = "20.09"; # Did you read the comment?
-}
--- a/hosts/prometheus/default.nix
+++ b/hosts/prometheus/default.nix
@ -0,0 +1,154 @@
+{ zentralwerk, config, lib, ... }:
+
+{
+  deployment = {
+    mem = 1024;
+  };
+
+  networking = {
+    hostName = "prometheus";
+    firewall = {
+      allowedTCPPorts = [
+        # nginx
+        80 443
+      ];
+      allowedUDPPorts = [
+        # services.prometheus.exporters.collectd.collectdBinary
+        25826
+      ];
+      enable = true;
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+    retentionTime = "7d";
+
+    alertmanager = {
+      enable = true;
+      webExternalUrl = "https://prometheus.serv.zentralwerk.org/alertmanager/";
+      listenAddress = "[::1]";
+      configuration = {
+        "route" = {
+          "group_by" = [ "instance" ];
+          "group_wait" = "1m";
+          "group_interval" = "1m";
+          "repeat_interval" = "7d";
+          "receiver" = "xmpp";
+        };
+        "receivers" = [{
+          "name" = "xmpp";
+          "webhook_configs" = [{
+            "url" = "http://127.0.0.1:9022/alert";
+          }];
+        }];
+      };
+    };
+
+    alertmanagers = [{
+      static_configs = [{
+        targets = [ "localhost:${toString config.services.prometheus.alertmanager.port}" ];
+      }];
+      path_prefix = "/alertmanager";
+    } {
+      static_configs = [{
+        targets = [ "localhost:9022" ];
+      }];
+    }];
+
+    enableReload = true;
+    ruleFiles = [ ./rules.yaml ];
+
+    scrapeConfigs = [{
+      # TODO: authorization?
+      job_name = "node";
+      scrape_interval = "1m";
+      static_configs =
+        let
+          zwNets = zentralwerk.lib.config.site.net;
+          fromNet = net: _:
+            map
+              (host:
+                "${host}.${net}.zentralwerk.org:9100"
+              )
+              (builtins.attrNames zwNets.${net}.hosts4);
+        in
+        [ {
+          targets =
+            fromNet "serv" (_: true);
+          labels.__meta_net = "net-serv";
+        } {
+          targets =
+            fromNet "flpk" (host: host != "flpk-gw");
+          labels.__meta_net = "net-flpk";
+        } {
+          targets =
+            fromNet "cluster" (host: builtins.elem host [
+              "server8"
+              "server9"
+              "server10"
+            ]);
+          labels.__meta_net = "net-flpk";
+        } {
+          targets = [ "localhost:${toString config.services.prometheus.exporters.collectd.port}" ];
+        } ];
+    }];
+
+    exporters = {
+      collectd = {
+        enable = true;
+        collectdBinary.enable = true;
+      };
+      # TODO: deploy with every nginx
+      nginx = {
+        enable = true;
+        openFirewall = true;
+      };
+    };
+
+    webExternalUrl = "https://prometheus.serv.zentralwerk.org/";
+  };
+
+  services.alert2muc = {
+    enable = true;
+    configFile = config.sops.secrets."alert2muc/config".path;
+  };
+
+  services.nginx = {
+    enable = true;
+
+    virtualHosts."prometheus.serv.zentralwerk.org" = {
+      # serverAliases = [ "registry.serv.zentralwerk.org" ];
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://localhost:${toString config.services.prometheus.port}";
+        extraConfig = ''
+          auth_basic "Prometheus";
+          auth_basic_user_file ${config.sops.secrets."nginx/httpAuth".path};
+        '';
+      };
+      locations."/alertmanager" = {
+        proxyPass = "http://localhost:${toString config.services.prometheus.alertmanager.port}";
+        extraConfig = ''
+          auth_basic "Prometheus";
+          auth_basic_user_file ${config.sops.secrets."nginx/httpAuth".path};
+        '';
+      };
+    };
+  };
+
+  sops = {
+    defaultSopsFile = ./secrets.yaml;
+    secrets."nginx/httpAuth".owner = config.systemd.services.nginx.serviceConfig.User;
+    secrets."alertmanager/xmpp-password".owner = config.systemd.services.prometheus-xmpp-alerts.serviceConfig.User;
+    secrets."alert2muc/config".owner = config.services.alert2muc.user;
+  };
+
+  system.stateVersion = "22.11";
+
+  systemd.services.prometheus-xmpp-alerts.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    User = "prometheus";
+  };
+}
--- a/hosts/prometheus/rules.yaml
+++ b/hosts/prometheus/rules.yaml
@ -0,0 +1,84 @@
+groups:
+  - name: default
+    rules:
+    - alert: oom_kills
+      expr: round(increase(node_vmstat_oom_kill[7d])) >= 1
+      for: 10m
+      labels:
+        severity: error
+      annotations:
+        summary: service gets oom killed
+
+    - alert: disk_almost_full
+      expr: collectd_df_df_complex{type="free"} < 1024^3
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: filesystem has less than 1GB of free space
+
+    - alert: disk_full
+      expr: collectd_df_df_complex{type="free"} < 100 * 1024^2
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: filesystem has less than 100MB of free space
+
+    - alert: disk_free
+      expr: predict_linear(collectd_df_df_complex{type="free"}[1h], 8*3600) < 0
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: filesystem will be full within 8h
+
+    - alert: load1
+      expr: avg_over_time(node_load1[15m]) > 50
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: instance has high load avg
+
+    - alert: systemd_unit_failed
+      expr: increase(node_systemd_unit_state{state="failed"}[7d]) >= 1
+      for: 10m
+      labels:
+        severity: error
+      annotations:
+        summary: service fails
+
+  - name: network
+    rules:
+    - alert: load1
+      expr: avg_over_time(collectd_load_0[15m]) > 1
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: network device has high load avg
+
+    - alert: memory_free
+      expr: collectd_memory{memory="free"} < 4 * 1024^2
+      for: 10m
+      labels:
+        severity: warn
+      annotations:
+        summary: memory full
+
+    - alert: throughput0
+      expr: increase(collectd_interface_if_octets_0_total[10m]) > 10 * 3600 * 1024^2
+      for: 2h
+      labels:
+        severity: warn
+      annotations:
+        summary: sustained throughput
+
+    - alert: throughput1
+      expr: increase(collectd_interface_if_octets_1_total[10m]) > 10 * 3600 * 1024^2
+      for: 2h
+      labels:
+        severity: warn
+      annotations:
+        summary: sustained throughput
--- a/hosts/prometheus/secrets.yaml
+++ b/hosts/prometheus/secrets.yaml
@ -0,0 +1,185 @@
+alertmanager:
+    xmpp-password: ENC[AES256_GCM,data:v+pRv/q4Z/ZT18PBxxKrq9P9QNbjN14edSeMCo5If6Y=,iv:oIdXFPdppgV7uTWY/eKSK9T2vDXW2Uur9iwnftQB+dQ=,tag:QD2MB3ZeMlvuGIMIpHI97Q==,type:str]
+alert2muc:
+    config: ENC[AES256_GCM,data:hdVL+/bC67dP/MDfL+Jd7tq09IuP0l3uigi+kFdV6H6KVVj5XcqtTWd7ZUe6s5lo+ZHWx6QGwliSASNFyLWhF2TV6PGFnT/qQyKmQoY+R9nxeuJ8A8wib21F6qVSoiTHMxg0R13d2/Y1LqZdpsa+2KNdSe1UiBD6/LaIDxMd9rQizVUrnSQ5IDm0BNPhdlFfEvRYJO9TWkgQ0sfmt5Q7,iv:1yLsZLtUtfhJ+1+1w5lBSZ4TsDzmGgUFyA5XpJ/WZH4=,tag:78F5sgrrMezSuh8xbqTLiQ==,type:str]
+nginx:
+    httpAuth: ENC[AES256_GCM,data:PS7icDVNB4g7XBMP7mMSbalkvQ==,iv:0GOfGl97k1AjkRxm2x2f4LpeQOuJcFqAHgdRrbceW6U=,tag:GX5L0wI5zwHwuls7ZOPlOQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age13xhxqulvswuckmpkmy2fgeqd5jx0ar8e2hst33leljt69r6hsvnsrdw63k
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxZGJwdVkyejVHb1hLeU9Y
+            QVJ1bURJV2xwV3FaKzF6YXZHTHFyekR4TEJBCkZoUGxwU0trVXdLM3E5cmdPcmtu
+            bE5NNkRNNlBtclpsbXRNaVJtcnVpTW8KLS0tIFpzR1RlM2ZzYi9wSHRQWFZ6VUxs
+            Tml4ZENJd3Y0cmtTdnQ0ZCtTY256Sm8KRKvkk5WDaC8THCqgoKe2cD+AzdAqtfMH
+            GynKYyQU3rgXl4r8K4XUEkEX8g3+SLitfbo35E66531Q/+yQc79V8Q==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age12aukzah0pt2rck52hwn08kezyxueqz2f49ld7hpyuzmu847vavdqkunn5c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1M1F2N3FEY2xiRFZaVlJv
+            cXUyU1NZYXpKVmlDT2hGM2ltNmVHS0E3SFhVCkcwWmZuT2RZVzg3aC9zQXI3aDhk
+            cFp4RHN2b1RRcXRSNXFudmYrY3oydVEKLS0tIE1JUXdpVlFkQzZKNFBVaFFKTkxR
+            dkpOeVJIVnJtNDM5M0RQaWRudUcxOG8KZnHCLuyPFdx4j1WY6fk8nqMeACmpYZzU
+            EpNqjoBswCkUnaRMVcj5lrHvHDjdbQ0Ypn3s/YvI4UBsXMnnv9UD7w==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-12-16T23:40:12Z"
+    mac: ENC[AES256_GCM,data:GXW0hMq84q5DrbV9YLNu5fvgAwFuBoq3cK7fD+FYHdnX1hq/6pTF2ksu7K7M9DE4yetXx7RKBWERm/1DShpV7Qe0jP/+kh31uwhCws6Ay0l14UDT52lL8uQ/jDHKv6ghG4/Q58u1Hvj/xZas34YmYxxJoyEzzCbMSfSgWh6GSMM=,iv:GuBCWI6s9yLhz3tEarZ1gbtXC+gUOVlbc43bezb9nbU=,tag:2bE8yOu87ooLdqjU5Ex1MQ==,type:str]
+    pgp:
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA6j84+xkv3y7AQ//bQe3Ci9UwmvIuNasPiFzRtI+872msVZWlthYHc0SkrU9
+            A8exnBo0uQbA6h5sA/so+CCt4wIxhMGikzapgTQZ1oP24fzvInES0aSuUsBDfys8
+            Mgp9lvRGCv5jaxhfF8gAeI46M0qvkIkbSInfaLjjFbQn9xFwPrejE4l8RQ8At4Di
+            NSWkvxNQlLH0qBVyXbFvToMhSYVZFsGoncHcNTOC4nEktnFNhptsjsvtA4u6aatv
+            QXSLrlDY5ng4hvjtDTvrzdS1UH5pE9gm9xgqdb95xiS24zwTUGbACP8sWVIUKPX/
+            E38/g6H1AmaVadK/t9F/fP5sTcBI2dMHxHXbndpOb2MxS2lJ/sA4rbFmaVHmblgR
+            J3w1g7JqPOANEneKJ4JWVDRWYwfgQ2IpB4EAetiDtN7KJTWyUgSrB0unNcmy+4cd
+            2H+/CM9ARDwnohPhypSIovjXeh8L1uHqniUXxlIUbXHoNugRybQZYMxZx3cLoH5a
+            WTTy+KJKLdRjeYxDyxVq00KCe9dXsxP/CTRpxZN+ejVAFvDqukzmI3yfSlgiX/M1
+            Ysaa9su1NiFU5h4xsY828Vw4TsNfeiUB/MkHlQQc0f09cd0Aq7Z7lnGs9oWJQOKX
+            bqI6Fjw3nY1QYkfivFZ3baXq4rbt7kTN5WoA+tPXntNVibp93A4X3jl53X5coPbS
+            UQGz3jOgf7Vm5LUQ0VAErMClgKhddv9u+g0XH/uunfRO1ULw4fLFxBmpq/18Tfdg
+            YgiTyJRW/qEqXGzbRl+k2oJz49rn6uJ+Fj6quwZHU67EwA==
+            =kTaA
+            -----END PGP MESSAGE-----
+          fp: A5EE826D645DBE35F9B0993358512AE87A69900F
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA8zMZ+ak7y/zARAA3O9dCE+hACJ84wcEQ+eOF9xlZo96cUbcI/v75+PX9mBG
+            js0ST9ykF8YFtAkAjpuS/B3dwPQY8PdA1fYX83Ke/tw4UeikF3e5kJ2SadlhryGE
+            XpV0qRp79QCHPMauVGYiMK9gLtkgZFGYRcy3GEIz8EDvRpgYnSKJNrJXjDOJhW++
+            WyjwytvSE3WHogP+mhOMgRIn+uYuqirw5c6jIq33ZS3pPuaGe4IR3YysNL8vI+z5
+            6QFa/Vzrh+T2gE6/iGyZ+/Kc0N2Rbg8YwEec5nFGKMAyRsK4nO6e2h+nAjoCR2Hn
+            3NZ4elSTJB+/yRCxcs+TECynPkFpjER49c0sHSvyaCvkoQmRc4kZChukbPEG4SDi
+            qMQnmYNfto5TgkM/2SDpzK+UFq3iYVt+lTySW2sdtn2Kv/oDy/iFfsqFuIix1Egu
+            rlwfSusdLFzvkzD0Lc8ynEevJAA2q7ZnVxSpY7EUgR+9RLfRn3m5KwHzvl20Ylvk
+            O3GU3tJSERY794WmCNiGlmz4q6G84WQTGFo/7e/fxGQJ1gz4btQdxnHMZFJywHzh
+            klO3ZxgFPWfKiBzI47ta/xHxUhcYwjVud3IG8rc/g8LSt4ZJ4aEBIRvrQk6m5v5h
+            32AtP2c3bLz0uSyQWZ4z5OarAxKoReb+7aYPU7BZpoqciElQkyza/hAdC6iEf17S
+            UQFWWEbUvBNW3hWYNhv1sVcIownmObMP5jbwJ/1UuGVBK4MfCCpLcoJgIeVH0m5h
+            LHdHmgwkKO5Rpn7iO8EDn69lBYM2mhbCDeigCaAubLAnsQ==
+            =XHIF
+            -----END PGP MESSAGE-----
+          fp: D4E89C6A0A58EE803EF708EFA9B23715F7AA3F1A
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcBMA45bZkLXmBFpAQgAtwj46Mv1ih23i4RUjW+WwqTrt+2d65JK4Kjm5oPvpGyO
+            hoYLuEV2V9dI1WoJgmp0+pUS/f8LxJdo1OxeZP82ctrOFPxcb1wLfoG+Vg94GPHx
+            wMtY9jNn0W3FX66a32AK0GNorSLl1miT7NRBimD2KAvoPSqKEGc1av4xDm9ue1fc
+            tdcFf8MJjAvaNtAfCvnQ4NqM+lJwPjNmH6CtAQlHHJMgHFuNXZvY8lYbGOw7JpKt
+            opGQD11FjMa5EUyxZfTk8vrPuhgTGknbj/hJOaRLJtTRjnjPfFkLULS9lwaJ0RxN
+            UVI0v6FfrBcC/rfRJC7Uda0UvDSaaUNAiYlSlQZ4c9JRAVKh3yUxGC4rEDP+ecGO
+            8VPvF+H5c3xJ6Qe+81Q1B+vm6rq8WoguilPlQLD3fc8C3vqqNTWpA0b0vVUXm/oq
+            /yeF6f4+jLkZ16LNzIpQ9uyq
+            =qkB4
+            -----END PGP MESSAGE-----
+          fp: A4B0F5A80C2E2448A97BEC25BB829C4DECA6CCB9
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMAwMCBBrc/JA6AQ/9G/+euKdVS9Jiqs9yRyqAujcPwPlYr85zAk+Rjzjo1tux
+            N07qtz6z7HRHgHhP4lGRw7PkvUkxdFB19nnF41jn4ohoXGJrSkHlKEXGcrCJP+Lm
+            8bIKAkTqXvN3qqJH6Rbd76T3wpLAQuq9pm4fsxXl1qHU2PMxehbhVZnwHtPT5mD9
+            YkJV0ZVW91tDeJT5Od93vPyD+z+vccS0kBTWVwlSG7I0fMwQsbQwkVtqrJWM2Foa
+            UPYHDgZEKIMEvpYwy33l9NHjrsLf/kul/xNtKxFCzcjOkw9k/pJdc6CVzX2rVhtp
+            shUOMdzFh/DYqdtwm1h7VVS8xGSdlcFb7nxgYGZuIY8QsWuRB/+j/p5vDmlE46P9
+            SvsuwFJnNc7bE8EHuU1GJGJdQUpyVv+nPGam7L3zLoLRggB8OBS07e/z+ORLogB8
+            I7AF6Pcx9g0AwZeeEczhBYQvFcGjuHGb6uJNMYLylxoNm8u197Fyu7On3l7/IJqX
+            q0w4ickZlkPySbx1OeqIEektiw69HwYhr3/E9B6O+0YL8JRJ5qVtDSw5cudhgIlB
+            b5AfMqt1l+KvjByalhCK6msfuOUDnMhbdjLvSOr8iDUXQ/ZGPtiPaJDJ2Htx32yh
+            zmhWfI7Ws0l/z8Ai939k5ssESccOAfsV4WaKcCcbAke581n+jYEwcTp4KQKzoLLS
+            UQE1vWLijpOjby3So2IMu9gBF0ZmN9EkCt12IP1oa3mMU2yZ5wV/VW1BLSezuVvE
+            v7/FuFnC3jvvYxLV9VUYKQ492PS8oLAwirxrL5a45IyrMA==
+            =of/K
+            -----END PGP MESSAGE-----
+          fp: 4F9F44A64CC2E438979329E1F122F05437696FCE
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA9XEenRNYVGHARAAir30MBOtYtXP3K0hN++LgCtLefPi6455kns0KWkPI8Jd
+            tIrn7h7/O4Znd4QO8Iz1ouyubeEvLbLjS4wMc4NIUNVoDWpmlWveHDgp9ddLDLUG
+            MYBsswVVx2SwCzsIQHADpAa0m2MSfmhTxwBP+lieFg5T3LQi0Jo30DFrDtdIKLbN
+            GPeHuDFdQ8zB2dyPXvSgsOxW9ZNDXAObwewsuEAWhQtkNvtIH62fH+wopjtEdZAX
+            pbHwcAZCOVciWhbKo3zKme1Jq6XQE8Of+w0mBoMeeVr+f54s+1DRE6vy2hH5QDBK
+            1oWRtWpoFrTKGO/KwN5QsehgP6FqZVaWiAMubAR17k2WGzlTQvIWoCxlpv+Mu7eN
+            tsqbLyEmMvmc1NoN06N2CP31KP1Z9bwpxPmiz9Ph8ZfzL9Qw68zJx+dFjrAJIEuo
+            T1KOuWkfPg6llE5Hs+1WqR4Kj0+uH7irNjRGCv6ruWVoYQXP/7dpnwcHBX/Is5IN
+            2gR4Btn4t+nBNCfVcvOq0CUdFGrOr/O0sUDX7Ob0Gqu7fY3nrZpTwGQ5okPFSpJE
+            1/OkWIUgAEHO1Y3X61nc2Iu5eI+jYmSaF12xjiHdKGwmkLfryt4taQBmxPJSN7ym
+            0uiXf0d8SRuqHbz8keBCjp85RV/y1XGX08yltgPXpU2fmv1k5dKErBtNeAIudD/S
+            UQERFu1HIYfCPzAxwI3Z+WbTFo+euO0dZ1Uaw8CVMGcaMTufNQg7H99d+td2pM+D
+            /W8QfZAIGSYDjYWn6YffgRvDwsNZeMpGRo57XNvwZiIu7g==
+            =xRwC
+            -----END PGP MESSAGE-----
+          fp: 4B12EFA69166CA8C23FC47E49CD3A46248B660CA
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcBMA/Z87ylQaotQAQf/ceQh3zHJqWDRP2Q1l5rz69DfEweCJkuCxewVICRKYVie
+            p4LzKUjnZp+8KB3LjHzSEuQXc0eqNXNmGu908uwbwG9Z+xiFj+CIe7KMZWW42gY+
+            I1/nOA0WRzOYevO3vlZSzfZgsN5tfFkQkrU4hMf6YFhuM1m3HOrz5P9pc7uJGETh
+            wHX7k0BSrUjmg8RYcJ+WIc2SPUqv/g0zceLSTE0Btpxg3XmrcHnvs/ThQ6afxHYN
+            K0QEAgIqWwzNU+1+1QkB+yFeAflY10Zbhv0K+WdTn7JzlJZrxyvY32x7lTfCH5c1
+            ycZy/AvGfk5ohkLtSx9f1DxpRCUlOS5TvP7I5X47p9JRAZAaCM9HCG4uMKZwkmEF
+            /Yf7fTJHnIPPY7neR+2qUUg/Y3Th1mNbijBtV0A8XLFpNtWpn8Qzylmak2amYIql
+            l6cwBxhl5N3vg7LGxUQQJfEO
+            =dfLg
+            -----END PGP MESSAGE-----
+          fp: 9EA68B7F21204979645182E4287B083353C3241C
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA9qJIVK2WMV7AQ/+JXfpbLEUdY8fSAmKo/MyaZi9K9cHDs7c4CnEU3Mi8enh
+            9j1/0KTzs99zb6gZKQ/z4MMCiNQ70jK5XcTOgrKtUKeZ0sH969TDamsxxD1Ocfxh
+            Ts+MGTE9C+odIBFvQDOAazkWbPGQ8EheCDauFk8FFZDBj8oK3Vw7hKUxYFMBTM/z
+            UBLAHkbO6Sv38AHmqsHwzsP0+YZgA9pployt28arYXlwX+I7tYAK1V74SkxIWSD1
+            4YHSJvHpos25/MS/PNR4SEQFSceQGfDuFCdwfkC6bKi7tdp3Af34q4v6OqA/iFnx
+            hErcjrXPmlHm/YR2gd7AcLPb7WolB8/j/txl1TAxkSwCRodqcQH8L3bYoA9XxkHP
+            7Yd5gfvivkQy0sjKF3dwpetu4bOdZUEwj+jY/54iHPECKTLK7TFGJ48A3v51Juw3
+            4uU4pPVCTkQnRnkknbhicvs2IzvgS/OfAJTUBKW8+3yPkWenQQfeje0VHSUfT9hO
+            KQ3zafhwGShqXPxbM0J9beigvL3iDE3U7YZYOfrryuHCveSkXobwQZY0Ylok52BX
+            9t12lOldOKBLy127V0sQeZ4eWRiKjoyHC46DFByWN03dn6yRXrcE/8QGOyUgu2PR
+            3SOEm1pnujVa5dhq0MVAxwgHH/+avI+HM0VHRsykYVVOR5O9ywpCAurgB0/wKJrS
+            UQEUAPEm/YDfg7no9GnD/tXCwIgjO1m+H8+Z8e3Mama4hPZV9fuSc4M8GQGhWqFY
+            jYvNgfR7UG/RsqAxoEA1hCoh0Jfu6uLX5/P9X0DQdM8WHA==
+            =KHuT
+            -----END PGP MESSAGE-----
+          fp: 53B26AEDC08246715E15504B236B6291555E8401
+        - created_at: "2022-11-27T00:27:03Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            wcFMA/YLzOYaRIJJARAAs33yYShXtNdy9N+7D0fs5EU7gWYmJfL/lETxkcmQjQg9
+            baX2Zqc2z96jg9TcqFuwbvV6xb/JrSGOs/Qusq1lGPhGaSNHo1KWY8EyDM/6v04x
+            bE9poMb3D0HzIpm+fpDkKZyU15f3Mye5rauQm8a6qgHfd0Aus1HNs9R7RjBeU6pc
+            esEOFOAvbKhg5FcY+E7BVa6nHAHACaK0jiTKNEWU3qfTAqYqarGXhALhujywHMxh
+            YBtu1EQBGimAT6orZCaBMaLbH4LL6ozFk50jHQSgDxti/J4GHDtN4GVzIzySwfkS
+            1S/8PYiM7UglxmdezxopOijzQD36DU0Sh6m44KWIWFPeN2P/Zkau16sfDQ49/K1w
+            sEoZK9RZlFXI+O0PsPHmUkewvUhHMbLF8oa4By8jS0I6rcR+zNPlXQ3Y44v5bEtk
+            y+9CaeJWg0mPf+rtqa5cwkbJhNgpMJM2k6sl5reczVzruYRkMcMO9QlIUF1Apxxa
+            Z6hkFS7TlY7GQ0mpeg7DgKzCOHzHxVPRetit5uH25zxluHzwxYF16G6K2fF+V71v
+            ERlryh/cJDuwAl64aXgIw1DBWMOwixJjO+qfyEUrb4taOXyv5wJ0p+ew6F6AeWue
+            RC1aHcfXN3QJsqXCRRqdI8ceu/Tfo5xTOOAqbNlDzM/kcEXj4rE8WS1cKNknLZHS
+            UQH9XIbJ70Nc5gWR35V7z2Lqos2TRp4WaeTbmr7z902wMDAl2GrjRoyo8EA7k2gG
+            13XKgeObbkdmE7dTwOZbhzrBDFn2pcyI1NsC85iX154pKQ==
+            =wWjU
+            -----END PGP MESSAGE-----
+          fp: 91EBE87016391323642A6803B966009D57E69CC6
+    unencrypted_suffix: _unencrypted
+    version: 3.7.3
--- a/hosts/public-access-proxy/default.nix
+++ b/hosts/public-access-proxy/default.nix
@ -65,7 +65,7 @@
        "autotopia.c3d2.de"
        "c3d2-web.serv.zentralwerk.org"
      ];
-      proxyTo.host = config.c3d2.hosts.c3d2-web.ip4;
+      proxyTo.host = zentralwerk.lib.config.site.net.flpk.hosts4.c3d2-web;
    } {
      hostNames = [
        "codimd.c3d2.de"
@ -133,6 +133,12 @@
    } {
      hostNames = [ "zengel.datenspuren.de" ];
      proxyTo.host = config.c3d2.hosts.zengel.ip4;
+    } {
+      hostNames = [ "owncast.c3d2.de" ];
+      proxyTo.host = config.c3d2.hosts.owncast.ip4;
+    } {
+      hostNames = [ "c3d2.social" ];
+      proxyTo.host = config.c3d2.hosts.mastodon.ip4;
    } ];
  };

@ -149,7 +155,7 @@
    wantedBy = [ "multi-user.target" ];
    path = with pkgs; [ socat ];
    script = ''
-      socat tcp6-listen:1965,fork "tcp6:[${zentralwerk.lib.config.site.net.serv.hosts6.dn42.c3d2-web}]:1965"
+      socat tcp6-listen:1965,fork "tcp6:[${zentralwerk.lib.config.site.net.flpk.hosts6.flpk.c3d2-web}]:1965"
    '';
    serviceConfig = {
      ProtectSystem = "strict";
--- a/hosts/public-access-proxy/proxy.nix
+++ b/hosts/public-access-proxy/proxy.nix
@ -1,50 +1,48 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:

-with lib;
-let cfg = config.services.proxy;
-
-    canonicalize = builtins.replaceStrings ["*" "." ":" "[" "]"] ["all" "_" "_" "" ""];
-
-in {
+let
+  cfg = config.services.proxy;
+  canonicalize = builtins.replaceStrings [ "*" "." ":" "[" "]" ] [ "all" "_" "_" "" "" ];
+in
+{

  options.services.proxy = {
-
-    enable = mkOption {
+    enable = lib.mkOption {
      default = false;
      description = "whether to enable proxy";
-      type = types.bool;
+      type = lib.types.bool;
    };

-    proxyHosts = mkOption {
-      type = types.listOf (types.submodule {
+    proxyHosts = lib.mkOption {
+      type = lib.types.listOf (lib.types.submodule {
        options = {
-          hostNames = mkOption {
-            type = types.listOf types.str;
+          hostNames = lib.mkOption {
+            type = with lib.types; listOf str;
            default = [ ];
            description = ''
              Proxy these hostNames.
            '';
          };
-          proxyTo = mkOption {
-            type = types.submodule {
+          proxyTo = lib.mkOption {
+            type = lib.types.submodule {
              options = {
-                host = mkOption {
-                  type = types.nullOr types.string;
+                host = lib.mkOption {
+                  type = with lib.types; nullOr string;
                  default = null;
                  description = ''
                    Host to forward traffic to.
                    Any hostname may only be used once
                  '';
                };
-                httpPort = mkOption {
-                  type = types.int;
+                httpPort = lib.mkOption {
+                  type = lib.types.int;
                  default = 80;
                  description = ''
                    Port to forward http to.
                  '';
                };
-                httpsPort = mkOption {
-                  type = types.int;
+                httpsPort = lib.mkOption {
+                  type = lib.types.int;
                  default = 443;
                  description = ''
                    Port to forward http to.
@ -57,8 +55,8 @@ in {
            '';
            default = { };
          };
-          matchArg = mkOption {
-            type = types.str;
+          matchArg = lib.mkOption {
+            type = lib.types.str;
            default = "";
            description = "Optional argument to HAProxy `req.ssl_sni -i`";
          };
@ -76,11 +74,9 @@ in {
        };
      }];
    };
-
  };

-  config = mkIf cfg.enable {
-
+  config = lib.mkIf cfg.enable {
    services.haproxy = {
      enable = true;
      config = ''
@ -101,10 +97,9 @@ in {
          option forwardfor
          http-request set-header X-Forwarded-Proto http
          http-request set-header X-Forwarded-Port 80
-          ${
-            concatMapStrings ({ proxyTo, hostNames, matchArg }:
-              optionalString (hostNames != [ ] && proxyTo.host != null) (
-                concatMapStrings (hostname: ''
+          ${lib.concatMapStrings ({ proxyTo, hostNames, matchArg }:
+              lib.optionalString (hostNames != [ ] && proxyTo.host != null) (
+                lib.concatMapStrings (hostname: ''
                  use-server ${canonicalize hostname}-http if { req.hdr(host) -i ${matchArg} ${hostname} }
                  server ${canonicalize hostname}-http ${proxyTo.host}:${
                    toString proxyTo.httpPort
@ -118,13 +113,13 @@ in {
          bind :::443 v4v6
          tcp-request inspect-delay 5s
          tcp-request content accept if { req.ssl_hello_type 1 }
-          ${concatMapStrings ({ proxyTo, hostNames, matchArg }:
-              concatMapStrings (hostname: ''
+          ${lib.concatMapStrings ({ proxyTo, hostNames, matchArg }:
+              lib.concatMapStrings (hostname: ''
                use_backend ${canonicalize proxyTo.host}-https if { req.ssl_sni -i ${matchArg} ${hostname} }
              '') hostNames
          ) cfg.proxyHosts}

-        ${concatMapStrings ({ proxyTo, hostNames, matchArg  }: ''
+        ${lib.concatMapStrings ({ proxyTo, ...  }: ''
          backend ${canonicalize proxyTo.host}-https
            server ${canonicalize proxyTo.host}-https ${proxyTo.host}:${
              toString proxyTo.httpsPort
--- a/hosts/public-access-proxy/stats.nix
+++ b/hosts/public-access-proxy/stats.nix
@ -1,4 +1,4 @@
-{ pkgs, lib, ... }:
+{ pkgs, ... }:
 {
  c3d2.hq.statistics.enable = true;

--- a/hosts/pulsebert/hardware-configuration.nix
+++ b/hosts/pulsebert/hardware-configuration.nix
@ -1,7 +1,7 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{ lib, ... }:

 {
  #imports =
--- a/hosts/radiobert/base.nix
+++ b/hosts/radiobert/base.nix
--- a/hosts/radiobert/default.nix
+++ b/hosts/radiobert/default.nix
@ -26,14 +26,13 @@

  boot = {
    loader = {
+      # generates entry for u-boot
+      generic-extlinux-compatible.enable = true;
      grub.enable = false;
      raspberryPi = {
-        enable = true;
+        # don't use rpi bootloader shit
+        enable = false;
        version = 4;
-        # TODO: this machine actually has u-boot setup but that's no
-        # longer working with nixos-22.05. DO NOT REBOOT! :-)
-        # https://github.com/NixOS/nixpkgs/pull/112677
-        # uboot.enable = true;
        firmwareConfig = ''
          gpu_mem=192
          dtparam=audio=on
@ -86,9 +85,9 @@
      pkgs.makeModulesClosure (x // { allowMissing = true; });
  };

-  nix = {
-    buildCores = 4;
-    maxJobs = 2;
+  nix.settings = {
+    cores = 4;
+    max-jobs = 2;
  };

  sops = {
@ -131,5 +130,14 @@
    '';
  };

+  systemd.extraConfig = ''
+    # Keep cores 2-3 exclusive for SDR processing
+    CPUAffinity=0-1
+  '';
+  systemd.services = {
+    soapysdr-server.serviceConfig.CPUAffinity = "2-2";
+    readsdb.serviceConfig.CPUAffinity = "3-3";
+  };
+
  system.stateVersion = "21.05"; # Did you read the comment?
 }
--- a/hosts/riscbert/default.nix
+++ b/hosts/riscbert/default.nix
@ -54,14 +54,14 @@
  '';

  nix = {
-    buildCores = 2;
-    maxJobs = 1;
-    # trustedUsers = [ "client" ];
-    extraOptions = ''
-      builders-use-substitutes = true
-    '';
    daemonCPUSchedPolicy = "idle";
    daemonIOSchedClass = "idle";
+    settings = {
+      builders-use-substitutes = true;
+      cores = 2;
+      max-jobs = 1;
+      # trusted-users = [ "client" ];
+    };
  };

  system.stateVersion = "22.05";
--- a/hosts/rpi-netboot/default.nix
+++ b/hosts/rpi-netboot/default.nix
@ -1,4 +1,4 @@
-{ hostRegistry, nixpkgs, config, lib, pkgs, modulesPath, ... }:
+{ hostRegistry, lib, pkgs, ... }:

 {
  c3d2 = {
--- a/hosts/scrape/default.nix
+++ b/hosts/scrape/default.nix
@ -1,4 +1,4 @@
-{ zentralwerk, pkgs, config, scrapers, ... }:
+{ pkgs, config, scrapers, ... }:

 let
  freifunkNodes = {
--- a/hosts/sdrweb/default.nix
+++ b/hosts/sdrweb/default.nix
@ -1,9 +1,11 @@
-{ zentralwerk, config, pkgs, ... }:
+{ config, pkgs, ... }:
 {
  deployment = {
    # needs to keep just its ssh key for sops-nix
    persistedShares = [ "/home" "/etc" "/var" ];
    mem = 4096;
+    vcpu = 16;
+    needForSpeed = true;
  };
  c3d2 = {
    isInHq = false;
@ -86,7 +88,4 @@
    inherit (pkgs.mucbot) password;
    muc = "luftraum@chat.c3d2.de/Hubschraubereinsatz";
  };
-
-  # noXlibs breaks cairo:
-  environment.noXlibs = false;
 }
--- a/hosts/server10/default.nix
+++ b/hosts/server10/default.nix
@ -1,4 +1,4 @@
-{ config, pkgs, ... }:
+{ config, ... }:

 {
  imports = [
@ -9,9 +9,10 @@
  c3d2 = {
    deployment.microvmBaseZfsDataset = "server10/vm";
    hq.statistics.enable = true;
+    simd.arch = "ivybridge";
  };

-  boot= {
+  boot = {
    loader.grub = {
      enable = true;
      version = 2;
@ -49,6 +50,7 @@
    "data-hoarder"
    "staging-data-hoarder"
  ];
+  skyflake.nomad.client.meta."c3d2.cpuSpeed" = "4";

  system.stateVersion = "21.11"; # Did you read the comment?
 }
--- a/hosts/server10/hardware-configuration.nix
+++ b/hosts/server10/hardware-configuration.nix
@ -1,4 +1,4 @@
-{ config, lib, pkgs, modulesPath, ... }:
+{ config, lib, modulesPath, ... }:

 {
  imports =
--- a/hosts/server10/microvm-staging.nix
+++ b/hosts/server10/microvm-staging.nix
@ -1,9 +1,7 @@
 { config, pkgs, ... }:
 let
  microvms = {
-    staging-data-hoarder = {
-      flakeref = "git+file:///tmp/dvb-nix-config";
-    };
+    staging-data-hoarder.flakeref = "git+file:///tmp/dvb-nix-config";
  };

  realizeFlake = with pkgs; "${writeScriptBin "realize-flake" ''
--- a/hosts/server8/default.nix
+++ b/hosts/server8/default.nix
@ -0,0 +1,46 @@
+{ ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+  ];
+
+  c3d2 = {
+    # deployment.microvmBaseZfsDataset = "tank/storage";
+    hq.statistics.enable = true;
+    simd.arch = "westmere";
+  };
+
+  boot = {
+    loader.grub = {
+      enable = true;
+      version = 2;
+      # Define on which hard drive you want to install Grub.
+      device = "/dev/disk/by-id/scsi-3600300570140a6102b0acad9825149f2"; # or "nodev" for efi only
+    };
+
+    # kernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
+    kernelParams = [
+      "preempt=none"
+      # No server/router runs any untrusted user code
+      "mitigations=off"
+    ];
+    tmpOnTmpfs = true;
+    tmpOnTmpfsSize = "80%";
+  };
+
+  networking = {
+    hostName = "server8";
+    hostId = "08080808";
+  };
+
+  services = {
+    openssh.enable = true;
+    smartd.enable = true;
+    zfs.autoScrub.enable = true;
+  };
+
+  skyflake.nomad.client.meta."c3d2.cpuSpeed" = "3";
+
+  system.stateVersion = "22.11";
+}
--- a/hosts/server8/hardware-configuration.nix
+++ b/hosts/server8/hardware-configuration.nix
@ -0,0 +1,61 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "uhci_hcd" "ehci_pci" "ahci" "megaraid_sas" "nvme" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "server8_root/nixos";
+      fsType = "zfs";
+    };
+
+  fileSystems."/var" =
+    { device = "server8_root/nixos/var";
+      fsType = "zfs";
+    };
+
+  fileSystems."/nix" =
+    { device = "server8_root/nixos/nix";
+      fsType = "zfs";
+    };
+
+  fileSystems."/nix/store" =
+    { device = "server8_root/nixos/nix/store";
+      fsType = "zfs";
+    };
+
+  fileSystems."/nix/var" =
+    { device = "server8_root/nixos/nix/var";
+      fsType = "zfs";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/984ea997-9591-4efb-8212-8381ad829d0b";
+      fsType = "ext2";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp5s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp5s0f1.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp5s0f2.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp5s0f3.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp8s0f0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp8s0f1.useDHCP = lib.mkDefault true;
+
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}
--- a/hosts/server9/default.nix
+++ b/hosts/server9/default.nix
@ -1,10 +1,17 @@
-{ config, pkgs, ... }:
+_:

 {
  imports = [
    ./hardware-configuration.nix
  ];
-  boot= {
+
+  c3d2 = {
+    deployment.microvmBaseZfsDataset = "tank/storage";
+    hq.statistics.enable = true;
+    simd.arch = "westmere";
+  };
+
+  boot = {
    loader.grub = {
      enable = true;
      version = 2;
@ -25,22 +32,24 @@
    hostName = "server9";
    hostId = "09090909";
  };
-  system.stateVersion = "21.11";
-  services.openssh.enable = true;

-  services.zfs.autoScrub.enable = true;
-  services.smartd.enable = true;
+  # required by libvirtd
+  security.polkit.enable = true;

-  c3d2 = {
-    deployment.microvmBaseZfsDataset = "tank/storage";
-    hq.statistics.enable = true;
+  services = {
+    openssh.enable = true;
+    smartd.enable = true;
+    zfs.autoScrub.enable = true;
  };

+  skyflake.nomad.client.meta."c3d2.cpuSpeed" = "3";
+
+  system.stateVersion = "21.11";
+
  # XXX: enable for zw-ev and poelzi-ha until we find a better solution
  virtualisation.libvirtd = {
    enable = true;
    onShutdown = "shutdown";
  };
-  # required by libvirtd
-  security.polkit.enable = true;
+
 }
--- a/hosts/server9/hardware-configuration.nix
+++ b/hosts/server9/hardware-configuration.nix
@ -1,7 +1,7 @@
 # Do not modify this file!  It was generated by ‘nixos-generate-config’
 # and may be overwritten by future invocations.  Please make changes
 # to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
+{ config, lib, modulesPath, ... }:

 {
  imports =
--- a/hosts/spaceapi/default.nix
+++ b/hosts/spaceapi/default.nix
@ -1,4 +1,5 @@
-{ zentralwerk, ... }:
+_:
+
 {
  c3d2.deployment = {
    server = "server10";
--- a/hosts/sshlog/default.nix
+++ b/hosts/sshlog/default.nix
@ -1,13 +1,12 @@
 { config, lib, pkgs, ... }:
 {
+  deployment = {
+    persistedShares = [ "/etc" "/home" "/var" ];
+    mem = 512;
+  };
  c3d2 = {
    isInHq = false;
    hq.statistics.enable = true;
-    deployment = {
-      server = "server9";
-      mounts = [ "etc" "home" "var" ];
-      mountBase = "/tank/storage/${config.networking.hostName}";
-    };
  };

  networking = {
--- a/hosts/stream/default.nix
+++ b/hosts/stream/default.nix
@ -1,36 +1,16 @@
-{ zentralwerk, config, pkgs, ... }:
+{ zentralwerk, config, lib, pkgs, ... }:
 let
  authFile = pkgs.writeText "htpasswd" "k-ot:sawCOTsl/fIUY";
-  mac = {
-    pub = "DE:91:C7:51:D1:C5";
-    serv = "C6:40:E0:21:9B:A4";
-  };
 in
 {
  networking.hostName = "stream";
  c3d2.hq.statistics.enable = true;
-  c3d2.deployment = {
-    server = "server9";
-    autoNetSetup = false;
-    mounts = [];
+  deployment = {
+    persistedShares = [ "/etc" "/home" "/var" ];
+    storage = "big";
+    mem = 4096;
+    networks = lib.mkForce [ "pub" "serv" ];
  };
-  microvm.mem = 2048;
-  microvm.interfaces = [ {
-    type = "tap";
-    id = "pub-stream";
-    mac = mac.pub;
-  } {
-    type = "tap";
-    id = "serv-stream";
-    mac = mac.serv;
-  } ];
-  microvm.shares = map (name: {
-    source = "/tank/storage/stream/${name}";
-    mountPoint = "/${name}";
-    tag = name;
-    proto = "virtiofs";
-    socket = "${name}.socket";
-  }) [ "etc" "home" "var" ];

  systemd.network = {
    enable = true;
@ -39,19 +19,14 @@ in
    # rest of the network so that I am reachable by
    # public-access-proxy.

-    links."00-serv" = {
-      matchConfig.MACAddress = mac.serv;
-      linkConfig.Name = "serv";
-    };
-
-    networks."00-serv" = {
-      matchConfig.MACAddress = mac.serv;
+    networks."30-serv" = {
      networkConfig.IPv6AcceptRA = false;
      # try harder disabling global ipv6
      networkConfig.LinkLocalAddressing = "no";
      addresses = [ {
        addressConfig.Address = "${config.c3d2.hosts.stream.ip4}/${toString zentralwerk.lib.config.site.net.serv.subnet4Len}";
      } ];
+      gateway = lib.mkForce [];
      routes = [ {
        routeConfig = {
          Destination = "172.20.0.0/14";
@ -61,15 +36,9 @@ in
    };

    # On the pub network I am a normal client.
-
-    links."00-pub" = {
-      matchConfig.MACAddress = mac.pub;
-      linkConfig.Name = "pub";
-    };
-
-    networks."01-pub" = {
-      matchConfig.MACAddress = mac.pub;
+    networks."30-pub" = {
      networkConfig.DHCP = "ipv4";
+      gateway = lib.mkForce [];
      networkConfig.IPv6AcceptRA = true;
    };
  };
--- a/hosts/ticker/default.nix
+++ b/hosts/ticker/default.nix
@ -1,6 +1,4 @@
-{ zentralwerk, config, lib, pkgs, ... }:
-
-with lib;
+{ zentralwerk, config, ... }:

 {
  system.stateVersion = "22.05";
--- a/hosts/zengel/default.nix
+++ b/hosts/zengel/default.nix
@ -1,7 +1,6 @@
-{ pkgs, ... }:
-let
-  domain = "zengel.datenspuren.de";
-in {
+{ config, pkgs, ... }:
+
+{
  networking.hostName = "zengel";
  microvm.mem = 1024;
  c3d2.deployment = {
@ -13,7 +12,7 @@ in {

  services.engelsystem = {
    enable = true;
-    domain = domain;
+    domain = "zengel.datenspuren.de";
    createDatabase = true;
    package = pkgs.engelsystem.override { php = pkgs.php74; };
    config = {
@ -50,7 +49,7 @@ in {
  services.phpfpm.phpPackage = pkgs.php74;
  services.nginx = {
    enable = true;
-    virtualHosts."${domain}" = {
+    virtualHosts."${config.services.engelsystem.domain}" = {
      default = true;
      forceSSL = true;
      enableACME = true;
--- a/modules/audio-server.nix
+++ b/modules/audio-server.nix
@ -35,7 +35,7 @@
        bluetoothSupport = true;
        advancedBluetoothCodecs = true;
        zeroconfSupport = true;
-      }).overrideAttrs (oldAttrs: {
+      }).overrideAttrs (_: {
        # one test times out
        doCheck = false;
      });
--- a/modules/backup.nix
+++ b/modules/backup.nix
@ -2,10 +2,10 @@

 {
  config.services.postgresqlBackup = {
-    enable = config.services.postgresql.enable;
+    inherit (config.services.postgresql) enable;
    backupAll = true;
    compression = "zstd";
-    # compressionLevel = 9; # TODO: only available with 21.11
+    compressionLevel = 9;
    pgdumpOptions = "--create --clean";
    startAt = "*-*-* 06:00:00";
  };
--- a/modules/c3d2.nix
+++ b/modules/c3d2.nix
@ -157,6 +157,18 @@ in
      };
    };

+    simd = {
+      enable = lib.mkEnableOption "optimized builds with simd instructions";
+      arch = lib.mkOption {
+        type = with lib.types; nullOr str;
+        default = null;
+        description = ''
+          Microarchitecture string for nixpkgs.hostPlatform.gcc.march and to generate system-features.
+          Can be determined with: gcc -march=native -Q --help=target | grep march
+        '';
+      };
+    };
+
    users = mkOption {
      type = types.attrsOf (types.submodule {
        options.sshKeys = mkOption {
@ -169,21 +181,21 @@ in

  config =
    let
-      adminKeys = (with builtins; lib.lists.flatten (
+      adminKeys = with builtins; lib.lists.flatten (
        map
          (getAttr "sshKeys")
          (attrValues cfg.users)
-      ));
+      );
      mkIfIsInHq = x: lib.mkIf cfg.isInHq (lib.mkDefault x);
    in
    {
      networking.hosts = lib.mkIf cfg.mergeHostsFile
        ((
          lib.attrsets.mapAttrs' (n: v: { name = v.ip4; value = [ "${n}.c3d2" ]; })
-            (lib.attrsets.filterAttrs (n: v: v.ip4 != null) cfg.hosts)
+            (lib.attrsets.filterAttrs (_: v: v.ip4 != null) cfg.hosts)
        ) // (
          lib.attrsets.mapAttrs' (n: v: { name = v.ip6; value = [ "${n}.c3d2" ]; })
-            (lib.attrsets.filterAttrs (n: v: v.ip6 != null) cfg.hosts)
+            (lib.attrsets.filterAttrs (_: v: v.ip6 != null) cfg.hosts)
        ));

      programs.nncp.settings = lib.optionalAttrs cfg.mergeNncpSettings cfg.nncp;
@ -383,9 +395,13 @@ in

      nix = {
        settings = {
-          auto-optimise-store = true;
+          builders-use-substitutes = true;
+          connect-timeout = 20;
+          experimental-features = "nix-command flakes";
+          fallback = true;
+          # don't self feed hydra
          trusted-public-keys = lib.mkIf (config.networking.hostName != "hydra") [
-            (builtins.readFile ../hosts/hydra/cache-pub.key)
+            "nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps="
          ];
          substituters = lib.mkIf (config.networking.hostName != "hydra") (
            lib.mkBefore [ "https://nix-serve.hq.c3d2.de" ]
@ -407,10 +423,6 @@ in
            url = "https://gitea.c3d2.de/C3D2/nix-config.git";
          };
        };
-        extraOptions = ''
-          experimental-features = nix-command flakes
-          builders-use-substitutes = true
-        '';
      };

      services.openssh = {
@ -423,32 +435,22 @@ in

      environment = {
        systemPackages = with pkgs; [
-          # Network fetchers
+          bmon
          curl
-          wget
+          ethtool
          git
-          # System monitors
          htop
          iotop
-          bmon
-          ripgrep
-          # Terminal managers
-          tmux
-          screen
-          # Editors
-          vim
-          # Pipeview
-          pv
-          # Network debugging
-          tcpdump
-          ethtool
          mtr
+          pv
+          ripgrep
+          screen
+          tcpdump
+          tmux
+          tree
+          vim
+          wget
        ];
-        variables = {
-          # TERM = "xterm-256color";
-        };
-        # breaks various package builds
-        noXlibs = lib.mkForce false;
      };

      programs = {
--- a/modules/cluster/default.nix
+++ b/modules/cluster/default.nix
@ -1,11 +1,12 @@
-{ zentralwerk, hostRegistry, config, lib, pkgs, ... }:
+{ zentralwerk, hostRegistry, config, lib, ... }:
 let
  inherit (config.networking) hostName;

  # hydra does *not* use this module because it only runs a nomad
  # server but no client and no microvms
-  servers = [ "server9" "server10" "hydra" ];
-  microvmServers = [ "server9" "server10" ];
+  servers = [ "server8" "server9" "server10" "hydra" ];
+  microvmServers = [ "server8" "server9" "server10" ];
+  storageServers = [ "server8" "server9" ];

  serverNet = server:
    builtins.foldl' (result: net:
@ -14,8 +15,6 @@ let
      then net
      else result
    ) null [ "cluster" "serv" ];
-
-  ipv4Addr = zentralwerk.lib.config.site.net.${serverNet hostName}.hosts4.${hostName};
 in {
  # Open firewall between cluster members
  networking.firewall.extraCommands = lib.concatMapStrings (server:
@ -31,21 +30,35 @@ in {

  # Cluster configuration
  skyflake = {
+    # debug = true;
    nodes = builtins.listToAttrs (
      map (name: {
        inherit name;
-        value.address = builtins.trace name hostRegistry.hosts.${name}.ip4;
+        value.address = hostRegistry.hosts.${name}.ip4;
      }) servers
    );
    nomad = {
      datacenter = "c3d2";
-      servers = servers;
+      inherit servers;
      # run tasks only on these:
      client.enable = builtins.elem hostName microvmServers;
+      client.meta =
+        lib.optionalAttrs (builtins.elem hostName storageServers) {
+          "c3d2.storage" = "big";
+        };
    };
    microvmUid = 997;

-    users.c3d2.sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
+    users.c3d2 = {
+      uid = 1001;
+      sshKeys = config.users.users.root.openssh.authorizedKeys.keys;
+    };
+    users.leon = {
+      uid = 1002;
+      sshKeys = with import ../../users.nix;
+        leon.sshKeys ++
+        astro.sshKeys;
+    };
    deploy.customizationModule = ./deployment.nix;

    storage.glusterfs = {
@ -53,7 +66,16 @@ in {
        servers = microvmServers;
        mountPoint = "/glusterfs/fast";
        source = "/var/glusterfs-fast";
+      } {
+        servers = storageServers;
+        mountPoint = "/glusterfs/big";
+        source = "/var/glusterfs-big";
      } ];
    };
  };
+
+  systemd.tmpfiles.rules = [
+    # additional gcroots
+    "L+ /nix/var/nix/gcroots/skyflake-microvms-big - - - - /glusterfs/big/gcroots"
+  ];
 }
--- a/modules/cluster/deployment-options.nix
+++ b/modules/cluster/deployment-options.nix
@ -1,22 +1,79 @@
-{ lib, ... }:
+{ zentralwerk, config, lib, ... }:
 # our custom options
 {
  options.deployment = with lib; {
    vcpu = mkOption {
      type = types.int;
-      default = 2;
+      default = 4;
    };
    mem = mkOption {
      type = types.int;
      default = 512;
    };
+    hypervisor = mkOption {
+      type = types.enum [
+        "qemu"
+        "cloud-hypervisor"
+        "firecracker"
+        "crosvm"
+        "kvmtool"
+      ];
+      default = "cloud-hypervisor";
+    };
    networks = mkOption {
      type = with types; listOf str;
-      default = [ "serv" ];
+      default = builtins.attrNames (
+        lib.filterAttrs (_: { hosts4, hosts6, ... }:
+          hosts4 ? ${config.networking.hostName} ||
+          lib.filterAttrs (_: hosts6:
+            hosts6 ? ${config.networking.hostName}
+          ) hosts6 != {}
+        ) zentralwerk.lib.config.site.net
+      );
    };
    persistedShares = mkOption {
      type = with types; listOf str;
      default = [ "/etc" "/home" "/var" ];
    };
+    storage = mkOption {
+      type = types.enum [ "fast" "big" ];
+      default = "fast";
+      description = ''
+        Which glusterfs volume to use for persistedShares
+      '';
+    };
+    extraShares = mkOption {
+      type = with types; listOf (submodule {
+        options = {
+          source = mkOption {
+            type = str;
+          };
+          mountPoint = mkOption {
+            type = str;
+          };
+        };
+      });
+      default = [];
+      description = ''
+        Extra shares. THESE MUST BE AVAILABLE ON ALL MICROVM HOSTS!
+      '';
+    };
+    needForSpeed = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Prefer deployment on Nomad clients with a higher c3d2.cpuSpeed
+      '';
+    };
+  };
+
+  config = {
+    # HACK: Avoid conflicts when building a NixOS configuration on Hydra
+    boot.loader.grub.enable = false;
+    fileSystems."/" = lib.mkDefault {
+      device = "rootfs";
+      fsType = "tmpfs";
+      options = [ "size=50%,mode=0755" ];
+    };
  };
 }
--- a/modules/cluster/deployment.nix
+++ b/modules/cluster/deployment.nix
@ -8,17 +8,9 @@ let
    flpk = "flpk-gw";
  };

-  nets = builtins.attrNames (
-    lib.filterAttrs (net: { hosts4, hosts6, ... }:
-      hosts4 ? ${hostName} ||
-      lib.filterAttrs (ctx: hosts6:
-        hosts6 ? ${hostName}
-      ) hosts6 != {}
-    ) zentralwerk.lib.config.site.net
-  );
-
  inherit (config.networking) hostName;
  inherit (config.system.build.skyflake-deployment) user repo vmName;
+  inherit (config.deployment) networks;

  generateMacAddress = net:
    let
@ -33,32 +25,48 @@ let
    then s
    else withoutLeadingSlash s';

+  writableStoreOverlayImage = "/var/tmp/${user}-${repo}-${vmName}-overlay.img";
+
 in
 {
  microvm = {
-    hypervisor = "cloud-hypervisor";
-    vcpu = config.deployment.vcpu;
-    mem = config.deployment.mem;
+    inherit (config.deployment) mem vcpu hypervisor;

-    shares = [ {
-      proto = "virtiofs";
-      tag = "ro-store";
-      source = "/nix/store";
-      mountPoint = "/nix/.ro-store";
-    } ] ++ map (mountPoint: {
-      proto = "virtiofs";
-      tag = builtins.replaceStrings [ "/" ] [ "-" ] (
-        withoutLeadingSlash mountPoint
-      );
-      source = "/storage/glusterfs/microvms/${user}/${repo}/${vmName}/${withoutLeadingSlash mountPoint}";
-      inherit mountPoint;
-    }) config.deployment.persistedShares;
-    # volumes = [ {
-    #   image = "/storage/glusterfs/microvms/${user}/${repo}/${vmName}/overlay.img";
-    #   mountPoint = "/";
-    #   size = 8 * 1024;
-    # } ];
-    # writableStoreOverlay = "/nix/.rw-store";
+    preStart = ''
+      # Discard old writable store overlay
+      rm -f "${writableStoreOverlayImage}"
+    '';
+
+    shares =
+      [ {
+        proto = "virtiofs";
+        tag = "ro-store";
+        source = "/nix/store";
+        mountPoint = "/nix/.ro-store";
+      } ]
+      ++
+      map (mountPoint: {
+        proto = "virtiofs";
+        tag = builtins.replaceStrings [ "/" ] [ "-" ] (
+          withoutLeadingSlash mountPoint
+        );
+        source = "/glusterfs/${config.deployment.storage}/microvms/${user}/${repo}/${vmName}/${withoutLeadingSlash mountPoint}";
+        inherit mountPoint;
+      }) config.deployment.persistedShares
+      ++
+      map ({ source, mountPoint }: {
+        proto = "virtiofs";
+        tag = builtins.replaceStrings [ "/" ] [ "-" ] (
+          withoutLeadingSlash mountPoint
+        );
+        inherit mountPoint source;
+      }) config.deployment.extraShares;
+    volumes = [ {
+      image = writableStoreOverlayImage;
+      mountPoint = config.microvm.writableStoreOverlay;
+      size = 8 * 1024;
+    } ];
+    writableStoreOverlay = "/nix/.rw-store";

    interfaces = map (net: {
      type = "tap";
@ -67,6 +75,21 @@ in
    }) config.deployment.networks;
  };

+  skyflake = {
+    nomadJob = {
+      constraints = lib.optionals (config.deployment.storage == "big") [ {
+        attribute = "\${meta.c3d2.storage}";
+        value = "big";
+      } ];
+      affinities = lib.optionals config.deployment.needForSpeed (builtins.genList (i: {
+        attribute = "\${meta.c3d2.cpuSpeed}";
+        operator = ">=";
+        value = toString (i + 1);
+        weight = 10 + i;
+      }) 10);
+    };
+  };
+
  networking = {
    useDHCP = false;
    dhcpcd.enable = false;
@ -81,7 +104,7 @@ in
        # rename interface to net name
        linkConfig.Name = net;
      };
-    }) {} nets;
+    }) {} networks;

    networks = builtins.foldl' (networks: net: networks // {
      "30-${net}" =
@ -112,6 +135,10 @@ builtins.filter (hosts6: hosts6 ? ${hostName}) (
              )
          );
        };
-    }) {} nets;
+    }) {} networks;
  };
+
+  # nix-gc breaks writable store overlays, devastating running
+  # MicroVMs
+  nix.gc.automatic = false;
 }
--- a/modules/default.nix
+++ b/modules/default.nix
@ -1,127 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-let
-  # TODO: move to flake
-  nixcloud-webservices = pkgs.fetchFromGitHub {
-    owner = "nixcloud";
-    repo = "nixcloud-webservices";
-    rev = "3a0767f0536fac811065eb87e6342f27eac085aa";
-    sha256 = "vC0vBu+0HchrevuWsmE7giouKnSt/q4F0TffwhuNJv8=";
-  };
-  inherit (import "${nixcloud-webservices}/pkgs" { inherit pkgs; }) nixcloud;
-
-  profilesDir = "/nix/var/nix/profiles/lxc";
-  inherit (config.lxc) containers;
-  inherit (config.nix) nixPath;
-
-  toLxcConfig' = path: a:
-    if builtins.isString a then ''
-      ${path} = ${a}
-    '' else if builtins.isInt a then ''
-      ${path} = ${toString a}
-    '' else if builtins.isAttrs a then
-      lib.concatMapStrings (name:
-        let path' = if path == "" then name else "${path}.${name}";
-        in toLxcConfig' path' (builtins.getAttr name a)) (builtins.attrNames a)
-    else if builtins.isList a then
-      lib.concatMapStrings (toLxcConfig' path) a
-    else
-      throw "Invalid LXC config value";
-  toLxcConfig = toLxcConfig' "";
-
-  lxc-rootfs = pkgs.runCommand "lxc-rootfs" { } ''
-    mkdir -p $out/share/lxc/rootfs/{dev,nix/store,proc,run,sys,tmp}
-  '';
-in {
-  options = with types; {
-    lxc.containers = mkOption {
-      type = attrs;
-      default = { };
-    };
-  };
-
-  config = mkIf (containers != { }) {
-    virtualisation.lxc.enable = true;
-    environment.systemPackages = [ nixcloud.container ];
-
-    virtualisation.lxc.defaultConfig = ''
-      lxc.id_map = u 0 100000 65536
-      lxc.id_map = g 0 100000 65536
-    '';
-    users.users.root.subGidRanges = [{
-      count = 65536;
-      startGid = 100000;
-    }];
-    users.users.root.subUidRanges = [{
-      count = 65536;
-      startUid = 100000;
-    }];
-
-    systemd.services = if true then
-      { }
-    else
-      builtins.foldl' (services: name:
-        let
-          systemDir = "/${profilesDir}/${name}/system";
-          lxcDefaults = {
-            lxc = {
-              uts.name = name;
-              rootfs.path = "/run/current-system/sw/share/lxc/rootfs";
-              mount.entry = [
-                "${systemDir}/init /init none bind,ro 0 0"
-                "/nix/store /nix/store none bind,ro 0 0"
-              ];
-              autodev = 1;
-              include = "/run/current-system/sw/share/lxc/config/common.conf";
-              apparmor.profile = "generated";
-              environment = "TERM=linux";
-            };
-          };
-          config = builtins.getAttr name containers;
-          lxcConfig = builtins.toFile "lxc-container-${name}.conf"
-            # TODO: more intelligent merging?
-            (toLxcConfig (lxcDefaults // config.lxc));
-
-          builder = {
-            description = "Build NixOS for lxc container ${name}";
-            wants = [ "nix-daemon.socket" ];
-            after = [ "nix-daemon.service" ];
-
-            path = with pkgs; [ coreutils nix ];
-
-            serviceConfig = {
-              Type = "oneshot";
-              RemainAfterExit = true;
-              Environment = [ "NIX_PATH=${builtins.concatStringsSep ":" nixPath}" ];
-            };
-
-            script = ''
-              mkdir -p ${profilesDir}/${name}
-
-              nix-env -p ${profilesDir}/${name}/system \
-                -I nixos-config=${config.nixos-config} \
-                -f '<nixpkgs/nixos>' \
-                --set -A system
-            '';
-          };
-          starter = {
-            description = "LXC container ${name}";
-            requires = [ "lxc-container-${name}-builder.service" ];
-            after = [ "lxc-container-${name}-builder.service" ];
-
-            path = with pkgs; [ lxc apparmor-parser ];
-
-            script = ''
-              mkdir -p /var/lib/lxc/${name}
-              ln -fs ${lxcConfig} /var/lib/lxc/${name}/config
-              lxc-start -F -n ${name}
-            '';
-          };
-        in services // {
-          "lxc-container-${name}-builder" = builder;
-          "lxc-container-${name}" = starter;
-        }) { } (builtins.attrNames containers);
-  };
-}
--- a/modules/logging.nix
+++ b/modules/logging.nix
@ -1,57 +0,0 @@
-{ hostRegistry, config, pkgs, lib, ... }:
-
-let
-
-  nginxGlobalLogging = ''
-    log_format graylog2_json escape=json '{ "timestamp": "$time_iso8601", '
-                     '"remote_addr": "$remote_addr", '
-                     '"body_bytes_sent": $body_bytes_sent, '
-                     '"request_time": $request_time, '
-                     '"response_status": $status, '
-                     '"request": "$request", '
-                     '"request_method": "$request_method", '
-                     '"host": "$host",'
-                     '"upstream_cache_status": "$upstream_cache_status",'
-                     '"upstream_addr": "$upstream_addr",'
-                     '"http_x_forwarded_for": "$http_x_forwarded_for",'
-                     '"http_referrer": "$http_referer", '
-                     '"http_user_agent": "$http_user_agent" }';
-
-    # replace the hostnames with the IP or hostname of your Graylog2 server
-    access_log syslog:server=graylog.server.org:12301 graylog2_json;
-    error_log syslog:server=graylog.server.org:12302;
-  '';
-
-in {
-  # add central logging
-  services.journalbeat = {
-    enable = false;
-    extraConfig = ''
-      journalbeat.inputs:
-        # Paths that should be crawled and fetched. Possible values files and directories.
-        # When setting a directory, all journals under it are merged.
-        # When empty starts to read from local journal.
-      - paths: []
-
-      journalbeat:
-        seek_position: cursor
-        cursor_seek_fallback: tail
-        write_cursor_state: true
-        cursor_flush_period: 5s
-        clean_field_names: true
-        convert_to_numbers: false
-        move_metadata_to_field: journal
-        default_type: journal
-        kernel: true
-      output.logstash:
-        # Boolean flag to enable or disable the output module.
-        enabled: true
-        hosts: ["${config.c3d2.hosts.logging.ip4}:5044"]
-    '';
-  };
-
-  services.prometheus.exporters.node = lib.mkIf (pkgs.system != "riscv64-linux") {
-    enable = true;
-    openFirewall = true;
-  };
-}
--- a/modules/lxc-container.nix
+++ b/modules/lxc-container.nix
@ -1,66 +0,0 @@
-{ hostRegistry, config, pkgs, lib, modulesPath, ... }:
-
-{
-  imports = [
-    (modulesPath + "/profiles/docker-container.nix")
-  ];
-
-  boot = {
-    isContainer = true;
-    loader = {
-      grub.enable = false;
-      # /sbin/init
-      initScript.enable = true;
-    };
-  };
-
-  environment.etc."resolv.conf".text = lib.concatMapStrings (ns: ''
-    nameserver ${ns}
-  '') config.networking.nameservers;
-
-  fileSystems."/" = {
-    fsType = "rootfs";
-    device = "rootfs";
-  };
-
-  nix = {
-    useSandbox = false;
-    maxJobs = lib.mkDefault 1;
-    buildCores = lib.mkDefault 4;
-  };
-
-  networking = {
-    interfaces.eth0 = {
-      useDHCP = false;
-      tempAddress = "disabled";
-    };
-    nameservers = with hostRegistry.hosts.dnscache; [
-      ip4
-      ip6
-      "9.9.9.9"
-    ];
-    networkmanager.dns = "unbound";
-    useDHCP = false;
-    useHostResolvConf = false;
-    useNetworkd = true;
-  };
-
-  services = {
-    # Required for remote deployment
-    openssh.enable = true;
-    resolved.enable = false;
-  };
-
-  # Create a few files early before packing tarball for Proxmox architecture/OS detection.
-  system.extraSystemBuilderCmds = ''
-    mkdir -m 0755 -p $out/bin
-    ln -s ${pkgs.bash}/bin/bash $out/bin/sh
-    mkdir -m 0755 -p $out/sbin
-    ln -s ../init $out/sbin/init
-  '';
-
-  systemd.network.networks."40-eth0".networkConfig = {
-    IPv6AcceptRA = true;
-    LinkLocalAddressing = "ipv6";
-  };
-}
--- a/modules/microvm-host.nix
+++ b/modules/microvm-host.nix
@ -59,13 +59,11 @@
    };

    nix.settings = {
+      min-free = 128000000;
+      max-free = 1000000000;
      # fetch github-prebuilt microvm-kernels
-      substituters = [
-        "https://microvm.cachix.org"
-      ];
-      trusted-public-keys = [
-        "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys="
-      ];
+      substituters = [ "https://microvm.cachix.org" ];
+      trusted-public-keys = [ "microvm.cachix.org-1:oXnBc6hRE3eX5rSYdRyMYXnfzcCxC7yKPTbZXALsqys=" ];
    };

    environment.systemPackages = [ (
--- a/modules/microvm.nix
+++ b/modules/microvm.nix
@ -1,4 +1,5 @@
 { zentralwerk, options, config, lib, pkgs, ... }:
+
 let
  defaultGateways = {
    serv = "serv-gw";
@ -19,22 +20,27 @@ let
      "${builtins.substring 0 1 hash}2:${c 2}:${c 4}:${c 6}:${c 8}:${c 10}";

  nets = builtins.attrNames (
-    lib.filterAttrs (net: { hosts4, hosts6, ... }:
+    lib.filterAttrs (_: { hosts4, hosts6, ... }:
      hosts4 ? ${hostName} ||
-      lib.filterAttrs (ctx: hosts6:
+      lib.filterAttrs (_: hosts6:
        hosts6 ? ${hostName}
      ) hosts6 != {}
    ) zentralwerk.lib.config.site.net
  );
+
+  arch-to-host = rec {
+    server9 = "westmere";
+    server10 = "ivybridge";
+    nomad = server9;
+  };
 in
 {
  options.c3d2.deployment = with lib; {
    server = mkOption {
-      type = types.enum [ "server9" "server10" "nomad" ];
+      type = types.enum [ "server9" "server10" ];
      default = null;
      description = ''
-        Server that is supposed to host this MicroVM,
-        or \"nomad\" for HA clustering.
+        Server that is supposed to host this MicroVM.
      '';
    };

@ -62,21 +68,10 @@ in
    };
  };

-  config.system.build = with pkgs; {
-    copyToServer = writeScript "copy-to-${server}" ''
-      #! ${runtimeShell} -e
-
-      nix copy --no-check-sigs --to ssh-ng://root@${serverFQDN} $@
-    '';
-
-    runOnServer = writeScript "run-on-${server}" ''
-      #! ${runtimeShell} -e
-
-      ssh root@${serverFQDN} -- $@
-    '';
-  };
-
  config = {
+    # autoupdates do not make sense inside MicroVMs with read-only /nix/store
+    c3d2.autoUpdate = false;
+
    boot.kernelParams = [
      "preempt=none"
      # No server/router runs any untrusted user code
@ -123,6 +118,9 @@ in
      useNetworkd = true;
    };

+    # nix store is mounted read only
+    nix.gc.automatic = false;
+
    systemd.network = lib.mkIf config.c3d2.deployment.autoNetSetup {
      links = builtins.foldl' (links: net: links // {
        "30-${net}" = {
@ -165,10 +163,20 @@ in
      }) {} nets;
    };

-    # autoupdates do not make sense inside MicroVMs with read-only /nix/store
-    c3d2.autoUpdate = false;
+    c3d2.simd.arch = arch-to-host.${config.c3d2.deployment.server};

-    # nix store is mounted read only
-    nix.gc.automatic = false;
+    system.build = with pkgs; {
+      copyToServer = writeScript "copy-to-${server}" ''
+        #! ${runtimeShell} -e
+
+        nix copy --no-check-sigs --to ssh-ng://root@${serverFQDN} $@
+      '';
+
+      runOnServer = writeScript "run-on-${server}" ''
+        #! ${runtimeShell} -e
+
+        ssh root@${serverFQDN} -- $@
+      '';
+    };
  };
 }
--- a/modules/monitoring.nix
+++ b/modules/monitoring.nix
@ -0,0 +1,9 @@
+{ pkgs, lib, ... }:
+
+{
+  services.prometheus.exporters.node = lib.mkIf (pkgs.system != "riscv64-linux") {
+    enable = true;
+    enabledCollectors = [ "ethtool" "systemd" ];
+    openFirewall = true;
+  };
+}
--- a/modules/nncp.nix
+++ b/modules/nncp.nix
@ -1,13 +1,10 @@
-{ config, lib, pkgs, ... }:
-with lib;
+{ config, lib, ... }:

 let
  nncpCfgFile = "/run/nncp.hjson";
  programCfg = lib.optionalAttrs (config.programs ? nncp) config.programs.nncp;
  callerCfg = config.services.nncp.caller;
  daemonCfg = config.services.nncp.daemon;
-  settingsFormat = pkgs.formats.json { };
-  jsonCfgFile = settingsFormat.generate "nncp.json" programCfg.settings;
  pkg = programCfg.package;
 in
 {
@ -15,13 +12,13 @@ in

    services.nncp = {
      caller = {
-        enable = mkEnableOption ''
+        enable = lib.mkEnableOption ''
          croned NNCP TCP daemon caller.
          The daemon will take configuration from
          <xref linkend="opt-programs.nncp.settings"/>
        '';
-        extraArgs = mkOption {
-          type = with types; listOf str;
+        extraArgs = lib.mkOption {
+          type = with lib.types; listOf str;
          description = "Extra command-line arguments to pass to caller.";
          default = [ ];
          example = [ "-autotoss" ];
@ -29,18 +26,18 @@ in
      };

      daemon = {
-        enable = mkEnableOption ''
+        enable = lib.mkEnableOption ''
          NNCP TCP synronization daemon.
          The daemon will take configuration from
          <xref linkend="opt-programs.nncp.settings"/>
        '';

        socketActivation = {
-          enable = mkEnableOption ''
+          enable = lib.mkEnableOption ''
            Whether to run nncp-daemon persistently or socket-activated.
          '';
-          listenStreams = mkOption {
-            type = with types; listOf str;
+          listenStreams = lib.mkOption {
+            type = with lib.types; listOf str;
            description = ''
              TCP sockets to bind to.
              See <xref linkend="opt-systemd.sockets._name_.listenStreams"/>.
@ -49,8 +46,8 @@ in
          };
        };

-        extraArgs = mkOption {
-          type = with types; listOf str;
+        extraArgs = lib.mkOption {
+          type = with lib.types; listOf str;
          description = "Extra command-line arguments to pass to daemon.";
          default = [ ];
          example = [ "-autotoss" ];
@ -60,15 +57,14 @@ in
    };
  };

-  config = mkIf (programCfg.enable or callerCfg.enable or daemonCfg.enable) {
-
+  config = lib.mkIf (programCfg.enable or callerCfg.enable or daemonCfg.enable) {
    assertions = [{
-      assertion = with builtins;
+      assertion =
        let
          callerCongfigured =
            let neigh = config.programs.nncp.settings.neigh or { };
-            in lib.lists.any (x: hasAttr "calls" x && x.calls != [ ])
-            (attrValues neigh);
+            in lib.lists.any (x: lib.hasAttr "calls" x && x.calls != [ ])
+            (lib.attrValues neigh);
        in !callerCfg.enable || callerCongfigured;
      message = "NNCP caller enabled but call configuration is missing";
    }];
@ -89,7 +85,7 @@ in
      };
    };

-    systemd.services."nncp-daemon" = mkIf daemonCfg.enable {
+    systemd.services."nncp-daemon" = lib.mkIf daemonCfg.enable {
      enable = !daemonCfg.socketActivation.enable;
      description = "NNCP TCP syncronization daemon.";
      documentation = [ "http://www.nncpgo.org/nncp_002ddaemon.html" ];
@ -106,7 +102,7 @@ in
      };
    };

-    systemd.services."nncp-daemon@" = mkIf daemonCfg.socketActivation.enable {
+    systemd.services."nncp-daemon@" = lib.mkIf daemonCfg.socketActivation.enable {
      description = "NNCP TCP syncronization daemon.";
      documentation = [ "http://www.nncpgo.org/nncp_002ddaemon.html" ];
      after = [ "network.target" ];
@ -123,7 +119,7 @@ in
      };
    };

-    systemd.sockets.nncp-daemon = mkIf daemonCfg.socketActivation.enable {
+    systemd.sockets.nncp-daemon = lib.mkIf daemonCfg.socketActivation.enable {
      inherit (daemonCfg.socketActivation) listenStreams;
      description = "socket for NNCP TCP syncronization.";
      conflicts = [ "nncp-daemon.service" ];
--- a/modules/plume.nix
+++ b/modules/plume.nix
@ -1,6 +1,5 @@
 { config, lib, pkgs, ... }:
 let
-  inherit (pkgs) plume;
  cfg = config.services.plume;
 in
 {
@ -33,7 +32,7 @@ in
    ids.uids.plume = 499;
    users.users.${cfg.user} = {
      uid = config.ids.uids.plume;
-      group = cfg.group;
+      inherit (cfg) group;
      home = "/var/lib/plume";
    };
    users.groups.${cfg.group} = {};
@ -52,11 +51,11 @@ in
      after = [ "postgresql.service" ];
      requires = [ "postgresql.service" ];
      wantedBy = [ "multi-user.target" ];
-      path = [ plume ];
+      path = [ pkgs.plume ];
      script = ''
        ln -sf ${cfg.envFile} .env
        mkdir -p static/media
-        for f in ${plume}/share/plume/static/*; do
+        for f in ${pkgs.plume}/share/plume/static/*; do
          n=$(basename "$f")
          if [ "$n" != media ]; then
            rm -f "static/$n"
@ -79,7 +78,7 @@ in
      #! ${pkgs.runtimeShell} -e

      plm() {
-        sudo -u ${config.services.plume.user} -- ${plume}/bin/plm $@
+        sudo -u ${config.services.plume.user} -- ${pkgs.plume}/bin/plm $@
      }

      plm migration run
--- a/modules/rpi-netboot.nix
+++ b/modules/rpi-netboot.nix
@ -1,4 +1,4 @@
-{ hostRegistry, config, pkgs, lib, ... }:
+{ config, pkgs, lib, ... }:
 {
  boot = {
    loader.generic-extlinux-compatible.enable = false;
--- a/overlay/default.nix
+++ b/overlay/default.nix
@ -1,146 +0,0 @@
-{ nixos-unstable
-, tracer
-, bevy-julia
-, bevy-mandelbrot
-}:
-
-final: prev:
-
-let
-  pkgs-unstable = nixos-unstable.legacyPackages.${prev.system};
-in
-with final; {
-  allcolors = with final; rustPlatform.buildRustPackage rec {
-    pname = "allcolors";
-    version = "0.1.0";
-    src = fetchFromGitHub {
-      owner = "polygon";
-      repo = "allcolors-rs";
-      rev = "023bd480245052357a7fd5f42181ff6e67d98b31";
-      sha256 = "sha256-whaV+k5xh01OQNOehwkEBUDpMWn47mvVihVwchBvWoE=";
-    };
-    cargoPatches = [ ./allcolors-cargo-update.patch ];
-    cargoSha256 = "sha256-RbfACA4hcyemGkw9bqjpIk393SBgBM939I95+grVI0c=";
-    nativeBuildInputs = [ copyDesktopItems ];
-    buildInputs = [
-      xorg.libX11
-      xorg.libXcursor
-      xorg.libXrandr
-      xorg.libXi
-      libGL
-      mesa
-    ];
-    postFixup = ''
-      patchelf --set-rpath ${lib.makeLibraryPath buildInputs} $out/bin/allcolors-rs
-    '';
-    desktopItems = [ (makeDesktopItem {
-      name = "allcolors";
-      desktopName = "Polygon's allcolors-rs";
-      categories = [ "Game" ];
-      exec = "allcolors-rs";
-    }) ];
-  };
-
-  # HACK: referenced by sdrweb
-  # TODO: remove with 22.11
-  alsaUtils = final.alsa-utils;
-
-  inherit (bevy-julia.packages.${system}) bevy_julia;
-  inherit (bevy-mandelbrot.packages.${system}) bevy_mandelbrot;
-
-  bmxd = callPackage ./bmxd.nix { };
-
-  dump1090-influxdb = callPackage ./dump1090-influxdb { };
-
-  dump1090_rs = callPackage ./dump1090_rs.nix { };
-
-  chromium = prev.chromium.override {
-    commandLineArgs = "--enable-features=VaapiVideoEncoder,VaapiVideoDecoder,CanvasOopRasterization --force-dark-mode";
-  };
-
-  # hydra flake
-  hydra = prev.hydra.overrideAttrs (oldAttrs: {
-    patches = oldAttrs.patches or [ ] ++ [
-      # gitea webhook support
-      (fetchpatch {
-        url = "https://github.com/NixOS/hydra/pull/1227/commits/750978a19232583e17620a1bd80435e957e7213a.patch";
-        sha256 = "sha256-86Li0YUSVUdnw6lt6kZ56ohDRKPD13SZzukqPU1np8U=";
-      })
-      # fix github webhook from orgs
-      (fetchpatch {
-        url = "https://github.com/NixOS/hydra/commit/4d664ecb0faaf51b21673f979b543ea4694c3f1b.patch";
-        sha256 = "sha256-lF5Rnz8r9ptyMLhcg/XnjiNhOK1KcLA7hi01ye4KgmI=";
-      })
-    ];
-  });
-
-  # hydra in nixpkgs
-  hydra_unstable = prev.hydra_unstable.overrideAttrs (oldAttrs: {
-    patches = oldAttrs.patches or [ ] ++ [
-      # gitea webhook support
-      (fetchpatch {
-        url = "https://github.com/NixOS/hydra/pull/1227/commits/750978a19232583e17620a1bd80435e957e7213a.patch";
-        sha256 = "sha256-86Li0YUSVUdnw6lt6kZ56ohDRKPD13SZzukqPU1np8U=";
-      })
-      # fix github webhook from orgs
-      (fetchpatch {
-        url = "https://github.com/NixOS/hydra/commit/4d664ecb0faaf51b21673f979b543ea4694c3f1b.patch";
-        sha256 = "sha256-lF5Rnz8r9ptyMLhcg/XnjiNhOK1KcLA7hi01ye4KgmI=";
-      })
-    ];
-  });
-
-  mlat-client = prev.python3Packages.callPackage ./mlat-client.nix { };
-
-  nixVersions =
-    prev.nixVersions // {
-      stable = prev.nixVersions.stable.overrideAttrs (oldAttrs: {
-        patches = oldAttrs.patches or [ ] ++ lib.optionals (lib.versionAtLeast prev.nixVersions.stable.version "2.9") [
-          # use github.com archive URL instead of api.github.com
-          (fetchpatch {
-            url = "https://github.com/NixOS/nix/pull/6997.patch";
-            sha256 = "sha256-JaQ6OY1RjxCYymkI/x0rmKH8XVXYs5psRwNJ4TPHPS0=";
-          })
-        ];
-      });
-    };
-
-  inherit (nixos-unstable.legacyPackages.${prev.targetPlatform.system}) nomad_1_3;
-
-  openssh = prev.openssh.overrideAttrs (oldAttrs: {
-    # takes 30 minutes
-    doCheck = false;
-  });
-
-  pile = prev.callPackage ./pile.nix { };
-
-  pi-sensors = prev.callPackage ./pi-sensors { };
-
-  plume = prev.callPackage ./plume { };
-
-  readsb = prev.callPackage ./readsb.nix { };
-
-  SimpleYggGen-CPP = prev.callPackage ./simpleygggen.nix { };
-
-  tracer-game =
-    let
-      broken = true;
-      reason = "haddock runs on affection for 10 hours and more";
-    in
-      if broken
-      then throw "tracer-game: ${reason}"
-      else tracer.packages.${system}.tracer-game;
-
-  # HACK: referenced by hydra-module.nix but removed from nixos-unstable in 2022-09
-  # TODO: remove with 22.11
-  utillinux = final.util-linux;
-
-  # vector-0.23 + mqtt-sink
-  vector = pkgs-unstable.callPackage ./vector {};
-
-  wander =
-    if prev ? wander
-    then builtins.trace "`wander` is now available on stable NixOS. Please remove from overlay!"
-      prev.wander
-    else pkgs-unstable.wander;
-}
--- a/overlays/allcolors-cargo-update.patch
+++ b/overlays/allcolors-cargo-update.patch
--- a/overlays/allcolors.nix
+++ b/overlays/allcolors.nix
@ -0,0 +1,42 @@
+{ lib
+, copyDesktopItems
+, fetchFromGitHub
+, rustPlatform
+, libGL
+, makeDesktopItem
+, mesa
+, xorg
+}:
+
+rustPlatform.buildRustPackage rec {
+  pname = "allcolors";
+  version = "0.1.0";
+  src = fetchFromGitHub {
+    owner = "polygon";
+    repo = "allcolors-rs";
+    rev = "023bd480245052357a7fd5f42181ff6e67d98b31";
+    sha256 = "sha256-whaV+k5xh01OQNOehwkEBUDpMWn47mvVihVwchBvWoE=";
+  };
+  cargoPatches = [ ./allcolors-cargo-update.patch ];
+  cargoSha256 = "sha256-RbfACA4hcyemGkw9bqjpIk393SBgBM939I95+grVI0c=";
+  nativeBuildInputs = [ copyDesktopItems ];
+  buildInputs = [
+    xorg.libX11
+    xorg.libXcursor
+    xorg.libXrandr
+    xorg.libXi
+    libGL
+    mesa
+  ];
+  postFixup = ''
+    patchelf --set-rpath ${lib.makeLibraryPath buildInputs} $out/bin/allcolors-rs
+  '';
+  desktopItems = [
+    (makeDesktopItem {
+      name = "allcolors";
+      desktopName = "Polygon's allcolors-rs";
+      categories = [ "Game" ];
+      exec = "allcolors-rs";
+    })
+  ];
+}
--- a/overlays/bmxd.nix
+++ b/overlays/bmxd.nix
@ -1,4 +1,4 @@
-{ stdenv, fetchgit, fetchpatch, }:
+{ stdenv, fetchgit }:

 stdenv.mkDerivation {
  pname = "bmxd";
--- a/overlays/default.nix
+++ b/overlays/default.nix
@ -0,0 +1,84 @@
+{  bevy-julia
+, bevy-mandelbrot
+, tracer
+}:
+
+final: prev:
+
+with final; {
+  allcolors = callPackage ./allcolors.nix { };
+
+  inherit (bevy-julia.packages.${system}) bevy_julia;
+  inherit (bevy-mandelbrot.packages.${system}) bevy_mandelbrot;
+
+  bmxd = callPackage ./bmxd.nix { };
+
+  boehmgc = prev.boehmgc.overrideAttrs(_: {
+    # tests fail under emulation
+    doCheck = false;
+  });
+
+  dump1090-influxdb = callPackage ./dump1090-influxdb { };
+
+  dump1090_rs = callPackage ./dump1090_rs.nix { };
+
+  chromium = prev.chromium.override {
+    commandLineArgs = "--enable-features=VaapiVideoEncoder,VaapiVideoDecoder,CanvasOopRasterization --force-dark-mode";
+  };
+
+  # hydra flake
+  hydra = prev.hydra.overrideAttrs (oldAttrs: {
+    patches = oldAttrs.patches or [ ] ++ [
+      # gitea webhook support
+      (fetchpatch {
+        url = "https://github.com/NixOS/hydra/pull/1227/commits/750978a19232583e17620a1bd80435e957e7213a.patch";
+        sha256 = "sha256-86Li0YUSVUdnw6lt6kZ56ohDRKPD13SZzukqPU1np8U=";
+      })
+      # fix github webhook from orgs
+      (fetchpatch {
+        url = "https://github.com/NixOS/hydra/commit/4d664ecb0faaf51b21673f979b543ea4694c3f1b.patch";
+        sha256 = "sha256-lF5Rnz8r9ptyMLhcg/XnjiNhOK1KcLA7hi01ye4KgmI=";
+      })
+    ];
+  });
+
+  # hydra in nixpkgs
+  hydra_unstable = prev.hydra_unstable.overrideAttrs (oldAttrs: {
+    patches = oldAttrs.patches or [ ] ++ [
+      # gitea webhook support
+      (fetchpatch {
+        url = "https://github.com/NixOS/hydra/pull/1227/commits/750978a19232583e17620a1bd80435e957e7213a.patch";
+        sha256 = "sha256-86Li0YUSVUdnw6lt6kZ56ohDRKPD13SZzukqPU1np8U=";
+      })
+      # fix github webhook from orgs
+      (fetchpatch {
+        url = "https://github.com/NixOS/hydra/commit/4d664ecb0faaf51b21673f979b543ea4694c3f1b.patch";
+        sha256 = "sha256-lF5Rnz8r9ptyMLhcg/XnjiNhOK1KcLA7hi01ye4KgmI=";
+      })
+    ];
+  });
+
+  mlat-client = python3Packages.callPackage ./mlat-client.nix { };
+
+  openssh = prev.openssh.overrideAttrs (_: {
+    # takes 30 minutes
+    doCheck = false;
+  });
+
+  pile = callPackage ./pile.nix { };
+
+  pi-sensors = callPackage ./pi-sensors { };
+
+  plume = callPackage ./plume { };
+
+  readsb = callPackage ./readsb.nix { };
+
+  simpleygggen-cpp = callPackage ./simpleygggen.nix { };
+
+  telme10 = callPackage ./telme10.nix { };
+
+  tracer-game =
+    if true
+    then throw "tracer-game: haddock runs on affection for 10 hours and more"
+    else tracer.packages.${system}.tracer-game;
+}
--- a/overlays/dump1090-influxdb/Gemfile
+++ b/overlays/dump1090-influxdb/Gemfile
--- a/overlays/dump1090-influxdb/Gemfile.lock
+++ b/overlays/dump1090-influxdb/Gemfile.lock
--- a/overlays/dump1090-influxdb/default.nix
+++ b/overlays/dump1090-influxdb/default.nix
--- a/overlays/dump1090-influxdb/gemset.nix
+++ b/overlays/dump1090-influxdb/gemset.nix
@ -1,19 +1,19 @@
 {
  gis-distance = {
-    groups = ["default"];
-    platforms = [];
+    groups = [ "default" ];
+    platforms = [ ];
    source = {
-      remotes = ["https://rubygems.org"];
+      remotes = [ "https://rubygems.org" ];
      sha256 = "1kgv1scv25b65d9xfricj1ayd2iry7imgk7qw4mryd91mhriibaf";
      type = "gem";
    };
    version = "1.1.0";
  };
  influxdb = {
-    groups = ["default"];
-    platforms = [];
+    groups = [ "default" ];
+    platforms = [ ];
    source = {
-      remotes = ["https://rubygems.org"];
+      remotes = [ "https://rubygems.org" ];
      sha256 = "1l2sjf8kaw3adjjg3l7zg1j735yxdfldf04gl9kjc3hbpdcd7d4w";
      type = "gem";
    };
--- a/overlays/dump1090-influxdb/main.rb
+++ b/overlays/dump1090-influxdb/main.rb
--- a/overlays/dump1090_rs.nix
+++ b/overlays/dump1090_rs.nix
@ -1,5 +1,7 @@
-{ fetchFromGitHub, rustPlatform
-, pkg-config, llvmPackages
+{ fetchFromGitHub
+, rustPlatform
+, pkg-config
+, llvmPackages
 , soapysdr-with-plugins
 }:

--- a/overlays/mlat-client.nix
+++ b/overlays/mlat-client.nix
--- a/overlays/pi-sensors/Cargo.lock
+++ b/overlays/pi-sensors/Cargo.lock
--- a/overlays/pi-sensors/Cargo.toml
+++ b/overlays/pi-sensors/Cargo.toml
--- a/overlays/pi-sensors/default.nix
+++ b/overlays/pi-sensors/default.nix
--- a/overlays/pi-sensors/src/dht.rs
+++ b/overlays/pi-sensors/src/dht.rs
--- a/Show More
+++ b/Show More
				`@ -1 +0,0 @@`
				`nix-serve.hq.c3d2.de:KZRGGnwOYzys6pxgM8jlur36RmkJQ/y8y62e52fj1ps=`