Merge remote-tracking branch 'origin/master' into mailtng

This commit is contained in:
Tassilo - 2022-12-18 19:56:04 +01:00
commit 16351819ca
111 changed files with 3126 additions and 1875 deletions

View File

@ -40,6 +40,7 @@ keys:
- &leon age1cm0cjk2764s4pv5g7e67as34g9xtcltex96ga87wckndw62wqqlsvkscqc
- &leoncloud age1aw9s4kcd6ys64ddzzfya9ajzln2tv8pm9uvz6d85v0r6eq4dudqq5vts86
- &mailtngbert age1lgjvtszpds9flpwsstxdht00c7zlk3mz7nlc5qftyt8rhfdm330qqmhl72
- &mastodon age1dcpd6u4psq3hehjyjrt3s7kzmnvxd20vsc8urjcdv6anr5v7ky2sq9rhtt
- &matemat age15vmz2evhnkn26fyt4vqvgztfrsr2s8qavd2m6zfjmkh84q2g75csnc5kr6
- &mediawiki age1xjvep7hsnfefgxvuwall8nq0486qu8yknhzwhf0cskw5xlpm8qws9txc56
- &mucbot age1cqeh03zq0hvz5l78r678q93ey5mlw49lqy4whvgqxgenudth7g6skee6kh
@ -58,6 +59,7 @@ keys:
- &stream age1j5csp5v5s2g8am47dd85kcke8986e0qc88f0vfgd3kmvwu8azg3smslk92
- &storage-ng age1qjvds58pedjdk9rj0yqfvad4xhpteapr9chvfucwcgwrsr8n7axqyhg2vu
- &ticker age1kdrpaqsy7gdnf80fpq6qrrc98nqjuzzlqx955uk2pkky3xcxky8sw9cdjl
- &prometheus age13xhxqulvswuckmpkmy2fgeqd5jx0ar8e2hst33leljt69r6hsvnsrdw63k
creation_rules:
- path_regex: config/[^/]+\.yaml$
@ -86,11 +88,13 @@ creation_rules:
- *leon
- *leoncloud
- *mailtngbert
- *mastodon
- *matemat
- *mediawiki
- *mucbot
- *nfsroot
- *oparl
- *prometheus
- *public-access-proxy
- *pulsebert
- *radiobert
@ -127,6 +131,12 @@ creation_rules:
age:
- *dn42
- *polygon-snowflake
- path_regex: hosts/dacbert/[^/]+\.yaml$
key_groups:
- pgp: *admins
age:
- *dacbert
- *polygon-snowflake
- path_regex: hosts/freifunk/[^/]+\.yaml$
key_groups:
- pgp: *admins
@ -163,6 +173,12 @@ creation_rules:
age:
- *mailtngbert
- *polygon-snowflake
- path_regex: hosts/mastodon/[^/]+\.yaml$
key_groups:
- pgp: *admins
age:
- *mastodon
- *polygon-snowflake
- path_regex: hosts/mediawiki/[^/]+\.yaml$
key_groups:
- pgp: *admins
@ -187,3 +203,9 @@ creation_rules:
age:
- *storage-ng
- *polygon-snowflake
- path_regex: hosts/prometheus/[^/]+\.yaml$
key_groups:
- pgp: *admins
age:
- *prometheus
- *polygon-snowflake

121
README.md
View File

@ -139,14 +139,123 @@ so the following is all that is needed on a MicroVM-hosting server:
microvm -Ru $hostname
```
## High Availability Deployment on Nomad
# Cluster deployment with Skyflake
First, stop and delete `/var/lib/microvm/$NAME` where the
systemd-managed MicroVMs live, or move the state to
`/glusterfs/fast/microvms/$NAME`.
## About
```sh
nix run .#nomad-$NAME
[Skyflake](https://github.com/astro/skyflake) provides Hyperconverged
Infrastructure to run NixOS MicroVMs on a cluster. Our setup unifies
networking with one bridge per VLAN. Persistent storage is replicated
with Glusterfs.
Recognize nixosConfiguration for our Skyflake deployment by the
`self.nixosModules.cluster-options` module being included.
## User interface
We use the less-privileged `c3d2@` user for deployment. This flake's
name on the cluster is `config`. Other flakes can coexist in the same
user so that we can run separately developed projects like
*dump-dvb*. *leon* and potentially other users can deploy Flakes and
MicroVMs without name clashes.
### Deploying
**git push** this repo to any machine in the cluster, preferably to
Hydra because there building won't disturb any services.
You don't deploy all MicroVMs at once. Instead, Skyflake allows you to
select NixOS systems by the branches you push to. **You must commit
before you push!**
**Example:** deploy nixosConfigurations `mucbot` and `sdrweb` (`HEAD` is your
current commit)
```bash
git push c3d2@hydra.serv.zentralwerk.org:config HEAD:mucbot HEAD:sdrweb
```
This will:
1. Build the configuration on Hydra, refusing the branch update on
broken builds (through a git hook)
2. Copy the MicroVM package and its dependencies to the binary cache
that is accessible to all nodes with Glusterfs
3. Submit one job per MicroVM into the Nomad cluster
*Deleting* a nixosConfiguration's branch will **stop** the MicroVM in Nomad.
### Updating
**TODO:** how would you like it?
### MicroVM status
```bash
ssh c3d2@hydra.serv.zentralwerk.org status
```
## Debugging for cluster admins
### Glusterfs
Glusterfs holds our MicroVMs' state. They *must always be mounted* or
brains are split.
```bash
gluster volume info
gluster volume status
```
#### Restart glusterd
```bash
systemctl restart glusterd
```
#### Remount volumes
```bash
systemctl restart /glusterfs/fast
systemctl restart /glusterfs/big
```
### Nomad
#### Check the cluster state
```shell
nomad server members
```
Nomad *servers* **coordinate** the cluster.
Nomad *clients* **run** the tasks.
#### Browse in the terminal
[wander](https://github.com/robinovitch61/wander) and
[damon](https://github.com/hashicorp/damon) are nice TUIs that are
preinstalled on our cluster nodes.
#### Browse with a browser
First, tunnel TCP port `:4646` from a cluster server:
```bash
ssh -L 4646:localhost:4646 root@server10.cluster.zentralwerk.org
```
Then, visit https://localhost:4646 for for full klickibunti.
#### Reset the Nomad state on a node
After upgrades, Nomad servers may fail rejoining the cluster. Do this
to make a *Nomad server* behave like a newborn:
```shell
systemctl stop nomad
rm -rf /var/lib/nomad/server/raft/
systemctl start nomad
```
# Secrets management

View File

@ -10,11 +10,11 @@
]
},
"locked": {
"lastModified": 1663176622,
"narHash": "sha256-ahmQXwS2P34x7PxXt8Ve2ZVKJHW6yP1m/nZoo8sHwmE=",
"ref": "master",
"rev": "b56ed86e45b2a8cdf811f2659644192a69ab5818",
"revCount": 293,
"lastModified": 1669920985,
"narHash": "sha256-Ff9FxYqYNVovOCDcECGKHoiqpIOUvF6/q17H+k06iXw=",
"ref": "refs/heads/master",
"rev": "5bef189c308df9dda1449a8305a7092fb5c77827",
"revCount": 298,
"type": "git",
"url": "https://gitea.nek0.eu/nek0/affection"
},
@ -23,9 +23,37 @@
"url": "https://gitea.nek0.eu/nek0/affection"
}
},
"alert2muc": {
"inputs": {
"naersk": [
"naersk"
],
"nixpkgs": [
"nixos"
],
"utils": [
"flake-utils"
]
},
"locked": {
"lastModified": 1671329943,
"narHash": "sha256-7gFF8z1ww+LoC9Pk5hflvnlrzIf8gp7UbL6o8Xyl4Dw=",
"ref": "refs/heads/main",
"rev": "0ae1024cc7bf45dceb03f089f3e5485c0a43b860",
"revCount": 14,
"type": "git",
"url": "https://gitea.c3d2.de/astro/alert2muc"
},
"original": {
"type": "git",
"url": "https://gitea.c3d2.de/astro/alert2muc"
}
},
"bevy-julia": {
"inputs": {
"naersk": "naersk",
"naersk": [
"naersk"
],
"nixpkgs": [
"nixos"
],
@ -91,11 +119,11 @@
]
},
"locked": {
"lastModified": 1668535353,
"narHash": "sha256-cVe++C4wg2CG80qjjSVG4H1udcRi7sbdU/xMGC5WSxc=",
"lastModified": 1670028698,
"narHash": "sha256-JjmRgUg5004snQL03r5n4TjJmyD4qiwKdPDmx/1GeFw=",
"ref": "main",
"rev": "a12dc753b3f3d145883e1de7a0c12407bbe97288",
"revCount": 111,
"rev": "908fb9c8d3d2b0fbe5a17895639ef478f81e3f8d",
"revCount": 153,
"type": "git",
"url": "https://gitea.c3d2.de/astro/caveman.git"
},
@ -113,11 +141,11 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1668493618,
"narHash": "sha256-Pw8dzHPI3My/nWthhWlD6nVLVPDAi9+NODvayVhKGKk=",
"lastModified": 1671171782,
"narHash": "sha256-G7FNiYVl/jOkvNEhLO8O7uq0MuhFLKGMsCxFjqATVc0=",
"owner": "nix-community",
"repo": "fenix",
"rev": "3e59a48c3171664ea0797f28273f9929a2335617",
"rev": "d3eaf97d81161bea9177cc80e07d26ba2d96569f",
"type": "github"
},
"original": {
@ -144,11 +172,11 @@
"harmonia": {
"flake": false,
"locked": {
"lastModified": 1668458039,
"narHash": "sha256-w4OCjMRPrPIY0dlUkiA1XPSmRstnNSRVmglw7hfSs3s=",
"lastModified": 1670319728,
"narHash": "sha256-7hsq6Sv06UcIjjlZTFlsYWDfGrc9u77OAr25SjnvZ4A=",
"owner": "helsinki-systems",
"repo": "harmonia",
"rev": "9b61645fce85a7abfe6e812f68df085d688f9711",
"rev": "f2476198fb8236c7c7eb432aab4472083cb9831d",
"type": "github"
},
"original": {
@ -175,7 +203,7 @@
"locked": {
"lastModified": 1657923513,
"narHash": "sha256-YzHPow09B9uSdybUxP5lQn2hXk90Q6oTDL6UXzD0/+k=",
"ref": "master",
"ref": "refs/heads/master",
"rev": "f7cf04a7ad47e388121f0771651fec0df91407f3",
"revCount": 61,
"type": "git",
@ -237,11 +265,11 @@
]
},
"locked": {
"lastModified": 1668518796,
"narHash": "sha256-DyteijJn0JZphJdQaHpPWxvkKrBPvCW3B53tCasDX8c=",
"lastModified": 1670897870,
"narHash": "sha256-nWLU48WlhVYm53cTUEDX8mZwZqdzObO6299hrCmdYcU=",
"owner": "astro",
"repo": "microvm.nix",
"rev": "5181933ca7bbaad37ceb82a848630ea3b30d522b",
"rev": "b12b4d426e1050479e9d571c81cda2b7ae0256da",
"type": "github"
},
"original": {
@ -251,35 +279,17 @@
}
},
"naersk": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1662220400,
"narHash": "sha256-9o2OGQqu4xyLZP9K6kNe1pTHnyPz0Wr3raGYnr9AIgY=",
"owner": "nix-community",
"repo": "naersk",
"rev": "6944160c19cb591eb85bbf9b2f2768a935623ed3",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "naersk",
"type": "github"
}
},
"naersk_2": {
"inputs": {
"nixpkgs": [
"nixos"
]
},
"locked": {
"lastModified": 1662220400,
"narHash": "sha256-9o2OGQqu4xyLZP9K6kNe1pTHnyPz0Wr3raGYnr9AIgY=",
"lastModified": 1671096816,
"narHash": "sha256-ezQCsNgmpUHdZANDCILm3RvtO1xH8uujk/+EqNvzIOg=",
"owner": "nix-community",
"repo": "naersk",
"rev": "6944160c19cb591eb85bbf9b2f2768a935623ed3",
"rev": "d998160d6a076cfe8f9741e56aeec7e267e3e114",
"type": "github"
},
"original": {
@ -288,28 +298,6 @@
"type": "github"
}
},
"naersk_3": {
"inputs": {
"nixpkgs": [
"ticker",
"nixpkgs"
]
},
"locked": {
"lastModified": 1659610603,
"narHash": "sha256-LYgASYSPYo7O71WfeUOaEUzYfzuXm8c8eavJcel+pfI=",
"owner": "nix-community",
"repo": "naersk",
"rev": "c6a45e4277fa58abd524681466d3450f896dc094",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "naersk",
"rev": "c6a45e4277fa58abd524681466d3450f896dc094",
"type": "github"
}
},
"newNixpkgs": {
"locked": {
"lastModified": 1647380550,
@ -329,7 +317,7 @@
"nix": {
"inputs": {
"lowdown-src": "lowdown-src",
"nixpkgs": "nixpkgs_2",
"nixpkgs": "nixpkgs",
"nixpkgs-regression": "nixpkgs-regression"
},
"locked": {
@ -349,43 +337,27 @@
},
"nixos": {
"locked": {
"lastModified": 1668459637,
"narHash": "sha256-HqnWCKujmtu8v0CjzOT0sr7m2AR7+vpbZJOp1R0rodY=",
"owner": "nixos",
"lastModified": 1671217927,
"narHash": "sha256-VjoidSKImZGWYqL1Z+ntfEzqeMjNX4zxcBr9PIqORwM=",
"owner": "SuperSandro2000",
"repo": "nixpkgs",
"rev": "16f4e04658c2ab10114545af2f39db17d51bd1bd",
"rev": "0a597dc83e81526de48488088c92349d7170445c",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-22.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixos-armv6": {
"locked": {
"lastModified": 1664701736,
"narHash": "sha256-Va3NyZ+uyZztu506qM+sLxd69DBzN5CdoCAu1lzVk0U=",
"owner": "rnhmjoj",
"repo": "nixpkgs",
"rev": "10b75bee02bc7c25e596847357c70b277c534588",
"type": "github"
},
"original": {
"owner": "rnhmjoj",
"ref": "pr-fix-armv6",
"owner": "SuperSandro2000",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixos-hardware": {
"locked": {
"lastModified": 1668334946,
"narHash": "sha256-omMbUj4r5DVBWh7KxkoO/Z/1V1shVR6Ls4jXNB4mr3U=",
"lastModified": 1671183612,
"narHash": "sha256-Q6so0tBGEb9Bhx++FP6cJQ+K83hOZ99ffmcdcWtDS14=",
"owner": "nixos",
"repo": "nixos-hardware",
"rev": "e0452b33ab0ef16ffe075e980644ed92a6a200bb",
"rev": "488931efb69a50307fa0d71e23e78c8706909416",
"type": "github"
},
"original": {
@ -394,37 +366,7 @@
"type": "github"
}
},
"nixos-unstable": {
"locked": {
"lastModified": 1668417584,
"narHash": "sha256-yeuEyxKPwsm5fIHN49L/syn9g5coxnPp3GsVquhrv5A=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "013fcdd106823416918004bb684c3c186d3c460f",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1663264531,
"narHash": "sha256-2ncO5chPXlTxaebDlhx7MhL0gOEIWxzSyfsl0r0hxQk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "454887a35de6317a30be284e8adc2d2f6d8a07c4",
"type": "github"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"nixpkgs-mobilizon": {
"nixos-mobilizon": {
"locked": {
"lastModified": 1664466500,
"narHash": "sha256-FvEUAKkf0PDZ2j2qIbI4+3oPTnuQq4HdX00iqBkvKOU=",
@ -440,6 +382,53 @@
"type": "github"
}
},
"nixos-unstable": {
"locked": {
"lastModified": 1671108576,
"narHash": "sha256-6ggOL6KoaELNA1562tnPjtAnQ9SwsKRTgeuaXvPzCwI=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "0f5996b524c91677891a432cc99c7567c7c402b1",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixos-unstable-simd": {
"locked": {
"lastModified": 1671211489,
"narHash": "sha256-0AbRULr8+U70TPLiMz7PK8mTp1lD+Ct8VKElZ0WmPzE=",
"owner": "SuperSandro2000",
"repo": "nixpkgs",
"rev": "8ef6d44870fbc02b5c43922b8dbe95aa8b53091b",
"type": "github"
},
"original": {
"owner": "SuperSandro2000",
"ref": "nixos-unstable-simd",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1645296114,
"narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-21.05-small",
"type": "indirect"
}
},
"nixpkgs-regression": {
"locked": {
"lastModified": 1643052045,
@ -455,21 +444,6 @@
"type": "indirect"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1645296114,
"narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-21.05-small",
"type": "indirect"
}
},
"oparl-scraper": {
"flake": false,
"locked": {
@ -490,11 +464,11 @@
"openwrt": {
"flake": false,
"locked": {
"lastModified": 1668297972,
"narHash": "sha256-i39KfzBoNiiScd1M8kV/5WrGzEgG6+PxsFxoW+DT9UQ=",
"lastModified": 1670552749,
"narHash": "sha256-ZhbNee6F+vJRirskdiYpXVRldJTHSbClLiYtJe4wgqE=",
"ref": "openwrt-21.02",
"rev": "079ce0413a1e3c19dd00be1b90de737c2bc09223",
"revCount": 51164,
"rev": "784565b6a0ad482b19c4851b0ce8ffeddb7010c1",
"revCount": 51264,
"type": "git",
"url": "https://git.openwrt.org/openwrt/openwrt.git"
},
@ -511,11 +485,11 @@
]
},
"locked": {
"lastModified": 1668458369,
"narHash": "sha256-Mv4F2AgOIpGyvfoHQtw/sDg3hU7O0K1P9ercnfrwnEQ=",
"lastModified": 1671207552,
"narHash": "sha256-7C3QXWM615TJJqNQlKS27HaQh/m6vctKhZ1fw9ex7xE=",
"owner": "astro",
"repo": "nix-openwrt-imagebuilder",
"rev": "818db40dd8a0a22b029823ba477c5c3301bd7534",
"rev": "a53edbc364df74d467d4f1e8d45dca6ee4e92ced",
"type": "github"
},
"original": {
@ -527,6 +501,7 @@
"root": {
"inputs": {
"affection-src": "affection-src",
"alert2muc": "alert2muc",
"bevy-julia": "bevy-julia",
"bevy-mandelbrot": "bevy-mandelbrot",
"caveman": "caveman",
@ -536,12 +511,12 @@
"heliwatch": "heliwatch",
"hydra-ca": "hydra-ca",
"microvm": "microvm",
"naersk": "naersk_2",
"naersk": "naersk",
"nixos": "nixos",
"nixos-armv6": "nixos-armv6",
"nixos-hardware": "nixos-hardware",
"nixos-mobilizon": "nixos-mobilizon",
"nixos-unstable": "nixos-unstable",
"nixpkgs-mobilizon": "nixpkgs-mobilizon",
"nixos-unstable-simd": "nixos-unstable-simd",
"oparl-scraper": "oparl-scraper",
"openwrt": "openwrt",
"openwrt-imagebuilder": "openwrt-imagebuilder",
@ -562,11 +537,11 @@
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1668182250,
"narHash": "sha256-PYGaOCiFvnJdVz+ZCaKF8geGdffXjJUNcMwaBHv0FT4=",
"lastModified": 1671029659,
"narHash": "sha256-2sqAgHcLWpNGVHG2LQIXgRMuqgCArOD9frQSrrXrwSA=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "45ec315e01dc8dd1146dfeb65f0ef6e5c2efed78",
"rev": "95671d53ea3063da5316784af9ef69c71219f320",
"type": "github"
},
"original": {
@ -586,11 +561,11 @@
]
},
"locked": {
"lastModified": 1668479979,
"narHash": "sha256-UI+JUCBaMpn+5Y1hSePmndbYX5zu0+bavlfzrhPrGEk=",
"lastModified": 1671157233,
"narHash": "sha256-gvQaOKaV1UK6IzsFzkVLsEavGxnAsQFT3zUqcg0RXLU=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "2342f70f7257046effc031333c4cfdea66c91d82",
"rev": "7da2f6b3a0c32f661cb2864d7fbd1d7e6f0c7543",
"type": "github"
},
"original": {
@ -604,7 +579,7 @@
"locked": {
"lastModified": 1665446321,
"narHash": "sha256-GuZr+BCAIe+UYmQrLHaVr8iRRajn5nSdWyqhjWDIX1Y=",
"ref": "master",
"ref": "refs/heads/master",
"rev": "3700761dd06f271ef26261ed2a90dce8c22b6dca",
"revCount": 61,
"type": "git",
@ -624,7 +599,7 @@
"locked": {
"lastModified": 1659890996,
"narHash": "sha256-xURgGoznCPmpX35dn5AXcyNYicVn5ruvUKxfIMMiu8o=",
"ref": "master",
"ref": "refs/heads/master",
"rev": "5ca106f648bef15d9954d956bda336eea28e8d75",
"revCount": 149,
"type": "git",
@ -645,11 +620,11 @@
]
},
"locked": {
"lastModified": 1668537992,
"narHash": "sha256-dktkznGkVUtOXyA19J4YoSiyhBkCinqH8LDnU2o/rmw=",
"lastModified": 1670188372,
"narHash": "sha256-lKjP3rYsmiXb9kEU+stcsm48dTEMC6Ed+rLEClEl+Vs=",
"owner": "astro",
"repo": "skyflake",
"rev": "4ccb72c616212ef2149458dade8d1199dc69477a",
"rev": "c78a3e8f64930bf5c48b0f75e577e4294d8750c6",
"type": "github"
},
"original": {
@ -663,16 +638,16 @@
"nixpkgs": [
"nixos"
],
"nixpkgs-22_05": [
"nixpkgs-stable": [
"nixos"
]
},
"locked": {
"lastModified": 1668311578,
"narHash": "sha256-nF6mwSbVyvnlIICWFZlADegWdTsgrk1pZnA/0VqByNw=",
"lastModified": 1670149631,
"narHash": "sha256-rwmtlxx45PvOeZNP51wql/cWjY3rqzIR3Oj2Y+V7jM0=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "39f0fe57f1ef78764c1abc1de145f091fee1bbbb",
"rev": "da98a111623101c64474a14983d83dad8f09f93d",
"type": "github"
},
"original": {
@ -732,7 +707,9 @@
"fenix": [
"fenix"
],
"naersk": "naersk_3",
"naersk": [
"naersk"
],
"nixpkgs": [
"nixos"
],
@ -743,7 +720,7 @@
"locked": {
"lastModified": 1666559258,
"narHash": "sha256-m4f5QQbE+usnEnM6CJ+nCgXcsi9mm0cCwFm2BhJ0pvQ=",
"ref": "master",
"ref": "refs/heads/master",
"rev": "22ecb2b375bebffdfb1af3435a4c4486e6dd923b",
"revCount": 108,
"type": "git",
@ -785,7 +762,7 @@
"locked": {
"lastModified": 1663279525,
"narHash": "sha256-lUq4CY//ISplh/4i33nOU7cchpxKrw5V8mVdRnHMBaA=",
"ref": "master",
"ref": "refs/heads/master",
"rev": "6d8d2cb1268d26add05baa3f21c325cfe051add3",
"revCount": 342,
"type": "git",
@ -830,11 +807,11 @@
]
},
"locked": {
"lastModified": 1668554331,
"narHash": "sha256-QExvepHzH2QtHjeu1ZG2gIWh3bOxwsNsO2SBVmGIlRo=",
"ref": "master",
"rev": "9cb7dbc38595e7c213d193f2adf3cbeffcded74b",
"revCount": 1669,
"lastModified": 1671215340,
"narHash": "sha256-e1+KFpcOxdz0yvThbHszq9+sXhH7SsFGP/dsIpUrvZY=",
"ref": "refs/heads/master",
"rev": "4f090527d4cc64305add16ba68c6f72a275a507b",
"revCount": 1689,
"type": "git",
"url": "https://gitea.c3d2.de/zentralwerk/network.git"
},

491
flake.nix
View File

@ -2,11 +2,12 @@
description = "C3D2 NixOS configurations";
inputs = {
nixos.url = "github:nixos/nixpkgs/nixos-22.05";
nixpkgs-mobilizon.url = "github:minijackson/nixpkgs/init-mobilizon";
nixos-unstable.url = "github:nixos/nixpkgs/nixos-unstable";
# use sandro's fork full with cherry-picked fixes
nixos.url = "github:SuperSandro2000/nixpkgs/nixos-22.11";
nixos-mobilizon.url = "github:minijackson/nixpkgs/init-mobilizon";
nixos-hardware.url = "github:nixos/nixos-hardware";
nixos-armv6.url = "github:rnhmjoj/nixpkgs/pr-fix-armv6";
nixos-unstable.url = "github:nixos/nixpkgs/nixos-unstable";
nixos-unstable-simd.url = "github:SuperSandro2000/nixpkgs/nixos-unstable-simd";
affection-src = {
url = "git+https://gitea.nek0.eu/nek0/affection";
@ -15,6 +16,14 @@
flake-utils.follows = "flake-utils";
};
};
alert2muc = {
url = "git+https://gitea.c3d2.de/astro/alert2muc";
inputs = {
naersk.follows = "naersk";
nixpkgs.follows = "nixos";
utils.follows = "flake-utils";
};
};
bevy-mandelbrot = {
# url = "github:matelab/bevy_mandelbrot";
url = "git+https://gitea.c3d2.de/astro/bevy-mandelbrot.git?ref=main";
@ -29,9 +38,7 @@
url = "git+https://gitea.c3d2.de/astro/bevy-julia.git?ref=main";
inputs = {
nixpkgs.follows = "nixos";
# breaks the build:
# naersk.follows = "naersk";
# naersk.inputs.nixpkgs.follows = "nixpkgs";
naersk.follows = "naersk";
rust-overlay.follows = "rust-overlay";
};
};
@ -129,7 +136,7 @@
url = "github:Mic92/sops-nix";
inputs = {
nixpkgs.follows = "nixos";
nixpkgs-22_05.follows = "nixos";
nixpkgs-stable.follows = "nixos";
};
};
spacemsg = {
@ -140,8 +147,7 @@
url = "git+https://gitea.c3d2.de/astro/ticker.git";
inputs = {
fenix.follows = "fenix";
# TODO: build regression in https://github.com/nix-community/naersk/commit/6944160c19cb591eb85bbf9b2f2768a935623ed3
# naersk.follows = "naersk";
naersk.follows = "naersk";
nixpkgs.follows = "nixos";
utils.follows = "flake-utils";
};
@ -173,14 +179,9 @@
};
};
outputs = inputs@{ self, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, caveman, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
outputs = inputs@{ self, alert2muc, fenix, harmonia, heliwatch, microvm, naersk, nixos, nixos-hardware, nixos-unstable, caveman, oparl-scraper, scrapers, secrets, skyflake, sshlogd, sops-nix, spacemsg, ticker, tigger, yammat, zentralwerk, ... }:
let
inherit (nixos) lib;
forAllSystems = lib.genAttrs [ "aarch64-linux" "x86_64-linux" ];
# all the input flakes for `nix copy` to the build machine,
# allowing --override-input
inputPaths = lib.escapeShellArgs (builtins.attrValues inputs);
extractZwHosts = { hosts4, hosts6, ... }:
lib.recursiveUpdate (
@ -207,348 +208,56 @@
extraHostRegistry.hosts = import ./host-registry.nix;
hostRegistry = lib.recursiveUpdate zwHostRegistry extraHostRegistry;
getHostAddr = name:
let
hostConf = hostRegistry.hosts."${name}";
in
if hostConf ? ip4
then hostConf.ip4
else if hostConf ? ip6
then hostConf.ip6
else throw "Host ${name} has no ip4 or ip6 address";
# Our custom NixOS builder
nixosSystem' =
{ nixpkgs ? inputs.nixos
, modules
, extraArgs ? { }
, specialArgs ? { }
, system ? "x86_64-linux"
}@args:
{ inherit args; } // nixpkgs.lib.nixosSystem {
inherit specialArgs system;
modules = [
({ ... }: {
_module.args = extraArgs // {
inherit hostRegistry inputs zentralwerk;
};
nixpkgs = {
overlays = [ self.overlays ];
};
})
self.nixosModules.c3d2
./modules/audio-server.nix
./modules/c3d2.nix
./modules/stats.nix
./modules/pi-sensors.nix
] ++ modules;
};
in {
overlay = import ./overlay {
inherit nixos-unstable;
overlays = import ./overlays {
inherit (inputs) tracer bevy-mandelbrot bevy-julia;
};
legacyPackages = lib.attrsets.mapAttrs (system: pkgs:
legacyPackages = lib.attrsets.mapAttrs (_: pkgs:
pkgs.appendOverlays [
fenix.overlay
fenix.overlays.default
naersk.overlay
self.overlay
self.overlays
]) nixos.legacyPackages;
packages = lib.attrsets.mapAttrs (system: pkgs:
let overlayPkgs = builtins.intersectAttrs (self.overlay {} {}) pkgs;
in overlayPkgs //
{
host-registry = pkgs.runCommand "host-registry" {
src = builtins.toFile "host-registry.nix" (
lib.generators.toPretty {} hostRegistry
);
} ''
ln -s $src $out
'';
packages = import ./packages.nix { inherit hostRegistry inputs lib microvm secrets self; };
list-upgradable = pkgs.writeScriptBin "list-upgradable" ''
#! ${pkgs.runtimeShell}
NORMAL="\033[0m"
RED="\033[0;31m"
YELLOW="\033[0;33m"
GREEN="\033[0;32m"
${pkgs.lib.concatMapStringsSep "\n" (name:
let
addr = getHostAddr name;
in nixos.lib.optionalString (addr != null) ''
echo -n -e "${name}: $RED"
RUNNING=$(ssh -o PreferredAuthentications=publickey -o StrictHostKeyChecking=accept-new root@"${addr}" "readlink /run/current-system")
if [ $? = 0 ] && [ -n "$RUNNING" ]; then
CURRENT=$(nix eval --raw ".#nixosConfigurations.${name}.config.system.build.toplevel" 2>/dev/null)
RUNNING_VER=$(basename $RUNNING|rev|cut -d - -f 1|rev)
RUNNING_DATE=$(echo $RUNNING_VER|cut -d . -f 3)
CURRENT_VER=$(basename $CURRENT|rev|cut -d - -f 1|rev)
CURRENT_DATE=$(echo $CURRENT_VER|cut -d . -f 3)
if [ "$RUNNING" = "$CURRENT" ]; then
echo -e "$GREEN"current"$NORMAL $RUNNING_VER"
elif [ $RUNNING_DATE -gt $CURRENT_DATE ]; then
echo -e "$GREEN"newer"$NORMAL $RUNNING_VER > $CURRENT_VER"
elif [ "$RUNNING_VER" = "$CURRENT_VER" ]; then
echo -e "$YELLOW"modified"$NORMAL $RUNNING_VER"
elif [ -n "$RUNNING_VER" ]; then
echo -e "$RED"outdated"$NORMAL $RUNNING_VER < $CURRENT_VER"
else
echo -e "$RED"error"$NORMAL $RUNNING_VER"
fi
fi
echo -n -e "$NORMAL"
'') (builtins.attrNames self.nixosConfigurations)}
'';
prebuild-all = pkgs.runCommand "prebuild-all" {
preferLocalBuild = true;
} ''
mkdir $out
${pkgs.lib.concatMapStrings (name: ''
ln -s ${self.nixosConfigurations."${name}".config.system.build.toplevel} name
'') (builtins.attrNames self.nixosConfigurations)}
'';
prebuild-all-remote = pkgs.writeScriptBin "prebuild-all" ''
#!${pkgs.runtimeShell} -e
nix copy --no-check-sigs --to ssh-ng://$1 ${inputPaths}
set -x
ssh $1 -- nix build -L --no-link ${
pkgs.lib.concatMapStringsSep " " (name:
"${self}#nixosConfigurations.${name}.config.system.build.toplevel"
) (builtins.attrNames self.nixosConfigurations)
}
'';
} //
builtins.foldl' (result: host: result // {
# TODO: check if the ethernet address is reachable and if not,
# execute wol on a machine in HQ.
"${host}-wake" = pkgs.writeScriptBin "${host}-wake" ''
#!${pkgs.runtimeShell}
exec ${pkgs.wol}/bin/wol ${hostRegistry.hosts."${host}".ether}
'';
}) {} (builtins.attrNames (nixos.lib.filterAttrs (_: { wol ? false, ... }: wol) hostRegistry.hosts)) //
builtins.foldl' (result: name:
let
host = getHostAddr name;
target = ''root@"${host}"'';
rebuildArg = "--flake ${self}#${name} --option extra-substituters https://nix-serve.hq.c3d2.de";
hostConfig = self.nixosConfigurations."${name}".config;
profile = hostConfig.system.build.toplevel;
# let /var/lib/microvm/*/flake point to the flake-update branch so that
# `microvm -u $NAME` updates to what hydra built today.
selfRef = "git+https://gitea.c3d2.de/c3d2/nix-config?ref=flake-update";
in result // {
# Generate a small script for copying this flake to the
# remote machine and bulding and switching there.
# Can be run with `nix run c3d2#…-nixos-rebuild switch`
"${name}-nixos-rebuild" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
#!${pkgs.runtimeShell} -ex
[[ $(ssh ${target} cat /etc/hostname) == ${name} ]]
nix copy --no-check-sigs --to ssh-ng://${target} ${inputPaths}
ssh ${target} nixos-rebuild ${rebuildArg} "$@"
'';
"${name}-nixos-rebuild-hydra" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
#!${pkgs.runtimeShell} -e
echo Copying Flakes
nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${inputPaths}
echo Building on Hydra
ssh root@hydra.serv.zentralwerk.org -- \
nix build -L -o /tmp/nixos-system-${name} \
${self}#nixosConfigurations.${name}.config.system.build.toplevel
echo Built. Obtaining link to data
TOPLEVEL=$(ssh root@hydra.serv.zentralwerk.org \
readlink /tmp/nixos-system-${name})
echo Checking target ${name}
ssh ${target} -- bash -e <<EOF
[[ \$(cat /etc/hostname) == ${name} ]]
echo Copying data from Hydra to ${name}
nix copy --from https://nix-serve.hq.c3d2.de \
$TOPLEVEL
echo Activation on ${name}: "$@"
nix-env -p /nix/var/nix/profiles/system --set $TOPLEVEL
$TOPLEVEL/bin/switch-to-configuration "$@"
EOF
'';
"${name}-nixos-rebuild-local" = pkgs.writeScriptBin "${name}-nixos-rebuild" ''
#!${pkgs.runtimeShell} -ex
[[ $1 == build || $(ssh ${target} cat /etc/hostname) == ${name} ]]
${pkgs.nixos-rebuild}/bin/nixos-rebuild ${rebuildArg} --target-host ${target} --use-remote-sudo "$@"
'';
"${name}-cleanup" = pkgs.writeScriptBin "${name}-cleanup" ''
#!${pkgs.runtimeShell} -ex
ssh ${target} "time nix-collect-garbage -d && time nix-store --optimise"
'';
"microvm-update-${name}" = pkgs.writeScriptBin "microvm-update-${name}" ''
#!${pkgs.runtimeShell} -e
${lib.optionalString (! builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]) ''
echo "MicroVM must be configured to proper server" >&2
exit 1
''}
${hostConfig.system.build.copyToServer} ${inputPaths}
${hostConfig.system.build.runOnServer} bash -e <<END
mkdir -p /var/lib/microvms/${name}
cd /var/lib/microvms/${name}
chown root:kvm .
chmod 0775 .
rm -f old
[ -e current ] && cp --no-dereference current old
nix build -L \
-o current \
${self}#nixosConfigurations.${name}.config.microvm.declaredRunner
echo '${selfRef}' > flake
[ -e old ] && nix store diff-closures ./old ./current
ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${name}
ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${name}
ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${name}
systemctl restart microvm@${name}.service
END
'';
"microvm-update-${name}-local" = pkgs.writeScriptBin "microvm-update-${name}" ''
#!${pkgs.runtimeShell} -e
${lib.optionalString (! builtins.elem (hostConfig.c3d2.deployment.server or null) [ "server9" "server10" ]) ''
echo "MicroVM must be configured to proper server" >&2
exit 1
''}
${hostConfig.system.build.copyToServer} ${hostConfig.microvm.declaredRunner}
${hostConfig.system.build.runOnServer} bash -e <<END
mkdir -p /var/lib/microvms/${name}
cd /var/lib/microvms/${name}
chown root:kvm .
chmod 0775 .
rm -f old
[ -e current ] && cp --no-dereference current old
ln -sfT ${hostConfig.microvm.declaredRunner} current
echo '${selfRef}' > flake
[ -e old ] && nix store diff-closures ./old ./current
ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${name}
ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${name}
ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${name}
systemctl restart microvm@${name}.service
END
'';
"nomad-${name}" = pkgs.writeScriptBin "nomad-${name}" ''
#!${pkgs.runtimeShell} -e
${lib.optionalString (hostConfig.c3d2.deployment.server or null == "nomad") ''
echo "MicroVM must be configured for nomad" >&2
exit 1
''}
echo Copying Flakes
nix copy --no-check-sigs --to ssh-ng://root@hydra.serv.zentralwerk.org ${secrets} ${self}
echo Building on Hydra
ssh root@hydra.serv.zentralwerk.org -- \
nix build -L -o /tmp/microvm-${name}.job \
${self}#nixosConfigurations.${name}.config.system.build.nomadJob
echo -n Built. Obtaining path...
JOB=$(ssh root@hydra.serv.zentralwerk.org -- \
readlink /tmp/microvm-${name}.job)
echo \ $JOB
for h in server9 server10 ; do
echo Sharing with $h
ssh root@$h.cluster.zentralwerk.org -- \
bash -e <<EOF &
nix copy --from https://nix-serve.hq.c3d2.de $JOB
mkdir -p /glusterfs/fast/microvms/${name}
chown microvm:kvm /glusterfs/fast/microvms/${name}
chmod 0775 /glusterfs/fast/microvms/${name}
mkdir -p /nix/var/nix/gcroots/microvm
rm -f /nix/var/nix/gcroots/microvm/${name}
ln -sfT $JOB /nix/var/nix/gcroots/microvm/${name}
EOF
done
wait
echo Now starting the job
ssh root@hydra.serv.zentralwerk.org -- \
nomad run -detach $JOB
'';
}) {} (builtins.attrNames self.nixosConfigurations) //
builtins.foldl' (result: host:
let
inherit (self.nixosConfigurations.${host}) config;
in
result // {
# boot any machine in a microvm
"${host}-vm" = (self.nixosConfigurations.${host}
.extendModules {
modules = [ {
microvm = {
mem = nixos.lib.mkForce 2048;
hypervisor = nixos.lib.mkForce "qemu";
socket = nixos.lib.mkForce null;
shares = nixos.lib.mkForce [ {
tag = "ro-store";
source = "/nix/store";
mountPoint = "/nix/.ro-store";
} ];
interfaces = nixos.lib.mkForce [ {
type = "user";
id = "eth0";
mac = "02:23:de:ad:be:ef";
} ];
};
boot.isContainer = lib.mkForce false;
users.users.root.password = "";
fileSystems."/".fsType = lib.mkForce "tmpfs";
services.getty.helpLine = ''
Log in as "root" with an empty password.
Use "reboot" to shut qemu down.
'';
} ] ++ lib.optionals (! config ? microvm) [
microvm.nixosModules.microvm
];
})
.config.microvm.declaredRunner;
"${host}-tftproot" =
if config.system.build ? tftproot
then config.system.build.tftproot
else throw "No tftproot for ${host}";
}
) {} (builtins.attrNames self.nixosConfigurations)
) self.legacyPackages;
nixosConfigurations = let
nixosSystem' =
# Our custom NixOS builder
{ nixpkgs ? inputs.nixos, modules, extraArgs ? {}, specialArgs ? { }, system ? "x86_64-linux" }:
nixpkgs.lib.nixosSystem {
inherit specialArgs system;
modules = [
({ pkgs, ... }: {
_module.args = extraArgs // {
inherit hostRegistry inputs zentralwerk;
};
nixpkgs = {
overlays = [ self.overlay ];
};
})
self.nixosModules.c3d2
./modules/audio-server.nix
./modules/c3d2.nix
./modules/stats.nix
./modules/pi-sensors.nix
] ++ modules;
};
in {
nixosConfigurations = {
auth = nixosSystem' {
modules = [
self.nixosModules.microvm
./hosts/auth
];
nixpkgs = inputs.nixos-unstable;
};
broker = nixosSystem' {
@ -590,7 +299,6 @@
self.nixosModules.microvm
./hosts/gitea
];
nixpkgs = inputs.nixos-unstable;
};
glotzbert = nixosSystem' {
@ -608,7 +316,6 @@
self.nixosModules.microvm
./hosts/hedgedoc
];
nixpkgs = inputs.nixos-unstable;
};
pulsebert = nixosSystem' {
@ -624,12 +331,11 @@
};
}
];
nixpkgs = inputs.nixos-unstable;
};
radiobert = nixosSystem' {
modules = [
({ modulesPath, ... }:
({ ... }:
{
nixpkgs.overlays = [ heliwatch.overlay ];
})
@ -674,7 +380,6 @@
};
schalter = nixosSystem' {
nixpkgs = inputs.nixos-armv6;
modules = [
"${nixos}/nixos/modules/installer/sd-card/sd-image-raspberrypi.nix"
({ lib, ... }: {
@ -751,7 +456,6 @@
self.nixosModules.microvm
./hosts/grafana
];
nixpkgs = nixos-unstable;
};
hydra = nixosSystem' {
@ -767,7 +471,6 @@
};
}
];
nixpkgs = nixos-unstable;
};
mucbot = nixosSystem' {
@ -805,14 +508,14 @@
stream = nixosSystem' {
modules = [
self.nixosModules.microvm
self.nixosModules.cluster-options
./hosts/stream
];
};
mobilizon = nixosSystem' {
# TODO: pending https://github.com/NixOS/nixpkgs/pull/119132
nixpkgs = inputs.nixpkgs-mobilizon;
nixpkgs = inputs.nixos-mobilizon;
modules = [
self.nixosModules.microvm
./hosts/mobilizon
@ -835,7 +538,7 @@
heliwatch.nixosModules.heliwatch
./hosts/sdrweb
];
nixpkgs = nixos-unstable;
# nixpkgs = nixos-unstable;
};
bind = nixosSystem' {
@ -862,6 +565,16 @@
];
};
server8 = nixosSystem' {
modules = [
./hosts/server8
self.nixosModules.cluster-network
self.nixosModules.cluster
skyflake.nixosModules.default
{ _module.args = { inherit self; }; }
];
};
server9 = nixosSystem' {
modules = [
./hosts/server9
@ -872,7 +585,6 @@
skyflake.nixosModules.default
{ _module.args = { inherit self; }; }
];
nixpkgs = nixos-unstable;
};
server10 = nixosSystem' {
@ -885,7 +597,6 @@
skyflake.nixosModules.default
{ _module.args = { inherit self; }; }
];
nixpkgs = nixos-unstable;
};
oparl = nixosSystem' {
@ -900,14 +611,14 @@
leon = nixosSystem' {
modules = [
self.nixosModules.microvm
self.nixosModules.cluster-options
./hosts/leon
];
};
leoncloud = nixosSystem' {
modules = [
self.nixosModules.microvm
self.nixosModules.cluster-options
./hosts/leoncloud
];
};
@ -935,7 +646,7 @@
};
}
];
nixpkgs = nixos-unstable;
# nixpkgs = nixos-unstable;
system = "x86_64-linux";
};
@ -951,15 +662,15 @@
self.nixosModules.microvm
./hosts/mediawiki
];
nixpkgs = nixos-unstable;
# nixpkgs = nixos-unstable;
};
gnunet = nixosSystem' {
modules = [
self.nixosModules.microvm
self.nixosModules.cluster-options
./hosts/gnunet
];
nixpkgs = nixos-unstable;
# nixpkgs = nixos-unstable;
};
zengel = nixosSystem' {
@ -971,7 +682,7 @@
sshlog = nixosSystem' {
modules = [
self.nixosModules.microvm
self.nixosModules.cluster-options
sshlogd.nixosModule
./hosts/sshlog
];
@ -992,9 +703,30 @@
];
};
owncast = nixosSystem' {
modules = [
self.nixosModules.cluster-options
./hosts/owncast
];
# nixpkgs = nixos-unstable;
};
prometheus = nixosSystem' {
modules = [
self.nixosModules.cluster-options
alert2muc.nixosModules.default
./hosts/prometheus
];
};
mastodon = nixosSystem' {
modules = [