From c41f5c56a617f5d2ad65cc63a4b84a86ad885847 Mon Sep 17 00:00:00 2001
From: Astro <astro@spaceboyz.net>
Date: Mon, 5 Jun 2023 01:17:05 +0200
Subject: [PATCH] nix/nixos-module/server/lxc-containers: make container config
 independend of host system

---
 nix/nixos-module/container/lxc-config.nix  | 111 +++++++++++++++++++
 nix/nixos-module/default.nix               |   1 +
 nix/nixos-module/server/lxc-containers.nix | 121 +--------------------
 nix/pkgs/default.nix                       |  15 ++-
 4 files changed, 132 insertions(+), 116 deletions(-)
 create mode 100644 nix/nixos-module/container/lxc-config.nix

diff --git a/nix/nixos-module/container/lxc-config.nix b/nix/nixos-module/container/lxc-config.nix
new file mode 100644
index 000000000..63bb2a3
--- /dev/null
+++ b/nix/nixos-module/container/lxc-config.nix
@@ -0,0 +1,111 @@
+{ config, lib, ... }:
+
+let
+
+  inherit (config.networking) hostName;
+
+  # linux iface name max length = 15
+  shortenNetName = name:
+    if builtins.match "priv(.*)" name != null
+    then "p" + builtins.substring 4 9 name
+    else if name == "coloradio"
+    then "cr"
+    else if name == "coloradio-gw"
+    then "cr-gw"
+    else name;
+
+  checkIfname = ifname: let
+      len = builtins.stringLength ifname;
+    in if len > 15
+    then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long."
+    else ifname;
+
+  # `lxc.net.*` formatter for lxc.container.conf files
+  netConfig = ctName: interfaces:
+    let
+      config = map (netName:
+        let
+          ifData = interfaces.${netName};
+        in {
+          type = ifData.type;
+          name = checkIfname netName;
+          flags = "up";
+          hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
+                   then ifData.hwaddr
+                   else "0A:14:48:xx:xx:xx";
+        } // (lib.optionalAttrs (ifData.type == "veth") {
+          veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}";
+          veth.mode = checkIfname "bridge";
+          link = checkIfname netName;
+        }) // (lib.optionalAttrs (ifData.type == "phys") {
+          link = checkIfname "ext-${netName}";
+        })
+      ) (builtins.attrNames interfaces);
+
+      attrNamesOrdered = attrs:
+        if attrs ? type
+        then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs)
+        else builtins.attrNames attrs;
+
+      serialize = name: x:
+        if builtins.isString x
+        then "${name} = ${x}\n"
+        else if builtins.isAttrs x
+        then builtins.concatStringsSep "" (
+          map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x)
+        )
+        else if builtins.isList x
+        then
+          let
+            enumerate = xs: n:
+              if xs == []
+              then []
+              else [ {
+                e = builtins.head xs;
+                i = n;
+              } ] ++ enumerate (builtins.tail xs) (n + 1);
+          in
+            builtins.concatStringsSep "" (
+              map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
+            )
+        else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
+    in
+      serialize "lxc.net" config;
+
+in
+{
+  system.build.lxcConfig = builtins.toFile "${hostName}.conf" ''
+    # For lxcfs and sane defaults
+    lxc.include = /etc/lxc/common.conf
+
+    lxc.uts.name = ${hostName}
+    # Handled by lxc@.service
+    lxc.start.auto = 0
+    lxc.rootfs.path = /var/lib/lxc/${hostName}/rootfs
+    lxc.init.cmd = "/init"
+
+    lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
+    lxc.mount.entry = none tmp tmpfs defaults 0 0
+    lxc.mount.auto = proc:mixed sys:ro cgroup:mixed
+
+    lxc.autodev = 1
+    lxc.tty.max = 0
+    lxc.pty.max = 8
+
+    lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio
+    security.privileged = false
+    lxc.apparmor.profile = lxc-container-default-with-mounting
+
+    lxc.cgroup.memory.limit_in_bytes = 1G
+    lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
+
+    # tuntap
+    lxc.cgroup.devices.allow = c 10:200 rw
+    lxc.cgroup2.devices.allow = c 10:200 rw
+    # ppp
+    lxc.cgroup.devices.allow = c 108:0 rwm
+    lxc.cgroup2.devices.allow = c 108:0 rwm
+
+    ${netConfig hostName config.site.hosts.${hostName}.physicalInterfaces}
+  '';
+}
diff --git a/nix/nixos-module/default.nix b/nix/nixos-module/default.nix
index 878e0e0..4d286b5 100644
--- a/nix/nixos-module/default.nix
+++ b/nix/nixos-module/default.nix
@@ -20,6 +20,7 @@ in {
     ./server/default.nix
   ] ++
   optionals (hostConfig.role == "container") [
+    ./container/lxc-config.nix
     ./container/defaults.nix
     ./container/dhcp-server.nix
     ./container/wireguard.nix
diff --git a/nix/nixos-module/server/lxc-containers.nix b/nix/nixos-module/server/lxc-containers.nix
index d31ba54..ef3b573 100644
--- a/nix/nixos-module/server/lxc-containers.nix
+++ b/nix/nixos-module/server/lxc-containers.nix
@@ -10,74 +10,6 @@ let
 
   enabled = containers != {};
 
-  # linux iface name max length = 15
-  shortenNetName = name:
-    if builtins.match "priv(.*)" name != null
-    then "p" + builtins.substring 4 9 name
-    else if name == "coloradio"
-    then "cr"
-    else if name == "coloradio-gw"
-    then "cr-gw"
-    else name;
-
-  checkIfname = ifname: let
-      len = builtins.stringLength ifname;
-    in if len > 15
-    then throw "Interface name ${ifname} is ${toString (len - 15)} chars too long."
-    else ifname;
-
-  # `lxc.net.*` formatter for lxc.container.conf files
-  netConfig = ctName: interfaces:
-    let
-      config = map (netName:
-        let
-          ifData = interfaces.${netName};
-        in {
-          type = ifData.type;
-          name = checkIfname netName;
-          flags = "up";
-          hwaddr = if ifData ? hwaddr && ifData.hwaddr != null
-                   then ifData.hwaddr
-                   else "0A:14:48:xx:xx:xx";
-        } // (lib.optionalAttrs (ifData.type == "veth") {
-          veth.pair = checkIfname "${shortenNetName ctName}-${shortenNetName netName}";
-          veth.mode = checkIfname "bridge";
-          link = checkIfname netName;
-        }) // (lib.optionalAttrs (ifData.type == "phys") {
-          link = checkIfname "ext-${netName}";
-        })
-      ) (builtins.attrNames interfaces);
-
-      attrNamesOrdered = attrs:
-        if attrs ? type
-        then [ "type" ] ++ lib.remove "type" (builtins.attrNames attrs)
-        else builtins.attrNames attrs;
-
-      serialize = name: x:
-        if builtins.isString x
-        then "${name} = ${x}\n"
-        else if builtins.isAttrs x
-        then builtins.concatStringsSep "" (
-          map (n: serialize "${name}.${n}" x.${n}) (attrNamesOrdered x)
-        )
-        else if builtins.isList x
-        then
-          let
-            enumerate = xs: n:
-              if xs == []
-              then []
-              else [ {
-                e = builtins.head xs;
-                i = n;
-              } ] ++ enumerate (builtins.tail xs) (n + 1);
-          in
-            builtins.concatStringsSep "" (
-              map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
-            )
-        else throw "Invalid data in lxc net config for ${name}: ${lib.generators.toPretty {} x}";
-    in
-      serialize "lxc.net" config;
-
   # User-facing script to build/update container NixOS systems
   build-script = pkgs.writeScriptBin "build-container" ''
     #! ${pkgs.runtimeShell} -e
@@ -98,6 +30,7 @@ let
             ${ctName})
               echo Using prebuilt system for container $c
               SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
+              CONFIG=${self.packages.x86_64-linux."${ctName}-lxc-config"}
               ;;
           '') (
             builtins.attrNames (
@@ -109,6 +42,8 @@ let
         echo Building $c
         nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
         SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
+        nix build -o /nix/var/nix/gcroots/lxc/$c.config zentralwerk-network#$c-lxc-config
+        CONFIG=$(readlink /nix/var/nix/gcroots/lxc/$c.config)
           ;;
       esac
 
@@ -121,6 +56,7 @@ let
         mkdir -p /var/lib/lxc/$c/rootfs/$d
       done
       ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
+      ln -fs $CONFIG /var/lib/lxc/$c/config
     done
 
     # Activate all the desired container after all of them are
@@ -166,10 +102,8 @@ in
 
   virtualisation.lxc = lib.mkIf enabled {
     enable = true;
-    # Container configs live in /etc so that they can be created
-    # through `environment.etc`.
     systemConfig = ''
-      lxc.lxcpath = /etc/lxc/containers
+      lxc.lxcpath = /var/lib/lxc
     '';
   };
 
@@ -180,50 +114,7 @@ in
     enable-script disable-script
   ];
 
-  # Create lxc.container.conf files
-  environment.etc =
-    builtins.foldl' (etc: ctName: etc // {
-      "lxc/containers/${ctName}/config" = {
-        enable = true;
-        source =
-          builtins.toFile "${ctName}.conf" ''
-            # For lxcfs and sane defaults
-            lxc.include = /etc/lxc/common.conf
-
-            lxc.uts.name = ${ctName}
-            # Handled by lxc@.service
-            lxc.start.auto = 0
-            lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs
-            lxc.init.cmd = "/init"
-
-            lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
-            lxc.mount.entry = none tmp tmpfs defaults 0 0
-            lxc.mount.auto = proc:mixed sys:ro cgroup:mixed
-
-            lxc.autodev = 1
-            lxc.tty.max = 0
-            lxc.pty.max = 8
-
-            lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio
-            security.privileged = false
-            lxc.apparmor.profile = lxc-container-default-with-mounting
-
-            lxc.cgroup.memory.limit_in_bytes = 1G
-            lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M
-
-            # tuntap
-            lxc.cgroup.devices.allow = c 10:200 rw
-            lxc.cgroup2.devices.allow = c 10:200 rw
-            # ppp
-            lxc.cgroup.devices.allow = c 108:0 rwm
-            lxc.cgroup2.devices.allow = c 108:0 rwm
-
-            ${netConfig ctName containers.${ctName}.physicalInterfaces}
-          '';
-      };
-    }) {
-      "lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
-    } (builtins.attrNames containers);
+  environment.etc."lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
 
   # Systemd service template for LXC containers
   systemd.services."lxc@" = {
diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix
index d2b402d..54a51c6 100644
--- a/nix/pkgs/default.nix
+++ b/nix/pkgs/default.nix
@@ -57,6 +57,19 @@ let
       )
     );
 
+  mkLxcConfig = hostName:
+    self.nixosConfigurations.${hostName}.config.system.build.lxcConfig;
+
+  lxc-configs =
+    builtins.foldl' (rootfs: hostName: rootfs // {
+      "${hostName}-lxc-config" = mkLxcConfig hostName;
+    }) {} (
+      builtins.attrNames (
+        nixpkgs.lib.filterAttrs (_: { role, ... }: role == "container")
+          config.site.hosts
+      )
+    );
+
   vm-packages =
     builtins.foldl' (rootfs: hostName: rootfs // {
       "${hostName}-vm" = self.nixosConfigurations.${hostName}.config.system.build.vm
@@ -117,7 +130,7 @@ let
     inherit self;
   };
 in
-rootfs-packages // vm-packages // device-templates // openwrt-packages // network-graphs // network-cypher-graphs // starlink // subnetplans // {
+rootfs-packages // lxc-configs // vm-packages // device-templates // openwrt-packages // network-graphs // network-cypher-graphs // starlink // subnetplans // {
   inherit export-openwrt-models export-config dns-slaves
     encrypt-secrets decrypt-secrets switch-to-production
     homepage gateway-report switch-report vlan-report