doc

README.md

@@ -38,6 +38,13 @@ Wir, ein kleiner Kreis von Menschen die das Netzwerk im Zentralwerk betreuen, ha
 - [ ] device-scripts auf Site Config umstellen
 - [ ] Site Config dumpen, Salt-Daten löschen
 
+### Server Setup
+
+checkout dieses repos
+/etc/nixos, registry
+input.key
+nixos-rebuild
+
 ### Nix Flake
 
 [Segfaulting nix?](https://github.com/NixOS/nix/issues/4178) Until

flake.nix

@@ -3,6 +3,8 @@
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs";
+    # `nix flake update --override-flake zentralwerk-network-key git+file:///...`
+    # to provide the GPG secret key
     zentralwerk-network-key.url = "git+https://gitea.c3d2.de/zentralwerk/network.git?dir=nix/key&ref=nix";
   };
 
@@ -22,6 +24,7 @@
           specialArgs.inputs = inputs;
         };
     in {
+      # Config, and utilities
       lib = nixpkgs.lib.extend (final: prev:
         import ./nix/lib {
           inherit self;
@@ -29,14 +32,18 @@
           pkgs = nixpkgs.legacyPackages.x86_64-linux;
         });
 
+      # Everything that can be built locally outside of NixOS
       packages = forAllSystems (system:
         import ./nix/pkgs { inherit self nixpkgs system; }
       );
 
+      # Configuration for nixosConfigurations
+      # (see nix/nixos-module/default.nix)
       nixosModule = { ... }: {
         imports = [ ./nix/nixos-module ];
       };
 
+      # NixOS host systems (servers, and containers)
       nixosConfigurations =
         builtins.mapAttrs (hostName: _: nixosConfig hostName) (
           nixpkgs.lib.filterAttrs (_: { role, ... }:
@@ -44,6 +51,7 @@
           ) self.lib.config.site.hosts
         );
 
+      # For `nix flake check`, and Hydra
       checks = self.packages;
     };
 }

nix/lib/config/options.nix

@@ -107,21 +107,32 @@ let
       hwaddr = mkOption {
         type = with types; nullOr str;
         default = null;
+        description = "Static MAC address";
       };
       type = mkOption {
         type = types.enum [ "phys" "veth" ];
+        description = ''
+          veth: Virtual ethernet to be attached to a bridge.
+
+          phys: (Physical) interface from a server moved into the
+            container. Do not use with VLAN interfaces because they
+            won't be moved back after lxc-stop.
+        '';
       };
       gw4 = mkOption {
         type = with types; nullOr str;
         default = null;
+        description = "IPv4 gateway";
       };
       gw6 = mkOption {
         type = with types; nullOr str;
         default = null;
+        description = "IPv6 gateway";
       };
       upstream = mkOption {
         type = with types; nullOr (submodule { options = upstreamOpts; });
         default = null;
+        description = "Upstream interface configuration";
       };
     };
   };
@@ -152,6 +163,7 @@ let
       interfaces = mkOption {
         default = {};
         type = with types; attrsOf (submodule interfaceOpts);
+        description = "Network interfaces";
       };
       isRouter = mkOption {
         type = types.bool;
@@ -178,10 +190,12 @@ let
       ospf.stubNets4 = mkOption {
         type = with types; listOf str;
         default = [];
+        description = "Additional IPv4 networks to announce";
       };
       ospf.stubNets6 = mkOption {
         type = with types; listOf str;
         default = [];
+        description = "Additional IPv6 networks to announce";
       };
       wireguard = mkOption {
         default = {};

nix/nixos-module/container/anon.nix

@@ -96,4 +96,6 @@ in
     externalInterface = firstTunnel;
     inherit (config.site.hosts.${hostName}) forwardPorts;
   };
+
+  # TODO: firewall
 }

nix/nixos-module/container/bird.nix

@@ -1,8 +1,10 @@
+# Routing daemon configuration
 { hostName, config, options, lib, ... }:
 
 let
   hostConf = config.site.hosts.${hostName};
 
+  # Configuring a gateway? If so, this is the associated net.
   gatewayNet =
     let
       m = builtins.match "(.+)-gw" hostName;
@@ -33,29 +35,32 @@ in
       }
 
       ${lib.optionalString (gatewayNet != null) ''
-      protocol radv {
+        # Router advertisements
-        rdnss ${config.site.net.serv.hosts6.dn42.dnscache};
+        protocol radv {
+          rdnss ${config.site.net.serv.hosts6.dn42.dnscache};
 
-        interface "${gatewayNet}" {
+          interface "${gatewayNet}" {
-          min ra interval 10;
+            min ra interval 10;
-          max ra interval 60;
+            max ra interval 60;
 
-          ${builtins.concatStringsSep "\n" (
+            ${builtins.concatStringsSep "\n" (
-            map (subnet6: ''
+              map (subnet6: ''
-              prefix ${subnet6} {
+                prefix ${subnet6} {
-                preferred lifetime 20;
+                  preferred lifetime 20;
-                valid lifetime 60;
+                  valid lifetime 60;
-              };
+                };
-            '') (builtins.attrValues config.site.net.${gatewayNet}.subnets6)
+              '') (builtins.attrValues config.site.net.${gatewayNet}.subnets6)
-          )}
+            )}
 
-          dnssl "${config.site.net.${gatewayNet}.domainName}";
+            dnssl "${config.site.net.${gatewayNet}.domainName}";
-        };
+          };
-      }
+        }
       ''}
 
+      # OSPFv2 for site-local IPv4
       protocol ospf v2 ZW4 {
         area 0 {
+          # Enabled on these networks
           networks {
             ${builtins.concatStringsSep " " (
               map (n: "  ${n};") config.site.ospf.networks4
@@ -64,6 +69,9 @@ in
           ${builtins.concatStringsSep "\n" (
               builtins.attrValues (
                 builtins.mapAttrs (net: _:
+                  # Enable OSPF only on networks with a secret. Others
+                  # are treated as a stubnet whose routes to
+                  # advertise.
                   if config.site.net.${net}.ospf.secret != null
                   then ''
                     interface "${net}" {
@@ -73,6 +81,7 @@ in
                   ''
                   else if config.site.net.${net}.subnet4 != null
                   then ''
+                    # Advertise route of network ${net}
                     stubnet ${config.site.net.${net}.subnet4} {};
                   ''
                   else ""
@@ -80,14 +89,18 @@ in
               )
           )}
           ${builtins.concatStringsSep "\n" (
-            map (stubnet4: "stubnet ${stubnet4} {};")
+            map (stubnet4: ''
-              hostConf.ospf.stubNets4
+              # Advertise additional route
+              stubnet ${stubnet4} {};
+            '') hostConf.ospf.stubNets4
           )}
         };
       }
 
+      # OSPFv3 for site-local IPv6
       protocol ospf v3 ZW6 {
         area 0 {
+          # Enabled on these networks
           networks {
             ${builtins.concatStringsSep " " (
               map (n: "  ${n};") config.site.ospf.networks6
@@ -96,6 +109,9 @@ in
           ${builtins.concatStringsSep "\n" (
               builtins.attrValues (
                 builtins.mapAttrs (net: _:
+                  # Enable OSPF only on networks with a secret. Others
+                  # are treated as a stubnet whose routes to
+                  # advertise.
                   if config.site.net.${net}.ospf.secret != null
                   then ''
                     interface "${net}" {
@@ -105,14 +121,19 @@ in
                     };
                   ''
                   else builtins.concatStringsSep "\n" (
-                    map (subnet6: "stubnet ${subnet6} {};")
+                    map (subnet6: ''
-                      (builtins.attrValues config.site.net.${net}.subnets6)
+                      # Advertise route of network ${net}
+                      stubnet ${subnet6} {};
+                    '') (builtins.attrValues config.site.net.${net}.subnets6)
                   )
                 ) hostConf.interfaces
               )
           )}
           ${builtins.concatStringsSep "\n" (
-            map (stubnet6: "stubnet ${stubnet6} {};")
+            map (stubnet6: ''
+              # Advertise additional route
+              stubnet ${stubnet6} {};
+            '')
               hostConf.ospf.stubNets6
           )}
         };

nix/nixos-module/container/dhcp-server.nix

@@ -1,3 +1,4 @@
+# ISC DHCP/IPv4 server configuration
 { hostName, config, lib, ... }:
 
 let

nix/nixos-module/default.nix

@@ -14,26 +14,26 @@ in {
     ./defaults.nix
     ./network.nix
     ./collectd.nix
-  ]
+  ] ++
-  ++ optionals (hostConfig.role == "server") [
+  optionals (hostConfig.role == "server") [
     ./server/lxc-containers.nix
     ./server/network.nix
-  ]
+  ] ++
-  ++ optionals (hostName == "server2") [
+  optionals (hostName == "server2") [
     ./server/server2.nix
-  ]
+  ] ++
-  ++ optionals (hostConfig.role == "container") [
+  optionals (hostConfig.role == "container") [
     ./container/defaults.nix
     ./container/dhcp-server.nix
     ./container/anon.nix
-  ] ++ optionals (
+  ] ++
-    hostConfig.role == "container" &&
+  optionals lib.config.site.hosts.${hostName}.isRouter [
-    lib.config.site.hosts.${hostName}.isRouter
-  ) [
     ./container/bird.nix
-  ] ++ optionals (builtins.match "upstream.*" hostName != null) [
+  ] ++
+  optionals (builtins.match "upstream.*" hostName != null) [
     ./container/upstream.nix
-  ] ++ optionals (hostName == "mgmt-gw") [
+  ] ++
+  optionals (hostName == "mgmt-gw") [
     ./container/mgmt-gw.nix
   ];
 }

nix/nixos-module/defaults.nix

@@ -35,11 +35,15 @@
 
   # for vm-packages
   virtualisation = lib.optionalAttrs (builtins.hasAttr "qemu" options.virtualisation) {
+    # larger than the defaults
     memorySize = 8192;
-    msize = 65536;
+    cores = 2;
-    cores = 4;
     diskSize = 8192;
+    # 9P performance optimization that quelches a qemu warning
+    msize = 65536;
+    # allow building packages
     writableStore = true;
+    # keep the store paths built inside the VM across reboots
     writableStoreUseTmpfs = false;
     qemu.options = [ "-enable-kvm" ];
   };

nix/nixos-module/server/lxc-containers.nix

@@ -1,6 +1,7 @@
 { hostName, self, config, lib, pkgs, ... }:
 
 let
+  # Containers that are run on this host
   containers =
     lib.filterAttrs (_: { role, model, location, ... }:
       role == "container" &&
@@ -10,6 +11,7 @@ let
 
   enabled = containers != {};
 
+  # `lxc.net.*` formatter for lxc.container.conf files
   netConfig = ctName: interfaces:
     let
       config = map (netName:
@@ -56,6 +58,7 @@ let
     in
       serialize "lxc.net" config;
 
+  # User-facing script to build/update container NixOS systems
   build-script = pkgs.writeScriptBin "build-container" ''
     #! ${pkgs.runtimeShell} -e
 
@@ -97,16 +100,16 @@ in
 {
   virtualisation.lxc = lib.mkIf enabled {
     enable = true;
+    # Container configs live in /etc so that they can be created
+    # through `environment.etc`.
     systemConfig = ''
       lxc.lxcpath = /etc/lxc/containers
-
-      # lxc.rootfs.backend = zfs
-      # lxc.bdev.zfs.root = vault/sys/atom/var/lib/lxc
     '';
   };
 
   environment.systemPackages = [ pkgs.lxc build-script ];
 
+  # Create lxc.container.conf files
   environment.etc =
     builtins.foldl' (etc: ctName: etc // {
       "lxc/containers/${ctName}/config" = {
@@ -148,6 +151,7 @@ in
       "lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
     } (builtins.attrNames containers);
 
+  # Systemd service template for LXC containers
   systemd.services."lxc@" = {
     description = "LXC container '%i'";
     after = [ "network.target" ];
@@ -185,6 +189,7 @@ in
     };
   };
 
+  # Starts all the containers after boot
   systemd.targets.lxc-containers = {
     wantedBy = [ "multi-user.target" ];
     wants = map (ctName: "lxc@${ctName}.service")

nix/nixos-module/server/network.nix

@@ -1,6 +1,8 @@
+# Server network configuration
 { hostName, self, config, lib, pkgs, ... }:
 
 let
+  # LXC containers on this host
   containers =
     lib.filterAttrs (_: { role, model, location, ... }:
       role == "container" &&
@@ -8,6 +10,7 @@ let
       location == hostName
     ) config.site.hosts;
 
+  # Every bridged veth network required by all containers
   bridgeNets =
     lib.lists.unique (
       builtins.concatMap ({ interfaces, ... }:
@@ -16,6 +19,7 @@ let
         )) (builtins.attrValues containers)
     );
 
+  # Every network (both veth+phys) required by all containers
   ctNets =
     lib.lists.unique (
       builtins.concatMap ({ interfaces, ... }:
@@ -27,7 +31,10 @@ in
 {
   networking.firewall = {
     enable = true;
-    allowedTCPPorts = [ 22 ];
+    allowedTCPPorts = [
+      # SSH
+      22
+    ];
   };
 
   systemd.network = {
@@ -38,9 +45,11 @@ in
         Kind = "bond";
         Name = "bond0";
       };
+      # LACP
       bond0.bondConfig.Mode = "802.3ad";
     } // (
       builtins.foldl' (result: net: result // {
+        # Bridges are named just like the corresponding net.
         "${net}".netdevConfig = {
           Kind = "bridge";
           Name = "${net}";
@@ -48,6 +57,8 @@ in
       }) {} bridgeNets
     ) // (
       builtins.foldl' (result: net: result // {
+        # External VLAN interfaces (to be attached to net bridges) are
+        # named with an "ext-" prefix.
         "ext-${net}" = {
           netdevConfig = {
             Kind = "vlan";
@@ -64,6 +75,7 @@ in
         networkConfig.Bond = "bond0";
       };
       en = {
+        # physical ethernet ports
         matchConfig.Name = "en*";
         networkConfig.Bond = "bond0";
       };
@@ -79,6 +91,8 @@ in
       "${net}" = {
         matchConfig.Name = net;
         networkConfig = {
+          # Disable all automatic addressing on bridges. It will delay
+          # networkd going into operational state.
           DHCP = lib.mkDefault "no";
           LinkLocalAddressing = lib.mkDefault "no";
         };
@@ -86,6 +100,7 @@ in
     }) {} bridgeNets) // builtins.foldl' (result: net: result // {
       "ext-${net}" = {
         matchConfig.Name = "ext-${net}";
+        # Attach eth*/bond0/VLAN to bridge
         networkConfig.Bridge = net;
       };
     }) {} ctNets;