diff --git a/README.md b/README.md index 37020cb..31a0e75 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,13 @@ Wir, ein kleiner Kreis von Menschen die das Netzwerk im Zentralwerk betreuen, ha - [ ] device-scripts auf Site Config umstellen - [ ] Site Config dumpen, Salt-Daten löschen +### Server Setup + +checkout dieses repos +/etc/nixos, registry +input.key +nixos-rebuild + ### Nix Flake [Segfaulting nix?](https://github.com/NixOS/nix/issues/4178) Until diff --git a/flake.nix b/flake.nix index ebcf792..a70df5c 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,8 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs"; + # `nix flake update --override-flake zentralwerk-network-key git+file:///...` + # to provide the GPG secret key zentralwerk-network-key.url = "git+https://gitea.c3d2.de/zentralwerk/network.git?dir=nix/key&ref=nix"; }; @@ -22,6 +24,7 @@ specialArgs.inputs = inputs; }; in { + # Config, and utilities lib = nixpkgs.lib.extend (final: prev: import ./nix/lib { inherit self; @@ -29,14 +32,18 @@ pkgs = nixpkgs.legacyPackages.x86_64-linux; }); + # Everything that can be built locally outside of NixOS packages = forAllSystems (system: import ./nix/pkgs { inherit self nixpkgs system; } ); + # Configuration for nixosConfigurations + # (see nix/nixos-module/default.nix) nixosModule = { ... }: { imports = [ ./nix/nixos-module ]; }; + # NixOS host systems (servers, and containers) nixosConfigurations = builtins.mapAttrs (hostName: _: nixosConfig hostName) ( nixpkgs.lib.filterAttrs (_: { role, ... }: @@ -44,6 +51,7 @@ ) self.lib.config.site.hosts ); + # For `nix flake check`, and Hydra checks = self.packages; }; } diff --git a/nix/lib/config/options.nix b/nix/lib/config/options.nix index 8ace91b..0e71334 100644 --- a/nix/lib/config/options.nix +++ b/nix/lib/config/options.nix @@ -107,21 +107,32 @@ let hwaddr = mkOption { type = with types; nullOr str; default = null; + description = "Static MAC address"; }; type = mkOption { type = types.enum [ "phys" "veth" ]; + description = '' + veth: Virtual ethernet to be attached to a bridge. + + phys: (Physical) interface from a server moved into the + container. Do not use with VLAN interfaces because they + won't be moved back after lxc-stop. + ''; }; gw4 = mkOption { type = with types; nullOr str; default = null; + description = "IPv4 gateway"; }; gw6 = mkOption { type = with types; nullOr str; default = null; + description = "IPv6 gateway"; }; upstream = mkOption { type = with types; nullOr (submodule { options = upstreamOpts; }); default = null; + description = "Upstream interface configuration"; }; }; }; @@ -152,6 +163,7 @@ let interfaces = mkOption { default = {}; type = with types; attrsOf (submodule interfaceOpts); + description = "Network interfaces"; }; isRouter = mkOption { type = types.bool; @@ -178,10 +190,12 @@ let ospf.stubNets4 = mkOption { type = with types; listOf str; default = []; + description = "Additional IPv4 networks to announce"; }; ospf.stubNets6 = mkOption { type = with types; listOf str; default = []; + description = "Additional IPv6 networks to announce"; }; wireguard = mkOption { default = {}; diff --git a/nix/nixos-module/container/anon.nix b/nix/nixos-module/container/anon.nix index 0844da7..f72c69b 100644 --- a/nix/nixos-module/container/anon.nix +++ b/nix/nixos-module/container/anon.nix @@ -96,4 +96,6 @@ in externalInterface = firstTunnel; inherit (config.site.hosts.${hostName}) forwardPorts; }; + + # TODO: firewall } diff --git a/nix/nixos-module/container/bird.nix b/nix/nixos-module/container/bird.nix index 3fff520..44863f8 100644 --- a/nix/nixos-module/container/bird.nix +++ b/nix/nixos-module/container/bird.nix @@ -1,8 +1,10 @@ +# Routing daemon configuration { hostName, config, options, lib, ... }: let hostConf = config.site.hosts.${hostName}; + # Configuring a gateway? If so, this is the associated net. gatewayNet = let m = builtins.match "(.+)-gw" hostName; @@ -33,29 +35,32 @@ in } ${lib.optionalString (gatewayNet != null) '' - protocol radv { - rdnss ${config.site.net.serv.hosts6.dn42.dnscache}; + # Router advertisements + protocol radv { + rdnss ${config.site.net.serv.hosts6.dn42.dnscache}; - interface "${gatewayNet}" { - min ra interval 10; - max ra interval 60; + interface "${gatewayNet}" { + min ra interval 10; + max ra interval 60; - ${builtins.concatStringsSep "\n" ( - map (subnet6: '' - prefix ${subnet6} { - preferred lifetime 20; - valid lifetime 60; - }; - '') (builtins.attrValues config.site.net.${gatewayNet}.subnets6) - )} + ${builtins.concatStringsSep "\n" ( + map (subnet6: '' + prefix ${subnet6} { + preferred lifetime 20; + valid lifetime 60; + }; + '') (builtins.attrValues config.site.net.${gatewayNet}.subnets6) + )} - dnssl "${config.site.net.${gatewayNet}.domainName}"; - }; - } + dnssl "${config.site.net.${gatewayNet}.domainName}"; + }; + } ''} + # OSPFv2 for site-local IPv4 protocol ospf v2 ZW4 { area 0 { + # Enabled on these networks networks { ${builtins.concatStringsSep " " ( map (n: " ${n};") config.site.ospf.networks4 @@ -64,6 +69,9 @@ in ${builtins.concatStringsSep "\n" ( builtins.attrValues ( builtins.mapAttrs (net: _: + # Enable OSPF only on networks with a secret. Others + # are treated as a stubnet whose routes to + # advertise. if config.site.net.${net}.ospf.secret != null then '' interface "${net}" { @@ -73,6 +81,7 @@ in '' else if config.site.net.${net}.subnet4 != null then '' + # Advertise route of network ${net} stubnet ${config.site.net.${net}.subnet4} {}; '' else "" @@ -80,14 +89,18 @@ in ) )} ${builtins.concatStringsSep "\n" ( - map (stubnet4: "stubnet ${stubnet4} {};") - hostConf.ospf.stubNets4 + map (stubnet4: '' + # Advertise additional route + stubnet ${stubnet4} {}; + '') hostConf.ospf.stubNets4 )} }; } + # OSPFv3 for site-local IPv6 protocol ospf v3 ZW6 { area 0 { + # Enabled on these networks networks { ${builtins.concatStringsSep " " ( map (n: " ${n};") config.site.ospf.networks6 @@ -96,6 +109,9 @@ in ${builtins.concatStringsSep "\n" ( builtins.attrValues ( builtins.mapAttrs (net: _: + # Enable OSPF only on networks with a secret. Others + # are treated as a stubnet whose routes to + # advertise. if config.site.net.${net}.ospf.secret != null then '' interface "${net}" { @@ -105,14 +121,19 @@ in }; '' else builtins.concatStringsSep "\n" ( - map (subnet6: "stubnet ${subnet6} {};") - (builtins.attrValues config.site.net.${net}.subnets6) + map (subnet6: '' + # Advertise route of network ${net} + stubnet ${subnet6} {}; + '') (builtins.attrValues config.site.net.${net}.subnets6) ) ) hostConf.interfaces ) )} ${builtins.concatStringsSep "\n" ( - map (stubnet6: "stubnet ${stubnet6} {};") + map (stubnet6: '' + # Advertise additional route + stubnet ${stubnet6} {}; + '') hostConf.ospf.stubNets6 )} }; diff --git a/nix/nixos-module/container/dhcp-server.nix b/nix/nixos-module/container/dhcp-server.nix index 2f07d5e..88d1b24 100644 --- a/nix/nixos-module/container/dhcp-server.nix +++ b/nix/nixos-module/container/dhcp-server.nix @@ -1,3 +1,4 @@ +# ISC DHCP/IPv4 server configuration { hostName, config, lib, ... }: let diff --git a/nix/nixos-module/default.nix b/nix/nixos-module/default.nix index e06f7dd..61aace0 100644 --- a/nix/nixos-module/default.nix +++ b/nix/nixos-module/default.nix @@ -14,26 +14,26 @@ in { ./defaults.nix ./network.nix ./collectd.nix - ] - ++ optionals (hostConfig.role == "server") [ + ] ++ + optionals (hostConfig.role == "server") [ ./server/lxc-containers.nix ./server/network.nix - ] - ++ optionals (hostName == "server2") [ + ] ++ + optionals (hostName == "server2") [ ./server/server2.nix - ] - ++ optionals (hostConfig.role == "container") [ + ] ++ + optionals (hostConfig.role == "container") [ ./container/defaults.nix ./container/dhcp-server.nix ./container/anon.nix - ] ++ optionals ( - hostConfig.role == "container" && - lib.config.site.hosts.${hostName}.isRouter - ) [ + ] ++ + optionals lib.config.site.hosts.${hostName}.isRouter [ ./container/bird.nix - ] ++ optionals (builtins.match "upstream.*" hostName != null) [ + ] ++ + optionals (builtins.match "upstream.*" hostName != null) [ ./container/upstream.nix - ] ++ optionals (hostName == "mgmt-gw") [ + ] ++ + optionals (hostName == "mgmt-gw") [ ./container/mgmt-gw.nix ]; } diff --git a/nix/nixos-module/defaults.nix b/nix/nixos-module/defaults.nix index f286093..85b619b 100644 --- a/nix/nixos-module/defaults.nix +++ b/nix/nixos-module/defaults.nix @@ -35,11 +35,15 @@ # for vm-packages virtualisation = lib.optionalAttrs (builtins.hasAttr "qemu" options.virtualisation) { + # larger than the defaults memorySize = 8192; - msize = 65536; - cores = 4; + cores = 2; diskSize = 8192; + # 9P performance optimization that quelches a qemu warning + msize = 65536; + # allow building packages writableStore = true; + # keep the store paths built inside the VM across reboots writableStoreUseTmpfs = false; qemu.options = [ "-enable-kvm" ]; }; diff --git a/nix/nixos-module/server/lxc-containers.nix b/nix/nixos-module/server/lxc-containers.nix index ab27fa3..367c51c 100644 --- a/nix/nixos-module/server/lxc-containers.nix +++ b/nix/nixos-module/server/lxc-containers.nix @@ -1,6 +1,7 @@ { hostName, self, config, lib, pkgs, ... }: let + # Containers that are run on this host containers = lib.filterAttrs (_: { role, model, location, ... }: role == "container" && @@ -10,6 +11,7 @@ let enabled = containers != {}; + # `lxc.net.*` formatter for lxc.container.conf files netConfig = ctName: interfaces: let config = map (netName: @@ -56,6 +58,7 @@ let in serialize "lxc.net" config; + # User-facing script to build/update container NixOS systems build-script = pkgs.writeScriptBin "build-container" '' #! ${pkgs.runtimeShell} -e @@ -97,16 +100,16 @@ in { virtualisation.lxc = lib.mkIf enabled { enable = true; + # Container configs live in /etc so that they can be created + # through `environment.etc`. systemConfig = '' lxc.lxcpath = /etc/lxc/containers - - # lxc.rootfs.backend = zfs - # lxc.bdev.zfs.root = vault/sys/atom/var/lib/lxc ''; }; environment.systemPackages = [ pkgs.lxc build-script ]; + # Create lxc.container.conf files environment.etc = builtins.foldl' (etc: ctName: etc // { "lxc/containers/${ctName}/config" = { @@ -148,6 +151,7 @@ in "lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf"; } (builtins.attrNames containers); + # Systemd service template for LXC containers systemd.services."lxc@" = { description = "LXC container '%i'"; after = [ "network.target" ]; @@ -185,6 +189,7 @@ in }; }; + # Starts all the containers after boot systemd.targets.lxc-containers = { wantedBy = [ "multi-user.target" ]; wants = map (ctName: "lxc@${ctName}.service") diff --git a/nix/nixos-module/server/network.nix b/nix/nixos-module/server/network.nix index fe5c02a..271cf8a 100644 --- a/nix/nixos-module/server/network.nix +++ b/nix/nixos-module/server/network.nix @@ -1,6 +1,8 @@ +# Server network configuration { hostName, self, config, lib, pkgs, ... }: let + # LXC containers on this host containers = lib.filterAttrs (_: { role, model, location, ... }: role == "container" && @@ -8,6 +10,7 @@ let location == hostName ) config.site.hosts; + # Every bridged veth network required by all containers bridgeNets = lib.lists.unique ( builtins.concatMap ({ interfaces, ... }: @@ -16,6 +19,7 @@ let )) (builtins.attrValues containers) ); + # Every network (both veth+phys) required by all containers ctNets = lib.lists.unique ( builtins.concatMap ({ interfaces, ... }: @@ -27,7 +31,10 @@ in { networking.firewall = { enable = true; - allowedTCPPorts = [ 22 ]; + allowedTCPPorts = [ + # SSH + 22 + ]; }; systemd.network = { @@ -38,9 +45,11 @@ in Kind = "bond"; Name = "bond0"; }; + # LACP bond0.bondConfig.Mode = "802.3ad"; } // ( builtins.foldl' (result: net: result // { + # Bridges are named just like the corresponding net. "${net}".netdevConfig = { Kind = "bridge"; Name = "${net}"; @@ -48,6 +57,8 @@ in }) {} bridgeNets ) // ( builtins.foldl' (result: net: result // { + # External VLAN interfaces (to be attached to net bridges) are + # named with an "ext-" prefix. "ext-${net}" = { netdevConfig = { Kind = "vlan"; @@ -64,6 +75,7 @@ in networkConfig.Bond = "bond0"; }; en = { + # physical ethernet ports matchConfig.Name = "en*"; networkConfig.Bond = "bond0"; }; @@ -79,6 +91,8 @@ in "${net}" = { matchConfig.Name = net; networkConfig = { + # Disable all automatic addressing on bridges. It will delay + # networkd going into operational state. DHCP = lib.mkDefault "no"; LinkLocalAddressing = lib.mkDefault "no"; }; @@ -86,6 +100,7 @@ in }) {} bridgeNets) // builtins.foldl' (result: net: result // { "ext-${net}" = { matchConfig.Name = "ext-${net}"; + # Attach eth*/bond0/VLAN to bridge networkConfig.Bridge = net; }; }) {} ctNets;