{ hostName, self, config, lib, pkgs, ... }: let # Containers that are run on this host containers = lib.filterAttrs (_: { role, model, location, ... }: role == "container" && model == "lxc" && location == hostName ) config.site.hosts; enabled = containers != {}; # `lxc.net.*` formatter for lxc.container.conf files netConfig = ctName: interfaces: let config = map (netName: let ifData = interfaces.${netName}; in { type = ifData.type; name = netName; flags = "up"; hwaddr = if ifData ? hwaddr then ifData.hwaddr else "0A:14:48:01:26:00"; } // (lib.optionalAttrs (ifData.type == "veth") { veth.pair = "${ctName}-${netName}"; veth.mode = "bridge"; link = "${netName}"; }) // (lib.optionalAttrs (ifData.type == "phys") { link = "ext-${netName}"; }) ) (builtins.attrNames interfaces); serialize = name: x: if builtins.isString x then "${name} = ${x}\n" else if builtins.isAttrs x then builtins.concatStringsSep "" ( map (n: serialize "${name}.${n}" x.${n}) (builtins.attrNames x) ) else if builtins.isList x then let enumerate = xs: n: if xs == [] then [] else [ { e = builtins.head xs; i = n; } ] ++ enumerate (builtins.tail xs) (n + 1); in builtins.concatStringsSep "" ( map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0) ) else throw "Invalid data in lxc net config: ${lib.generators.toPretty {} x}"; in serialize "lxc.net" config; # User-facing script to build/update container NixOS systems build-script = pkgs.writeScriptBin "build-container" '' #! ${pkgs.runtimeShell} -e if [ -z "$1" ]; then echo "Usage: $0 [containers ...]" exit 1 fi mkdir -p /nix/var/nix/gcroots/lxc for c in $@; do echo Building $c nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c) echo Installing $c for d in \ bin dev etc home mnt \ nix/store nix/var \ proc root run sys tmp var usr ; \ do mkdir -p /var/lib/lxc/$c/rootfs/$d done ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init set +e active=$(systemctl is-active lxc@$c) if [[ "$active" = active ]] ; then echo Activating $c systemctl reload lxc@$c else echo Starting $c systemctl start lxc@$c fi set -e done ''; in { virtualisation.lxc = lib.mkIf enabled { enable = true; # Container configs live in /etc so that they can be created # through `environment.etc`. systemConfig = '' lxc.lxcpath = /etc/lxc/containers ''; }; environment.systemPackages = [ pkgs.lxc build-script ]; # Create lxc.container.conf files environment.etc = builtins.foldl' (etc: ctName: etc // { "lxc/containers/${ctName}/config" = { enable = true; source = let inherit (containers.${ctName}) interfaces; in builtins.toFile "${ctName}.conf" '' # For lxcfs and sane defaults lxc.include = /etc/lxc/common.conf lxc.uts.name = ${ctName} # Handled by lxc@.service lxc.start.auto = 0 lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs lxc.init.cmd = "/init" lxc.mount.entry = /nix/store nix/store none bind,ro 0 0 lxc.mount.entry = none tmp tmpfs defaults 0 0 lxc,mount.auto = proc:mixed sys:ro cgroup:mixed lxc.autodev = 1 lxc.tty.max = 0 lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio sys_time mknod lxc.apparmor.profile = unchanged security.privileged = false lxc.cgroup.memory.limit_in_bytes = 1G lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M # tuntap lxc.cgroup.devices.allow = c 10:200 rw ${netConfig ctName interfaces} ''; }; }) { "lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf"; } (builtins.attrNames containers); # Systemd service template for LXC containers systemd.services."lxc@" = { description = "LXC container '%i'"; after = [ "network.target" ]; serviceConfig = { Type = "simple"; ExecStart = let script = pkgs.writeScript "start-lxc-container.sh" '' #! ${pkgs.runtimeShell} -e if [ ! -e /var/lib/lxc/$1/rootfs ]; then echo "Container $1 does not exist. Run: build-container $1" exit 1 fi exec ${pkgs.lxc}/bin/lxc-start -F -C -n $1 ''; in "${script} %i"; ExecStop = "${pkgs.lxc}/bin/lxc-stop -n %i"; ExecReload = let script = pkgs.writeScript "reload-lxc-container.sh" '' #! ${pkgs.runtimeShell} -e SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init)) exec ${pkgs.lxc}/bin/lxc-attach -n $1 $SYSTEM/bin/switch-to-configuration switch ''; in "${script} %i"; KillMode = "mixed"; OOMPolicy = "kill"; Restart = "always"; RestartSec = "30s"; }; }; # Starts all the containers after boot systemd.targets.lxc-containers = { wantedBy = [ "multi-user.target" ]; wants = map (ctName: "lxc@${ctName}.service") (builtins.attrNames containers); }; }