{ hostName, self, config, lib, pkgs, ... }: let containers = # TODO: remove 1 line lib.filterAttrs (ctName: _: ctName == "upstream1") ( lib.filterAttrs (_: { role, model, location, ... }: role == "container" && model == "lxc" && location == hostName ) config.site.hosts ); enabled = containers != {}; netConfig = ctName: interfaces: let config = map (netName: let ifData = interfaces.${netName}; in { type = ifData.type; name = netName; flags = "up"; hwaddr = if ifData ? hwaddr then ifData.hwaddr else "0A:14:48:01:26:00"; } // (lib.optionalAttrs (ifData.type == "veth") { veth.pair = "${ctName}-${netName}"; veth.mode = "bridge"; link = "${netName}"; }) // (lib.optionalAttrs (ifData.type == "phys") { link = "ext-${netName}"; }) ) (builtins.attrNames interfaces); serialize = name: x: if builtins.isString x then "${name} = ${x}\n" else if builtins.isAttrs x then builtins.concatStringsSep "" ( map (n: serialize "${name}.${n}" x.${n}) (builtins.attrNames x) ) else if builtins.isList x then let enumerate = xs: n: if xs == [] then [] else [ { e = builtins.head xs; i = n; } ] ++ enumerate (builtins.tail xs) (n + 1); in builtins.concatStringsSep "" ( map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0) ) else throw "Invalid data in lxc net config: ${lib.generators.toPretty {} x}"; in serialize "lxc.net" config; in { virtualisation.lxc = lib.mkIf enabled { enable = true; systemConfig = '' lxc.lxcpath = /etc/lxc/containers # lxc.rootfs.backend = zfs # lxc.bdev.zfs.root = vault/sys/atom/var/lib/lxc ''; }; environment.systemPackages = [ pkgs.lxc ]; environment.etc = builtins.foldl' (etc: ctName: etc // { "lxc/containers/${ctName}/config" = { enable = true; source = let inherit (containers.${ctName}) interfaces; in builtins.toFile "${ctName}.conf" '' # For lxcfs and sane defaults lxc.include = /etc/lxc/common.conf lxc.uts.name = ${ctName} # Handled by lxc@.service lxc.start.auto = 0 lxc.rootfs.path = /var/lib/lxc/${ctName}/rootfs lxc.init.cmd = "/init" lxc.mount.entry = /nix/store nix/store none bind,ro 0 0 lxc.mount.entry = none tmp tmpfs defaults 0 0 lxc,mount.auto = proc:mixed sys:ro cgroup:mixed lxc.autodev = 1 lxc.tty.max = 0 lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio sys_time mknod lxc.apparmor.profile = unchanged security.privileged = false lxc.cgroup.memory.limit_in_bytes = 1G lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M # tuntap lxc.cgroup.devices.allow = c 10:200 rw ${netConfig ctName interfaces} ''; }; }) { "lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf"; } (builtins.attrNames containers); systemd.services."lxc-rootfs@" = { description = "rootfs for '%i'"; wants = [ "nix-daemon.service" ]; path = [ config.nix.package pkgs.util-linux pkgs.git ]; scriptArgs = "%i"; script = '' mkdir -p /nix/var/nix/gcroots/lxc [ ! -e /nix/var/nix/gcroots/lxc/$1 ] && flock /tmp/lxc-rootfs-build.lock -c \ "nix build -o /nix/var/nix/gcroots/lxc/$1 zentralwerk-network#$1-rootfs" SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$1) mkdir -p /var/lib/lxc/$1/rootfs/{bin,dev,etc,home,mnt,nix/store,nix/var,proc,root,run,sys,tmp,var,usr} ln -fs $SYSTEM/init /var/lib/lxc/$1/rootfs/init exit 0 ''; serviceConfig.Type = "oneshot"; }; systemd.services."lxc@" = { description = "LXC container '%i'"; wants = [ "systemd-networkd.service" ]; requires = [ "lxc-rootfs@%i.service" ]; after = [ "lxc-rootfs@%i.service" ]; serviceConfig = { Type = "simple"; ExecStart = let script = pkgs.writeScript "start-lxc-container.sh" '' #! ${pkgs.runtimeShell} -e [ -e /var/lib/lxc/$1/rootfs ] exec ${pkgs.lxc}/bin/lxc-start -F -C -n $1 ''; in "${script} %i"; ExecStop = "${pkgs.lxc}/bin/lxc-stop -n %i"; ExecReload = let script = pkgs.writeScript "reload-lxc-container.sh" '' #! ${pkgs.runtimeShell} -e SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init)) exec ${pkgs.lxc}/bin/lxc-attach -n $1 $SYSTEM/activate ''; in "${script} %i"; KillMode = "mixed"; OOMPolicy = "kill"; Restart = "always"; RestartSec = "30s"; }; }; systemd.targets.lxc-containers = { wantedBy = [ "multi-user.target" ]; wants = map (ctName: "lxc@${ctName}.service") (builtins.attrNames containers); }; }