network/nix/nixos-module/server/lxc-containers.nix

{ hostName, self, config, lib, pkgs, ... }:

let
  containers =
    # TODO: remove 1 line
    lib.filterAttrs (ctName: _: ctName == "upstream1") (
    lib.filterAttrs (_: { role, model, location, ... }:
      role == "container" &&
      model == "lxc" &&
      location == hostName
    ) config.site.hosts
    );

  enabled = containers != {};

  netConfig = ctName: interfaces:
    let
      config = map (netName:
        let
          ifData = interfaces.${netName};
        in {
          type = ifData.type;
          name = netName;
          flags = "up";
          hwaddr = if ifData ? hwaddr
                   then ifData.hwaddr
                   else "0A:14:48:01:26:00";
        } // (lib.optionalAttrs (ifData.type == "veth") {
          veth.pair = "${ctName}-${netName}";
          veth.mode = "bridge";
          link = "${netName}";
        }) // (lib.optionalAttrs (ifData.type == "phys") {
          link = "ext-${netName}";
        })
      ) (builtins.attrNames interfaces);

      serialize = name: x:
        if builtins.isString x
        then "${name} = ${x}\n"
        else if builtins.isAttrs x
        then builtins.concatStringsSep "" (
          map (n: serialize "${name}.${n}" x.${n}) (builtins.attrNames x)
        )
        else if builtins.isList x
        then
          let
            enumerate = xs: n:
              if xs == []
              then []
              else [ {
                e = builtins.head xs;
                i = n;
              } ] ++ enumerate (builtins.tail xs) (n + 1);
          in
            builtins.concatStringsSep "" (
              map ({ e, i }: serialize "${name}.${toString i}" e) (enumerate x 0)
            )
        else throw "Invalid data in lxc net config: ${lib.generators.toPretty {} x}";
    in
      serialize "lxc.net" config;
in
{
  virtualisation.lxc = lib.mkIf enabled {
    enable = true;
    systemConfig = ''
      lxc.lxcpath = /etc/lxc/containers

      # lxc.rootfs.backend = zfs
      # lxc.bdev.zfs.root = vault/sys/atom/var/lib/lxc
    '';
  };

  environment.systemPackages = [ pkgs.lxc ];

  environment.etc =
    builtins.foldl' (etc: ctName: etc // {
      "lxc/containers/${ctName}/rootfs" = {
        source = self.packages.x86_64-linux."${ctName}-rootfs";
      };
      "lxc/containers/${ctName}/config" = {
        enable = true;
        source =
          let
            inherit (containers.${ctName}) interfaces;
          in builtins.toFile "${ctName}.conf" ''
            # For lxcfs and sane defaults
            lxc.include = /etc/lxc/common.conf

            lxc.uts.name = ${ctName}
            # Handled by lxc@.service
            lxc.start.auto = 0
            lxc.rootfs.path = /etc/lxc/containers/${ctName}/rootfs
            lxc.init.cmd = "/init"

            lxc.mount.entry = /nix/store nix/store none bind,ro 0 0
            lxc.mount.entry = none nix/var tmpfs defaults 0 0
            lxc.mount.entry = none bin tmpfs defaults 0 0
            #lxc.mount.entry = none dev tmpfs defaults 0 0
            lxc.mount.entry = none root tmpfs defaults 0 0
            lxc.mount.entry = none tmp tmpfs defaults 0 0
            lxc.mount.entry = none var tmpfs defaults 0 0
            lxc.mount.entry = none home tmpfs defaults 0 0
            lxc.mount.entry = none usr tmpfs defaults 0 0
            lxc.mount.entry = none run tmpfs defaults 0 0
            lxc.mount.entry = none etc tmpfs defaults 0 0
            lxc,mount.auto = proc:mixed sys:ro cgroup:mixed

            lxc.autodev = 1
            lxc.tty.max = 0

            lxc.cap.drop = sys_module sys_time sys_nice sys_pacct sys_rawio sys_time mknod
            lxc.apparmor.profile = unchanged
            security.privileged = false

            lxc.cgroup.memory.limit_in_bytes = 1G
            lxc.cgroup.memory.kmem.tcp.limit_in_bytes = 128M

            # tuntap
            lxc.cgroup.devices.allow = c 10:200 rw

            ${netConfig ctName interfaces}
          '';
      };
    }) {
      "lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";
    } (builtins.attrNames containers);

  systemd.targets.lxc-containers = {
    wantedBy = [ "multi-user.target" ];
    wants = map (ctName: "lxc@${ctName}.service") (builtins.attrNames containers);
  };
  systemd.services."lxc@" = {
    description = "LXC container '%i'";
    wants = [ "systemd-networkd.service" ];
    serviceConfig = {
      Type = "simple";
      ExecStart = "${pkgs.lxc}/bin/lxc-start -F -C -n %i";
      ExecStop = "${pkgs.lxc}/bin/lxc-stop -n %i";
      KillMode = "mixed";
      OOMPolicy = "kill";
      Restart = "always";
      RestartSec = "5s";
    };
  };
}