network/nix/nixos-module/server/lxc-containers.nix

{ self, config, lib, pkgs, ... }:

let
  # Containers that are run on this host
  containers =
    lib.filterAttrs (_: { role, model, ... }:
      role == "container" &&
      model == "lxc"
    ) config.site.hosts;

  enabled = containers != {};

  # User-facing script to build/update container NixOS systems
  build-script = pkgs.writeScriptBin "build-container" ''
    #! ${pkgs.runtimeShell} -e

    if [ -z "$1" ]; then
      echo "Usage: $0 [containers ...]"
      exit 1
    fi

    mkdir -p /nix/var/nix/gcroots/lxc

    for c in $@; do
      unset SYSTEM

      case "$c" in
        ${builtins.concatStringsSep "\n" (
          map (ctName: ''
            ${ctName})
              echo Using prebuilt system for container $c
              SYSTEM=${self.packages.x86_64-linux."${ctName}-rootfs"}
              CONFIG=${self.packages.x86_64-linux."${ctName}-lxc-config"}
              ;;
          '') (
            builtins.attrNames (
              lib.filterAttrs (_: { prebuilt, ... }: prebuilt)
                containers
            ))
        )}
        *)
        echo Building $c
        nix build -o /nix/var/nix/gcroots/lxc/$c zentralwerk-network#$c-rootfs
        SYSTEM=$(readlink /nix/var/nix/gcroots/lxc/$c)
        nix build -o /nix/var/nix/gcroots/lxc/$c.config zentralwerk-network#$c-lxc-config
        CONFIG=$(readlink /nix/var/nix/gcroots/lxc/$c.config)
          ;;
      esac

      echo Installing $c
      for d in \
        bin dev etc home mnt \
        nix/store nix/var \
        proc root run sys tmp var usr ; \
      do
        mkdir -p /var/lib/lxc/$c/rootfs/$d
      done
      ln -fs $SYSTEM/init /var/lib/lxc/$c/rootfs/init
      ln -fs $CONFIG /var/lib/lxc/$c/config
    done

    # Activate all the desired container after all of them are
    # built
    set +e
    for c in $@; do
      active=$(systemctl is-active lxc@$c)
      if [[ "$active" = active ]] ; then
        echo Activating $c
        systemctl reload lxc@$c || (
          echo Reload failed. Restarting $c
          systemctl restart lxc@$c
        )
      else
        echo Starting $c
        systemctl start lxc@$c
      fi
    done
    set -e
  '';

  enable-script = pkgs.writeScriptBin "enable-containers" ''
    touch /etc/start-containers
    systemctl start lxc-containers.target
  '';

  disable-script = pkgs.writeScriptBin "disable-containers" ''
    rm /etc/start-containers
    systemctl stop lxc-containers.target lxc@\*.service
  '';
in
{
  boot.kernel.sysctl = lib.mkIf enabled {
    "fs.inotify.max_queued_events" = 1048576;
    "fs.inotify.max_user_instances" = 1048576;
    "fs.inotify.max_user_watches" = 1048576;
    "vm.max_map_count" = 262144;
    "kernel.dmesg_restrict" = 1;
    "net.ipv4.neigh.default.gc_thresh3" = 8192;
    "net.ipv6.neigh.default.gc_thresh3" = 8192;
    "kernel.keys.maxkeys" = 2000;
  };

  virtualisation.lxc = lib.mkIf enabled {
    enable = true;
    systemConfig = ''
      lxc.lxcpath = /var/lib/lxc
    '';
  };

  environment.systemPackages = [
    # `lxc-attach` et al
    pkgs.lxc build-script
    # User scripts
    enable-script disable-script
  ];

  environment.etc."lxc/common.conf".source = "${pkgs.lxc}/share/lxc/config/common.conf";

  # Systemd service template for LXC containers
  systemd.services."lxc@" = {
    description = "LXC container '%i'";
    after = [ "network.target" ];
    unitConfig.ConditionPathExists = [
      "/var/lib/lxc/%i/rootfs/init"
      "/etc/start-containers"
    ];
    serviceConfig = with pkgs; {
      Type = "simple";
      ExecStart = "${lxc}/bin/lxc-start -F -C -n %i";
      ExecStop = "${lxc}/bin/lxc-stop -n %i";
      ExecReload =
        let
          script = writeScript "reload-lxc-container.sh" ''
            #! ${runtimeShell} -e

            SYSTEM=$(dirname $(readlink /var/lib/lxc/$1/rootfs/init))
            exec ${lxc}/bin/lxc-attach -n $1 $SYSTEM/bin/switch-to-configuration switch
          '';
        in
          "${script} %i";
      KillMode = "mixed";
      OOMPolicy = "kill";
      Restart = "always";
      RestartSec = "1s";
    };
    # Prevent restart on host nixos-rebuild switch
    restartIfChanged = false;
  };

  # Starts all the containers after boot
  systemd.targets.lxc-containers = {
    wantedBy = [ "multi-user.target" ];
    wants = map (ctName: "lxc@${ctName}.service")
      (builtins.attrNames containers);
  };
}